0xrsydn/sts2-bot
1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(dataset): migrate game history and captures into a trainable corpus

Browse source 
Turn the data we already have into an open, educational dataset, so the trace we
are about to start collecting has somewhere to go.

`migrate.py` produces:
  * runs.jsonl          37 runs with outcome, killer, seed and final deck
  * decisions.jsonl     1052 decisions, each with its own outcome attached
  * states_index.jsonl  346 unique observations
  * states/             content-addressed gzipped blobs

Content addressing matters: measured, only 56% of captures are unique, so 44% of
storage is duplicates. 3.28 MB raw -> 0.42 MB stored.

The card-reward rows keep the REJECTED options, so this is a ranking dataset
rather than a classification one, and the per-fight `damage_taken` /
`turns_taken` pair is the dense reward signal a combat policy is judged on.

What it deliberately does NOT do: reconstruct per-step combat state/action pairs.
The session logs record the action but not the observation, and captures exist
only for combat, so a step has a state with no action or an action with no state
-- never both. Inventing them would poison the corpus. The gap is declared in
manifest.json instead, and collect.py will close it going forward.

Integrity checking is a separate entry point (`--check-only`) because `--verify`
alone rebuilds first and so can only ever see data that is correct by
construction -- a smoke test pretending to be a check. All six invariants were
verified by deliberately breaking the dataset and confirming the checker fails.
This commit is contained in:
0xrsydn 2026-09-22 06:06:21 +07:00
commit 8fae007e50
352 changed files with 2164 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

482
migrate.py Normal file
View file

@ -0,0 +1,482 @@
#!/usr/bin/env python3
"""
migrate.py -- turn the data we already have into an open, trainable dataset.
Run: python3 migrate.py [--out dataset] [--verify]
WHAT THIS MIGRATES
Game run history (37 files) -> runs.jsonl + decisions.jsonl
Live captures (600+ files) -> states/ (content-addressed, gzipped)
WHAT IT CANNOT MIGRATE, AND WHY
Per-step combat state/action pairs. The session logs record the action, the
source and the reason, but NOT the observation, and the captures are only
written for combat. So a combat step has a state with no action, or an
action with no state -- never both. Those pairs are NOT reconstructed here.
Inventing them would poison the corpus, so the collector (collect.py)
records them going forward instead.
DESIGN RULES
1. Lossless raw, derived views. Nothing is dropped, nothing is edited.
2. Content-addressed states, so the 44% duplicate rate costs nothing.
3. Every inferred field is marked. `deck_reconstructed` is not the same
thing as an observed deck and the corpus must say which it is.
4. Provenance on every row: game build, mod commit, source file.
5. No secrets, no absolute paths, no personal data in the output.
"""
from __future__ import annotations
import argparse
import gzip
import hashlib
import json
import os
import pathlib
import shutil
import sys
from collections import Counter
SCHEMA_RUN = "sts2.run/1"
SCHEMA_DECISION = "sts2.decision/1"
SCHEMA_MANIFEST = "sts2.manifest/1"
GAME_BUILD = "0.107.1"
GAME_COMMIT = "59260271"
MOD_COMMIT = "55e0648"
HOME = pathlib.Path.home()
RUN_HISTORY = (
HOME
/ "Library/Application Support/SlayTheSpire2/steam"
/ "76561198141226155/modded/profile1/saves/history"
)
CAPTURE_DIR = pathlib.Path("capture")
# --------------------------------------------------------------------------
# helpers
# --------------------------------------------------------------------------
def short_id(text: str, n: int = 12) -> str:
return hashlib.sha256(text.encode()).hexdigest()[:n]
def strip_prefix(value, prefix: str) -> str:
"""'CARD.STRIKE_IRONCLAD' -> 'STRIKE_IRONCLAD'. Never mangles non-matching."""
text = str(value or "")
return text[len(prefix):] if text.startswith(prefix) else text
def write_jsonl(path: pathlib.Path, rows) -> int:
n = 0
with path.open("w") as fh:
for row in rows:
fh.write(json.dumps(row, sort_keys=True, separators=(",", ":")) + "\n")
n += 1
return n
def verify(out: pathlib.Path) -> int:
"""Check an existing dataset. Returns 0 when clean, else a count of failures.
Separate from the migrate path on purpose: verifying a dataset that was just
regenerated proves nothing, because it is correct by construction. This is
the entry point that can actually catch corruption.
"""
print()
print("verifying round-trip and invariants...")
bad = 0
def fail(msg):
nonlocal bad
bad += 1
print(f" FAIL {msg}")
for name in ("runs.jsonl", "decisions.jsonl", "states_index.jsonl"):
if not (out / name).exists():
fail(f"missing {name}")
return bad
run_rows = [json.loads(l) for l in (out / "runs.jsonl").read_text().splitlines()]
dec_rows = [json.loads(l) for l in (out / "decisions.jsonl").read_text().splitlines()]
st_rows = [json.loads(l) for l in (out / "states_index.jsonl").read_text().splitlines()]
# state blobs must exist and hash back to their digest
for rec in st_rows:
blob = out / rec["path"]
if not blob.exists():
fail(f"missing state {rec['path']}")
continue
if hashlib.sha256(gzip.decompress(blob.read_bytes())).hexdigest() \
!= rec["digest"].split(":")[1]:
fail(f"hash mismatch {rec['path']}")
for row in dec_rows:
# a reward decision that picked something must pick exactly one option
if row["kind"].endswith("reward") and row.get("chosen"):
n_picked = sum(1 for o in row["options"] if o.get("picked"))
if n_picked != 1:
fail(f"{row['decision_id']}: {n_picked} picked options")
# `chosen` must be one of `options` -- guards the mapping, not the parse
if row.get("chosen") and row["chosen"] not in \
[o.get("id") for o in row["options"]]:
fail(f"{row['decision_id']}: chosen not in options")
# no secrets and no absolute paths in a corpus meant to be published
blob = (out / "decisions.jsonl").read_text() + (out / "runs.jsonl").read_text()
if "/Users/" in blob or "/home/" in blob:
fail("absolute path leaked into the corpus")
for needle in ("sk-", "API_KEY", "Bearer "):
if needle in blob:
fail(f"possible secret leaked: {needle!r}")
# every row must carry its own outcome, and reference a real run
ids = {r["run_id"] for r in run_rows}
for row in dec_rows:
if "outcome" not in row:
fail(f"{row['decision_id']}: no outcome attached")
if row["run_id"] not in ids:
fail(f"{row['decision_id']}: unknown run_id")
print(f" {'OK' if not bad else str(bad) + ' PROBLEMS'}: "
f"{len(run_rows)} runs, {len(dec_rows)} decisions, {len(st_rows)} states")
return bad
# --------------------------------------------------------------------------
# run history -> runs.jsonl + decisions.jsonl
# --------------------------------------------------------------------------
def deck_as_counts(cards) -> list[dict]:
"""Collapse a card list to [{'id':..,'count':..}], sorted for stability."""
counts = Counter(strip_prefix((c or {}).get("id"), "CARD.") for c in (cards or []))
return [{"id": k, "count": v} for k, v in sorted(counts.items()) if k]
def migrate_runs(out: pathlib.Path, verbose: bool = True):
runs, decisions = [], []
files = sorted(RUN_HISTORY.glob("*.run"))
if not files:
print(f" no run files under {RUN_HISTORY}")
return runs, decisions
for path in files:
try:
d = json.loads(path.read_text())
except Exception as exc: # noqa: BLE001
print(f" SKIP {path.name}: {exc}")
continue
run_id = path.stem
player = (d.get("players") or [{}])[0]
history = d.get("map_point_history") or []
# `map_point_history` is a list of acts, each a list of map points.
acts = [a for a in history if isinstance(a, list)]
total_points = sum(len(a) for a in acts)
killed_by = strip_prefix(d.get("killed_by_encounter"), "ENCOUNTER.")
killed_by_event = strip_prefix(d.get("killed_by_event"), "EVENT.")
runs.append({
"schema": SCHEMA_RUN,
"run_id": run_id,
"seed": d.get("seed"),
"character": strip_prefix(player.get("character"), "CHARACTER."),
"ascension": d.get("ascension"),
"game_mode": d.get("game_mode"),
"build_id": d.get("build_id"),
"platform": d.get("platform_type"),
"modifiers": d.get("modifiers"),
"start_time": d.get("start_time"),
"run_time_seconds": d.get("run_time"),
"was_abandoned": d.get("was_abandoned"),
"win": bool(d.get("win")),
"killed_by": killed_by or None,
"killed_by_event": killed_by_event or None,
"progress": {"acts_entered": len(acts), "map_points": total_points},
"final": {
"deck": deck_as_counts(player.get("deck")),
"deck_size": len(player.get("deck") or []),
"relics": [strip_prefix((r or {}).get("id"), "RELIC.")
for r in (player.get("relics") or [])],
"potions": [strip_prefix((p or {}).get("id"), "POTION.")
for p in (player.get("potions") or [])],
"max_potion_slots": player.get("max_potion_slot_count"),
},
"source": {"kind": "game_run_history", "file": path.name},
})
# ---- per-map-point decisions ------------------------------------
outcome = {
"run_win": bool(d.get("win")),
"run_map_points": total_points,
"run_killed_by": killed_by or None,
"run_acts_entered": len(acts),
}
for act_i, act in enumerate(acts):
for pt_i, point in enumerate(act):
if not isinstance(point, dict):
continue
stats = (point.get("player_stats") or [{}])[0]
rooms = point.get("rooms") or []
room = rooms[0] if rooms else {}
where = f"{run_id}:act{act_i}:pt{pt_i}"
# Context at this map point. NOTE: the game does not record the
# deck as it stood here, so `deck` is NOT included. What is
# recorded is the observed delta, which the consumer can fold
# forward from the known starting deck. Marked as observed.
context = {
"act": act_i,
"map_point": pt_i,
"map_point_type": strip_prefix(point.get("map_point_type"), "MAP_POINT."),
"hp": stats.get("current_hp"),
"max_hp": stats.get("max_hp"),
"gold": stats.get("current_gold"),
"encounter": strip_prefix(room.get("model_id"), "ENCOUNTER.") or None,
"room_type": strip_prefix(room.get("room_type"), "ROOM.") or None,
"monsters": [strip_prefix(m, "MONSTER.")
for m in (room.get("monster_ids") or [])],
}
# Per-fight reward signal. This is the dense signal the corpus
# exists for: damage and turns are what a policy actually pays.
fight = {
"damage_taken": stats.get("damage_taken"),
"hp_healed": stats.get("hp_healed"),
"turns_taken": room.get("turns_taken"),
"gold_gained": stats.get("gold_gained"),
}
def emit(kind, options, chosen, extra=None):
row = {
"schema": SCHEMA_DECISION,
"decision_id": f"{where}:{kind}",
"run_id": run_id,
"kind": kind,
"context": context,
"options": options,
"chosen": chosen,
"fight": fight,
"outcome": outcome,
"source": {"kind": "game_run_history", "file": path.name},
}
if extra:
row.update(extra)
decisions.append(row)
# card reward: every offered card, with the pick flag
ccs = stats.get("card_choices") or []
if ccs:
options = [{
"id": strip_prefix((cc.get("card") or {}).get("id"), "CARD."),
"picked": bool(cc.get("was_picked")),
"floor_added": (cc.get("card") or {}).get("floor_added_to_deck"),
} for cc in ccs]
picked = [o["id"] for o in options if o["picked"]]
emit("card_reward", options, picked[0] if picked else None,
{"skipped": not picked})
if stats.get("relic_choices"):
options = [{
"id": strip_prefix((rc.get("relic") or {}).get("id"), "RELIC."),
"picked": bool(rc.get("was_picked")),
} for rc in stats["relic_choices"]]
picked = [o["id"] for o in options if o["picked"]]
emit("relic_reward", options, picked[0] if picked else None,
{"skipped": not picked})
if stats.get("potion_choices"):
options = [{
"id": strip_prefix((pc.get("potion") or {}).get("id"), "POTION."),
"picked": bool(pc.get("was_picked")),
} for pc in stats["potion_choices"]]
picked = [o["id"] for o in options if o["picked"]]
emit("potion_reward", options, picked[0] if picked else None,
{"skipped": not picked})
if stats.get("event_choices"):
# Shape varies by event; keep it raw rather than guessing.
emit("event", stats["event_choices"], None,
{"raw": True,
"note": "event option shape varies; stored unparsed"})
if stats.get("rest_site_choices"):
emit("rest_site",
[{"id": str(c)} for c in stats["rest_site_choices"]],
str(stats["rest_site_choices"][0]))
if stats.get("upgraded_cards"):
emit("upgrade",
[{"id": strip_prefix(c, "CARD.")} for c in stats["upgraded_cards"]],
strip_prefix(stats["upgraded_cards"][0], "CARD."))
if stats.get("cards_removed"):
emit("remove",
[{"id": strip_prefix(c, "CARD.")} for c in stats["cards_removed"]],
strip_prefix(stats["cards_removed"][0], "CARD."))
if stats.get("cards_transformed"):
emit("transform",
[{"id": strip_prefix(c, "CARD.")} for c in stats["cards_transformed"]],
strip_prefix(stats["cards_transformed"][0], "CARD."))
if stats.get("cards_enchanted"):
emit("enchant",
[{"id": strip_prefix(c, "CARD.")} for c in stats["cards_enchanted"]],
strip_prefix(stats["cards_enchanted"][0], "CARD."))
if stats.get("ancient_choice"):
emit("ancient", stats["ancient_choice"], None, {"raw": True})
bought = []
for key in ("bought_relics", "bought_colorless", "bought_potions"):
for item in (stats.get(key) or []):
bought.append({"item": item, "kind": key})
if bought:
emit("shop_purchase", bought, None, {"raw": True})
if verbose:
print(f" {run_id} pts={total_points:<3} win={bool(d.get('win'))!s:<5} "
f"killed_by={killed_by or '-'}")
return runs, decisions
# --------------------------------------------------------------------------
# captures -> content-addressed states/
# --------------------------------------------------------------------------
def migrate_states(out: pathlib.Path, verbose: bool = True) -> dict:
store = out / "states"
store.mkdir(parents=True, exist_ok=True)
index, seen, total_raw, total_gz = [], {}, 0, 0
files = sorted(CAPTURE_DIR.glob("*.json"))
for path in files:
raw = path.read_bytes()
digest = hashlib.sha256(raw).hexdigest()
total_raw += len(raw)
if digest in seen:
seen[digest]["seen_in"].append(path.name)
continue
try:
obs = json.loads(raw)
except Exception: # noqa: BLE001
continue
blob = gzip.compress(raw, 6)
total_gz += len(blob)
sub = store / digest[:2]
sub.mkdir(exist_ok=True)
(sub / f"{digest}.json.gz").write_bytes(blob)
state_type = obs.get("state_type")
seen[digest] = {
"digest": f"sha256:{digest}",
"path": f"states/{digest[:2]}/{digest}.json.gz",
"bytes_raw": len(raw),
"bytes_gz": len(blob),
"state_type": state_type,
"menu_screen": obs.get("menu_screen"),
"seen_in": [path.name],
}
index.append(seen[digest])
write_jsonl(out / "states_index.jsonl", index)
if verbose:
print(f" {len(files)} capture files -> {len(index)} unique states "
f"({100*len(index)/max(1,len(files)):.0f}% unique)")
print(f" {total_raw/1e6:.2f} MB raw -> {total_gz/1e6:.2f} MB stored")
return {"files": len(files), "unique": len(index),
"bytes_raw": total_raw, "bytes_gz": total_gz}
# --------------------------------------------------------------------------
# main
# --------------------------------------------------------------------------
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--out", default="dataset", help="output directory")
ap.add_argument("--verify", action="store_true",
help="re-read the output and check it round-trips")
ap.add_argument("--check-only", action="store_true",
help="verify an EXISTING dataset without regenerating it. "
"--verify alone rebuilds first, so it can only ever "
"see data that is correct by construction.")
args = ap.parse_args()
out = pathlib.Path(args.out)
if args.check_only:
if not out.exists():
print(f"no dataset at {out}/; run without --check-only first")
return 2
return verify(out)
if out.exists():
shutil.rmtree(out)
out.mkdir(parents=True)
print("migrating run history...")
runs, decisions = migrate_runs(out)
n_runs = write_jsonl(out / "runs.jsonl", runs)
n_dec = write_jsonl(out / "decisions.jsonl", decisions)
print()
print("migrating captures...")
state_stats = migrate_states(out)
kinds = Counter(d["kind"] for d in decisions)
print()
print("=== dataset ===")
print(f" runs.jsonl {n_runs:>6} rows")
print(f" decisions.jsonl {n_dec:>6} rows")
print(f" states_index.jsonl{state_stats['unique']:>6} rows")
print()
print(" decisions by kind:")
for kind, n in kinds.most_common():
print(f" {kind:<18}{n:>6}")
manifest = {
"schema": SCHEMA_MANIFEST,
"generated_by": "migrate.py",
"provenance": {
"game_build": GAME_BUILD,
"game_commit": GAME_COMMIT,
"mod_commit": MOD_COMMIT,
},
"counts": {"runs": n_runs, "decisions": n_dec,
"unique_states": state_stats["unique"],
"capture_files": state_stats["files"]},
"decisions_by_kind": dict(kinds),
"limitations": [
"No per-step combat state/action pairs: the logs record the action "
"but not the observation, and captures exist only for combat. "
"collect.py records these going forward.",
"Per-map-point deck composition is not recorded by the game. The "
"corpus stores observed deltas (cards_gained, cards_removed, "
"upgraded_cards) and the final deck; folding them forward is left "
"to the consumer.",
"0 wins in this corpus. The reward signal has no positive class.",
],
"license_note": "Slay the Spire 2 is (c) Mega Crit. Game-derived "
"identifiers are included for research and education.",
}
(out / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True))
if args.verify:
rc = verify(out)
if rc:
return rc
print()
print(f"written to {out}/")
return 0
if __name__ == "__main__":
sys.exit(main())