From efd7d2e423cd0e310ec18bce5ba490d63511625e Mon Sep 17 00:00:00 2001
From: 0xrsydn <dev@rasyidanaf.com>
Date: Mon, 21 Sep 2026 17:09:43 +0700
Subject: [PATCH] Add batch eval and card-skip A/B scripts

eval_batch.sh: N back-to-back sessions with per-session summaries.
ab_card_skip.sh: A/B the jev vs combined card-reward skip policy and
compare deck size and progress from the game's run history.
---
 ab_card_skip.sh | 76 +++++++++++++++++++++++++++++++++++++++++++++++++
 eval_batch.sh   | 25 ++++++++++++++++
 2 files changed, 101 insertions(+)
 create mode 100755 ab_card_skip.sh
 create mode 100755 eval_batch.sh

diff --git a/ab_card_skip.sh b/ab_card_skip.sh
new file mode 100755
index 0000000..6372654
--- /dev/null
+++ b/ab_card_skip.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# A/B the card-reward skip policy.
+#
+#   ./ab_card_skip.sh <sessions-per-arm> <steps-per-session>
+#
+# Runs N sessions with --card-skip-policy jev, then N with --card-skip-policy
+# combined, and prints a comparison of deck size and progress.
+#
+# Everything else is identical between the arms, so any difference is the
+# policy. (The earlier before/after comparison was confounded: several bug
+# fixes landed at the same time as the policy change.)
+set -u
+
+# Fail fast if the bot cannot start a run (e.g. a pending Timeline epoch).
+if ! python3 run.py --steps 1 >/tmp/preflight.log 2>&1; then
+  cat /tmp/preflight.log
+  exit 2
+fi
+
+N="${1:-5}"
+STEPS="${2:-600}"
+
+run_arm() {
+  local policy="$1" tag="$2"
+  for i in $(seq 1 "$N"); do
+    echo "--- $tag $i/$N ---"
+    python3 run.py --steps "$STEPS" --pause 0.3 \
+      --card-skip-policy "$policy" > "/tmp/ab_${tag}_${i}.log" 2>&1
+    echo "    exit=$? takes=$(grep -c 'select_card_reward' "/tmp/ab_${tag}_${i}.log") skips=$(grep -c 'skip_card_reward' "/tmp/ab_${tag}_${i}.log")"
+  done
+}
+
+echo "### ARM A: jev decides skip"
+run_arm jev jev
+
+echo
+echo "### ARM B: combined (jev + a floor)"
+run_arm combined combined
+
+echo
+python3 - <<'PY'
+import json, glob, os, statistics as st
+
+H = os.path.expanduser(
+    "~/Library/Application Support/SlayTheSpire2/steam/"
+    "76561198141226155/modded/profile1/saves/history"
+)
+
+rows = []
+for f in glob.glob(H + "/*.run"):
+    d = json.load(open(f))
+    pl = (d.get("players") or [{}])[0]
+    mph = d.get("map_point_history") or []
+    rows.append({
+        "t": d.get("start_time", 0),
+        "killed": d.get("killed_by_encounter", "?").replace("ENCOUNTER.", ""),
+        "deck": len(pl.get("deck") or []),
+        "pts": sum(len(a) for a in mph if isinstance(a, list)),
+    })
+rows.sort(key=lambda r: r["t"])
+
+# The last 2N runs are this experiment's.
+total = 2 * int(os.environ.get("AB_N", "5"))
+tail = rows[-total:] if total <= len(rows) else rows
+armA, armB = tail[:len(tail)//2], tail[len(tail)//2:]
+
+print(f"{'arm':<12} {'n':>2} {'deck avg':>9} {'pts avg':>8} {'reached act 2':>14}")
+for name, grp in (("jev", armA), ("combined", armB)):
+    if not grp:
+        continue
+    decks = [r["deck"] for r in grp]
+    pts = [r["pts"] for r in grp]
+    act2 = sum(1 for r in grp if r["pts"] > 17)
+    print(f"{name:<12} {len(grp):>2} {st.mean(decks):>9.1f} {st.mean(pts):>8.1f} "
+          f"{act2}/{len(grp):>12}")
+PY
diff --git a/eval_batch.sh b/eval_batch.sh
new file mode 100755
index 0000000..df275e7
--- /dev/null
+++ b/eval_batch.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Run N sessions back to back and collect the resulting run records.
+# The bot handles menu/character-select navigation itself, so no setup is needed.
+set -u
+
+# Fail fast if the bot cannot start a run (e.g. a pending Timeline epoch).
+if ! python3 run.py --steps 1 >/tmp/preflight.log 2>&1; then
+  cat /tmp/preflight.log
+  exit 2
+fi
+SESSIONS="${1:-4}"
+STEPS="${2:-600}"
+for i in $(seq 1 "$SESSIONS"); do
+  echo "=== session $i/$SESSIONS ==="
+  python3 run.py --steps "$STEPS" --pause 0.3 > "/tmp/eval_${i}.log" 2>&1
+  echo "  exit=$? takes=$(grep -c 'select_card_reward' "/tmp/eval_${i}.log") skips=$(grep -c 'skip_card_reward' "/tmp/eval_${i}.log")"
+  python3 - <<'PY'
+import sts2
+try:
+    d = sts2.state(); p = d.get('player') or {}
+    print(f"  now: {d.get('state_type')} run={d.get('run')} hp={p.get('hp')}/{p.get('max_hp')}")
+except Exception as e:
+    print(f"  state read failed: {e}")
+PY
+done