Add batch eval and card-skip A/B scripts

eval_batch.sh: N back-to-back sessions with per-session summaries.
ab_card_skip.sh: A/B the jev vs combined card-reward skip policy and
compare deck size and progress from the game's run history.
This commit is contained in:
0xrsydn 2026-09-21 17:09:43 +07:00
commit efd7d2e423
2 changed files with 101 additions and 0 deletions

76
ab_card_skip.sh Executable file
View file

@ -0,0 +1,76 @@
#!/usr/bin/env bash
# A/B the card-reward skip policy.
#
# ./ab_card_skip.sh <sessions-per-arm> <steps-per-session>
#
# Runs N sessions with --card-skip-policy jev, then N with --card-skip-policy
# combined, and prints a comparison of deck size and progress.
#
# Everything else is identical between the arms, so any difference is the
# policy. (The earlier before/after comparison was confounded: several bug
# fixes landed at the same time as the policy change.)
set -u
# Fail fast if the bot cannot start a run (e.g. a pending Timeline epoch).
if ! python3 run.py --steps 1 >/tmp/preflight.log 2>&1; then
cat /tmp/preflight.log
exit 2
fi
N="${1:-5}"
STEPS="${2:-600}"
run_arm() {
local policy="$1" tag="$2"
for i in $(seq 1 "$N"); do
echo "--- $tag $i/$N ---"
python3 run.py --steps "$STEPS" --pause 0.3 \
--card-skip-policy "$policy" > "/tmp/ab_${tag}_${i}.log" 2>&1
echo " exit=$? takes=$(grep -c 'select_card_reward' "/tmp/ab_${tag}_${i}.log") skips=$(grep -c 'skip_card_reward' "/tmp/ab_${tag}_${i}.log")"
done
}
echo "### ARM A: jev decides skip"
run_arm jev jev
echo
echo "### ARM B: combined (jev + a floor)"
run_arm combined combined
echo
python3 - <<'PY'
import json, glob, os, statistics as st
H = os.path.expanduser(
"~/Library/Application Support/SlayTheSpire2/steam/"
"76561198141226155/modded/profile1/saves/history"
)
rows = []
for f in glob.glob(H + "/*.run"):
d = json.load(open(f))
pl = (d.get("players") or [{}])[0]
mph = d.get("map_point_history") or []
rows.append({
"t": d.get("start_time", 0),
"killed": d.get("killed_by_encounter", "?").replace("ENCOUNTER.", ""),
"deck": len(pl.get("deck") or []),
"pts": sum(len(a) for a in mph if isinstance(a, list)),
})
rows.sort(key=lambda r: r["t"])
# The last 2N runs are this experiment's.
total = 2 * int(os.environ.get("AB_N", "5"))
tail = rows[-total:] if total <= len(rows) else rows
armA, armB = tail[:len(tail)//2], tail[len(tail)//2:]
print(f"{'arm':<12} {'n':>2} {'deck avg':>9} {'pts avg':>8} {'reached act 2':>14}")
for name, grp in (("jev", armA), ("combined", armB)):
if not grp:
continue
decks = [r["deck"] for r in grp]
pts = [r["pts"] for r in grp]
act2 = sum(1 for r in grp if r["pts"] > 17)
print(f"{name:<12} {len(grp):>2} {st.mean(decks):>9.1f} {st.mean(pts):>8.1f} "
f"{act2}/{len(grp):>12}")
PY

25
eval_batch.sh Executable file
View file

@ -0,0 +1,25 @@
#!/usr/bin/env bash
# Run N sessions back to back and collect the resulting run records.
# The bot handles menu/character-select navigation itself, so no setup is needed.
set -u
# Fail fast if the bot cannot start a run (e.g. a pending Timeline epoch).
if ! python3 run.py --steps 1 >/tmp/preflight.log 2>&1; then
cat /tmp/preflight.log
exit 2
fi
SESSIONS="${1:-4}"
STEPS="${2:-600}"
for i in $(seq 1 "$SESSIONS"); do
echo "=== session $i/$SESSIONS ==="
python3 run.py --steps "$STEPS" --pause 0.3 > "/tmp/eval_${i}.log" 2>&1
echo " exit=$? takes=$(grep -c 'select_card_reward' "/tmp/eval_${i}.log") skips=$(grep -c 'skip_card_reward' "/tmp/eval_${i}.log")"
python3 - <<'PY'
import sts2
try:
d = sts2.state(); p = d.get('player') or {}
print(f" now: {d.get('state_type')} run={d.get('run')} hp={p.get('hp')}/{p.get('max_hp')}")
except Exception as e:
print(f" state read failed: {e}")
PY
done