Add batch eval and card-skip A/B scripts
eval_batch.sh: N back-to-back sessions with per-session summaries. ab_card_skip.sh: A/B the jev vs combined card-reward skip policy and compare deck size and progress from the game's run history.
This commit is contained in:
parent
41e3ea4a2b
commit
efd7d2e423
2 changed files with 101 additions and 0 deletions
76
ab_card_skip.sh
Executable file
76
ab_card_skip.sh
Executable file
|
|
@ -0,0 +1,76 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# A/B the card-reward skip policy.
|
||||||
|
#
|
||||||
|
# ./ab_card_skip.sh <sessions-per-arm> <steps-per-session>
|
||||||
|
#
|
||||||
|
# Runs N sessions with --card-skip-policy jev, then N with --card-skip-policy
|
||||||
|
# combined, and prints a comparison of deck size and progress.
|
||||||
|
#
|
||||||
|
# Everything else is identical between the arms, so any difference is the
|
||||||
|
# policy. (The earlier before/after comparison was confounded: several bug
|
||||||
|
# fixes landed at the same time as the policy change.)
|
||||||
|
set -u
|
||||||
|
|
||||||
|
# Fail fast if the bot cannot start a run (e.g. a pending Timeline epoch).
|
||||||
|
if ! python3 run.py --steps 1 >/tmp/preflight.log 2>&1; then
|
||||||
|
cat /tmp/preflight.log
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
N="${1:-5}"
|
||||||
|
STEPS="${2:-600}"
|
||||||
|
|
||||||
|
run_arm() {
|
||||||
|
local policy="$1" tag="$2"
|
||||||
|
for i in $(seq 1 "$N"); do
|
||||||
|
echo "--- $tag $i/$N ---"
|
||||||
|
python3 run.py --steps "$STEPS" --pause 0.3 \
|
||||||
|
--card-skip-policy "$policy" > "/tmp/ab_${tag}_${i}.log" 2>&1
|
||||||
|
echo " exit=$? takes=$(grep -c 'select_card_reward' "/tmp/ab_${tag}_${i}.log") skips=$(grep -c 'skip_card_reward' "/tmp/ab_${tag}_${i}.log")"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "### ARM A: jev decides skip"
|
||||||
|
run_arm jev jev
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "### ARM B: combined (jev + a floor)"
|
||||||
|
run_arm combined combined
|
||||||
|
|
||||||
|
echo
|
||||||
|
python3 - <<'PY'
|
||||||
|
import json, glob, os, statistics as st
|
||||||
|
|
||||||
|
H = os.path.expanduser(
|
||||||
|
"~/Library/Application Support/SlayTheSpire2/steam/"
|
||||||
|
"76561198141226155/modded/profile1/saves/history"
|
||||||
|
)
|
||||||
|
|
||||||
|
rows = []
|
||||||
|
for f in glob.glob(H + "/*.run"):
|
||||||
|
d = json.load(open(f))
|
||||||
|
pl = (d.get("players") or [{}])[0]
|
||||||
|
mph = d.get("map_point_history") or []
|
||||||
|
rows.append({
|
||||||
|
"t": d.get("start_time", 0),
|
||||||
|
"killed": d.get("killed_by_encounter", "?").replace("ENCOUNTER.", ""),
|
||||||
|
"deck": len(pl.get("deck") or []),
|
||||||
|
"pts": sum(len(a) for a in mph if isinstance(a, list)),
|
||||||
|
})
|
||||||
|
rows.sort(key=lambda r: r["t"])
|
||||||
|
|
||||||
|
# The last 2N runs are this experiment's.
|
||||||
|
total = 2 * int(os.environ.get("AB_N", "5"))
|
||||||
|
tail = rows[-total:] if total <= len(rows) else rows
|
||||||
|
armA, armB = tail[:len(tail)//2], tail[len(tail)//2:]
|
||||||
|
|
||||||
|
print(f"{'arm':<12} {'n':>2} {'deck avg':>9} {'pts avg':>8} {'reached act 2':>14}")
|
||||||
|
for name, grp in (("jev", armA), ("combined", armB)):
|
||||||
|
if not grp:
|
||||||
|
continue
|
||||||
|
decks = [r["deck"] for r in grp]
|
||||||
|
pts = [r["pts"] for r in grp]
|
||||||
|
act2 = sum(1 for r in grp if r["pts"] > 17)
|
||||||
|
print(f"{name:<12} {len(grp):>2} {st.mean(decks):>9.1f} {st.mean(pts):>8.1f} "
|
||||||
|
f"{act2}/{len(grp):>12}")
|
||||||
|
PY
|
||||||
25
eval_batch.sh
Executable file
25
eval_batch.sh
Executable file
|
|
@ -0,0 +1,25 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Run N sessions back to back and collect the resulting run records.
|
||||||
|
# The bot handles menu/character-select navigation itself, so no setup is needed.
|
||||||
|
set -u
|
||||||
|
|
||||||
|
# Fail fast if the bot cannot start a run (e.g. a pending Timeline epoch).
|
||||||
|
if ! python3 run.py --steps 1 >/tmp/preflight.log 2>&1; then
|
||||||
|
cat /tmp/preflight.log
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
SESSIONS="${1:-4}"
|
||||||
|
STEPS="${2:-600}"
|
||||||
|
for i in $(seq 1 "$SESSIONS"); do
|
||||||
|
echo "=== session $i/$SESSIONS ==="
|
||||||
|
python3 run.py --steps "$STEPS" --pause 0.3 > "/tmp/eval_${i}.log" 2>&1
|
||||||
|
echo " exit=$? takes=$(grep -c 'select_card_reward' "/tmp/eval_${i}.log") skips=$(grep -c 'skip_card_reward' "/tmp/eval_${i}.log")"
|
||||||
|
python3 - <<'PY'
|
||||||
|
import sts2
|
||||||
|
try:
|
||||||
|
d = sts2.state(); p = d.get('player') or {}
|
||||||
|
print(f" now: {d.get('state_type')} run={d.get('run')} hp={p.get('hp')}/{p.get('max_hp')}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f" state read failed: {e}")
|
||||||
|
PY
|
||||||
|
done
|
||||||
Loading…
Add table
Add a link
Reference in a new issue