efd7d2e423
eval_batch.sh: N back-to-back sessions with per-session summaries. ab_card_skip.sh: A/B the jev vs combined card-reward skip policy and compare deck size and progress from the game's run history.
76 lines
2.4 KiB
Bash
Executable file
76 lines
2.4 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# A/B the card-reward skip policy.
|
|
#
|
|
# ./ab_card_skip.sh <sessions-per-arm> <steps-per-session>
|
|
#
|
|
# Runs N sessions with --card-skip-policy jev, then N with --card-skip-policy
|
|
# combined, and prints a comparison of deck size and progress.
|
|
#
|
|
# Everything else is identical between the arms, so any difference is the
|
|
# policy. (The earlier before/after comparison was confounded: several bug
|
|
# fixes landed at the same time as the policy change.)
|
|
set -u
|
|
|
|
# Fail fast if the bot cannot start a run (e.g. a pending Timeline epoch).
|
|
if ! python3 run.py --steps 1 >/tmp/preflight.log 2>&1; then
|
|
cat /tmp/preflight.log
|
|
exit 2
|
|
fi
|
|
|
|
N="${1:-5}"
|
|
STEPS="${2:-600}"
|
|
|
|
run_arm() {
|
|
local policy="$1" tag="$2"
|
|
for i in $(seq 1 "$N"); do
|
|
echo "--- $tag $i/$N ---"
|
|
python3 run.py --steps "$STEPS" --pause 0.3 \
|
|
--card-skip-policy "$policy" > "/tmp/ab_${tag}_${i}.log" 2>&1
|
|
echo " exit=$? takes=$(grep -c 'select_card_reward' "/tmp/ab_${tag}_${i}.log") skips=$(grep -c 'skip_card_reward' "/tmp/ab_${tag}_${i}.log")"
|
|
done
|
|
}
|
|
|
|
echo "### ARM A: jev decides skip"
|
|
run_arm jev jev
|
|
|
|
echo
|
|
echo "### ARM B: combined (jev + a floor)"
|
|
run_arm combined combined
|
|
|
|
echo
|
|
python3 - <<'PY'
|
|
import json, glob, os, statistics as st
|
|
|
|
H = os.path.expanduser(
|
|
"~/Library/Application Support/SlayTheSpire2/steam/"
|
|
"76561198141226155/modded/profile1/saves/history"
|
|
)
|
|
|
|
rows = []
|
|
for f in glob.glob(H + "/*.run"):
|
|
d = json.load(open(f))
|
|
pl = (d.get("players") or [{}])[0]
|
|
mph = d.get("map_point_history") or []
|
|
rows.append({
|
|
"t": d.get("start_time", 0),
|
|
"killed": d.get("killed_by_encounter", "?").replace("ENCOUNTER.", ""),
|
|
"deck": len(pl.get("deck") or []),
|
|
"pts": sum(len(a) for a in mph if isinstance(a, list)),
|
|
})
|
|
rows.sort(key=lambda r: r["t"])
|
|
|
|
# The last 2N runs are this experiment's.
|
|
total = 2 * int(os.environ.get("AB_N", "5"))
|
|
tail = rows[-total:] if total <= len(rows) else rows
|
|
armA, armB = tail[:len(tail)//2], tail[len(tail)//2:]
|
|
|
|
print(f"{'arm':<12} {'n':>2} {'deck avg':>9} {'pts avg':>8} {'reached act 2':>14}")
|
|
for name, grp in (("jev", armA), ("combined", armB)):
|
|
if not grp:
|
|
continue
|
|
decks = [r["deck"] for r in grp]
|
|
pts = [r["pts"] for r in grp]
|
|
act2 = sum(1 for r in grp if r["pts"] > 17)
|
|
print(f"{name:<12} {len(grp):>2} {st.mean(decks):>9.1f} {st.mean(pts):>8.1f} "
|
|
f"{act2}/{len(grp):>12}")
|
|
PY
|