fix(eval): end each A/B session with its run, and attribute runs to arms
Two defects made the earlier A/B numbers meaningless:
* Without `--stop-on-run-end` a session contained several runs, so runs could
not be attributed to an arm.
* The script then read "the last 2N runs", which silently compared runs from
earlier experiments rather than the two arms.
Pass `--stop-on-run-end`, and attribute each run record to its arm by diffing
the history directory around each session.
Also raise the step budget and warn on a thin sample: measured, a run still
going at step 600 produces NO run record, so a short budget biases the sample
toward runs that died early -- exactly the wrong bias for this question.
This commit is contained in:
0xrsydn
parent
471c77b353
commit
d676bf067e
1 changed files with 170 additions and 36 deletions
202
ab_card_skip.sh
202
ab_card_skip.sh
|
|
@ -3,74 +3,208 @@
|
|||
#
|
||||
# ./ab_card_skip.sh <sessions-per-arm> <steps-per-session>
|
||||
#
|
||||
# Runs N sessions with --card-skip-policy jev, then N with --card-skip-policy
|
||||
# combined, and prints a comparison of deck size and progress.
|
||||
# Everything except --card-skip-policy is identical between the arms.
|
||||
#
|
||||
# Everything else is identical between the arms, so any difference is the
|
||||
# policy. (The earlier before/after comparison was confounded: several bug
|
||||
# fixes landed at the same time as the policy change.)
|
||||
# Three things this script does that earlier versions did not, each because an
|
||||
# earlier attempt produced meaningless numbers:
|
||||
#
|
||||
# 1. --stop-on-run-end, so ONE SESSION = ONE RUN. Without it the bot dies,
|
||||
# returns to the menu and starts a fresh run inside the same session, and a
|
||||
# session yields several run records that cannot be attributed to an arm.
|
||||
# 2. Each run record is attributed to the arm that produced it, by diffing the
|
||||
# history directory around each session. The old version just took "the
|
||||
# last 2N runs", which silently compared old runs from earlier experiments.
|
||||
# 3. Every attributed run carries a hash of the decision layer, and the report
|
||||
# compares only runs from ONE revision. A change to brain.py / facts.py /
|
||||
# jev.py / run.py changes how both arms behave, so runs collected before it
|
||||
# are not comparable with runs collected after it -- and the working copy is
|
||||
# usually dirty, so a commit id would not distinguish the revisions.
|
||||
set -u
|
||||
|
||||
N="${1:-5}"
|
||||
# Steps per session. This must be generous: measured, a run that is still going
|
||||
# at step 600 produces NO run record, so the sample silently fills with runs
|
||||
# that died early -- exactly the wrong bias. --stop-on-run-end is the real stop
|
||||
# condition; this is only a safety cap.
|
||||
STEPS="${2:-4000}"
|
||||
HIST="$HOME/Library/Application Support/SlayTheSpire2/steam/76561198141226155/modded/profile1/saves/history"
|
||||
RESULTS=/tmp/ab_results.tsv
|
||||
|
||||
# The revision every arm in this invocation is collected from. Content hash, not
|
||||
# a commit id: the decision layer is normally edited in place, uncommitted.
|
||||
#
|
||||
# Recomputed BEFORE EVERY SESSION, not once: this file is edited in place while
|
||||
# experiments run, which is the whole reason the guard exists. A hash taken once
|
||||
# at startup would be stamped onto every row regardless, and because $RESULTS is
|
||||
# truncated below, the mixed-revision warning could never fire.
|
||||
CODE_FILES=(brain.py facts.py jev.py run.py sts2.py)
|
||||
code_hash() {
|
||||
cat "${CODE_FILES[@]}" | { shasum -a 256 2>/dev/null || sha256sum; } | cut -c1-12
|
||||
}
|
||||
CODE_HASH=$(code_hash)
|
||||
ABORTED=0
|
||||
echo "code revision: $CODE_HASH (${CODE_FILES[*]})"
|
||||
|
||||
# Fail fast if the bot cannot start a run (e.g. a pending Timeline epoch).
|
||||
if ! python3 run.py --steps 1 >/tmp/preflight.log 2>&1; then
|
||||
cat /tmp/preflight.log
|
||||
exit 2
|
||||
fi
|
||||
|
||||
N="${1:-5}"
|
||||
STEPS="${2:-600}"
|
||||
: > "$RESULTS"
|
||||
|
||||
snapshot() { ls "$HIST" 2>/dev/null | sort; }
|
||||
|
||||
run_arm() {
|
||||
local policy="$1" tag="$2"
|
||||
local policy="$1"
|
||||
for i in $(seq 1 "$N"); do
|
||||
echo "--- $tag $i/$N ---"
|
||||
local before after attempt=0 rc=1 session_hash
|
||||
before=$(snapshot)
|
||||
|
||||
# An arm collected across two revisions is not an arm. Stop the experiment
|
||||
# rather than stamp rows with a revision that did not produce them.
|
||||
session_hash=$(code_hash)
|
||||
if [ "$session_hash" != "$CODE_HASH" ]; then
|
||||
echo
|
||||
echo "!! ABORT: the decision layer changed mid-experiment."
|
||||
echo "!! started on $CODE_HASH, now $session_hash"
|
||||
echo "!! Rows already collected are in $RESULTS and are NOT comparable"
|
||||
echo "!! with anything collected on the other revision."
|
||||
echo "!! Re-run BOTH arms from one revision."
|
||||
ABORTED=1
|
||||
return 1
|
||||
fi
|
||||
|
||||
while [ "$attempt" -lt 3 ]; do
|
||||
attempt=$((attempt + 1))
|
||||
echo "--- $policy $i/$N (attempt $attempt) rev=$session_hash ---"
|
||||
python3 run.py --steps "$STEPS" --pause 0.3 \
|
||||
--card-skip-policy "$policy" > "/tmp/ab_${tag}_${i}.log" 2>&1
|
||||
echo " exit=$? takes=$(grep -c 'select_card_reward' "/tmp/ab_${tag}_${i}.log") skips=$(grep -c 'skip_card_reward' "/tmp/ab_${tag}_${i}.log")"
|
||||
--card-skip-policy "$policy" --stop-on-run-end \
|
||||
> "/tmp/ab_${policy}_${i}.log" 2>&1
|
||||
rc=$?
|
||||
[ "$rc" -eq 0 ] && break
|
||||
[ "$rc" -ne 3 ] && break # 2 = blocked, not worth retrying
|
||||
echo " retrying after model failures..."
|
||||
sleep 20
|
||||
done
|
||||
|
||||
after=$(snapshot)
|
||||
local new_runs session
|
||||
new_runs=$(comm -13 <(echo "$before") <(echo "$after") | tr '\n' ' ')
|
||||
# The session id the bot printed, so an attribution row joins to the same
|
||||
# decisions.jsonl rows and to sessions.jsonl. Without it the two ends of the
|
||||
# join carry different identifiers and nothing lines up.
|
||||
session=$(sed -n 's/^session: //p' "/tmp/ab_${policy}_${i}.log" | head -1)
|
||||
echo " exit=$rc session=${session:-?} takes=$(grep -c 'select_card_reward' "/tmp/ab_${policy}_${i}.log") skips=$(grep -c 'skip_card_reward' "/tmp/ab_${policy}_${i}.log") runs='${new_runs}'"
|
||||
|
||||
# Stamped with the hash that produced THIS run, not the startup hash.
|
||||
for r in $new_runs; do
|
||||
printf '%s\t%s\t%s\t%s\n' "$policy" "$r" "$session_hash" "${session:-?}" >> "$RESULTS"
|
||||
done
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
echo "### ARM A: jev decides skip"
|
||||
run_arm jev jev
|
||||
|
||||
run_arm jev
|
||||
echo
|
||||
if [ "$ABORTED" -eq 0 ]; then
|
||||
echo "### ARM B: combined (jev + a floor)"
|
||||
run_arm combined combined
|
||||
run_arm combined
|
||||
else
|
||||
echo "### ARM B: skipped (aborted mid-experiment)"
|
||||
fi
|
||||
|
||||
echo
|
||||
python3 - <<'PY'
|
||||
import json, glob, os, statistics as st
|
||||
POLICY=combined RESULTS="$RESULTS" CODE_HASH="$CODE_HASH" HIST="$HIST" python3 - <<'PY'
|
||||
import json, os, statistics as st
|
||||
|
||||
H = os.path.expanduser(
|
||||
"~/Library/Application Support/SlayTheSpire2/steam/"
|
||||
"76561198141226155/modded/profile1/saves/history"
|
||||
)
|
||||
# The same directory the shell snapshotted. A second hardcoded copy of this path
|
||||
# silently reported "no runs recorded" for sessions that did produce runs.
|
||||
hist = os.environ["HIST"]
|
||||
results = os.environ["RESULTS"]
|
||||
this_hash = os.environ["CODE_HASH"]
|
||||
|
||||
rows = []
|
||||
for f in glob.glob(H + "/*.run"):
|
||||
d = json.load(open(f))
|
||||
with open(results) as fh:
|
||||
for line in fh:
|
||||
parts = line.rstrip("\n").split("\t")
|
||||
if len(parts) < 2 or not parts[1]:
|
||||
continue
|
||||
rows.append((parts[0], parts[1],
|
||||
parts[2] if len(parts) > 2 and parts[2] else "unstamped",
|
||||
parts[3] if len(parts) > 3 and parts[3] else "?"))
|
||||
|
||||
hashes = sorted({h for _, _, h, _ in rows})
|
||||
print(f"code revision(s) in {results}: {', '.join(hashes) if hashes else '(none)'}")
|
||||
if len(hashes) > 1:
|
||||
print()
|
||||
print("!! MIXED REVISIONS. A change to the decision layer changes how BOTH")
|
||||
print("!! arms behave, so a run collected before it is not comparable with")
|
||||
print("!! one collected after it. Only ONE revision is scored below; re-run")
|
||||
print("!! BOTH arms on one revision before interpreting anything.")
|
||||
# Score the revision this invocation collected from when it has rows; otherwise
|
||||
# the most recently appended one. Never "the largest hash": hex is not ordered.
|
||||
if this_hash in hashes:
|
||||
current = this_hash
|
||||
elif rows:
|
||||
current = rows[-1][2]
|
||||
else:
|
||||
current = this_hash
|
||||
print(f"scoring revision: {current}")
|
||||
print()
|
||||
|
||||
arms = {"jev": [], "combined": []}
|
||||
sessions = {"jev": 0, "combined": 0}
|
||||
for policy, run, code, session in rows:
|
||||
if policy not in arms or code != current:
|
||||
continue
|
||||
sessions[policy] += 1
|
||||
path = os.path.join(hist, run)
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
d = json.load(open(path))
|
||||
pl = (d.get("players") or [{}])[0]
|
||||
mph = d.get("map_point_history") or []
|
||||
rows.append({
|
||||
"t": d.get("start_time", 0),
|
||||
arms[policy].append({
|
||||
"killed": d.get("killed_by_encounter", "?").replace("ENCOUNTER.", ""),
|
||||
"deck": len(pl.get("deck") or []),
|
||||
"pts": sum(len(a) for a in mph if isinstance(a, list)),
|
||||
"session": session,
|
||||
})
|
||||
rows.sort(key=lambda r: r["t"])
|
||||
|
||||
# The last 2N runs are this experiment's.
|
||||
total = 2 * int(os.environ.get("AB_N", "5"))
|
||||
tail = rows[-total:] if total <= len(rows) else rows
|
||||
armA, armB = tail[:len(tail)//2], tail[len(tail)//2:]
|
||||
|
||||
print(f"{'arm':<12} {'n':>2} {'deck avg':>9} {'pts avg':>8} {'reached act 2':>14}")
|
||||
for name, grp in (("jev", armA), ("combined", armB)):
|
||||
print(f"{'arm':<10} {'sess':>4} {'runs':>4} {'deck':>6} {'pts':>6} {'added/pt':>9} {'act2':>6}")
|
||||
print("-" * 54)
|
||||
for name, grp in arms.items():
|
||||
if not grp:
|
||||
print(f"{name:<10} {sessions[name]:>4} {0:>4} (no runs recorded)")
|
||||
continue
|
||||
decks = [r["deck"] for r in grp]
|
||||
pts = [r["pts"] for r in grp]
|
||||
act2 = sum(1 for r in grp if r["pts"] > 17)
|
||||
print(f"{name:<12} {len(grp):>2} {st.mean(decks):>9.1f} {st.mean(pts):>8.1f} "
|
||||
f"{act2}/{len(grp):>12}")
|
||||
added = max(0.0, st.mean(decks) - 10) / max(1e-9, st.mean(pts))
|
||||
print(f"{name:<10} {sessions[name]:>4} {len(grp):>4} {st.mean(decks):>6.1f} "
|
||||
f"{st.mean(pts):>6.1f} {added:>9.2f} {act2:>4}/{len(grp)}")
|
||||
|
||||
print()
|
||||
print("sess = sessions that produced a COMPLETED run record")
|
||||
print("runs = run records attributed to this arm, on the scored revision")
|
||||
print("added/pt = non-starting cards per map point reached <- the metric that")
|
||||
print(" matters, because deck size at death scales with run length")
|
||||
print("act2 = runs past the Act 1 boss (17 map points)")
|
||||
|
||||
thin = [n for n, g in arms.items() if len(g) < 5]
|
||||
if thin:
|
||||
print()
|
||||
print(f"!! THIN SAMPLE: {', '.join(thin)} have fewer than 5 runs.")
|
||||
print("!! Sessions that hit the step cap mid-run produce NO record, so a thin")
|
||||
print("!! sample is biased toward runs that died early. Raise STEPS and re-run.")
|
||||
print()
|
||||
for name, grp in arms.items():
|
||||
if grp:
|
||||
print(f"{name} killed by: {[r['killed'] for r in grp]}")
|
||||
print(f"{name} sessions : {[r['session'] for r in grp]}")
|
||||
print()
|
||||
print("sessions are the join key: grep one in capture/decisions.jsonl for that")
|
||||
print("session's decisions, and in capture/sessions.jsonl for its outcome row.")
|
||||
PY
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue