From 7ebf63ed427375fe6dc2a91581629f52a23b8a58 Mon Sep 17 00:00:00 2001
From: hermes <hermes@rasyidanaf.com>
Date: Fri, 16 Oct 2026 06:09:00 +0000
Subject: [PATCH] test(agents): a live variant of the ownership-changes case

The `changes` case names the fixture releases (2026-01-30, 2026-02-27),
which the live snapshot does not have, so under --live/--npx it failed
agents that correctly declined to answer about missing releases and
passed ones that called the tool anyway. A case's `live` fields now
replace the mock ones; this one asks about the two most recent releases.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V3syn2fN2LNo5pvTSqc6eh
---
 docs/SMOKE.md               | 2 ++
 tests/agents/agent-smoke.sh | 4 +++-
 tests/agents/cases.json     | 6 +++++-
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/docs/SMOKE.md b/docs/SMOKE.md
index 008933e..682b791 100644
--- a/docs/SMOKE.md
+++ b/docs/SMOKE.md
@@ -111,6 +111,8 @@ each run keeps a copy next to its output as `<case>.<run>.mcp-stderr`.
 - `IDX_AGENT_MODEL_<AGENT>` picks a model (for example
   `IDX_AGENT_MODEL_CLAUDE=sonnet`). `IDX_AGENT_EXEC_PREFIX` prefixes every
   agent command, for hosts that inject API keys through a wrapper.
+- With `--live` (and `--npx`), a case's `live` fields replace its mock ones,
+  for prompts that name data only the fixtures have, such as release dates.
 - The run fails only when an agent fails a case in every run; use `--runs` to
   measure pass rates. `--keep` keeps raw agent output and `results.jsonl`.
 - `nix develop .#agents` adds Claude Code, Codex, and opencode from nixpkgs
diff --git a/tests/agents/agent-smoke.sh b/tests/agents/agent-smoke.sh
index 01a1462..7fca924 100755
--- a/tests/agents/agent-smoke.sh
+++ b/tests/agents/agent-smoke.sh
@@ -339,7 +339,9 @@ for agent in "${agents[@]}"; do
     fi
     "setup_$agent"
     for id in "${case_ids[@]}"; do
-        case_json="$(jq -c --arg id "$id" '.[] | select(.id == $id)' "$cases_file")"
+        # A case's "live" fields replace the mock ones, e.g. release dates.
+        case_json="$(jq -c --arg id "$id" --argjson live "$live" '.[] | select(.id == $id)
+            | if $live == 1 then . + (.live // {}) else . end | del(.live)' "$cases_file")"
         prompt="$(jq -r '.prompt' <<< "$case_json") Use the idx MCP tools."
         for ((run = 1; run <= runs; run++)); do
             out="$work/runs/$agent/$id.$run"
diff --git a/tests/agents/cases.json b/tests/agents/cases.json
index 37c1816..0d7ac64 100644
--- a/tests/agents/cases.json
+++ b/tests/agents/cases.json
@@ -33,6 +33,10 @@
     "id": "changes",
     "prompt": "Which shareholders changed between the 2026-01-30 and 2026-02-27 KSEI releases?",
     "expect_tools": [{"name": "get_ownership_changes", "args": {"from": "2026-01-30", "to": "2026-02-27"}}],
-    "expect_text": []
+    "expect_text": [],
+    "live": {
+      "prompt": "Which shareholders changed between the two most recent KSEI releases?",
+      "expect_tools": [{"name": "get_ownership_changes"}]
+    }
   }
 ]