diff --git a/docs/SMOKE.md b/docs/SMOKE.md index 008933e..682b791 100644 --- a/docs/SMOKE.md +++ b/docs/SMOKE.md @@ -111,6 +111,8 @@ each run keeps a copy next to its output as `..mcp-stderr`. - `IDX_AGENT_MODEL_` picks a model (for example `IDX_AGENT_MODEL_CLAUDE=sonnet`). `IDX_AGENT_EXEC_PREFIX` prefixes every agent command, for hosts that inject API keys through a wrapper. +- With `--live` (and `--npx`), a case's `live` fields replace its mock ones, + for prompts that name data only the fixtures have, such as release dates. - The run fails only when an agent fails a case in every run; use `--runs` to measure pass rates. `--keep` keeps raw agent output and `results.jsonl`. - `nix develop .#agents` adds Claude Code, Codex, and opencode from nixpkgs diff --git a/tests/agents/agent-smoke.sh b/tests/agents/agent-smoke.sh index 01a1462..7fca924 100755 --- a/tests/agents/agent-smoke.sh +++ b/tests/agents/agent-smoke.sh @@ -339,7 +339,9 @@ for agent in "${agents[@]}"; do fi "setup_$agent" for id in "${case_ids[@]}"; do - case_json="$(jq -c --arg id "$id" '.[] | select(.id == $id)' "$cases_file")" + # A case's "live" fields replace the mock ones, e.g. release dates. + case_json="$(jq -c --arg id "$id" --argjson live "$live" '.[] | select(.id == $id) + | if $live == 1 then . + (.live // {}) else . end | del(.live)' "$cases_file")" prompt="$(jq -r '.prompt' <<< "$case_json") Use the idx MCP tools." for ((run = 1; run <= runs; run++)); do out="$work/runs/$agent/$id.$run" diff --git a/tests/agents/cases.json b/tests/agents/cases.json index 37c1816..0d7ac64 100644 --- a/tests/agents/cases.json +++ b/tests/agents/cases.json @@ -33,6 +33,10 @@ "id": "changes", "prompt": "Which shareholders changed between the 2026-01-30 and 2026-02-27 KSEI releases?", "expect_tools": [{"name": "get_ownership_changes", "args": {"from": "2026-01-30", "to": "2026-02-27"}}], - "expect_text": [] + "expect_text": [], + "live": { + "prompt": "Which shareholders changed between the two most recent KSEI releases?", + "expect_tools": [{"name": "get_ownership_changes"}] + } } ]