0xrsydn/idx-cli
1
mirror of https://github.com/0xrsydn/idx-cli.git synced 2026-11-07 09:45:36 +00:00
Code Issues Projects Releases 1 Packages Wiki Activity

test(agents): a live variant of the ownership-changes case

Browse source 
The `changes` case names the fixture releases (2026-01-30, 2026-02-27),
which the live snapshot does not have, so under --live/--npx it failed
agents that correctly declined to answer about missing releases and
passed ones that called the tool anyway. A case's `live` fields now
replace the mock ones; this one asks about the two most recent releases.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V3syn2fN2LNo5pvTSqc6eh
This commit is contained in:
hermes 2026-10-16 06:09:00 +00:00
commit 7ebf63ed42
3 changed files with 10 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
tests/agents/agent-smoke.sh
View file

@ -339,7 +339,9 @@ for agent in "${agents[@]}"; do
fi
"setup_$agent"
for id in "${case_ids[@]}"; do
case_json="$(jq -c --arg id "$id" '.[] | select(.id == $id)' "$cases_file")"
# A case's "live" fields replace the mock ones, e.g. release dates.
case_json="$(jq -c --arg id "$id" --argjson live "$live" '.[] | select(.id == $id)
| if $live == 1 then . + (.live // {}) else . end | del(.live)' "$cases_file")"
prompt="$(jq -r '.prompt' <<< "$case_json") Use the idx MCP tools."
for ((run = 1; run <= runs; run++)); do
out="$work/runs/$agent/$id.$run"

6
tests/agents/cases.json
View file

@ -33,6 +33,10 @@
"id": "changes",
"prompt": "Which shareholders changed between the 2026-01-30 and 2026-02-27 KSEI releases?",
"expect_tools": [{"name": "get_ownership_changes", "args": {"from": "2026-01-30", "to": "2026-02-27"}}],
"expect_text": []
"expect_text": [],
"live": {
"prompt": "Which shareholders changed between the two most recent KSEI releases?",
"expect_tools": [{"name": "get_ownership_changes"}]
}
}
]