mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-11-07 09:45:36 +00:00
test(mcp): agent smoke test across five coding agents, tool snapshot
- tests/agents/agent-smoke.sh runs tests/agents/cases.json through Claude Code, Codex, pi, Hermes Agent, and opencode headless against a local idx-mcp build, each with an isolated config, and checks the tools called, their key arguments, and (mock data) facts in the answer. Opt-in; never part of the gate. - crates/idx-mcp/tests/golden/tools.json pins tool names, descriptions, and input schemas, so changes to what agents see are reviewed. - nix develop .#agents adds Claude Code, Codex, and opencode from nixpkgs after the host PATH. First run: 30/30 (5 agents x 6 cases, mock data). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
386a63c288
commit
9d8914b2c9
7 changed files with 758 additions and 1 deletions
38
tests/agents/cases.json
Normal file
38
tests/agents/cases.json
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
[
|
||||
{
|
||||
"id": "shareholders",
|
||||
"prompt": "Who are the shareholders above 5% of BBCA?",
|
||||
"expect_tools": [{"name": "get_shareholders", "args": {"ticker": "BBCA"}}],
|
||||
"expect_text": ["(?i)dwimuria", "54[.,]9"]
|
||||
},
|
||||
{
|
||||
"id": "quote",
|
||||
"prompt": "What is the latest BBCA share price?",
|
||||
"expect_tools": [{"name": "get_quotes", "args": {"symbols": ["BBCA"]}}],
|
||||
"expect_text": ["(?i)bbca"]
|
||||
},
|
||||
{
|
||||
"id": "compare",
|
||||
"prompt": "Compare the fundamentals of BBCA and BBRI.",
|
||||
"expect_tools": [{"name": "compare_stocks", "args": {"symbols": ["BBCA", "BBRI"]}}],
|
||||
"expect_text": ["(?i)bbri"]
|
||||
},
|
||||
{
|
||||
"id": "technical",
|
||||
"prompt": "Give me a technical analysis of BBCA: trend, RSI, and MACD.",
|
||||
"expect_tools": [{"name": "analyze_stock", "args": {"symbol": "BBCA"}}],
|
||||
"expect_text": ["(?i)rsi"]
|
||||
},
|
||||
{
|
||||
"id": "investor",
|
||||
"prompt": "Which IDX stocks does Anthoni Salim hold above 1%?",
|
||||
"expect_tools": [{"any": ["get_investor_holdings", "search_investors"]}],
|
||||
"expect_text": ["(?i)bbca"]
|
||||
},
|
||||
{
|
||||
"id": "changes",
|
||||
"prompt": "Which shareholders changed between the 2026-01-30 and 2026-02-27 KSEI releases?",
|
||||
"expect_tools": [{"name": "get_ownership_changes", "args": {"from": "2026-01-30", "to": "2026-02-27"}}],
|
||||
"expect_text": []
|
||||
}
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue