mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-11-07 09:45:36 +00:00
test(mcp): agent smoke test across five coding agents, tool snapshot
- tests/agents/agent-smoke.sh runs tests/agents/cases.json through Claude Code, Codex, pi, Hermes Agent, and opencode headless against a local idx-mcp build, each with an isolated config, and checks the tools called, their key arguments, and (mock data) facts in the answer. Opt-in; never part of the gate. - crates/idx-mcp/tests/golden/tools.json pins tool names, descriptions, and input schemas, so changes to what agents see are reviewed. - nix develop .#agents adds Claude Code, Codex, and opencode from nixpkgs after the host PATH. First run: 30/30 (5 agents x 6 cases, mock data). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
386a63c288
commit
9d8914b2c9
7 changed files with 758 additions and 1 deletions
|
|
@ -181,6 +181,31 @@ fn lists_grouped_tools_and_hides_sync_unless_allowed() {
|
|||
assert_eq!(client.tool_names(), expected);
|
||||
}
|
||||
|
||||
/// What agents see (names, descriptions, input schemas) is part of the
|
||||
/// contract: changes must show up as a reviewed diff. Regenerate with
|
||||
/// `UPDATE_GOLDEN=1`.
|
||||
#[test]
|
||||
fn tool_definitions_match_golden() {
|
||||
let mut client = Client::start(&["--allow-sync"], false);
|
||||
let mut tools = client.request("tools/list", json!({}))["tools"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.clone();
|
||||
tools.sort_by(|a, b| a["name"].as_str().cmp(&b["name"].as_str()));
|
||||
let rendered = serde_json::to_string_pretty(&tools).unwrap() + "\n";
|
||||
|
||||
let golden = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/golden/tools.json");
|
||||
if std::env::var_os("UPDATE_GOLDEN").is_some() {
|
||||
std::fs::create_dir_all(golden.parent().unwrap()).unwrap();
|
||||
std::fs::write(&golden, &rendered).unwrap();
|
||||
}
|
||||
let expected = std::fs::read_to_string(&golden).expect("tests/golden/tools.json");
|
||||
assert!(
|
||||
rendered == expected,
|
||||
"tool definitions changed; review and rerun with UPDATE_GOLDEN=1:\n{rendered}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_tools_wrap_data_with_freshness() {
|
||||
let mut client = Client::start(&[], false);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue