test(mcp): agent smoke test across five coding agents, tool snapshot

- tests/agents/agent-smoke.sh runs tests/agents/cases.json through
  Claude Code, Codex, pi, Hermes Agent, and opencode headless against a
  local idx-mcp build, each with an isolated config, and checks the
  tools called, their key arguments, and (mock data) facts in the
  answer. Opt-in; never part of the gate.
- crates/idx-mcp/tests/golden/tools.json pins tool names, descriptions,
  and input schemas, so changes to what agents see are reviewed.
- nix develop .#agents adds Claude Code, Codex, and opencode from
  nixpkgs after the host PATH.

First run: 30/30 (5 agents x 6 cases, mock data).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
hermes 2026-10-14 20:59:56 +00:00
commit 9d8914b2c9
7 changed files with 758 additions and 1 deletions

View file

@ -181,6 +181,31 @@ fn lists_grouped_tools_and_hides_sync_unless_allowed() {
assert_eq!(client.tool_names(), expected);
}
/// What agents see (names, descriptions, input schemas) is part of the
/// contract: changes must show up as a reviewed diff. Regenerate with
/// `UPDATE_GOLDEN=1`.
#[test]
fn tool_definitions_match_golden() {
let mut client = Client::start(&["--allow-sync"], false);
let mut tools = client.request("tools/list", json!({}))["tools"]
.as_array()
.unwrap()
.clone();
tools.sort_by(|a, b| a["name"].as_str().cmp(&b["name"].as_str()));
let rendered = serde_json::to_string_pretty(&tools).unwrap() + "\n";
let golden = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/golden/tools.json");
if std::env::var_os("UPDATE_GOLDEN").is_some() {
std::fs::create_dir_all(golden.parent().unwrap()).unwrap();
std::fs::write(&golden, &rendered).unwrap();
}
let expected = std::fs::read_to_string(&golden).expect("tests/golden/tools.json");
assert!(
rendered == expected,
"tool definitions changed; review and rerun with UPDATE_GOLDEN=1:\n{rendered}"
);
}
#[test]
fn live_tools_wrap_data_with_freshness() {
let mut client = Client::start(&[], false);