test(mcp): agent smoke test across five coding agents, tool snapshot

- tests/agents/agent-smoke.sh runs tests/agents/cases.json through
  Claude Code, Codex, pi, Hermes Agent, and opencode headless against a
  local idx-mcp build, each with an isolated config, and checks the
  tools called, their key arguments, and (mock data) facts in the
  answer. Opt-in; never part of the gate.
- crates/idx-mcp/tests/golden/tools.json pins tool names, descriptions,
  and input schemas, so changes to what agents see are reviewed.
- nix develop .#agents adds Claude Code, Codex, and opencode from
  nixpkgs after the host PATH.

First run: 30/30 (5 agents x 6 cases, mock data).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
hermes 2026-10-14 20:59:56 +00:00
commit 9d8914b2c9
7 changed files with 758 additions and 1 deletions

View file

@ -280,6 +280,7 @@
shellcheck -s sh ${installerSrc}/install.sh
shellcheck ${installerSrc}/scripts/install-sh-test.sh
shellcheck ${./scripts/live-smoke.sh}
shellcheck ${./tests/agents/agent-smoke.sh}
touch "$out"
'';
publisher-test = pkgs.runCommand "idx-publisher-test" {
@ -296,6 +297,23 @@
'';
};
# `nix develop .#agents`: the default shell plus coding-agent CLIs for
# tests/agents/agent-smoke.sh. They are appended to PATH, so agents
# already installed (and logged in) on the host take precedence.
# pi and Hermes Agent are not in the pinned nixpkgs; use host installs.
devShells.agents =
let
agentPkgs = import nixpkgs {
inherit system;
config.allowUnfreePredicate = pkg: lib.getName pkg == "claude-code";
};
in
self.devShells.${system}.default.overrideAttrs (old: {
shellHook = (old.shellHook or "") + ''
export PATH="$PATH:${lib.makeBinPath (with agentPkgs; [ claude-code codex opencode ])}"
'';
});
devShells.default = pkgs.mkShell {
inputsFrom = [ idxPackage ];
packages = with pkgs; [