mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-11-07 09:45:36 +00:00
test(mcp): agent smoke test across five coding agents, tool snapshot
- tests/agents/agent-smoke.sh runs tests/agents/cases.json through Claude Code, Codex, pi, Hermes Agent, and opencode headless against a local idx-mcp build, each with an isolated config, and checks the tools called, their key arguments, and (mock data) facts in the answer. Opt-in; never part of the gate. - crates/idx-mcp/tests/golden/tools.json pins tool names, descriptions, and input schemas, so changes to what agents see are reviewed. - nix develop .#agents adds Claude Code, Codex, and opencode from nixpkgs after the host PATH. First run: 30/30 (5 agents x 6 cases, mock data). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
386a63c288
commit
9d8914b2c9
7 changed files with 758 additions and 1 deletions
18
flake.nix
18
flake.nix
|
|
@ -280,6 +280,7 @@
|
|||
shellcheck -s sh ${installerSrc}/install.sh
|
||||
shellcheck ${installerSrc}/scripts/install-sh-test.sh
|
||||
shellcheck ${./scripts/live-smoke.sh}
|
||||
shellcheck ${./tests/agents/agent-smoke.sh}
|
||||
touch "$out"
|
||||
'';
|
||||
publisher-test = pkgs.runCommand "idx-publisher-test" {
|
||||
|
|
@ -296,6 +297,23 @@
|
|||
'';
|
||||
};
|
||||
|
||||
# `nix develop .#agents`: the default shell plus coding-agent CLIs for
|
||||
# tests/agents/agent-smoke.sh. They are appended to PATH, so agents
|
||||
# already installed (and logged in) on the host take precedence.
|
||||
# pi and Hermes Agent are not in the pinned nixpkgs; use host installs.
|
||||
devShells.agents =
|
||||
let
|
||||
agentPkgs = import nixpkgs {
|
||||
inherit system;
|
||||
config.allowUnfreePredicate = pkg: lib.getName pkg == "claude-code";
|
||||
};
|
||||
in
|
||||
self.devShells.${system}.default.overrideAttrs (old: {
|
||||
shellHook = (old.shellHook or "") + ''
|
||||
export PATH="$PATH:${lib.makeBinPath (with agentPkgs; [ claude-code codex opencode ])}"
|
||||
'';
|
||||
});
|
||||
|
||||
devShells.default = pkgs.mkShell {
|
||||
inputsFrom = [ idxPackage ];
|
||||
packages = with pkgs; [
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue