test(agents): run agents through npx -y idx-mcp, as users configure it

- agent-smoke `--npx <spec>` registers `npx -y <spec>` (the published package
  or an `npm pack` tarball) with a fresh npm cache, instead of the local build,
  so each agent's own npx handling and a cold install are tested.
- `nix develop .#agents-fhs` (Linux) runs agents in an FHS sandbox: Hermes
  replaces a bare `npx` with its own generic-Linux Node, which NixOS cannot
  load. `agent-fhs <cmd>` runs anything inside it.
- Hermes: only the idx toolset (with its terminal it repaired a broken
  server mid-run and passed), the Node toolchain fetched before the first
  case (`hermes -z` waits ~15s for MCP tools), JSON config.yaml (as profiles
  write it), linked plugins for plugin model providers, and a copy of its
  mcp-stderr.log per run.
- A failed run names its cause inline, since output is deleted without --keep.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WB13omP6h7YeyJx7GNtUnF
This commit is contained in:
hermes 2026-10-16 04:40:40 +00:00
commit ca218ef2ea
3 changed files with 121 additions and 30 deletions

View file

@ -160,8 +160,27 @@
./scripts/build-ownership-snapshot.sh
];
};
# `nix develop .#agents-fhs` (Linux): the agents shell, with every agent
# run inside an FHS sandbox. Hermes runs a bare `npx` with its own
# generic-Linux Node, which NixOS cannot load, so testing the
# `npx -y idx-mcp` setup users have needs this on NixOS.
# `agent-fhs <cmd>` runs any command (e.g. an interactive `hermes`) inside it.
agentsFhsShell =
let
agentFhs = pkgs.buildFHSEnv {
name = "agent-fhs";
targetPkgs = p: [ p.stdenv.cc.cc.lib p.zlib p.openssl ];
runScript = pkgs.writeShellScript "agent-fhs-run" ''exec "''${@:-bash}"'';
};
in
self.devShells.${system}.agents.overrideAttrs (old: {
shellHook = (old.shellHook or "") + ''
export PATH="${agentFhs}/bin:$PATH"
export IDX_AGENT_EXEC_PREFIX="''${IDX_AGENT_EXEC_PREFIX:+$IDX_AGENT_EXEC_PREFIX }agent-fhs"
'';
});
in
{
lib.recursiveUpdate {
packages.default = idxPackage;
packages.idx-mcp = mcpPackage;
packages.ownership-publisher = ownershipPublisher;
@ -352,6 +371,6 @@
RUST_BACKTRACE = "1";
};
};
}
} (lib.optionalAttrs pkgs.stdenv.isLinux { devShells.agents-fhs = agentsFhsShell; })
);
}