test(mcp): agent smoke test across five coding agents, tool snapshot

- tests/agents/agent-smoke.sh runs tests/agents/cases.json through
  Claude Code, Codex, pi, Hermes Agent, and opencode headless against a
  local idx-mcp build, each with an isolated config, and checks the
  tools called, their key arguments, and (mock data) facts in the
  answer. Opt-in; never part of the gate.
- crates/idx-mcp/tests/golden/tools.json pins tool names, descriptions,
  and input schemas, so changes to what agents see are reviewed.
- nix develop .#agents adds Claude Code, Codex, and opencode from
  nixpkgs after the host PATH.

First run: 30/30 (5 agents x 6 cases, mock data).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

AGENTS.md

@@ -22,6 +22,7 @@ crates/idx-cli/             binary `idx`
 └── tests/                  cli.rs (behavior), golden.rs + golden/ (output parity)
 crates/idx-mcp/             binary `idx-mcp`: stdio MCP tools (server.rs), inputs (params.rs)
 └── tests/mcp.rs            JSON-RPC over stdio against the mock provider
+tests/agents/               opt-in agent smoke test: real coding agents against idx-mcp
 scripts/                    smoke, installer, and ownership snapshot publisher
 ```
 
@@ -52,4 +53,5 @@ Docs:
 nix flake check -L                       # canonical gate (fmt, clippy, tests, package)
 nix develop --command cargo test <name>  # focused loop
 scripts/live-smoke.sh --compare <old>    # live drift check, A/B against an older build
+tests/agents/agent-smoke.sh              # after changing idx-mcp tools: real agents, opt-in
 ```

crates/idx-mcp/tests/golden/tools.json

@@ -0,0 +1,333 @@
+[
+  {
+    "description": "Analyze one IDX ticker: technical signals (trend, RSI, MACD, moving averages) and fundamental growth, valuation, and risk ratings. Analyses that fail are listed under `errors`; the call fails only if every analysis fails.",
+    "inputSchema": {
+      "$defs": {
+        "AnalysisKind": {
+          "enum": [
+            "technical",
+            "growth",
+            "valuation",
+            "risk"
+          ],
+          "type": "string"
+        }
+      },
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "properties": {
+        "kinds": {
+          "description": "Analyses to run (default: all).",
+          "items": {
+            "$ref": "#/$defs/AnalysisKind"
+          },
+          "type": [
+            "array",
+            "null"
+          ]
+        },
+        "symbol": {
+          "description": "IDX ticker, e.g. \"BBCA\".",
+          "type": "string"
+        }
+      },
+      "required": [
+        "symbol"
+      ],
+      "type": "object"
+    },
+    "name": "analyze_stock"
+  },
+  {
+    "description": "Compare fundamental reports (growth, valuation, risk) across IDX tickers. Tickers that fail are listed under `failures`.",
+    "inputSchema": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "properties": {
+        "symbols": {
+          "description": "IDX tickers, e.g. [\"BBCA\", \"BBRI\"]; the `.JK` suffix is optional.",
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        }
+      },
+      "required": [
+        "symbols"
+      ],
+      "type": "object"
+    },
+    "name": "compare_stocks"
+  },
+  {
+    "description": "Company data for one IDX ticker: profile, financial statements, earnings, analyst sentiment, insights, and news. Sections that fail are listed under `errors`; the call fails only if every section fails.",
+    "inputSchema": {
+      "$defs": {
+        "CompanySection": {
+          "enum": [
+            "profile",
+            "financials",
+            "earnings",
+            "sentiment",
+            "insights",
+            "news"
+          ],
+          "type": "string"
+        }
+      },
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "properties": {
+        "sections": {
+          "description": "Sections to fetch (default: all).",
+          "items": {
+            "$ref": "#/$defs/CompanySection"
+          },
+          "type": [
+            "array",
+            "null"
+          ]
+        },
+        "symbol": {
+          "description": "IDX ticker, e.g. \"BBCA\".",
+          "type": "string"
+        }
+      },
+      "required": [
+        "symbol"
+      ],
+      "type": "object"
+    },
+    "name": "get_company"
+  },
+  {
+    "description": "Every IDX holding (above 1%) of one investor, matched by name.",
+    "inputSchema": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "properties": {
+        "entity": {
+          "description": "Investor name or part of it, e.g. \"DWIMURIA\"; the closest match is used.",
+          "type": "string"
+        }
+      },
+      "required": [
+        "entity"
+      ],
+      "type": "object"
+    },
+    "name": "get_investor_holdings"
+  },
+  {
+    "description": "Holder changes (new, exit, increase, decrease) between two KSEI release dates, optionally for one ticker.",
+    "inputSchema": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "properties": {
+        "from": {
+          "description": "Earlier release date, YYYY-MM-DD (see ownership_status).",
+          "type": "string"
+        },
+        "limit": {
+          "default": null,
+          "description": "Maximum rows returned (default 200); `total` reports the full count.",
+          "format": "uint",
+          "minimum": 0,
+          "type": [
+            "integer",
+            "null"
+          ]
+        },
+        "ticker": {
+          "default": null,
+          "description": "Only this ticker's changes.",
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "to": {
+          "description": "Later release date, YYYY-MM-DD.",
+          "type": "string"
+        }
+      },
+      "required": [
+        "from",
+        "to"
+      ],
+      "type": "object"
+    },
+    "name": "get_ownership_changes"
+  },
+  {
+    "description": "OHLCV price history for one IDX ticker.",
+    "inputSchema": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "properties": {
+        "interval": {
+          "default": null,
+          "description": "Candle size (default \"1d\").",
+          "enum": [
+            "1d",
+            "1wk",
+            "1mo"
+          ],
+          "type": "string"
+        },
+        "period": {
+          "default": null,
+          "description": "Lookback window (default \"3mo\").",
+          "enum": [
+            "1d",
+            "5d",
+            "1mo",
+            "3mo",
+            "6mo",
+            "1y",
+            "2y",
+            "5y"
+          ],
+          "type": "string"
+        },
+        "symbol": {
+          "description": "IDX ticker, e.g. \"BBCA\".",
+          "type": "string"
+        }
+      },
+      "required": [
+        "symbol"
+      ],
+      "type": "object"
+    },
+    "name": "get_price_history"
+  },
+  {
+    "description": "Latest quotes (price, change, volume, 52-week range, market cap) for one or more IDX tickers.",
+    "inputSchema": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "properties": {
+        "symbols": {
+          "description": "IDX tickers, e.g. [\"BBCA\", \"BBRI\"]; the `.JK` suffix is optional.",
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        }
+      },
+      "required": [
+        "symbols"
+      ],
+      "type": "object"
+    },
+    "name": "get_quotes"
+  },
+  {
+    "description": "Shareholders above 1% of an IDX ticker from the latest KSEI release, with concentration metrics (top1, top3, HHI, free float).",
+    "inputSchema": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "properties": {
+        "ticker": {
+          "description": "IDX ticker, e.g. \"BBCA\".",
+          "type": "string"
+        }
+      },
+      "required": [
+        "ticker"
+      ],
+      "type": "object"
+    },
+    "name": "get_shareholders"
+  },
+  {
+    "description": "Ownership database status: whether it has data, and the imported KSEI release dates.",
+    "inputSchema": {
+      "properties": {},
+      "type": "object"
+    },
+    "name": "ownership_status"
+  },
+  {
+    "description": "Download or refresh the KSEI ownership snapshot (several MB) into the local database.",
+    "inputSchema": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "properties": {
+        "force": {
+          "default": false,
+          "description": "Re-download even when the local snapshot is current.",
+          "type": "boolean"
+        }
+      },
+      "type": "object"
+    },
+    "name": "ownership_sync"
+  },
+  {
+    "description": "Run a stock screen (top/worst performers, high dividend, low P/E, 52-week highs/lows, high volume, large cap).",
+    "inputSchema": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "properties": {
+        "filter": {
+          "default": null,
+          "description": "Screen to run (default \"top-performers\").",
+          "enum": [
+            "top-performers",
+            "worst-performers",
+            "high-dividend",
+            "low-pe",
+            "52w-high",
+            "52w-low",
+            "high-volume",
+            "large-cap"
+          ],
+          "type": "string"
+        },
+        "limit": {
+          "default": null,
+          "description": "Maximum results (default 50).",
+          "format": "uint",
+          "minimum": 0,
+          "type": [
+            "integer",
+            "null"
+          ]
+        },
+        "region": {
+          "default": null,
+          "description": "Market region (default \"id\").",
+          "enum": [
+            "id",
+            "us",
+            "sg",
+            "hk",
+            "jp"
+          ],
+          "type": "string"
+        }
+      },
+      "type": "object"
+    },
+    "name": "screen_stocks"
+  },
+  {
+    "description": "Search investor names in the KSEI ownership database.",
+    "inputSchema": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "properties": {
+        "limit": {
+          "default": null,
+          "description": "Maximum results (default 20).",
+          "format": "uint",
+          "minimum": 0,
+          "type": [
+            "integer",
+            "null"
+          ]
+        },
+        "query": {
+          "description": "Full-text query over investor names.",
+          "type": "string"
+        }
+      },
+      "required": [
+        "query"
+      ],
+      "type": "object"
+    },
+    "name": "search_investors"
+  }
+]

crates/idx-mcp/tests/mcp.rs

@@ -181,6 +181,31 @@ fn lists_grouped_tools_and_hides_sync_unless_allowed() {
     assert_eq!(client.tool_names(), expected);
 }
 
+/// What agents see (names, descriptions, input schemas) is part of the
+/// contract: changes must show up as a reviewed diff. Regenerate with
+/// `UPDATE_GOLDEN=1`.
+#[test]
+fn tool_definitions_match_golden() {
+    let mut client = Client::start(&["--allow-sync"], false);
+    let mut tools = client.request("tools/list", json!({}))["tools"]
+        .as_array()
+        .unwrap()
+        .clone();
+    tools.sort_by(|a, b| a["name"].as_str().cmp(&b["name"].as_str()));
+    let rendered = serde_json::to_string_pretty(&tools).unwrap() + "\n";
+
+    let golden = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/golden/tools.json");
+    if std::env::var_os("UPDATE_GOLDEN").is_some() {
+        std::fs::create_dir_all(golden.parent().unwrap()).unwrap();
+        std::fs::write(&golden, &rendered).unwrap();
+    }
+    let expected = std::fs::read_to_string(&golden).expect("tests/golden/tools.json");
+    assert!(
+        rendered == expected,
+        "tool definitions changed; review and rerun with UPDATE_GOLDEN=1:\n{rendered}"
+    );
+}
+
 #[test]
 fn live_tools_wrap_data_with_freshness() {
     let mut client = Client::start(&[], false);

docs/SMOKE.md

@@ -8,7 +8,7 @@ offline with pinned tools:
 | Check | Covers |
 | --- | --- |
 | `fmt`, `clippy` | formatting and lints (`--all-targets`) |
-| `test` | unit/module tests, `tests/cli.rs` CLI integration tests, and `tests/golden.rs` output parity |
+| `test` | unit/module tests, `tests/cli.rs` CLI integration tests, `tests/golden.rs` output parity, and `crates/idx-mcp/tests/mcp.rs` MCP stdio tests |
 | `default`, `package` | release build and `cargo package` verification |
 | `packaged-cli` | the wrapped Nix release binary emits JSON, generates completions, and returns a JSON `CACHEMISS` error with exit 3 offline |
 | `ownership-publisher`, `publisher-test` | packaged publisher entry point and its regression scenarios |
@@ -25,6 +25,11 @@ user-visible output changed. Regenerate with
 `UPDATE_GOLDEN=1 cargo test --test golden` only when the change is intended,
 and commit the golden diff with the reason.
 
+`crates/idx-mcp/tests/mcp.rs` drives `idx-mcp` over stdio JSON-RPC with the
+mock provider. `tests/golden/tools.json` there pins what agents see (tool
+names, descriptions, input schemas); regenerate it the same way after an
+intended change.
+
 For a focused loop, run one check or filter Cargo tests:
 
 ```bash
@@ -66,3 +71,35 @@ and keeps per-case `.stdout`, `.stderr`, and `.validation` logs there.
 
 `scripts/audit-msn-fundamentals.sh` is a heavier sweep of valuation output
 across the whole IDX MSN symbol map, for provider-health audits.
+
+## Agent smoke test
+
+The MCP tests prove the protocol works; they cannot show whether a real
+coding agent picks the right tools. `tests/agents/agent-smoke.sh` runs each
+case in `tests/agents/cases.json` through Claude Code, Codex, pi, Hermes Agent,
+and opencode in headless mode against a local `idx-mcp` build, then checks
+the tools each agent called (and their key arguments) and, with mock data, key
+facts in the answer. It is opt-in: it needs agent logins and model credits,
+and agent behavior varies between runs, so it is never part of the gate.
+
+```bash
+nix develop .#agents --command tests/agents/agent-smoke.sh            # every agent found
+nix develop .#agents --command tests/agents/agent-smoke.sh --agents claude,codex --runs 3
+tests/agents/agent-smoke.sh --live --case shareholders --keep          # real data
+```
+
+- Each agent gets an isolated config that registers only the local server;
+  your own agent settings and MCP servers are not loaded or changed. Logins
+  are copied from the agent's home (Codex `auth.json`, pi and Hermes auth
+  files) into a temp dir that is deleted afterwards unless `--keep`.
+- Tools are auto-approved for the run (Claude `dontAsk` with
+  `mcp__idx__*`, Codex `default_tools_approval_mode="approve"`); Claude and pi
+  also get no built-in tools, so they can only answer through MCP.
+- `IDX_AGENT_MODEL_<AGENT>` picks a model (for example
+  `IDX_AGENT_MODEL_CLAUDE=sonnet`). `IDX_AGENT_EXEC_PREFIX` prefixes every
+  agent command, for hosts that inject API keys through a wrapper.
+- The run fails only when an agent fails a case in every run; use `--runs` to
+  measure pass rates. `--keep` keeps raw agent output and `results.jsonl`.
+- `nix develop .#agents` adds Claude Code, Codex, and opencode from nixpkgs
+  after your PATH, so installed versions win. pi and Hermes Agent must be
+  installed on the host.

flake.nix

@@ -280,6 +280,7 @@
             shellcheck -s sh ${installerSrc}/install.sh
             shellcheck ${installerSrc}/scripts/install-sh-test.sh
             shellcheck ${./scripts/live-smoke.sh}
+            shellcheck ${./tests/agents/agent-smoke.sh}
             touch "$out"
           '';
           publisher-test = pkgs.runCommand "idx-publisher-test" {
@@ -296,6 +297,23 @@
           '';
         };
 
+        # `nix develop .#agents`: the default shell plus coding-agent CLIs for
+        # tests/agents/agent-smoke.sh. They are appended to PATH, so agents
+        # already installed (and logged in) on the host take precedence.
+        # pi and Hermes Agent are not in the pinned nixpkgs; use host installs.
+        devShells.agents =
+          let
+            agentPkgs = import nixpkgs {
+              inherit system;
+              config.allowUnfreePredicate = pkg: lib.getName pkg == "claude-code";
+            };
+          in
+          self.devShells.${system}.default.overrideAttrs (old: {
+            shellHook = (old.shellHook or "") + ''
+              export PATH="$PATH:${lib.makeBinPath (with agentPkgs; [ claude-code codex opencode ])}"
+            '';
+          });
+
         devShells.default = pkgs.mkShell {
           inputsFrom = [ idxPackage ];
           packages = with pkgs; [

tests/agents/agent-smoke.sh

@@ -0,0 +1,304 @@
+#!/usr/bin/env bash
+# Agent smoke test: runs real coding agents headless against a local idx-mcp
+# build and checks which MCP tools they call. Opt-in: it needs agent logins
+# and model credits, and agent behavior is nondeterministic. It is not part
+# of `nix flake check`. See docs/SMOKE.md.
+set -Eeuo pipefail
+
+usage() {
+    cat <<'EOF'
+Usage: tests/agents/agent-smoke.sh [options]
+
+Runs each case in tests/agents/cases.json through each agent, with an
+isolated agent config that registers only the local idx-mcp, and checks the
+tools called (and, with mock data, key facts in the answer).
+
+Options:
+  --agents <list>   Comma-separated: claude,codex,pi,hermes,opencode
+                    (default: every one found on PATH)
+  --case <id>       Run only this case (repeatable)
+  --runs <n>        Runs per agent and case (default: 1)
+  --live            Real upstream data and the published ownership snapshot
+                    instead of mock fixtures; answer text is not checked
+  --keep            Keep the work directory (raw agent output, results.jsonl)
+  --help            Show this help
+
+Environment:
+  IDX_AGENT_EXEC_PREFIX   Command prefix for every agent, e.g. a secrets
+                          wrapper: "with-ai-secrets ai-tools"
+  IDX_AGENT_MODEL_<AGENT> Model override, e.g. IDX_AGENT_MODEL_CLAUDE=sonnet,
+                          IDX_AGENT_MODEL_OPENCODE=opencode-go/glm-5.3
+  IDX_AGENT_TIMEOUT       Seconds per agent run (default: 300)
+
+Exit status: 0 when every agent passed every case at least once, 1 when some
+agent failed a case in all runs, 2 on usage or setup errors.
+EOF
+}
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cases_file="$repo_root/tests/agents/cases.json"
+all_agents=(claude codex pi hermes opencode)
+agents=()
+only_cases=()
+runs=1
+live=0
+keep=0
+timeout_secs="${IDX_AGENT_TIMEOUT:-300}"
+read -r -a exec_prefix <<< "${IDX_AGENT_EXEC_PREFIX:-}"
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --agents) IFS=, read -r -a agents <<< "${2:?}"; shift 2 ;;
+        --case) only_cases+=("${2:?}"); shift 2 ;;
+        --runs) runs="${2:?}"; shift 2 ;;
+        --live) live=1; shift ;;
+        --keep) keep=1; shift ;;
+        --help|-h) usage; exit 0 ;;
+        *) usage >&2; exit 2 ;;
+    esac
+done
+
+[[ "$runs" =~ ^[1-9][0-9]*$ ]] || { echo "--runs must be a positive integer" >&2; exit 2; }
+for tool in jq cargo timeout; do
+    command -v "$tool" >/dev/null || { echo "agent-smoke: $tool is required" >&2; exit 2; }
+done
+
+if [[ ${#agents[@]} -eq 0 ]]; then
+    for agent in "${all_agents[@]}"; do
+        command -v "$agent" >/dev/null && agents+=("$agent")
+    done
+fi
+[[ ${#agents[@]} -gt 0 ]] || { echo "agent-smoke: no agent CLI found on PATH" >&2; exit 2; }
+for agent in "${agents[@]}"; do
+    [[ " ${all_agents[*]} " == *" $agent "* ]] || { echo "unknown agent: $agent" >&2; exit 2; }
+    command -v "$agent" >/dev/null || { echo "agent-smoke: $agent not found on PATH" >&2; exit 2; }
+done
+
+work="$(mktemp -d "${TMPDIR:-/tmp}/idx-agent-smoke.XXXXXX")"
+cleanup() {
+    if [[ "$keep" == 1 ]]; then
+        echo "agent-smoke: kept $work"
+    elif [[ "$work" == */idx-agent-smoke.* ]]; then
+        rm -rf -- "$work"
+    fi
+}
+trap cleanup EXIT
+
+# --- idx-mcp under test ----------------------------------------------------
+
+echo "agent-smoke: building idx-mcp ($([[ $live == 1 ]] && echo live || echo mock) data)"
+features=()
+[[ "$live" == 1 ]] || features=(--features idx-core/testing)
+(cd "$repo_root" && cargo build -q --locked -p idx-cli -p idx-mcp "${features[@]}")
+idx="$repo_root/target/debug/idx"
+
+data_env=(
+    "XDG_CONFIG_HOME=$work/idx/config"
+    "XDG_CACHE_HOME=$work/idx/cache"
+    "XDG_DATA_HOME=$work/idx/data"
+)
+if [[ "$live" == 1 ]]; then
+    env "${data_env[@]}" "$idx" -o json ownership sync >/dev/null
+else
+    data_env+=(IDX_USE_MOCK_PROVIDER=1)
+    fixtures="$repo_root/crates/idx-core/tests/fixtures"
+    # The importer derives the release from the IDX file name.
+    cp "$fixtures/ksei_above1_20260831_excerpt.xlsx" "$work/peng-2026-08-00017-satu-persen.xlsx"
+    for file in "$work/peng-2026-08-00017-satu-persen.xlsx" \
+        "$fixtures/ksei_balancepos_20260130_excerpt.txt" \
+        "$fixtures/ksei_balancepos_20260227_excerpt.txt"; do
+        env "${data_env[@]}" "$idx" -o json ownership import --file "$file" >/dev/null
+    done
+fi
+
+# One launcher for every agent: isolated data dirs, and the idx-core crate
+# as working directory (the mock provider reads tests/fixtures from there).
+launcher="$work/idx-mcp"
+{
+    echo '#!/usr/bin/env bash'
+    printf 'export %q\n' "${data_env[@]}"
+    printf 'cd %q\n' "$repo_root/crates/idx-core"
+    printf 'exec %q "$@"\n' "$repo_root/target/debug/idx-mcp"
+} > "$launcher"
+chmod +x "$launcher"
+
+# --- per-agent setup, run, and tool-call extraction ---------------------------
+# Each run_<agent> writes raw output to $out.raw, the answer to $out.answer, and
+# one {"name","args"} JSON object per idx tool call to $out.calls, with names
+# normalized to the bare tool name.
+
+copy_if_present() { [[ -f "$1" ]] && cp "$1" "$2/"; return 0; }
+
+setup_claude() {
+    printf '{"mcpServers":{"idx":{"type":"stdio","command":"%s","args":[]}}}\n' "$launcher" > "$work/claude-mcp.json"
+}
+run_claude() {
+    local prompt="$1" out="$2" model=()
+    [[ -n "${IDX_AGENT_MODEL_CLAUDE:-}" ]] && model=(--model "$IDX_AGENT_MODEL_CLAUDE")
+    "${exec_prefix[@]}" claude -p "$prompt" "${model[@]}" \
+        --mcp-config "$work/claude-mcp.json" --strict-mcp-config \
+        --setting-sources project --no-session-persistence --tools "" \
+        --permission-mode dontAsk --allowedTools "mcp__idx__*" \
+        --output-format stream-json --verbose --max-turns 10 < /dev/null > "$out.raw"
+    jq -c 'select(.type=="assistant") | .message.content[]? | select(.type=="tool_use")
+           | select(.name | startswith("mcp__idx__"))
+           | {name: (.name | ltrimstr("mcp__idx__")), args: .input}' "$out.raw" > "$out.calls"
+    jq -r 'select(.type=="result") | .result // ""' "$out.raw" > "$out.answer"
+}
+
+setup_codex() {
+    mkdir -p "$work/codex"
+    copy_if_present "${CODEX_HOME:-$HOME/.codex}/auth.json" "$work/codex"
+}
+run_codex() {
+    local prompt="$1" out="$2" model=()
+    [[ -n "${IDX_AGENT_MODEL_CODEX:-}" ]] && model=(-m "$IDX_AGENT_MODEL_CODEX")
+    CODEX_HOME="$work/codex" "${exec_prefix[@]}" codex exec --json --ephemeral \
+        --skip-git-repo-check -s read-only "${model[@]}" \
+        -c "mcp_servers.idx.command=\"$launcher\"" -c 'mcp_servers.idx.args=[]' \
+        -c 'mcp_servers.idx.default_tools_approval_mode="approve"' \
+        "$prompt" < /dev/null > "$out.raw"
+    jq -c 'select(.type=="item.completed" and .item.type=="mcp_tool_call" and .item.server=="idx")
+           | select(.item.status=="completed") | {name: .item.tool, args: .item.arguments}' "$out.raw" > "$out.calls"
+    jq -r 'select(.type=="item.completed" and .item.type=="agent_message") | .item.text' "$out.raw" > "$out.answer"
+}
+
+setup_pi() {
+    local src="${PI_CODING_AGENT_DIR:-$HOME/.pi/agent}"
+    mkdir -p "$work/pi"
+    for file in auth.json models.json settings.json; do copy_if_present "$src/$file" "$work/pi"; done
+    # "direct" exposes each MCP tool by name (the default "codemode" wraps them).
+    jq -n --arg cmd "$launcher" '{mcpServers: {idx: {command: $cmd, exposure: "direct"}}}' > "$work/pi/mcp.json"
+}
+run_pi() {
+    local prompt="$1" out="$2" model=()
+    [[ -n "${IDX_AGENT_MODEL_PI:-}" ]] && model=(--model "$IDX_AGENT_MODEL_PI")
+    # No built-in tools, so the agent can only reach the data through MCP.
+    PI_CODING_AGENT_DIR="$work/pi" "${exec_prefix[@]}" pi -p --mode json --no-session \
+        --no-builtin-tools --no-skills --no-context-files "${model[@]}" "$prompt" < /dev/null > "$out.raw"
+    jq -c 'select(.type=="tool_execution_start") | select(.toolName | startswith("mcp__idx__"))
+           | {name: (.toolName | ltrimstr("mcp__idx__")), args: .args}' "$out.raw" > "$out.calls"
+    jq -r 'select(.type=="agent_end") | .messages[-1].content[]? | select(.type=="text") | .text' "$out.raw" > "$out.answer"
+}
+
+setup_hermes() {
+    local src="${HERMES_HOME:-$HOME/.hermes}"
+    mkdir -p "$work/hermes"
+    copy_if_present "$src/auth.json" "$work/hermes"
+    copy_if_present "$src/.env" "$work/hermes"
+    {
+        [[ -f "$src/config.yaml" ]] && sed -n '/^model:/,/^[a-z_]*:/p' "$src/config.yaml" | sed '$d'
+        printf 'mcp_servers:\n  idx:\n    command: %s\n    enabled: true\n' "$launcher"
+    } > "$work/hermes/config.yaml"
+}
+run_hermes() {
+    local prompt="$1" out="$2" model=() session
+    [[ -n "${IDX_AGENT_MODEL_HERMES:-}" ]] && model=(-m "$IDX_AGENT_MODEL_HERMES")
+    HERMES_HOME="$work/hermes" "${exec_prefix[@]}" hermes -z "$prompt" "${model[@]}" \
+        --usage-file "$out.usage.json" < /dev/null > "$out.answer"
+    # -z prints only the answer; tool calls come from the session export.
+    session="$(jq -r '.session_id // empty' "$out.usage.json")"
+    [[ -n "$session" ]] || return 1
+    HERMES_HOME="$work/hermes" hermes sessions export --format jsonl --session-id "$session" \
+        "$out.raw" --yes < /dev/null >/dev/null
+    # Hermes may call MCP tools directly or through its tool_call wrapper.
+    jq -c '.messages[]? | .tool_calls[]? | .function
+           | (.arguments | (try fromjson catch {})) as $args
+           | if .name == "tool_call" then ($args.calls[]? | {name, args: .arguments}) else {name, args: $args} end
+           | select(.name | startswith("mcp__idx__"))
+           | .name |= ltrimstr("mcp__idx__")' "$out.raw" > "$out.calls"
+}
+
+setup_opencode() {
+    mkdir -p "$work/opencode/cwd"
+    jq -n --arg cmd "$launcher" \
+        '{"$schema": "https://opencode.ai/config.json", mcp: {idx: {type: "local", command: [$cmd], enabled: true}}}' \
+        > "$work/opencode/cwd/opencode.json"
+}
+run_opencode() {
+    local prompt="$1" out="$2" model=() home="$work/opencode"
+    [[ -n "${IDX_AGENT_MODEL_OPENCODE:-}" ]] && model=(-m "$IDX_AGENT_MODEL_OPENCODE")
+    (cd "$home/cwd" && XDG_CONFIG_HOME="$home/config" XDG_DATA_HOME="$home/data" \
+        XDG_CACHE_HOME="$home/cache" XDG_STATE_HOME="$home/state" \
+        "${exec_prefix[@]}" opencode run --pure --format json "${model[@]}" "$prompt" < /dev/null > "$out.raw")
+    jq -c 'select(.type=="tool_use") | .part | select(.tool | startswith("idx_"))
+           | select(.state.status=="completed") | {name: (.tool | ltrimstr("idx_")), args: .state.input}' "$out.raw" > "$out.calls"
+    jq -r 'select(.type=="text") | .part.text' "$out.raw" > "$out.answer"
+}
+
+# --- checks -----------------------------------------------------------------
+
+# Prints failure reasons for one run; no output means it passed.
+check_run() {
+    local case_json="$1" out="$2"
+    jq -rn --argjson case "$case_json" --slurpfile calls "$out.calls" '
+        $case.expect_tools[] as $want
+        | if $want.any then
+            select([$calls[] | .name] | any(. as $n | $want.any | index($n)) | not)
+            | "no call to any of \($want.any | join(", "))"
+          else
+            ([$calls[] | select(.name == $want.name)]) as $matches
+            | if ($matches | length) == 0 then "no call to \($want.name)"
+              elif ($want.args // {}) as $args | ($matches | any(.args | contains($args))) | not
+              then "\($want.name) called without \($want.args | tojson): \($matches | map(.args) | tojson)"
+              else empty end
+          end'
+    if [[ "$live" == 0 ]]; then
+        local pattern
+        while IFS= read -r pattern; do
+            grep -qP -- "$pattern" "$out.answer" || echo "answer does not match /$pattern/"
+        done < <(jq -r '.expect_text[]?' <<< "$case_json")
+    fi
+}
+
+# --- run --------------------------------------------------------------------
+
+mapfile -t case_ids < <(jq -r '.[].id' "$cases_file")
+if [[ ${#only_cases[@]} -gt 0 ]]; then
+    for id in "${only_cases[@]}"; do
+        [[ " ${case_ids[*]} " == *" $id "* ]] || { echo "unknown case: $id" >&2; exit 2; }
+    done
+    case_ids=("${only_cases[@]}")
+fi
+
+results="$work/results.jsonl"
+: > "$results"
+echo "agent-smoke: agents=${agents[*]} cases=${#case_ids[@]} runs=$runs"
+for agent in "${agents[@]}"; do
+    "setup_$agent"
+    for id in "${case_ids[@]}"; do
+        case_json="$(jq -c --arg id "$id" '.[] | select(.id == $id)' "$cases_file")"
+        prompt="$(jq -r '.prompt' <<< "$case_json") Use the idx MCP tools."
+        for ((run = 1; run <= runs; run++)); do
+            out="$work/runs/$agent/$id.$run"
+            mkdir -p "$(dirname "$out")"
+            : > "$out.calls"; : > "$out.answer"
+            started=$SECONDS
+            if timeout "$timeout_secs" bash -c "set -eo pipefail; $(declare -f "run_$agent"); $(declare -p work launcher exec_prefix); run_$agent \"\$1\" \"\$2\"" _ "$prompt" "$out" 2> "$out.stderr"; then
+                failures="$(check_run "$case_json" "$out")"
+            else
+                failures="agent exited with status $? (see $out.stderr)"
+            fi
+            status=pass
+            [[ -z "$failures" ]] || status=fail
+            jq -nc --arg agent "$agent" --arg case "$id" --argjson run "$run" --arg status "$status" \
+                --arg failures "$failures" --argjson secs "$((SECONDS - started))" \
+                --slurpfile calls "$out.calls" \
+                '{agent: $agent, case: $case, run: $run, status: $status, secs: $secs,
+                  tools: [$calls[] | .name], failures: ($failures | split("\n") | map(select(. != "")))}' >> "$results"
+            printf '  %-9s %-13s run %d  %s  %3ss  %s\n' "$agent" "$id" "$run" "$status" "$((SECONDS - started))" \
+                "$(jq -r '[.[] | .name] | join(",")' <(jq -s . "$out.calls"))"
+            [[ -z "$failures" ]] || printf '      - %s\n' "${failures//$'\n'/$'\n      - '}"
+        done
+    done
+done
+
+echo
+echo "agent-smoke: pass rate per agent and case"
+jq -rs 'group_by(.agent)[] | "\(.[0].agent): " + (group_by(.case) | map("\(.[0].case) \(map(select(.status=="pass")) | length)/\(length)") | join("  "))' "$results"
+consistent_failures="$(jq -s '[group_by([.agent, .case])[] | select(all(.status == "fail"))] | length' "$results")"
+if [[ "$consistent_failures" -gt 0 ]]; then
+    echo "agent-smoke: FAIL ($consistent_failures agent/case pairs failed every run)"
+    exit 1
+fi
+echo "agent-smoke: PASS"

tests/agents/cases.json

@@ -0,0 +1,38 @@
+[
+  {
+    "id": "shareholders",
+    "prompt": "Who are the shareholders above 5% of BBCA?",
+    "expect_tools": [{"name": "get_shareholders", "args": {"ticker": "BBCA"}}],
+    "expect_text": ["(?i)dwimuria", "54[.,]9"]
+  },
+  {
+    "id": "quote",
+    "prompt": "What is the latest BBCA share price?",
+    "expect_tools": [{"name": "get_quotes", "args": {"symbols": ["BBCA"]}}],
+    "expect_text": ["(?i)bbca"]
+  },
+  {
+    "id": "compare",
+    "prompt": "Compare the fundamentals of BBCA and BBRI.",
+    "expect_tools": [{"name": "compare_stocks", "args": {"symbols": ["BBCA", "BBRI"]}}],
+    "expect_text": ["(?i)bbri"]
+  },
+  {
+    "id": "technical",
+    "prompt": "Give me a technical analysis of BBCA: trend, RSI, and MACD.",
+    "expect_tools": [{"name": "analyze_stock", "args": {"symbol": "BBCA"}}],
+    "expect_text": ["(?i)rsi"]
+  },
+  {
+    "id": "investor",
+    "prompt": "Which IDX stocks does Anthoni Salim hold above 1%?",
+    "expect_tools": [{"any": ["get_investor_holdings", "search_investors"]}],
+    "expect_text": ["(?i)bbca"]
+  },
+  {
+    "id": "changes",
+    "prompt": "Which shareholders changed between the 2026-01-30 and 2026-02-27 KSEI releases?",
+    "expect_tools": [{"name": "get_ownership_changes", "args": {"from": "2026-01-30", "to": "2026-02-27"}}],
+    "expect_text": []
+  }
+]