diff --git a/AGENTS.md b/AGENTS.md index e3851bd..5b6a9fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,6 +22,7 @@ crates/idx-cli/ binary `idx` └── tests/ cli.rs (behavior), golden.rs + golden/ (output parity) crates/idx-mcp/ binary `idx-mcp`: stdio MCP tools (server.rs), inputs (params.rs) └── tests/mcp.rs JSON-RPC over stdio against the mock provider +tests/agents/ opt-in agent smoke test: real coding agents against idx-mcp scripts/ smoke, installer, and ownership snapshot publisher ``` @@ -52,4 +53,5 @@ Docs: nix flake check -L # canonical gate (fmt, clippy, tests, package) nix develop --command cargo test # focused loop scripts/live-smoke.sh --compare # live drift check, A/B against an older build +tests/agents/agent-smoke.sh # after changing idx-mcp tools: real agents, opt-in ``` diff --git a/crates/idx-mcp/tests/golden/tools.json b/crates/idx-mcp/tests/golden/tools.json new file mode 100644 index 0000000..b8c7505 --- /dev/null +++ b/crates/idx-mcp/tests/golden/tools.json @@ -0,0 +1,333 @@ +[ + { + "description": "Analyze one IDX ticker: technical signals (trend, RSI, MACD, moving averages) and fundamental growth, valuation, and risk ratings. Analyses that fail are listed under `errors`; the call fails only if every analysis fails.", + "inputSchema": { + "$defs": { + "AnalysisKind": { + "enum": [ + "technical", + "growth", + "valuation", + "risk" + ], + "type": "string" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "kinds": { + "description": "Analyses to run (default: all).", + "items": { + "$ref": "#/$defs/AnalysisKind" + }, + "type": [ + "array", + "null" + ] + }, + "symbol": { + "description": "IDX ticker, e.g. \"BBCA\".", + "type": "string" + } + }, + "required": [ + "symbol" + ], + "type": "object" + }, + "name": "analyze_stock" + }, + { + "description": "Compare fundamental reports (growth, valuation, risk) across IDX tickers. Tickers that fail are listed under `failures`.", + "inputSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "symbols": { + "description": "IDX tickers, e.g. [\"BBCA\", \"BBRI\"]; the `.JK` suffix is optional.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "symbols" + ], + "type": "object" + }, + "name": "compare_stocks" + }, + { + "description": "Company data for one IDX ticker: profile, financial statements, earnings, analyst sentiment, insights, and news. Sections that fail are listed under `errors`; the call fails only if every section fails.", + "inputSchema": { + "$defs": { + "CompanySection": { + "enum": [ + "profile", + "financials", + "earnings", + "sentiment", + "insights", + "news" + ], + "type": "string" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "sections": { + "description": "Sections to fetch (default: all).", + "items": { + "$ref": "#/$defs/CompanySection" + }, + "type": [ + "array", + "null" + ] + }, + "symbol": { + "description": "IDX ticker, e.g. \"BBCA\".", + "type": "string" + } + }, + "required": [ + "symbol" + ], + "type": "object" + }, + "name": "get_company" + }, + { + "description": "Every IDX holding (above 1%) of one investor, matched by name.", + "inputSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "entity": { + "description": "Investor name or part of it, e.g. \"DWIMURIA\"; the closest match is used.", + "type": "string" + } + }, + "required": [ + "entity" + ], + "type": "object" + }, + "name": "get_investor_holdings" + }, + { + "description": "Holder changes (new, exit, increase, decrease) between two KSEI release dates, optionally for one ticker.", + "inputSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "from": { + "description": "Earlier release date, YYYY-MM-DD (see ownership_status).", + "type": "string" + }, + "limit": { + "default": null, + "description": "Maximum rows returned (default 200); `total` reports the full count.", + "format": "uint", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "ticker": { + "default": null, + "description": "Only this ticker's changes.", + "type": [ + "string", + "null" + ] + }, + "to": { + "description": "Later release date, YYYY-MM-DD.", + "type": "string" + } + }, + "required": [ + "from", + "to" + ], + "type": "object" + }, + "name": "get_ownership_changes" + }, + { + "description": "OHLCV price history for one IDX ticker.", + "inputSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "interval": { + "default": null, + "description": "Candle size (default \"1d\").", + "enum": [ + "1d", + "1wk", + "1mo" + ], + "type": "string" + }, + "period": { + "default": null, + "description": "Lookback window (default \"3mo\").", + "enum": [ + "1d", + "5d", + "1mo", + "3mo", + "6mo", + "1y", + "2y", + "5y" + ], + "type": "string" + }, + "symbol": { + "description": "IDX ticker, e.g. \"BBCA\".", + "type": "string" + } + }, + "required": [ + "symbol" + ], + "type": "object" + }, + "name": "get_price_history" + }, + { + "description": "Latest quotes (price, change, volume, 52-week range, market cap) for one or more IDX tickers.", + "inputSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "symbols": { + "description": "IDX tickers, e.g. [\"BBCA\", \"BBRI\"]; the `.JK` suffix is optional.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "symbols" + ], + "type": "object" + }, + "name": "get_quotes" + }, + { + "description": "Shareholders above 1% of an IDX ticker from the latest KSEI release, with concentration metrics (top1, top3, HHI, free float).", + "inputSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "ticker": { + "description": "IDX ticker, e.g. \"BBCA\".", + "type": "string" + } + }, + "required": [ + "ticker" + ], + "type": "object" + }, + "name": "get_shareholders" + }, + { + "description": "Ownership database status: whether it has data, and the imported KSEI release dates.", + "inputSchema": { + "properties": {}, + "type": "object" + }, + "name": "ownership_status" + }, + { + "description": "Download or refresh the KSEI ownership snapshot (several MB) into the local database.", + "inputSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "force": { + "default": false, + "description": "Re-download even when the local snapshot is current.", + "type": "boolean" + } + }, + "type": "object" + }, + "name": "ownership_sync" + }, + { + "description": "Run a stock screen (top/worst performers, high dividend, low P/E, 52-week highs/lows, high volume, large cap).", + "inputSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "filter": { + "default": null, + "description": "Screen to run (default \"top-performers\").", + "enum": [ + "top-performers", + "worst-performers", + "high-dividend", + "low-pe", + "52w-high", + "52w-low", + "high-volume", + "large-cap" + ], + "type": "string" + }, + "limit": { + "default": null, + "description": "Maximum results (default 50).", + "format": "uint", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "region": { + "default": null, + "description": "Market region (default \"id\").", + "enum": [ + "id", + "us", + "sg", + "hk", + "jp" + ], + "type": "string" + } + }, + "type": "object" + }, + "name": "screen_stocks" + }, + { + "description": "Search investor names in the KSEI ownership database.", + "inputSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "limit": { + "default": null, + "description": "Maximum results (default 20).", + "format": "uint", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "query": { + "description": "Full-text query over investor names.", + "type": "string" + } + }, + "required": [ + "query" + ], + "type": "object" + }, + "name": "search_investors" + } +] diff --git a/crates/idx-mcp/tests/mcp.rs b/crates/idx-mcp/tests/mcp.rs index 9581448..555839e 100644 --- a/crates/idx-mcp/tests/mcp.rs +++ b/crates/idx-mcp/tests/mcp.rs @@ -181,6 +181,31 @@ fn lists_grouped_tools_and_hides_sync_unless_allowed() { assert_eq!(client.tool_names(), expected); } +/// What agents see (names, descriptions, input schemas) is part of the +/// contract: changes must show up as a reviewed diff. Regenerate with +/// `UPDATE_GOLDEN=1`. +#[test] +fn tool_definitions_match_golden() { + let mut client = Client::start(&["--allow-sync"], false); + let mut tools = client.request("tools/list", json!({}))["tools"] + .as_array() + .unwrap() + .clone(); + tools.sort_by(|a, b| a["name"].as_str().cmp(&b["name"].as_str())); + let rendered = serde_json::to_string_pretty(&tools).unwrap() + "\n"; + + let golden = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/golden/tools.json"); + if std::env::var_os("UPDATE_GOLDEN").is_some() { + std::fs::create_dir_all(golden.parent().unwrap()).unwrap(); + std::fs::write(&golden, &rendered).unwrap(); + } + let expected = std::fs::read_to_string(&golden).expect("tests/golden/tools.json"); + assert!( + rendered == expected, + "tool definitions changed; review and rerun with UPDATE_GOLDEN=1:\n{rendered}" + ); +} + #[test] fn live_tools_wrap_data_with_freshness() { let mut client = Client::start(&[], false); diff --git a/docs/SMOKE.md b/docs/SMOKE.md index 6512288..374abe0 100644 --- a/docs/SMOKE.md +++ b/docs/SMOKE.md @@ -8,7 +8,7 @@ offline with pinned tools: | Check | Covers | | --- | --- | | `fmt`, `clippy` | formatting and lints (`--all-targets`) | -| `test` | unit/module tests, `tests/cli.rs` CLI integration tests, and `tests/golden.rs` output parity | +| `test` | unit/module tests, `tests/cli.rs` CLI integration tests, `tests/golden.rs` output parity, and `crates/idx-mcp/tests/mcp.rs` MCP stdio tests | | `default`, `package` | release build and `cargo package` verification | | `packaged-cli` | the wrapped Nix release binary emits JSON, generates completions, and returns a JSON `CACHEMISS` error with exit 3 offline | | `ownership-publisher`, `publisher-test` | packaged publisher entry point and its regression scenarios | @@ -25,6 +25,11 @@ user-visible output changed. Regenerate with `UPDATE_GOLDEN=1 cargo test --test golden` only when the change is intended, and commit the golden diff with the reason. +`crates/idx-mcp/tests/mcp.rs` drives `idx-mcp` over stdio JSON-RPC with the +mock provider. `tests/golden/tools.json` there pins what agents see (tool +names, descriptions, input schemas); regenerate it the same way after an +intended change. + For a focused loop, run one check or filter Cargo tests: ```bash @@ -66,3 +71,35 @@ and keeps per-case `.stdout`, `.stderr`, and `.validation` logs there. `scripts/audit-msn-fundamentals.sh` is a heavier sweep of valuation output across the whole IDX MSN symbol map, for provider-health audits. + +## Agent smoke test + +The MCP tests prove the protocol works; they cannot show whether a real +coding agent picks the right tools. `tests/agents/agent-smoke.sh` runs each +case in `tests/agents/cases.json` through Claude Code, Codex, pi, Hermes Agent, +and opencode in headless mode against a local `idx-mcp` build, then checks +the tools each agent called (and their key arguments) and, with mock data, key +facts in the answer. It is opt-in: it needs agent logins and model credits, +and agent behavior varies between runs, so it is never part of the gate. + +```bash +nix develop .#agents --command tests/agents/agent-smoke.sh # every agent found +nix develop .#agents --command tests/agents/agent-smoke.sh --agents claude,codex --runs 3 +tests/agents/agent-smoke.sh --live --case shareholders --keep # real data +``` + +- Each agent gets an isolated config that registers only the local server; + your own agent settings and MCP servers are not loaded or changed. Logins + are copied from the agent's home (Codex `auth.json`, pi and Hermes auth + files) into a temp dir that is deleted afterwards unless `--keep`. +- Tools are auto-approved for the run (Claude `dontAsk` with + `mcp__idx__*`, Codex `default_tools_approval_mode="approve"`); Claude and pi + also get no built-in tools, so they can only answer through MCP. +- `IDX_AGENT_MODEL_` picks a model (for example + `IDX_AGENT_MODEL_CLAUDE=sonnet`). `IDX_AGENT_EXEC_PREFIX` prefixes every + agent command, for hosts that inject API keys through a wrapper. +- The run fails only when an agent fails a case in every run; use `--runs` to + measure pass rates. `--keep` keeps raw agent output and `results.jsonl`. +- `nix develop .#agents` adds Claude Code, Codex, and opencode from nixpkgs + after your PATH, so installed versions win. pi and Hermes Agent must be + installed on the host. diff --git a/flake.nix b/flake.nix index 9c53422..e6e6a4e 100644 --- a/flake.nix +++ b/flake.nix @@ -280,6 +280,7 @@ shellcheck -s sh ${installerSrc}/install.sh shellcheck ${installerSrc}/scripts/install-sh-test.sh shellcheck ${./scripts/live-smoke.sh} + shellcheck ${./tests/agents/agent-smoke.sh} touch "$out" ''; publisher-test = pkgs.runCommand "idx-publisher-test" { @@ -296,6 +297,23 @@ ''; }; + # `nix develop .#agents`: the default shell plus coding-agent CLIs for + # tests/agents/agent-smoke.sh. They are appended to PATH, so agents + # already installed (and logged in) on the host take precedence. + # pi and Hermes Agent are not in the pinned nixpkgs; use host installs. + devShells.agents = + let + agentPkgs = import nixpkgs { + inherit system; + config.allowUnfreePredicate = pkg: lib.getName pkg == "claude-code"; + }; + in + self.devShells.${system}.default.overrideAttrs (old: { + shellHook = (old.shellHook or "") + '' + export PATH="$PATH:${lib.makeBinPath (with agentPkgs; [ claude-code codex opencode ])}" + ''; + }); + devShells.default = pkgs.mkShell { inputsFrom = [ idxPackage ]; packages = with pkgs; [ diff --git a/tests/agents/agent-smoke.sh b/tests/agents/agent-smoke.sh new file mode 100755 index 0000000..7d5f4fc --- /dev/null +++ b/tests/agents/agent-smoke.sh @@ -0,0 +1,304 @@ +#!/usr/bin/env bash +# Agent smoke test: runs real coding agents headless against a local idx-mcp +# build and checks which MCP tools they call. Opt-in: it needs agent logins +# and model credits, and agent behavior is nondeterministic. It is not part +# of `nix flake check`. See docs/SMOKE.md. +set -Eeuo pipefail + +usage() { + cat <<'EOF' +Usage: tests/agents/agent-smoke.sh [options] + +Runs each case in tests/agents/cases.json through each agent, with an +isolated agent config that registers only the local idx-mcp, and checks the +tools called (and, with mock data, key facts in the answer). + +Options: + --agents Comma-separated: claude,codex,pi,hermes,opencode + (default: every one found on PATH) + --case Run only this case (repeatable) + --runs Runs per agent and case (default: 1) + --live Real upstream data and the published ownership snapshot + instead of mock fixtures; answer text is not checked + --keep Keep the work directory (raw agent output, results.jsonl) + --help Show this help + +Environment: + IDX_AGENT_EXEC_PREFIX Command prefix for every agent, e.g. a secrets + wrapper: "with-ai-secrets ai-tools" + IDX_AGENT_MODEL_ Model override, e.g. IDX_AGENT_MODEL_CLAUDE=sonnet, + IDX_AGENT_MODEL_OPENCODE=opencode-go/glm-5.3 + IDX_AGENT_TIMEOUT Seconds per agent run (default: 300) + +Exit status: 0 when every agent passed every case at least once, 1 when some +agent failed a case in all runs, 2 on usage or setup errors. +EOF +} + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cases_file="$repo_root/tests/agents/cases.json" +all_agents=(claude codex pi hermes opencode) +agents=() +only_cases=() +runs=1 +live=0 +keep=0 +timeout_secs="${IDX_AGENT_TIMEOUT:-300}" +read -r -a exec_prefix <<< "${IDX_AGENT_EXEC_PREFIX:-}" + +while [[ $# -gt 0 ]]; do + case "$1" in + --agents) IFS=, read -r -a agents <<< "${2:?}"; shift 2 ;; + --case) only_cases+=("${2:?}"); shift 2 ;; + --runs) runs="${2:?}"; shift 2 ;; + --live) live=1; shift ;; + --keep) keep=1; shift ;; + --help|-h) usage; exit 0 ;; + *) usage >&2; exit 2 ;; + esac +done + +[[ "$runs" =~ ^[1-9][0-9]*$ ]] || { echo "--runs must be a positive integer" >&2; exit 2; } +for tool in jq cargo timeout; do + command -v "$tool" >/dev/null || { echo "agent-smoke: $tool is required" >&2; exit 2; } +done + +if [[ ${#agents[@]} -eq 0 ]]; then + for agent in "${all_agents[@]}"; do + command -v "$agent" >/dev/null && agents+=("$agent") + done +fi +[[ ${#agents[@]} -gt 0 ]] || { echo "agent-smoke: no agent CLI found on PATH" >&2; exit 2; } +for agent in "${agents[@]}"; do + [[ " ${all_agents[*]} " == *" $agent "* ]] || { echo "unknown agent: $agent" >&2; exit 2; } + command -v "$agent" >/dev/null || { echo "agent-smoke: $agent not found on PATH" >&2; exit 2; } +done + +work="$(mktemp -d "${TMPDIR:-/tmp}/idx-agent-smoke.XXXXXX")" +cleanup() { + if [[ "$keep" == 1 ]]; then + echo "agent-smoke: kept $work" + elif [[ "$work" == */idx-agent-smoke.* ]]; then + rm -rf -- "$work" + fi +} +trap cleanup EXIT + +# --- idx-mcp under test ---------------------------------------------------- + +echo "agent-smoke: building idx-mcp ($([[ $live == 1 ]] && echo live || echo mock) data)" +features=() +[[ "$live" == 1 ]] || features=(--features idx-core/testing) +(cd "$repo_root" && cargo build -q --locked -p idx-cli -p idx-mcp "${features[@]}") +idx="$repo_root/target/debug/idx" + +data_env=( + "XDG_CONFIG_HOME=$work/idx/config" + "XDG_CACHE_HOME=$work/idx/cache" + "XDG_DATA_HOME=$work/idx/data" +) +if [[ "$live" == 1 ]]; then + env "${data_env[@]}" "$idx" -o json ownership sync >/dev/null +else + data_env+=(IDX_USE_MOCK_PROVIDER=1) + fixtures="$repo_root/crates/idx-core/tests/fixtures" + # The importer derives the release from the IDX file name. + cp "$fixtures/ksei_above1_20260831_excerpt.xlsx" "$work/peng-2026-08-00017-satu-persen.xlsx" + for file in "$work/peng-2026-08-00017-satu-persen.xlsx" \ + "$fixtures/ksei_balancepos_20260130_excerpt.txt" \ + "$fixtures/ksei_balancepos_20260227_excerpt.txt"; do + env "${data_env[@]}" "$idx" -o json ownership import --file "$file" >/dev/null + done +fi + +# One launcher for every agent: isolated data dirs, and the idx-core crate +# as working directory (the mock provider reads tests/fixtures from there). +launcher="$work/idx-mcp" +{ + echo '#!/usr/bin/env bash' + printf 'export %q\n' "${data_env[@]}" + printf 'cd %q\n' "$repo_root/crates/idx-core" + printf 'exec %q "$@"\n' "$repo_root/target/debug/idx-mcp" +} > "$launcher" +chmod +x "$launcher" + +# --- per-agent setup, run, and tool-call extraction --------------------------- +# Each run_ writes raw output to $out.raw, the answer to $out.answer, and +# one {"name","args"} JSON object per idx tool call to $out.calls, with names +# normalized to the bare tool name. + +copy_if_present() { [[ -f "$1" ]] && cp "$1" "$2/"; return 0; } + +setup_claude() { + printf '{"mcpServers":{"idx":{"type":"stdio","command":"%s","args":[]}}}\n' "$launcher" > "$work/claude-mcp.json" +} +run_claude() { + local prompt="$1" out="$2" model=() + [[ -n "${IDX_AGENT_MODEL_CLAUDE:-}" ]] && model=(--model "$IDX_AGENT_MODEL_CLAUDE") + "${exec_prefix[@]}" claude -p "$prompt" "${model[@]}" \ + --mcp-config "$work/claude-mcp.json" --strict-mcp-config \ + --setting-sources project --no-session-persistence --tools "" \ + --permission-mode dontAsk --allowedTools "mcp__idx__*" \ + --output-format stream-json --verbose --max-turns 10 < /dev/null > "$out.raw" + jq -c 'select(.type=="assistant") | .message.content[]? | select(.type=="tool_use") + | select(.name | startswith("mcp__idx__")) + | {name: (.name | ltrimstr("mcp__idx__")), args: .input}' "$out.raw" > "$out.calls" + jq -r 'select(.type=="result") | .result // ""' "$out.raw" > "$out.answer" +} + +setup_codex() { + mkdir -p "$work/codex" + copy_if_present "${CODEX_HOME:-$HOME/.codex}/auth.json" "$work/codex" +} +run_codex() { + local prompt="$1" out="$2" model=() + [[ -n "${IDX_AGENT_MODEL_CODEX:-}" ]] && model=(-m "$IDX_AGENT_MODEL_CODEX") + CODEX_HOME="$work/codex" "${exec_prefix[@]}" codex exec --json --ephemeral \ + --skip-git-repo-check -s read-only "${model[@]}" \ + -c "mcp_servers.idx.command=\"$launcher\"" -c 'mcp_servers.idx.args=[]' \ + -c 'mcp_servers.idx.default_tools_approval_mode="approve"' \ + "$prompt" < /dev/null > "$out.raw" + jq -c 'select(.type=="item.completed" and .item.type=="mcp_tool_call" and .item.server=="idx") + | select(.item.status=="completed") | {name: .item.tool, args: .item.arguments}' "$out.raw" > "$out.calls" + jq -r 'select(.type=="item.completed" and .item.type=="agent_message") | .item.text' "$out.raw" > "$out.answer" +} + +setup_pi() { + local src="${PI_CODING_AGENT_DIR:-$HOME/.pi/agent}" + mkdir -p "$work/pi" + for file in auth.json models.json settings.json; do copy_if_present "$src/$file" "$work/pi"; done + # "direct" exposes each MCP tool by name (the default "codemode" wraps them). + jq -n --arg cmd "$launcher" '{mcpServers: {idx: {command: $cmd, exposure: "direct"}}}' > "$work/pi/mcp.json" +} +run_pi() { + local prompt="$1" out="$2" model=() + [[ -n "${IDX_AGENT_MODEL_PI:-}" ]] && model=(--model "$IDX_AGENT_MODEL_PI") + # No built-in tools, so the agent can only reach the data through MCP. + PI_CODING_AGENT_DIR="$work/pi" "${exec_prefix[@]}" pi -p --mode json --no-session \ + --no-builtin-tools --no-skills --no-context-files "${model[@]}" "$prompt" < /dev/null > "$out.raw" + jq -c 'select(.type=="tool_execution_start") | select(.toolName | startswith("mcp__idx__")) + | {name: (.toolName | ltrimstr("mcp__idx__")), args: .args}' "$out.raw" > "$out.calls" + jq -r 'select(.type=="agent_end") | .messages[-1].content[]? | select(.type=="text") | .text' "$out.raw" > "$out.answer" +} + +setup_hermes() { + local src="${HERMES_HOME:-$HOME/.hermes}" + mkdir -p "$work/hermes" + copy_if_present "$src/auth.json" "$work/hermes" + copy_if_present "$src/.env" "$work/hermes" + { + [[ -f "$src/config.yaml" ]] && sed -n '/^model:/,/^[a-z_]*:/p' "$src/config.yaml" | sed '$d' + printf 'mcp_servers:\n idx:\n command: %s\n enabled: true\n' "$launcher" + } > "$work/hermes/config.yaml" +} +run_hermes() { + local prompt="$1" out="$2" model=() session + [[ -n "${IDX_AGENT_MODEL_HERMES:-}" ]] && model=(-m "$IDX_AGENT_MODEL_HERMES") + HERMES_HOME="$work/hermes" "${exec_prefix[@]}" hermes -z "$prompt" "${model[@]}" \ + --usage-file "$out.usage.json" < /dev/null > "$out.answer" + # -z prints only the answer; tool calls come from the session export. + session="$(jq -r '.session_id // empty' "$out.usage.json")" + [[ -n "$session" ]] || return 1 + HERMES_HOME="$work/hermes" hermes sessions export --format jsonl --session-id "$session" \ + "$out.raw" --yes < /dev/null >/dev/null + # Hermes may call MCP tools directly or through its tool_call wrapper. + jq -c '.messages[]? | .tool_calls[]? | .function + | (.arguments | (try fromjson catch {})) as $args + | if .name == "tool_call" then ($args.calls[]? | {name, args: .arguments}) else {name, args: $args} end + | select(.name | startswith("mcp__idx__")) + | .name |= ltrimstr("mcp__idx__")' "$out.raw" > "$out.calls" +} + +setup_opencode() { + mkdir -p "$work/opencode/cwd" + jq -n --arg cmd "$launcher" \ + '{"$schema": "https://opencode.ai/config.json", mcp: {idx: {type: "local", command: [$cmd], enabled: true}}}' \ + > "$work/opencode/cwd/opencode.json" +} +run_opencode() { + local prompt="$1" out="$2" model=() home="$work/opencode" + [[ -n "${IDX_AGENT_MODEL_OPENCODE:-}" ]] && model=(-m "$IDX_AGENT_MODEL_OPENCODE") + (cd "$home/cwd" && XDG_CONFIG_HOME="$home/config" XDG_DATA_HOME="$home/data" \ + XDG_CACHE_HOME="$home/cache" XDG_STATE_HOME="$home/state" \ + "${exec_prefix[@]}" opencode run --pure --format json "${model[@]}" "$prompt" < /dev/null > "$out.raw") + jq -c 'select(.type=="tool_use") | .part | select(.tool | startswith("idx_")) + | select(.state.status=="completed") | {name: (.tool | ltrimstr("idx_")), args: .state.input}' "$out.raw" > "$out.calls" + jq -r 'select(.type=="text") | .part.text' "$out.raw" > "$out.answer" +} + +# --- checks ----------------------------------------------------------------- + +# Prints failure reasons for one run; no output means it passed. +check_run() { + local case_json="$1" out="$2" + jq -rn --argjson case "$case_json" --slurpfile calls "$out.calls" ' + $case.expect_tools[] as $want + | if $want.any then + select([$calls[] | .name] | any(. as $n | $want.any | index($n)) | not) + | "no call to any of \($want.any | join(", "))" + else + ([$calls[] | select(.name == $want.name)]) as $matches + | if ($matches | length) == 0 then "no call to \($want.name)" + elif ($want.args // {}) as $args | ($matches | any(.args | contains($args))) | not + then "\($want.name) called without \($want.args | tojson): \($matches | map(.args) | tojson)" + else empty end + end' + if [[ "$live" == 0 ]]; then + local pattern + while IFS= read -r pattern; do + grep -qP -- "$pattern" "$out.answer" || echo "answer does not match /$pattern/" + done < <(jq -r '.expect_text[]?' <<< "$case_json") + fi +} + +# --- run -------------------------------------------------------------------- + +mapfile -t case_ids < <(jq -r '.[].id' "$cases_file") +if [[ ${#only_cases[@]} -gt 0 ]]; then + for id in "${only_cases[@]}"; do + [[ " ${case_ids[*]} " == *" $id "* ]] || { echo "unknown case: $id" >&2; exit 2; } + done + case_ids=("${only_cases[@]}") +fi + +results="$work/results.jsonl" +: > "$results" +echo "agent-smoke: agents=${agents[*]} cases=${#case_ids[@]} runs=$runs" +for agent in "${agents[@]}"; do + "setup_$agent" + for id in "${case_ids[@]}"; do + case_json="$(jq -c --arg id "$id" '.[] | select(.id == $id)' "$cases_file")" + prompt="$(jq -r '.prompt' <<< "$case_json") Use the idx MCP tools." + for ((run = 1; run <= runs; run++)); do + out="$work/runs/$agent/$id.$run" + mkdir -p "$(dirname "$out")" + : > "$out.calls"; : > "$out.answer" + started=$SECONDS + if timeout "$timeout_secs" bash -c "set -eo pipefail; $(declare -f "run_$agent"); $(declare -p work launcher exec_prefix); run_$agent \"\$1\" \"\$2\"" _ "$prompt" "$out" 2> "$out.stderr"; then + failures="$(check_run "$case_json" "$out")" + else + failures="agent exited with status $? (see $out.stderr)" + fi + status=pass + [[ -z "$failures" ]] || status=fail + jq -nc --arg agent "$agent" --arg case "$id" --argjson run "$run" --arg status "$status" \ + --arg failures "$failures" --argjson secs "$((SECONDS - started))" \ + --slurpfile calls "$out.calls" \ + '{agent: $agent, case: $case, run: $run, status: $status, secs: $secs, + tools: [$calls[] | .name], failures: ($failures | split("\n") | map(select(. != "")))}' >> "$results" + printf ' %-9s %-13s run %d %s %3ss %s\n' "$agent" "$id" "$run" "$status" "$((SECONDS - started))" \ + "$(jq -r '[.[] | .name] | join(",")' <(jq -s . "$out.calls"))" + [[ -z "$failures" ]] || printf ' - %s\n' "${failures//$'\n'/$'\n - '}" + done + done +done + +echo +echo "agent-smoke: pass rate per agent and case" +jq -rs 'group_by(.agent)[] | "\(.[0].agent): " + (group_by(.case) | map("\(.[0].case) \(map(select(.status=="pass")) | length)/\(length)") | join(" "))' "$results" +consistent_failures="$(jq -s '[group_by([.agent, .case])[] | select(all(.status == "fail"))] | length' "$results")" +if [[ "$consistent_failures" -gt 0 ]]; then + echo "agent-smoke: FAIL ($consistent_failures agent/case pairs failed every run)" + exit 1 +fi +echo "agent-smoke: PASS" diff --git a/tests/agents/cases.json b/tests/agents/cases.json new file mode 100644 index 0000000..37c1816 --- /dev/null +++ b/tests/agents/cases.json @@ -0,0 +1,38 @@ +[ + { + "id": "shareholders", + "prompt": "Who are the shareholders above 5% of BBCA?", + "expect_tools": [{"name": "get_shareholders", "args": {"ticker": "BBCA"}}], + "expect_text": ["(?i)dwimuria", "54[.,]9"] + }, + { + "id": "quote", + "prompt": "What is the latest BBCA share price?", + "expect_tools": [{"name": "get_quotes", "args": {"symbols": ["BBCA"]}}], + "expect_text": ["(?i)bbca"] + }, + { + "id": "compare", + "prompt": "Compare the fundamentals of BBCA and BBRI.", + "expect_tools": [{"name": "compare_stocks", "args": {"symbols": ["BBCA", "BBRI"]}}], + "expect_text": ["(?i)bbri"] + }, + { + "id": "technical", + "prompt": "Give me a technical analysis of BBCA: trend, RSI, and MACD.", + "expect_tools": [{"name": "analyze_stock", "args": {"symbol": "BBCA"}}], + "expect_text": ["(?i)rsi"] + }, + { + "id": "investor", + "prompt": "Which IDX stocks does Anthoni Salim hold above 1%?", + "expect_tools": [{"any": ["get_investor_holdings", "search_investors"]}], + "expect_text": ["(?i)bbca"] + }, + { + "id": "changes", + "prompt": "Which shareholders changed between the 2026-01-30 and 2026-02-27 KSEI releases?", + "expect_tools": [{"name": "get_ownership_changes", "args": {"from": "2026-01-30", "to": "2026-02-27"}}], + "expect_text": [] + } +]