diff --git a/tests/agents/agent-smoke.sh b/tests/agents/agent-smoke.sh index b8f4728..f5b4b7d 100755 --- a/tests/agents/agent-smoke.sh +++ b/tests/agents/agent-smoke.sh @@ -111,8 +111,12 @@ data_env=( "XDG_CACHE_HOME=$work/idx/cache" "XDG_DATA_HOME=$work/idx/data" ) +latest_release='' previous_release='' if [[ "$live" == 1 ]]; then env "${data_env[@]}" "$idx" -o json ownership sync >/dev/null + # Live cases name releases as {latest_release} and {previous_release}. + mapfile -t releases < <(env "${data_env[@]}" "$idx" -o json ownership releases | jq -r '.[].as_of_date') + latest_release="${releases[0]:-}" previous_release="${releases[1]:-}" else data_env+=(IDX_USE_MOCK_PROVIDER=1) fixtures="$repo_root/crates/idx-core/tests/fixtures" @@ -136,6 +140,9 @@ if [[ -n "$npx_spec" ]]; then if [[ "$npx_spec" == *.tgz && ! -f "$npx_spec" ]]; then echo "agent-smoke: tarball not found: $npx_spec" >&2; exit 2 fi + # In the dev shell target/debug is on PATH; drop it so a missing package + # bin cannot fall through to the local build. + PATH="$(tr ':' '\n' <<< "$PATH" | grep -vxF "$repo_root/target/debug" | paste -sd: -)" npx_args=(-y "$npx_spec") [[ ! -f "$npx_spec" ]] || npx_args=(-y -p "file:$(realpath "$npx_spec")" idx-mcp) server_json="$(jq -nc --argjson args "$(printf '%s\n' "${npx_args[@]}" | jq -Rnc '[inputs]')" '{command: "npx", args: $args, @@ -226,7 +233,8 @@ setup_hermes() { # Model providers can be user plugins. A symlink: cleanup removes only the link. [[ ! -d "$src/plugins" ]] || ln -s "$src/plugins" "$work/hermes/plugins" local server - server="$(jq -c '. + {enabled: true}' <<< "$server_json")" + # A cold `npx` install can outlast the default connect timeout. + server="$(jq -c '. + {enabled: true, connect_timeout: 300}' <<< "$server_json")" if jq -e 'type == "object"' "$src/config.yaml" >/dev/null 2>&1; then # Hermes also writes config.yaml as JSON (e.g. in profiles). jq --argjson s "$server" '{model, mcp_servers: {idx: $s}} | del(.. | nulls)' \ @@ -240,11 +248,15 @@ setup_hermes() { } > "$work/hermes/config.yaml" fi # A bare `npx` runs on Hermes' own Node, fetched into a fresh HERMES_HOME - # on first use (~20s); `hermes -z` waits only ~15s for MCP tools, so fetch - # it first or the first case runs without them. + # on first use (~20s), and the cold package install and binary download + # come on top. `hermes -z` waits only ~15s for MCP tools whatever + # connect_timeout says, so fetch Node and install the server first: `mcp + # test` starts it through Hermes' own npx and waits for connect_timeout. if [[ -n "$npx_spec" ]]; then HERMES_HOME="$work/hermes" "${exec_prefix[@]}" hermes pm install npm >/dev/null \ || { echo "agent-smoke: hermes pm install npm failed" >&2; exit 2; } + HERMES_HOME="$work/hermes" "${exec_prefix[@]}" hermes mcp test idx < /dev/null > "$work/hermes-warmup.log" 2>&1 \ + || echo "agent-smoke: hermes mcp test idx failed (see $work/hermes-warmup.log)" >&2 fi } run_hermes() { @@ -362,7 +374,10 @@ for agent in "${agents[@]}"; do for id in "${case_ids[@]}"; do # A case's "live" fields replace the mock ones, e.g. release dates. case_json="$(jq -c --arg id "$id" --argjson live "$live" '.[] | select(.id == $id) - | if $live == 1 then . + (.live // {}) else . end | del(.live)' "$cases_file")" + | if $live == 1 then . + (.live // {}) else . end | del(.live) + | walk(if type == "string" then gsub("\\{latest_release\\}"; $latest) + | gsub("\\{previous_release\\}"; $previous) else . end)' \ + --arg latest "$latest_release" --arg previous "$previous_release" "$cases_file")" prompt="$(jq -r '.prompt' <<< "$case_json") Use the idx MCP tools." for ((run = 1; run <= runs; run++)); do out="$work/runs/$agent/$id.$run" @@ -391,6 +406,19 @@ for agent in "${agents[@]}"; do [[ -z "$failures" ]] || printf ' - %s\n' "${failures//$'\n'/$'\n - '}" done done + if [[ -n "$npx_spec" ]]; then + # Proof that this agent's own npx installed the package binary, rather + # than the server coming from somewhere else on PATH. + status=pass failures= + if ! compgen -G "$npm_config_cache/_npx/*/node_modules/idx-mcp/bin/idx-mcp" > /dev/null; then + status=fail failures="no idx-mcp binary under $npm_config_cache/_npx" + fi + jq -nc --arg agent "$agent" --arg status "$status" --arg failures "$failures" \ + '{agent: $agent, case: "npx-install", run: 1, status: $status, secs: 0, tools: [], + failures: ([$failures] | map(select(. != "")))}' >> "$results" + printf ' %-9s %-13s %s\n' "$agent" npx-install "$status" + [[ -z "$failures" ]] || printf ' - %s\n' "$failures" + fi done echo diff --git a/tests/agents/cases.json b/tests/agents/cases.json index 0d7ac64..fda0c8e 100644 --- a/tests/agents/cases.json +++ b/tests/agents/cases.json @@ -36,7 +36,7 @@ "expect_text": [], "live": { "prompt": "Which shareholders changed between the two most recent KSEI releases?", - "expect_tools": [{"name": "get_ownership_changes"}] + "expect_tools": [{"name": "get_ownership_changes", "args": {"from": "{previous_release}", "to": "{latest_release}"}}] } } ]