test(agents): close ways an --npx run could pass or fail wrongly

From a Codex review of the harness:
- Drop target/debug from PATH in --npx mode (the dev shell adds it), so
  a missing package bin cannot fall through to the local build, and fail
  an agent whose npm cache holds no installed idx-mcp binary.
- The live `changes` case must diff the two latest releases, read from
  the synced snapshot, not any two dates.
- Hermes: `hermes -z` waits ~15s for MCP tools regardless of
  connect_timeout, so a slow cold install lost the first case. Warm up
  with `hermes mcp test idx`, which installs through Hermes' own npx and
  honors connect_timeout (now 300s).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V3syn2fN2LNo5pvTSqc6eh

tests/agents/agent-smoke.sh

@@ -111,8 +111,12 @@ data_env=(
     "XDG_CACHE_HOME=$work/idx/cache"
     "XDG_DATA_HOME=$work/idx/data"
 )
+latest_release='' previous_release=''
 if [[ "$live" == 1 ]]; then
     env "${data_env[@]}" "$idx" -o json ownership sync >/dev/null
+    # Live cases name releases as {latest_release} and {previous_release}.
+    mapfile -t releases < <(env "${data_env[@]}" "$idx" -o json ownership releases | jq -r '.[].as_of_date')
+    latest_release="${releases[0]:-}" previous_release="${releases[1]:-}"
 else
     data_env+=(IDX_USE_MOCK_PROVIDER=1)
     fixtures="$repo_root/crates/idx-core/tests/fixtures"
@@ -136,6 +140,9 @@ if [[ -n "$npx_spec" ]]; then
     if [[ "$npx_spec" == *.tgz && ! -f "$npx_spec" ]]; then
         echo "agent-smoke: tarball not found: $npx_spec" >&2; exit 2
     fi
+    # In the dev shell target/debug is on PATH; drop it so a missing package
+    # bin cannot fall through to the local build.
+    PATH="$(tr ':' '\n' <<< "$PATH" | grep -vxF "$repo_root/target/debug" | paste -sd: -)"
     npx_args=(-y "$npx_spec")
     [[ ! -f "$npx_spec" ]] || npx_args=(-y -p "file:$(realpath "$npx_spec")" idx-mcp)
     server_json="$(jq -nc --argjson args "$(printf '%s\n' "${npx_args[@]}" | jq -Rnc '[inputs]')" '{command: "npx", args: $args,
@@ -226,7 +233,8 @@ setup_hermes() {
     # Model providers can be user plugins. A symlink: cleanup removes only the link.
     [[ ! -d "$src/plugins" ]] || ln -s "$src/plugins" "$work/hermes/plugins"
     local server
-    server="$(jq -c '. + {enabled: true}' <<< "$server_json")"
+    # A cold `npx` install can outlast the default connect timeout.
+    server="$(jq -c '. + {enabled: true, connect_timeout: 300}' <<< "$server_json")"
     if jq -e 'type == "object"' "$src/config.yaml" >/dev/null 2>&1; then
         # Hermes also writes config.yaml as JSON (e.g. in profiles).
         jq --argjson s "$server" '{model, mcp_servers: {idx: $s}} | del(.. | nulls)' \
@@ -240,11 +248,15 @@ setup_hermes() {
         } > "$work/hermes/config.yaml"
     fi
     # A bare `npx` runs on Hermes' own Node, fetched into a fresh HERMES_HOME
-    # on first use (~20s); `hermes -z` waits only ~15s for MCP tools, so fetch
-    # it first or the first case runs without them.
+    # on first use (~20s), and the cold package install and binary download
+    # come on top. `hermes -z` waits only ~15s for MCP tools whatever
+    # connect_timeout says, so fetch Node and install the server first: `mcp
+    # test` starts it through Hermes' own npx and waits for connect_timeout.
     if [[ -n "$npx_spec" ]]; then
         HERMES_HOME="$work/hermes" "${exec_prefix[@]}" hermes pm install npm >/dev/null \
             || { echo "agent-smoke: hermes pm install npm failed" >&2; exit 2; }
+        HERMES_HOME="$work/hermes" "${exec_prefix[@]}" hermes mcp test idx < /dev/null > "$work/hermes-warmup.log" 2>&1 \
+            || echo "agent-smoke: hermes mcp test idx failed (see $work/hermes-warmup.log)" >&2
     fi
 }
 run_hermes() {
@@ -362,7 +374,10 @@ for agent in "${agents[@]}"; do
     for id in "${case_ids[@]}"; do
         # A case's "live" fields replace the mock ones, e.g. release dates.
         case_json="$(jq -c --arg id "$id" --argjson live "$live" '.[] | select(.id == $id)
-            | if $live == 1 then . + (.live // {}) else . end | del(.live)' "$cases_file")"
+            | if $live == 1 then . + (.live // {}) else . end | del(.live)
+            | walk(if type == "string" then gsub("\\{latest_release\\}"; $latest)
+                   | gsub("\\{previous_release\\}"; $previous) else . end)' \
+            --arg latest "$latest_release" --arg previous "$previous_release" "$cases_file")"
         prompt="$(jq -r '.prompt' <<< "$case_json") Use the idx MCP tools."
         for ((run = 1; run <= runs; run++)); do
             out="$work/runs/$agent/$id.$run"
@@ -391,6 +406,19 @@ for agent in "${agents[@]}"; do
             [[ -z "$failures" ]] || printf '      - %s\n' "${failures//$'\n'/$'\n      - '}"
         done
     done
+    if [[ -n "$npx_spec" ]]; then
+        # Proof that this agent's own npx installed the package binary, rather
+        # than the server coming from somewhere else on PATH.
+        status=pass failures=
+        if ! compgen -G "$npm_config_cache/_npx/*/node_modules/idx-mcp/bin/idx-mcp" > /dev/null; then
+            status=fail failures="no idx-mcp binary under $npm_config_cache/_npx"
+        fi
+        jq -nc --arg agent "$agent" --arg status "$status" --arg failures "$failures" \
+            '{agent: $agent, case: "npx-install", run: 1, status: $status, secs: 0, tools: [],
+              failures: ([$failures] | map(select(. != "")))}' >> "$results"
+        printf '  %-9s %-13s        %s\n' "$agent" npx-install "$status"
+        [[ -z "$failures" ]] || printf '      - %s\n' "$failures"
+    fi
 done
 
 echo

tests/agents/cases.json

@@ -36,7 +36,7 @@
     "expect_text": [],
     "live": {
       "prompt": "Which shareholders changed between the two most recent KSEI releases?",
-      "expect_tools": [{"name": "get_ownership_changes"}]
+      "expect_tools": [{"name": "get_ownership_changes", "args": {"from": "{previous_release}", "to": "{latest_release}"}}]
     }
   }
 ]