fix(ownership): report a Cloudflare challenge as its own error and result

When Cloudflare answers idx.co.id with a challenge, curl fails with only
`curl: (22)`, which exits 4 like any other HTTP failure, and every profile
is tried twice. A challenge needs a browser, so no profile can solve it. The
source is monthly, so the publisher only needs one run per month to get
through: treat a challenge as "retry tomorrow" and alert on stale data.

- Add IdxError::CloudflareChallenge (CLOUDFLARECHALLENGE, exit 6), detected
  from the cf-mitigated header. curl reports it on stderr through --write-out,
  because --fail drops the response.
- A URL with a challenge gets one round of profiles; other failures keep two.
  This also covers `stocks flow --source idx`, which now exits 6 on a challenge.
- -v logs which profile fetched each URL.
- Discovery keeps error types, so a challenge survives an announcement miss.
  A source answering with something unparseable still is not called a challenge.
- `ownership discover --require-data-page` fails when the data page cannot be
  read. Default discover is unchanged: it warns and lists older announcements.
- The publisher and builder use that flag, so a failed data page can no longer
  end in a false "up-to-date". A challenge ends in `RESULT: blocked stage=<stage>`,
  exit 6; every other failure stays `RESULT: FAILED`.
- The systemd example treats exit 6 as success and runs the freshness check, so
  the unit fails on FAILED or on stale data.
- live-smoke reports a challenge as CHALLENGED, a failure and never a pass.

Golden change: completions-bash.golden gains `--require-data-page` in the
discover options, one word on one line. It is the direct result of the new flag;
no other golden file changes.

Live check on 2026-11-05 against release/v0.5.0: 19 passed, 2 failed. Both
failures are real challenges (flow-idx, discover), each exit 6 with empty stdout.
The import, releases and ticker cases did not run, because discover did not pass.

Refs #83

scripts/build-latest-ownership-snapshot.sh

@@ -171,7 +171,11 @@ MANIFEST_PATH="$OUTPUT_DIR/ownership-snapshot-manifest.json"
 DB_PATH="$XDG_DATA_HOME/idx/ownership.db"
 
 printf 'Discovering latest supported IDX/KSEI above-1%% report...\n'
-"$IDX_BIN" -o json ownership discover --family above1 --limit 50 > "$DISCOVERY_JSON"
+# -v logs which curl profile passed. --require-data-page: an announcement-only
+# answer can miss the newest report, so it must not be built as if current.
+# A Cloudflare challenge exits 6 here and on import; the publisher relies on that
+# status reaching it, so nothing below may catch these failures.
+"$IDX_BIN" -v -o json ownership discover --family above1 --limit 50 --require-data-page > "$DISCOVERY_JSON"
 
 # NUL-separated, so a field containing newlines (a report title) cannot
 # shift the fields after it. Written to a file: command substitution drops
@@ -230,12 +234,12 @@ if (( HISTORY > 0 )); then
     fi
     printf 'Importing %d earlier monthly report(s) for history...\n' "${#history_urls[@]}"
     for url in "${history_urls[@]}"; do
-        "$IDX_BIN" ownership import --url "$url"
+        "$IDX_BIN" -v ownership import --url "$url"
     done
 fi
 
 printf 'Importing discovered report into isolated DB...\n'
-"$IDX_BIN" ownership import --url "$DISCOVERED_PDF_URL"
+"$IDX_BIN" -v ownership import --url "$DISCOVERED_PDF_URL"
 
 printf 'Inspecting imported release metadata...\n'
 "$IDX_BIN" -o json ownership releases > "$RELEASES_JSON"

scripts/live-smoke.sh

@@ -162,6 +162,13 @@ run_case() {
     read -r -a args <<< "$cmd"
     run_bin new "$BIN_PATH" "$log" "${args[@]}"
     status=$?
+    # A challenge is neither a pass nor drift, and asking the old binary as well
+    # would only add requests while the site is challenging this client.
+    if grep -q 'CLOUDFLARECHALLENGE' "$log.stderr"; then
+        FAILURES+=("$name: CHALLENGED, Cloudflare answered with a challenge (see $log.stderr)")
+        echo "     CHALLENGED (Cloudflare)"
+        return
+    fi
     if [[ -n "$COMPARE_BIN" ]]; then
         run_bin old "$COMPARE_BIN" "$log.old" "${args[@]}"
         old_status=$?

scripts/publish-ownership-snapshot-test.sh

@@ -151,9 +151,21 @@ if [[ "${1:-}" == "version" ]]; then
     exit 0
 fi
 if [[ "$*" == *"ownership discover"* ]]; then
+    if [[ "$*" != *"--require-data-page"* ]]; then
+        echo "fake idx: discovery must require the data page" >&2
+        exit 99
+    fi
+    if [[ -n "${FAKE_DISCOVER_EXIT:-}" ]]; then
+        echo "fake idx: discovery failed with ${FAKE_DISCOVER_EXIT}" >&2
+        exit "$FAKE_DISCOVER_EXIT"
+    fi
     printf '%s\n' "${FAKE_DISCOVER_JSON:?FAKE_DISCOVER_JSON not set}"
     exit 0
 fi
+if [[ "$*" == *"ownership import"* && -n "${FAKE_IMPORT_EXIT:-}" ]]; then
+    echo "fake idx: import failed with ${FAKE_IMPORT_EXIT}" >&2
+    exit "$FAKE_IMPORT_EXIT"
+fi
 echo "fake idx: unexpected args: $*" >&2
 exit 99
 FAKE_IDX
@@ -182,6 +194,10 @@ while [[ $# -gt 0 ]]; do
 done
 [[ -n "$out" ]] || { echo "stub builder: missing --output-dir" >&2; exit 2; }
 printf 'history=%s max_bytes=%s\n' "$history" "$max_bytes" >>"${FAKE_BUILDER_CALLS:?}"
+if [[ -n "${FAKE_BUILDER_EXIT:-}" ]]; then
+    echo "stub builder: failed with ${FAKE_BUILDER_EXIT}" >&2
+    exit "$FAKE_BUILDER_EXIT"
+fi
 if [[ "${FAKE_BUILDER_TOO_LARGE:-0}" == "1" ]]; then
     echo "snapshot is 15290368 bytes (6 monthly releases), above --max-snapshot-bytes $max_bytes; lower --history" >&2
     exit 1
@@ -625,6 +641,82 @@ else
     fail "--keep-workdir prints nothing after RESULT (trailing='$after_result')"
 fi
 
+# ===========================================================================
+# 9c. a Cloudflare challenge is "blocked"; every other failure stays FAILED
+# ===========================================================================
+reset_remote
+run_publisher
+manifest_before="$(cat "$state/remote/assets/ownership-snapshot-manifest.json")"
+calls_before="$(builder_calls)"
+
+export FAKE_DISCOVER_EXIT="6"
+run_publisher
+unset FAKE_DISCOVER_EXIT
+if [[ "$PUBLISH_STATUS" == "6" ]] && [[ "$PUBLISH_LAST" == "RESULT: blocked stage=discover" ]]; then
+    pass "challenged discovery is blocked with its own exit status"
+else
+    fail "challenged discovery is blocked (status=$PUBLISH_STATUS last='$PUBLISH_LAST')"
+fi
+if [[ "$(builder_calls)" == "$calls_before" ]]; then
+    pass "challenged discovery does not run the builder"
+else
+    fail "challenged discovery does not run the builder"
+fi
+
+export FAKE_DISCOVER_EXIT="4" FAKE_BUILDER_EXIT="4"
+run_publisher
+unset FAKE_DISCOVER_EXIT FAKE_BUILDER_EXIT
+if [[ "$PUBLISH_STATUS" == "4" ]] && [[ "$PUBLISH_LAST" == "RESULT: FAILED stage=build-snapshot (exit 4)" ]]; then
+    pass "a failed data page is FAILED, never up-to-date"
+else
+    fail "a failed data page is FAILED (status=$PUBLISH_STATUS last='$PUBLISH_LAST')"
+fi
+
+export FAKE_BUILDER_EXIT="6"
+run_publisher --force
+unset FAKE_BUILDER_EXIT
+if [[ "$PUBLISH_STATUS" == "6" ]] && [[ "$PUBLISH_LAST" == "RESULT: blocked stage=build-snapshot" ]]; then
+    pass "a challenge during the build is blocked"
+else
+    fail "a challenge during the build is blocked (status=$PUBLISH_STATUS last='$PUBLISH_LAST')"
+fi
+
+# A real parser failure from the build (idx exits 1) stays loud.
+export FAKE_BUILDER_EXIT="1"
+run_publisher --force
+unset FAKE_BUILDER_EXIT
+if [[ "$PUBLISH_STATUS" == "1" ]] && [[ "$PUBLISH_LAST" == "RESULT: FAILED stage=build-snapshot (exit 1)" ]]; then
+    pass "a parser failure during the build is FAILED"
+else
+    fail "a parser failure during the build is FAILED (status=$PUBLISH_STATUS last='$PUBLISH_LAST')"
+fi
+
+if [[ "$(cat "$state/remote/assets/ownership-snapshot-manifest.json")" == "$manifest_before" ]]; then
+    pass "blocked and failed runs leave the published manifest alone"
+else
+    fail "blocked and failed runs leave the published manifest alone"
+fi
+
+# The real builder has to hand idx's exit status back to the publisher.
+real_builder_status() {
+    local status=0
+    FAKE_DISCOVER_JSON='[{"family":"above_one_percent","status":"supported","listing_page_url":"https://idx.test/","query_url":"https://idx.test/","pdf_url":"https://idx.test/a.xlsx","title":"t","publish_date":"2026-09-02","original_filename":"a.xlsx","format":"xlsx","as_of_date":"2026-08-31"}]' \
+        bash "$repo_root/scripts/build-latest-ownership-snapshot.sh" \
+        --idx-bin "$bin_dir/idx" --output-dir "$work/real-build" --history 0 >/dev/null 2>&1 || status=$?
+    printf '%s' "$status"
+}
+for case in "FAKE_DISCOVER_EXIT 6" "FAKE_IMPORT_EXIT 6" "FAKE_DISCOVER_EXIT 4" "FAKE_IMPORT_EXIT 1"; do
+    read -r var code <<<"$case"
+    export "$var=$code"
+    got="$(real_builder_status)"
+    unset "$var"
+    if [[ "$got" == "$code" ]]; then
+        pass "real builder keeps exit $code from $var"
+    else
+        fail "real builder keeps exit $code from $var (got $got)"
+    fi
+done
+
 # ===========================================================================
 # 10. auth/network errors never become a trustworthy up-to-date
 # ===========================================================================

scripts/publish-ownership-snapshot.sh

@@ -23,8 +23,13 @@ artifact is missing, and a retry repairs the publication.
 The last line is always one of:
   RESULT: published <as-of>
   RESULT: up-to-date <as-of>
+  RESULT: blocked stage=<stage>
   RESULT: FAILED stage=<stage> (exit <code>)
 
+A Cloudflare challenge from IDX is "blocked", exit 6: nothing is wrong with the
+run, so retry on the next schedule and alert on stale data instead. Every other
+failure is FAILED.
+
 Options:
   --idx-bin <path>       idx binary to use (default: ./target/debug/idx)
   --output-dir <dir>     Directory to write the copied snapshot and manifest
@@ -66,6 +71,14 @@ fail() {
     exit "$status"
 }
 
+# idx's exit status for a Cloudflare challenge.
+BLOCKED_STATUS=6
+
+blocked() {
+    printf 'RESULT: blocked stage=%s\n' "$STAGE" >&2
+    exit "$BLOCKED_STATUS"
+}
+
 # Unexpected command failures (set -e). errtrace is deliberately off: with
 # it, a failure inside $(...) would run this in the subshell and the parent.
 on_error() {
@@ -291,8 +304,14 @@ remote_asset_sha256() {
 # asked for more history than the source has does not rebuild forever.
 # Returns non-zero when discovery is unusable; callers then build.
 compute_requested_coverage() {
-    local json latest available wanted
-    if ! json="$("$IDX_BIN" -o json ownership discover --family above1 --limit 50 2>/dev/null)"; then
+    local json latest available wanted status=0
+    # --require-data-page: the newest reports are only on the data page, so
+    # without it a failed page would read as "up-to-date" against older results.
+    json="$("$IDX_BIN" -v -o json ownership discover --family above1 --limit 50 --require-data-page)" || status=$?
+    if (( status == BLOCKED_STATUS )); then
+        blocked
+    fi
+    if (( status != 0 )); then
         return 1
     fi
 
@@ -410,7 +429,14 @@ if [[ "$KEEP_WORKDIR" == "1" ]]; then
 fi
 
 printf 'Preparing latest ownership snapshot artifacts...\n'
-"$BUILDER" "${build_args[@]}"
+build_status=0
+"$BUILDER" "${build_args[@]}" || build_status=$?
+if (( build_status == BLOCKED_STATUS )); then
+    blocked
+fi
+if (( build_status != 0 )); then
+    fail "$build_status" "snapshot build failed"
+fi
 
 STAGED_MANIFEST_PATH="$PUBLISH_WORKDIR/ownership-snapshot-manifest.json"
 if [[ ! -f "$STAGED_MANIFEST_PATH" ]]; then