From b236c7f9d935b586749d41ca40cfe29a9630e90a Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 5 Nov 2026 08:54:58 +0000 Subject: [PATCH] fix(ownership): report a Cloudflare challenge as its own error and result When Cloudflare answers idx.co.id with a challenge, curl fails with only `curl: (22)`, which exits 4 like any other HTTP failure, and every profile is tried twice. A challenge needs a browser, so no profile can solve it. The source is monthly, so the publisher only needs one run per month to get through: treat a challenge as "retry tomorrow" and alert on stale data. - Add IdxError::CloudflareChallenge (CLOUDFLARECHALLENGE, exit 6), detected from the cf-mitigated header. curl reports it on stderr through --write-out, because --fail drops the response. - A URL with a challenge gets one round of profiles; other failures keep two. This also covers `stocks flow --source idx`, which now exits 6 on a challenge. - -v logs which profile fetched each URL. - Discovery keeps error types, so a challenge survives an announcement miss. A source answering with something unparseable still is not called a challenge. - `ownership discover --require-data-page` fails when the data page cannot be read. Default discover is unchanged: it warns and lists older announcements. - The publisher and builder use that flag, so a failed data page can no longer end in a false "up-to-date". A challenge ends in `RESULT: blocked stage=`, exit 6; every other failure stays `RESULT: FAILED`. - The systemd example treats exit 6 as success and runs the freshness check, so the unit fails on FAILED or on stale data. - live-smoke reports a challenge as CHALLENGED, a failure and never a pass. Golden change: completions-bash.golden gains `--require-data-page` in the discover options, one word on one line. It is the direct result of the new flag; no other golden file changes. Live check on 2026-11-05 against release/v0.5.0: 19 passed, 2 failed. Both failures are real challenges (flow-idx, discover), each exit 6 with empty stdout. The import, releases and ticker cases did not run, because discover did not pass. Refs #83 --- .../idx-ownership-snapshot-publish.service | 5 + crates/idx-cli/src/cli/ownership.rs | 11 +- crates/idx-cli/src/main.rs | 1 + crates/idx-cli/tests/cli.rs | 423 ++++++++++++++++++ .../tests/golden/completions-bash.golden | 2 +- crates/idx-core/src/curl_impersonate.rs | 84 +++- crates/idx-core/src/error.rs | 7 + crates/idx-core/src/investor.rs | 7 +- crates/idx-core/src/lib.rs | 12 + crates/idx-core/src/ownership/remote.rs | 77 +++- docs/ARCHITECTURE.md | 26 +- docs/DESIGN.md | 1 + docs/OWNERSHIP_PUBLISH.md | 28 +- docs/SMOKE.md | 2 + flake.nix | 1 + scripts/build-latest-ownership-snapshot.sh | 10 +- scripts/live-smoke.sh | 7 + scripts/publish-ownership-snapshot-test.sh | 92 ++++ scripts/publish-ownership-snapshot.sh | 32 +- web/src/content/docs/agents/index.md | 2 +- web/src/content/docs/cli/output.md | 1 + web/src/content/docs/cli/ownership.md | 1 + web/src/content/docs/cli/reference.md | 19 +- 23 files changed, 797 insertions(+), 54 deletions(-) diff --git a/contrib/systemd/idx-ownership-snapshot-publish.service b/contrib/systemd/idx-ownership-snapshot-publish.service index 51fb6ee..d0b9638 100644 --- a/contrib/systemd/idx-ownership-snapshot-publish.service +++ b/contrib/systemd/idx-ownership-snapshot-publish.service @@ -9,4 +9,9 @@ Environment=HOME=/var/lib/idx-ownership-snapshot EnvironmentFile=/etc/idx-ownership-snapshot.env # Build once with `nix build github:0xrsydn/idx-cli#ownership-publisher -o /opt/idx-ownership-publisher`. # Idempotent: exits 0 with "RESULT: up-to-date" when nothing newer exists. +# Exit 6 is "RESULT: blocked" (a Cloudflare challenge from IDX): tomorrow's run +# retries, so it is not a failure unless the freshness check below finds stale data. +# Wire OnFailure= to a notification: it fires on "RESULT: FAILED" and on stale data. +SuccessExitStatus=6 ExecStart=/opt/idx-ownership-publisher/bin/idx-ownership-publish --output-dir /var/lib/idx-ownership-snapshot/current --repo 0xrsydn/idx-cli --release-tag ownership-snapshot-current --history 2 +ExecStartPost=/opt/idx-ownership-publisher/bin/idx-ownership-freshness --max-age-days 40 diff --git a/crates/idx-cli/src/cli/ownership.rs b/crates/idx-cli/src/cli/ownership.rs index 125952f..696a369 100644 --- a/crates/idx-cli/src/cli/ownership.rs +++ b/crates/idx-cli/src/cli/ownership.rs @@ -14,6 +14,7 @@ use idx_core::investor::{ ConcentrationRank, HolderSource, ImportSource, Investor, parse_report_family, }; use idx_core::ownership::graph; +use idx_core::ownership::remote::DataPagePolicy; use idx_core::ownership::snapshot; use idx_core::ownership::types::{ChangeType, FlowSignal, HolderRow, OwnershipSource}; @@ -88,6 +89,9 @@ pub struct DiscoverArgs { /// Maximum number of discovered report URLs to print. #[arg(long, default_value_t = 6)] pub limit: usize, + /// Fail if the IDX data page cannot be read, instead of warning and listing older announcements. The newest above-1% reports are only on that page. + #[arg(long)] + pub require_data_page: bool, } #[derive(Debug, Args)] @@ -284,7 +288,12 @@ fn handle_discover( } let family = parse_report_family(&args.family)?; - let reports = investor.discover(family, args.limit)?; + let data_page = if args.require_data_page { + DataPagePolicy::Required + } else { + DataPagePolicy::BestEffort + }; + let reports = investor.discover(family, args.limit, data_page)?; if config.output.is_json() { return json::print_json(&reports); diff --git a/crates/idx-cli/src/main.rs b/crates/idx-cli/src/main.rs index f5a7dec..eaf37f5 100644 --- a/crates/idx-cli/src/main.rs +++ b/crates/idx-cli/src/main.rs @@ -25,6 +25,7 @@ fn run() -> Result<(), IdxError> { Err(err) => return Err(argument_error(err)), }; runtime::set_quiet(cli.quiet); + runtime::set_verbose(cli.verbose > 0); let config = match IdxConfig::load_with_cli(&cli) { Ok(config) => config, Err(err) => { diff --git a/crates/idx-cli/tests/cli.rs b/crates/idx-cli/tests/cli.rs index a74683e..c86b9d3 100644 --- a/crates/idx-cli/tests/cli.rs +++ b/crates/idx-cli/tests/cli.rs @@ -4,6 +4,8 @@ use std::net::TcpListener; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::thread; use assert_cmd::Command; @@ -1110,6 +1112,427 @@ fn ownership_releases_uses_xdg_data_home_default_db_path() { ); } +/// Answers every request like Cloudflare's challenge and counts them. Without +/// `mitigated` it is a plain 403 from the origin. Returns the URL and the count. +fn spawn_403_server(mitigated: bool) -> (String, Arc) { + let extra = if mitigated { + "cf-mitigated: challenge\r\n" + } else { + "" + }; + let body = "Just a moment..."; + spawn_counting_server(format!( + "HTTP/1.1 403 Forbidden\r\nServer: cloudflare\r\n{extra}Content-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + )) +} + +/// Answers every request with `response`, counting them. +fn spawn_counting_server(response: String) -> (String, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind local test server"); + let addr = listener.local_addr().expect("local addr"); + let hits = Arc::new(AtomicUsize::new(0)); + let counted = Arc::clone(&hits); + + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { return }; + let mut buf = [0u8; 2048]; + let _ = stream.read(&mut buf); + counted.fetch_add(1, Ordering::SeqCst); + let _ = stream.write_all(response.as_bytes()); + } + }); + + (format!("http://{addr}"), hits) +} + +fn discover_with( + name: &str, + announcements: &str, + page: &str, + extra: &[&str], +) -> std::process::Output { + test_bin(name) + .env("IDX_CURL_IMPERSONATE_BIN", "curl") + .env("IDX_OWNERSHIP_ANNOUNCEMENT_API_URL", announcements) + .env("IDX_OWNERSHIP_DATA_PAGE_URL", page) + .args(["-o", "json", "ownership", "discover"]) + .args(extra) + .output() + .expect("ownership discover output") +} + +/// The one JSON error on stderr, after any diagnostic JSON lines. +fn stderr_error(output: &std::process::Output) -> Value { + let stderr = String::from_utf8_lossy(&output.stderr); + let line = stderr.lines().last().unwrap_or_default(); + serde_json::from_str(line).unwrap_or_else(|_| panic!("stderr is not JSON: {stderr}")) +} + +#[test] +fn ownership_discover_names_a_cloudflare_challenge_after_one_round() { + let (announcements, announcement_hits) = spawn_403_server(true); + let (page, page_hits) = spawn_403_server(true); + + let output = discover_with("ownership-discover-challenge", &announcements, &page, &[]); + + assert_eq!(output.status.code(), Some(6)); + assert!(output.stdout.is_empty()); + let error = stderr_error(&output); + assert_eq!(error["code"], "CLOUDFLARECHALLENGE"); + assert!( + error["message"] + .as_str() + .unwrap() + .contains("cf-mitigated: challenge"), + "{error}" + ); + assert!( + !error["message"] + .as_str() + .unwrap() + .contains("idx-curl-probe"), + "{error}" + ); + assert_eq!(announcement_hits.load(Ordering::SeqCst), 1); + assert_eq!(page_hits.load(Ordering::SeqCst), 1); +} + +#[test] +fn ownership_discover_plain_403_stays_http_and_keeps_retrying() { + let (announcements, announcement_hits) = spawn_403_server(false); + let (page, page_hits) = spawn_403_server(false); + + let output = discover_with("ownership-discover-plain-403", &announcements, &page, &[]); + + assert_eq!(output.status.code(), Some(4)); + assert!(output.stdout.is_empty()); + assert_eq!(stderr_error(&output)["code"], "HTTP"); + assert_eq!(announcement_hits.load(Ordering::SeqCst), 2); + assert_eq!(page_hits.load(Ordering::SeqCst), 2); +} + +fn announcement_fixture_url() -> String { + let raw = r#"{ + "Items": [{ + "PublishDate": "2026-03-10T12:09:09", + "Title": "Pemegang Saham di atas 1% (KSEI)", + "AnnouncementType": "", + "Code": "Semua Emiten Saham", + "Attachments": [{ + "PDFFilename": "above1.pdf", + "FullSavePath": "https://www.idx.co.id/StaticData/above1.pdf", + "IsAttachment": 1, + "OriginalFilename": "above1_lamp1.pdf" + }], + "PdfPath": "" + }], + "ItemCount": 1, + "PageSize": 10, + "PageNumber": 1, + "PageCount": 1 + }"#; + spawn_single_response_server("application/json", raw) +} + +#[test] +fn ownership_discover_warns_on_a_challenged_data_page_by_default() { + let (page, _) = spawn_403_server(true); + + let output = discover_with( + "ownership-discover-partial", + &announcement_fixture_url(), + &page, + &[], + ); + + assert!(output.status.success(), "{output:?}"); + assert!( + String::from_utf8_lossy(&output.stderr).contains("cf-mitigated: challenge"), + "{output:?}" + ); +} + +#[test] +fn ownership_discover_require_data_page_fails_on_a_challenged_page() { + let (page, _) = spawn_403_server(true); + + let output = discover_with( + "ownership-discover-require-challenge", + &announcement_fixture_url(), + &page, + &["--require-data-page"], + ); + + assert_eq!(output.status.code(), Some(6)); + assert!(output.stdout.is_empty(), "{output:?}"); + assert_eq!(stderr_error(&output)["code"], "CLOUDFLARECHALLENGE"); +} + +#[test] +fn ownership_discover_require_data_page_keeps_other_page_failures_distinct() { + let unreachable = discover_with( + "ownership-discover-require-unreachable", + &announcement_fixture_url(), + UNREACHABLE_DATA_PAGE_URL, + &["--require-data-page"], + ); + assert_eq!(unreachable.status.code(), Some(4)); + assert!(unreachable.stdout.is_empty(), "{unreachable:?}"); + assert_eq!(stderr_error(&unreachable)["code"], "HTTP"); + + // The page's validator looks for this word, so this loads but has no reports. + let empty_page = spawn_single_response_server("text/html", "Prospectus"); + let unparsed = discover_with( + "ownership-discover-require-unparsed", + &announcement_fixture_url(), + &empty_page, + &["--require-data-page"], + ); + assert_eq!(unparsed.status.code(), Some(1)); + assert!(unparsed.stdout.is_empty(), "{unparsed:?}"); + assert_eq!(stderr_error(&unparsed)["code"], "PARSEERROR"); +} + +#[test] +fn ownership_discover_require_data_page_passes_with_the_page() { + let page = fs::read_to_string(fixture_path("idx_share_ownership_page_excerpt.html")) + .expect("read share ownership page fixture"); + let page_url = spawn_single_response_server("text/html", page); + + let output = discover_with( + "ownership-discover-require-ok", + "http://127.0.0.1:9/announcements", + &page_url, + &["--require-data-page", "--limit", "1"], + ); + + assert!(output.status.success(), "{output:?}"); + let reports: Value = serde_json::from_slice(&output.stdout).expect("parse discover json"); + assert_eq!(reports[0]["as_of_date"], "2026-08-31"); +} + +#[test] +fn ownership_discover_logs_the_passing_profile_only_when_verbose() { + let page = fs::read_to_string(fixture_path("idx_share_ownership_page_excerpt.html")) + .expect("read share ownership page fixture"); + let run = |name: &str, flags: &[&str]| { + test_bin(name) + .env("IDX_CURL_IMPERSONATE_BIN", "curl") + .env( + "IDX_OWNERSHIP_ANNOUNCEMENT_API_URL", + "http://127.0.0.1:9/announcements", + ) + .env( + "IDX_OWNERSHIP_DATA_PAGE_URL", + spawn_single_response_server("text/html", page.clone()), + ) + .args(flags) + .args(["-o", "json", "ownership", "discover", "--limit", "1"]) + .output() + .expect("ownership discover output") + }; + + let verbose = run("ownership-discover-verbose", &["-v"]); + assert!(verbose.status.success(), "{verbose:?}"); + assert!( + String::from_utf8_lossy(&verbose.stderr).contains("fetched with curl"), + "{verbose:?}" + ); + + let default = run("ownership-discover-not-verbose", &[]); + assert!(default.status.success(), "{default:?}"); + assert!( + !String::from_utf8_lossy(&default.stderr).contains("fetched with"), + "{default:?}" + ); +} + +/// A directory holding only curl-impersonate profiles, which are real curl +/// routed to a local port, so the code's own fallback list picks them up. +#[cfg(unix)] +fn fake_profiles(name: &str, profiles: &[(&str, &str)]) -> PathBuf { + let curl = std::env::split_paths(&std::env::var_os("PATH").expect("PATH")) + .map(|dir| dir.join("curl")) + .find(|path| path.is_file()) + .expect("curl on PATH"); + let dir = test_env_dir(name).join("profiles"); + fs::create_dir_all(&dir).expect("create profile dir"); + for (profile, server) in profiles { + let port = server.rsplit(':').next().expect("server port"); + let path = dir.join(profile); + fs::write( + &path, + format!( + "#!/bin/sh\nexec {} --connect-to ::127.0.0.1:{port} \"$@\"\n", + curl.display() + ), + ) + .expect("write fake profile"); + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod fake profile"); + } + dir +} + +/// Discovery using the profile fallback list, not a pinned binary. +#[cfg(unix)] +fn discover_with_profiles(name: &str, profiles: &Path, flags: &[&str]) -> std::process::Output { + test_bin(name) + .env_remove("IDX_CURL_IMPERSONATE_BIN") + .env("PATH", profiles) + .env("IDX_OWNERSHIP_ANNOUNCEMENT_API_URL", "http://idx.test/api") + .env("IDX_OWNERSHIP_DATA_PAGE_URL", "http://idx.test/page") + .args(flags) + .args(["-o", "json", "ownership", "discover"]) + .output() + .expect("ownership discover output") +} + +#[cfg(unix)] +#[test] +fn challenged_profiles_get_one_round_per_url() { + let (server, hits) = spawn_403_server(true); + let profiles = fake_profiles( + "fallback-challenged", + &[("curl_chrome142", &server), ("curl_firefox144", &server)], + ); + + let output = discover_with_profiles("fallback-challenged-run", &profiles, &[]); + + assert_eq!(output.status.code(), Some(6), "{output:?}"); + assert_eq!(stderr_error(&output)["code"], "CLOUDFLARECHALLENGE"); + // Two URLs, each tried once by each of two profiles. + assert_eq!(hits.load(Ordering::SeqCst), 4); +} + +#[cfg(unix)] +#[test] +fn plain_403_profiles_keep_both_rounds() { + let (server, hits) = spawn_403_server(false); + let profiles = fake_profiles( + "fallback-plain-403", + &[("curl_chrome142", &server), ("curl_firefox144", &server)], + ); + + let output = discover_with_profiles("fallback-plain-403-run", &profiles, &[]); + + assert_eq!(output.status.code(), Some(4), "{output:?}"); + assert_eq!(stderr_error(&output)["code"], "HTTP"); + // Two URLs, two profiles, two rounds. + assert_eq!(hits.load(Ordering::SeqCst), 8); +} + +#[cfg(unix)] +#[test] +fn a_profile_can_pass_after_another_is_challenged_and_is_logged() { + let (challenged, _) = spawn_403_server(true); + let page = fs::read_to_string(fixture_path("idx_share_ownership_page_excerpt.html")) + .expect("read share ownership page fixture"); + let (passing, _) = spawn_counting_server(format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{page}", + page.len() + )); + let profiles = fake_profiles( + "fallback-passes", + &[ + ("curl_chrome142", &challenged), + ("curl_firefox144", &passing), + ], + ); + + let output = discover_with_profiles("fallback-passes-run", &profiles, &["-v"]); + + assert!(output.status.success(), "{output:?}"); + let reports: Value = serde_json::from_slice(&output.stdout).expect("parse discover json"); + assert_eq!(reports[0]["as_of_date"], "2026-08-31"); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("IDX share ownership data page discovery: fetched with curl_firefox144"), + "{output:?}" + ); +} + +#[cfg(unix)] +#[test] +fn a_challenge_outranks_a_later_profile_failing() { + let (challenged, _) = spawn_403_server(true); + let profiles = fake_profiles( + "fallback-later-failure", + &[ + ("curl_chrome142", &challenged), + ("curl_firefox144", UNREACHABLE_DATA_PAGE_PORT), + ], + ); + + let output = discover_with_profiles("fallback-later-failure-run", &profiles, &[]); + + assert_eq!(output.status.code(), Some(6), "{output:?}"); + assert_eq!(stderr_error(&output)["code"], "CLOUDFLARECHALLENGE"); +} + +#[test] +fn ownership_discover_keeps_the_challenge_past_an_announcement_miss() { + let announcements = spawn_single_response_server("application/json", r#"{"Items":[]}"#); + let (page, _) = spawn_403_server(true); + + let output = discover_with("discover-miss-and-challenge", &announcements, &page, &[]); + + assert_eq!(output.status.code(), Some(6), "{output:?}"); + assert_eq!(stderr_error(&output)["code"], "CLOUDFLARECHALLENGE"); +} + +#[test] +fn ownership_discover_does_not_blame_a_challenge_for_a_changed_page() { + let (announcements, _) = spawn_403_server(true); + // The page's validator looks for this word, so this loads but has no reports. + let page = spawn_single_response_server("text/html", "Prospectus"); + + let output = discover_with("discover-challenge-and-parse", &announcements, &page, &[]); + + assert_eq!(output.status.code(), Some(4), "{output:?}"); + assert_eq!(stderr_error(&output)["code"], "HTTP"); +} + +/// idx.co.id download URLs always go through curl, so a wrapper points real curl +/// at the local server, keeping this off the live network. +#[cfg(unix)] +#[test] +fn ownership_import_url_names_a_cloudflare_challenge() { + let (server, hits) = spawn_403_server(true); + let port = server.rsplit(':').next().expect("server port"); + let root = test_env_dir("ownership-import-challenge"); + let wrapper = root.join("curl-wrapper"); + fs::write( + &wrapper, + format!("#!/bin/sh\nexec curl --connect-to ::127.0.0.1:{port} \"$@\"\n"), + ) + .expect("write curl wrapper"); + fs::set_permissions(&wrapper, fs::Permissions::from_mode(0o755)).expect("chmod curl wrapper"); + + let output = bin_with_root(&root) + .env("IDX_CURL_IMPERSONATE_BIN", &wrapper) + .args([ + "-o", + "json", + "ownership", + "import", + "--url", + "http://www.idx.co.id/Media/fixture/peng-2026-09-00024-satu-persen.xlsx", + ]) + .output() + .expect("ownership import output"); + + assert_eq!(output.status.code(), Some(6), "{output:?}"); + assert!(output.stdout.is_empty()); + assert_eq!(stderr_error(&output)["code"], "CLOUDFLARECHALLENGE"); + assert_eq!(hits.load(Ordering::SeqCst), 1); +} + +/// A fake profile routed here fails to connect (nothing listens on port 9). +#[cfg(unix)] +const UNREACHABLE_DATA_PAGE_PORT: &str = "http://127.0.0.1:9"; + /// Keeps discovery tests off the live IDX data page (nothing listens on port 9). const UNREACHABLE_DATA_PAGE_URL: &str = "http://127.0.0.1:9/data-kepemilikan-saham/"; diff --git a/crates/idx-cli/tests/golden/completions-bash.golden b/crates/idx-cli/tests/golden/completions-bash.golden index aff5968..65973e0 100644 --- a/crates/idx-cli/tests/golden/completions-bash.golden +++ b/crates/idx-cli/tests/golden/completions-bash.golden @@ -1610,7 +1610,7 @@ _idx() { return 0 ;; idx__ownership__discover) - opts="-o -q -v -h --family --limit --output --no-color --quiet --verbose --offline --no-cache --help" + opts="-o -q -v -h --family --limit --require-data-page --output --no-color --quiet --verbose --offline --no-cache --help" if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 diff --git a/crates/idx-core/src/curl_impersonate.rs b/crates/idx-core/src/curl_impersonate.rs index a157b09..6f3cabd 100644 --- a/crates/idx-core/src/curl_impersonate.rs +++ b/crates/idx-core/src/curl_impersonate.rs @@ -44,9 +44,11 @@ pub fn run_owned(stage: &str, args: &[String]) -> Result { run_binary(stage, &binary, args) } -/// Browser profiles tried, in order, when a site's bot protection rejects -/// some TLS fingerprints but not others (IDX's Cloudflare rules block -/// `curl_chrome142` on some pages while Firefox/Safari profiles pass). +/// Browser profiles tried, in order, because a site's bot protection can reject +/// some TLS fingerprints but not others. On IDX which profiles pass changes over +/// time and by path, and at times none does (on 2026-11-05 Chrome, Firefox and +/// Safari profiles all got a challenge). A profile only changes the fingerprint, +/// so it cannot solve a challenge that is active for this client. const FALLBACK_PROFILES: &[&str] = &[ "curl_chrome142", "curl_firefox144", @@ -60,9 +62,47 @@ const FALLBACK_PROFILES: &[&str] = &[ ]; const FALLBACK_ROUNDS: usize = 2; +/// Marks the line curl adds to stderr describing the response. `--fail` drops +/// the response itself, and stdout must stay the body alone. +const PROBE_PREFIX: &str = "idx-curl-probe"; +const PROBE_WRITE_OUT: &str = + "%{stderr}idx-curl-probe status=%{http_code} cf-mitigated=%header{cf-mitigated}\\n"; + +/// What the probe line says about a response. +#[derive(Debug, PartialEq, Eq)] +struct ResponseProbe { + status: String, + challenged: bool, +} + +/// Separates the probe line from curl's own stderr. Without a probe line (the +/// caller did not ask for one) stderr comes back unchanged. +fn split_probe(stderr: &[u8]) -> (Option, Vec) { + let mut probe = None; + let mut rest = Vec::with_capacity(stderr.len()); + for line in stderr.split_inclusive(|byte| *byte == b'\n') { + let parsed = std::str::from_utf8(line) + .ok() + .and_then(|text| text.trim_end().strip_prefix(PROBE_PREFIX)) + .and_then(|fields| fields.strip_prefix(" status=")) + .and_then(|fields| fields.split_once(" cf-mitigated=")); + match parsed { + Some((status, mitigated)) => { + probe = Some(ResponseProbe { + status: status.to_string(), + challenged: mitigated.trim().eq_ignore_ascii_case("challenge"), + }); + } + None => rest.extend_from_slice(line), + } + } + (probe, rest) +} + /// Run curl with each available impersonation profile until `accept` approves /// the response body. `IDX_CURL_IMPERSONATE_BIN`, when set, is the only -/// profile used. Returns the first accepted output, or the last error. +/// profile used. Returns the first accepted output, or an error: a Cloudflare +/// challenge from any profile, otherwise the last error. pub fn run_owned_with_fallback( stage: &str, args: &[String], @@ -83,17 +123,39 @@ pub fn run_owned_with_fallback( ))); } + run_profiles(stage, &profiles, args, accept) +} + +/// A challenge can differ by profile, so one round tries them all, but a repeat +/// straight away is unlikely to change the answer: it ends after that round. +fn run_profiles( + stage: &str, + profiles: &[String], + args: &[String], + accept: &dyn Fn(&[u8]) -> Result<(), IdxError>, +) -> Result { + let mut probed_args = vec!["--write-out".to_string(), PROBE_WRITE_OUT.to_string()]; + probed_args.extend_from_slice(args); + let mut last_error = None; + let mut challenge = None; for _ in 0..FALLBACK_ROUNDS { - for binary in &profiles { - match run_binary(stage, binary, args) { + for binary in profiles { + match run_binary(stage, binary, &probed_args) { Ok(output) => match accept(&output.stdout) { - Ok(()) => return Ok(output), + Ok(()) => { + crate::runtime::verbose(format!("{stage}: fetched with {binary}")); + return Ok(output); + } Err(err) => last_error = Some(err), }, + Err(err @ IdxError::CloudflareChallenge(_)) => challenge = Some(err), Err(err) => last_error = Some(err), } } + if let Some(err) = challenge { + return Err(err); + } } Err(last_error.expect("at least one profile was tried")) } @@ -166,12 +228,20 @@ fn run_binary_limited( .wait() .map_err(|e| IdxError::Http(format!("failed to wait for {binary} for {stage}: {e}")))?; let stderr = stderr_reader.join().unwrap_or_default(); + let (probe, stderr) = split_probe(&stderr); let output = Output { status, stdout, stderr, }; + if let Some(probe) = probe.filter(|probe| probe.challenged) { + return Err(IdxError::CloudflareChallenge(format!( + "{stage}: Cloudflare answered {binary} with a challenge (HTTP {}, cf-mitigated: challenge), which curl cannot solve; retry later", + probe.status + ))); + } + if output.status.success() { return Ok(output); } diff --git a/crates/idx-core/src/error.rs b/crates/idx-core/src/error.rs index d991c6b..f4d6a57 100644 --- a/crates/idx-core/src/error.rs +++ b/crates/idx-core/src/error.rs @@ -34,6 +34,10 @@ pub enum IdxError { InvalidInput(String), #[error("no market data: {0}")] NoMarketData(String), + /// Cloudflare answered with a challenge only a browser can solve, on every + /// client tried. Nothing is wrong with the request, so a later run may pass. + #[error("cloudflare challenge: {0}")] + CloudflareChallenge(String), } #[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] @@ -53,6 +57,7 @@ pub enum ErrorCode { PdfParseError, InvalidInput, NoMarketData, + CloudflareChallenge, } /// The stable wire form frontends emit, e.g. `SYMBOLNOTFOUND`. @@ -80,6 +85,7 @@ impl IdxError { Self::PdfParseError(_) => ErrorCode::PdfParseError, Self::InvalidInput(_) => ErrorCode::InvalidInput, Self::NoMarketData(_) => ErrorCode::NoMarketData, + Self::CloudflareChallenge(_) => ErrorCode::CloudflareChallenge, } } @@ -95,6 +101,7 @@ impl IdxError { | ErrorCode::Offline | ErrorCode::AuthError => 4, ErrorCode::Unsupported => 5, + ErrorCode::CloudflareChallenge => 6, ErrorCode::Io | ErrorCode::ParseError | ErrorCode::DatabaseError diff --git a/crates/idx-core/src/investor.rs b/crates/idx-core/src/investor.rs index 6a13ca8..1d31ef8 100644 --- a/crates/idx-core/src/investor.rs +++ b/crates/idx-core/src/investor.rs @@ -14,7 +14,9 @@ use sha2::{Digest, Sha256}; use crate::config::CoreConfig; use crate::error::IdxError; -use crate::ownership::remote::{self, DiscoveredOwnershipPdf, OwnershipReportFamily}; +use crate::ownership::remote::{ + self, DataPagePolicy, DiscoveredOwnershipPdf, OwnershipReportFamily, +}; use crate::ownership::snapshot::{self, OwnershipSyncResult}; use crate::ownership::types::{ ChangeRow, ConcentrationMetrics, CrossHolderRow, Entity, EntityHoldings, GraphEdge, GraphNode, @@ -264,8 +266,9 @@ impl Investor { &self, family: Option, limit: usize, + data_page: DataPagePolicy, ) -> Result, IdxError> { - remote::discover_idx_ownership_reports(family, limit) + remote::discover_idx_ownership_reports(family, limit, data_page) } /// Installs or refreshes the maintained snapshot. `manifest` overrides diff --git a/crates/idx-core/src/lib.rs b/crates/idx-core/src/lib.rs index bd01c1c..5b79bc4 100644 --- a/crates/idx-core/src/lib.rs +++ b/crates/idx-core/src/lib.rs @@ -27,6 +27,7 @@ pub mod runtime { static QUIET: AtomicBool = AtomicBool::new(false); static JSON: AtomicBool = AtomicBool::new(false); + static VERBOSE: AtomicBool = AtomicBool::new(false); pub fn set_quiet(quiet: bool) { QUIET.store(quiet, Ordering::Relaxed); @@ -36,6 +37,10 @@ pub mod runtime { QUIET.load(Ordering::Relaxed) } + pub fn set_verbose(verbose: bool) { + VERBOSE.store(verbose, Ordering::Relaxed); + } + /// In JSON mode diagnostics are JSON lines, so stderr stays /// machine-readable alongside a JSON error. pub fn set_json(json: bool) { @@ -61,6 +66,13 @@ pub mod runtime { emit("info", message); } + /// Detail worth keeping in a job's log but not worth showing by default. + pub fn verbose(message: impl Display) { + if VERBOSE.load(Ordering::Relaxed) { + emit("info", message); + } + } + pub fn warn(message: impl Display) { emit("warning", message); } diff --git a/crates/idx-core/src/ownership/remote.rs b/crates/idx-core/src/ownership/remote.rs index 8960dde..55ed7ee 100644 --- a/crates/idx-core/src/ownership/remote.rs +++ b/crates/idx-core/src/ownership/remote.rs @@ -281,12 +281,24 @@ pub fn select_latest_ownership_reports( Ok(attachments) } +/// Whether discovery may answer without the data page, the only place the +/// newest above-1% reports are listed since June 2026. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DataPagePolicy { + /// Warn and return what the announcements list, which can miss the newest + /// reports. + BestEffort, + /// Fail with the data page's error, so old results cannot pass as current. + Required, +} + pub fn discover_idx_ownership_reports( family_filter: Option, limit: usize, + data_page: DataPagePolicy, ) -> Result, IdxError> { let mut discovered = Vec::new(); - let mut errors = Vec::new(); + let mut errors: Vec = Vec::new(); for query in DISCOVERY_QUERIES .iter() @@ -303,7 +315,7 @@ pub fn discover_idx_ownership_reports( .and_then(|page| select_latest_ownership_reports(&page, &query_url, query.family)) { Ok(mut reports) => discovered.append(&mut reports), - Err(err) => errors.push(err.to_string()), + Err(err) => errors.push(err), } } @@ -320,10 +332,10 @@ pub fn discover_idx_ownership_reports( // The page loaded (it has its file list) but nothing parsed: // the payload shape changed. Say so instead of silently // falling back to announcement-only results. - page_error = Some(format!( + page_error = Some(IdxError::ParseError(format!( "IDX share ownership data page {} loaded but no report entries could be parsed; the page layout may have changed", share_ownership_page_url() - )); + ))); } page_reports .retain(|report| family_filter.is_none_or(|family| family == report.family)); @@ -343,7 +355,13 @@ pub fn discover_idx_ownership_reports( }); discovered.extend(page_reports); } - Err(err) => page_error = Some(err.to_string()), + Err(err) => page_error = Some(err), + } + + if data_page == DataPagePolicy::Required + && let Some(err) = page_error + { + return Err(err); } discovered.sort_by(|left, right| { @@ -369,21 +387,50 @@ pub fn discover_idx_ownership_reports( errors.extend(page_error); if discovered.is_empty() { - let detail = if errors.is_empty() { - "no matching ownership reports found".to_string() - } else { - errors.join("; ") - }; - return Err(IdxError::Http(format!( - "failed to discover IDX ownership reports from {} or {}: {detail}", - announcement_listing_url(), - share_ownership_page_url() - ))); + return Err(discovery_failure( + &errors, + &announcement_listing_url(), + &share_ownership_page_url(), + )); } Ok(discovered) } +/// The error for a discovery that found nothing. A challenge names the cause +/// unless a source answered with something unparseable: a retry cannot fix that, +/// so it stays a plain failure. Announcement misses do not hide a challenge. +fn discovery_failure(errors: &[IdxError], listing_url: &str, page_url: &str) -> IdxError { + let challenged = errors + .iter() + .any(|err| matches!(err, IdxError::CloudflareChallenge(_))); + let unparseable = errors + .iter() + .any(|err| matches!(err, IdxError::ParseError(_))); + + let detail = if errors.is_empty() { + "no matching ownership reports found".to_string() + } else { + errors + .iter() + .map(|err| match err { + IdxError::CloudflareChallenge(message) => message.clone(), + other => other.to_string(), + }) + .collect::>() + .join("; ") + }; + let message = format!( + "failed to discover IDX ownership reports from {listing_url} or {page_url}: {detail}" + ); + + if challenged && !unparseable { + IdxError::CloudflareChallenge(message) + } else { + IdxError::Http(message) + } +} + pub fn download_idx_pdf(url: &str, target: &Path) -> Result<(), IdxError> { let bytes = fetch_bytes( "IDX ownership PDF download", diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e9f3215..e506bf4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -144,7 +144,7 @@ WIB as the fallback. - Cloudflare rejects most TLS fingerprints (in October 2026 only `curl_safari184` passed), so the request uses curl-impersonate's profile fallback with a 30 s limit. A challenge page fails that profile, not the - parse. + parse; if no profile passes, the error is `CLOUDFLARECHALLENGE`. - Each session is cached whole (every stock) by date: 30 days once past, 5 minutes for today. `Flow::foreign` walks weekdays without requesting weekends; offline, `--days` skips uncached sessions while a `--from/--to` @@ -190,10 +190,23 @@ Only the above-1% XLSX is `supported`; the daily above-5% and investor-type work are listed as `unsupported`. Both URLs can be overridden for tests with `IDX_OWNERSHIP_ANNOUNCEMENT_API_URL` and `IDX_OWNERSHIP_DATA_PAGE_URL`. -IDX sits behind Cloudflare, which accepts some curl-impersonate TLS profiles and -rejects others per path (for example `curl_chrome142` gets 403 on the data page). -IDX fetches therefore try several profiles until the response validates (JSON, page -payload, `%PDF`, or `PK` zip magic). `IDX_CURL_IMPERSONATE_BIN` pins a single profile. +IDX sits behind Cloudflare, which can accept some curl-impersonate TLS profiles and +reject others, varying by path and over time. IDX fetches therefore try several +profiles until the response validates (JSON, page payload, `%PDF`, or `PK` zip +magic). `IDX_CURL_IMPERSONATE_BIN` pins a single profile. + +At times every profile gets a challenge (`cf-mitigated: challenge`, seen on +2026-11-05), which only a browser can solve. curl reports that header on stderr +through `--write-out`, because `--fail` drops the response. A profile that gets a +challenge fails with `CLOUDFLARECHALLENGE` (exit 6): the remaining profiles still +run once, but a URL with a challenge gets no second round. Other failures keep two +rounds. `-v` logs which profile fetched each URL. + +Discovery that finds nothing reports `CLOUDFLARECHALLENGE` if a source was +challenged, unless another source answered with something unparseable, which a retry +cannot fix. By default a failed data page is only a warning when announcements +returned results, and those can miss the newest reports; +`ownership discover --require-data-page` makes it an error. ### Parser Pipeline ``` @@ -240,7 +253,8 @@ CLI flags > environment variables > config file > defaults bidirectional overrides replaced before it reaches a terminal table. - Exit codes by class: `0` success, `2` `INVALIDINPUT`/`CONFIGERROR` and clap usage errors, `3` `SYMBOLNOTFOUND`/`NOMARKETDATA`/`CACHEMISS`, `4` `HTTP`/`RATELIMITED`/ - `PROVIDERUNAVAILABLE`/`OFFLINE`/`AUTHERROR`, `5` `UNSUPPORTED`, `1` everything else + `PROVIDERUNAVAILABLE`/`OFFLINE`/`AUTHERROR`, `5` `UNSUPPORTED`, `6` `CLOUDFLARECHALLENGE`, + `1` everything else - A totally failed `stocks compare` emits one structured error in JSON mode; per-symbol warnings are retained for partial success, not printed ahead of a total-failure error. ## CLI Output and Offline Cache diff --git a/docs/DESIGN.md b/docs/DESIGN.md index fbb6a02..8a18789 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -186,6 +186,7 @@ is another 20-30% smaller than pretty JSON. | 3 | not found or no data | `SYMBOLNOTFOUND`, `NOMARKETDATA`, `CACHEMISS` | | 4 | upstream unavailable, retry later | `HTTP`, `RATELIMITED`, `PROVIDERUNAVAILABLE`, `OFFLINE`, `AUTHERROR` | | 5 | unsupported | `UNSUPPORTED` | + | 6 | Cloudflare challenge, retry later | `CLOUDFLARECHALLENGE` | - JSON keeps today's top-level shape: no `{data, meta}` wrapper, so `jq` paths stay short and JSONL works per record. Freshness metadata (`Fetched`) is diff --git a/docs/OWNERSHIP_PUBLISH.md b/docs/OWNERSHIP_PUBLISH.md index 81a8929..2ed8408 100644 --- a/docs/OWNERSHIP_PUBLISH.md +++ b/docs/OWNERSHIP_PUBLISH.md @@ -48,8 +48,10 @@ check needs no token. ## What It Does -1. `idx -o json ownership discover --family above1 --limit 50` (newest supported - report first; XLSX from the Data Kepemilikan Saham page since June 2026) +1. `idx -v -o json ownership discover --family above1 --limit 50 --require-data-page` + (newest supported report first; XLSX from the Data Kepemilikan Saham page since + June 2026, so a run that cannot read that page stops rather than miss the newest + report). `-v` logs which curl-impersonate profile fetched each URL. 2. verifies the discovered report is the supported import path 3. imports it plus `--history ` earlier months, oldest first, into an isolated temp DB @@ -91,12 +93,24 @@ The publisher is idempotent and prints exactly one final line: | --- | --- | --- | | `RESULT: published ` | 0 | a newer snapshot, or larger history, was uploaded | | `RESULT: up-to-date ` | 0 | nothing to upload | +| `RESULT: blocked stage=` | 6 | IDX answered with a Cloudflare challenge in `discover` or `build-snapshot`; nothing was uploaded | | `RESULT: FAILED stage= (exit N)` | N | failed in `arguments`, `build`, `preflight`, `discover`, `build-snapshot`, `stage-output`, `upload`, or `publish` | `idx-ownership-freshness` is the independent alarm: it exits 1 when the published `latest_as_of_date` is older than `--max-age-days` (default 40). The source lands 2-3 days after month end, so 40 days only trips when publishing has stopped. +`blocked` means retry on the next run, not that something broke. The source is +monthly, so one run getting through per month is enough, and stale data is what +should alert: + +| Result | Snapshot | Action | +| --- | --- | --- | +| `published` or `up-to-date` | fresh | none | +| `blocked` | fresh | none; the next daily run retries | +| `blocked` | stale | alert: retry, or import the file by hand | +| `FAILED` | either | alert at once | + ## History Coverage `--history ` (maximum 1000) requests the latest report plus up to `n` earlier @@ -135,16 +149,18 @@ on a self-hosted machine that IDX accepts: - `contrib/systemd/idx-ownership-snapshot-publish.service` and `.timer` run the packaged publisher daily at 06:00 UTC with `GH_TOKEN` from - `/etc/idx-ownership-snapshot.env`. Wire `OnFailure=` to a notification. + `/etc/idx-ownership-snapshot.env`. The service treats exit 6 (`blocked`) as success + and then runs the freshness check, so the unit fails on `FAILED` or on stale data. + An unreadable manifest also fails it. Wire `OnFailure=` to a notification. - On NixOS, add this repo as a flake input and use `inputs.idx-cli.packages.${system}.ownership-publisher`; `nix flake update idx-cli` picks up fixes. `.github/workflows/publish-ownership-snapshot.yml` is a manual (`workflow_dispatch`) fallback that runs the same packaged publisher with inputs -passed as environment variables. +passed as environment variables. There a `blocked` run shows as a failed job. Observed XLSX upload times (HTTP Last-Modified) were the 2nd-3rd of each month around 02:00-04:00 UTC; a daily idempotent run makes the exact day irrelevant. -If the monthly source has not appeared yet, the run should fail loudly and retry -later rather than publish stale data. +If the monthly source has not appeared yet, the run should not publish stale data: +it ends `up-to-date` until the new report is listed, and a later daily run picks it up. diff --git a/docs/SMOKE.md b/docs/SMOKE.md index 5f56439..75c2921 100644 --- a/docs/SMOKE.md +++ b/docs/SMOKE.md @@ -61,6 +61,8 @@ scripts/live-smoke.sh --dry-run - `ownership`: `discover`, then imports the newest supported report into a temp DB and lists `releases`. IDX sits behind Cloudflare and returns `403` to many datacenter IPs, including GitHub-hosted runners; run it from a network IDX accepts. + A case that gets a Cloudflare challenge (`CLOUDFLARECHALLENGE`) prints `CHALLENGED` + and counts as a failure, never as a pass. `--compare ` runs every case with both binaries back to back, each with its own config and cache, and fails on a different exit code, any added, diff --git a/flake.nix b/flake.nix index b3b9b45..43a07e6 100644 --- a/flake.nix +++ b/flake.nix @@ -157,6 +157,7 @@ fileset = lib.fileset.unions [ ./scripts/publish-ownership-snapshot-test.sh ./scripts/publish-ownership-snapshot.sh + ./scripts/build-latest-ownership-snapshot.sh ./scripts/build-ownership-snapshot.sh ]; }; diff --git a/scripts/build-latest-ownership-snapshot.sh b/scripts/build-latest-ownership-snapshot.sh index c09fd75..96b2341 100755 --- a/scripts/build-latest-ownership-snapshot.sh +++ b/scripts/build-latest-ownership-snapshot.sh @@ -171,7 +171,11 @@ MANIFEST_PATH="$OUTPUT_DIR/ownership-snapshot-manifest.json" DB_PATH="$XDG_DATA_HOME/idx/ownership.db" printf 'Discovering latest supported IDX/KSEI above-1%% report...\n' -"$IDX_BIN" -o json ownership discover --family above1 --limit 50 > "$DISCOVERY_JSON" +# -v logs which curl profile passed. --require-data-page: an announcement-only +# answer can miss the newest report, so it must not be built as if current. +# A Cloudflare challenge exits 6 here and on import; the publisher relies on that +# status reaching it, so nothing below may catch these failures. +"$IDX_BIN" -v -o json ownership discover --family above1 --limit 50 --require-data-page > "$DISCOVERY_JSON" # NUL-separated, so a field containing newlines (a report title) cannot # shift the fields after it. Written to a file: command substitution drops @@ -230,12 +234,12 @@ if (( HISTORY > 0 )); then fi printf 'Importing %d earlier monthly report(s) for history...\n' "${#history_urls[@]}" for url in "${history_urls[@]}"; do - "$IDX_BIN" ownership import --url "$url" + "$IDX_BIN" -v ownership import --url "$url" done fi printf 'Importing discovered report into isolated DB...\n' -"$IDX_BIN" ownership import --url "$DISCOVERED_PDF_URL" +"$IDX_BIN" -v ownership import --url "$DISCOVERED_PDF_URL" printf 'Inspecting imported release metadata...\n' "$IDX_BIN" -o json ownership releases > "$RELEASES_JSON" diff --git a/scripts/live-smoke.sh b/scripts/live-smoke.sh index 98dbfb9..7e8a591 100755 --- a/scripts/live-smoke.sh +++ b/scripts/live-smoke.sh @@ -162,6 +162,13 @@ run_case() { read -r -a args <<< "$cmd" run_bin new "$BIN_PATH" "$log" "${args[@]}" status=$? + # A challenge is neither a pass nor drift, and asking the old binary as well + # would only add requests while the site is challenging this client. + if grep -q 'CLOUDFLARECHALLENGE' "$log.stderr"; then + FAILURES+=("$name: CHALLENGED, Cloudflare answered with a challenge (see $log.stderr)") + echo " CHALLENGED (Cloudflare)" + return + fi if [[ -n "$COMPARE_BIN" ]]; then run_bin old "$COMPARE_BIN" "$log.old" "${args[@]}" old_status=$? diff --git a/scripts/publish-ownership-snapshot-test.sh b/scripts/publish-ownership-snapshot-test.sh index 796b564..6c3f055 100755 --- a/scripts/publish-ownership-snapshot-test.sh +++ b/scripts/publish-ownership-snapshot-test.sh @@ -151,9 +151,21 @@ if [[ "${1:-}" == "version" ]]; then exit 0 fi if [[ "$*" == *"ownership discover"* ]]; then + if [[ "$*" != *"--require-data-page"* ]]; then + echo "fake idx: discovery must require the data page" >&2 + exit 99 + fi + if [[ -n "${FAKE_DISCOVER_EXIT:-}" ]]; then + echo "fake idx: discovery failed with ${FAKE_DISCOVER_EXIT}" >&2 + exit "$FAKE_DISCOVER_EXIT" + fi printf '%s\n' "${FAKE_DISCOVER_JSON:?FAKE_DISCOVER_JSON not set}" exit 0 fi +if [[ "$*" == *"ownership import"* && -n "${FAKE_IMPORT_EXIT:-}" ]]; then + echo "fake idx: import failed with ${FAKE_IMPORT_EXIT}" >&2 + exit "$FAKE_IMPORT_EXIT" +fi echo "fake idx: unexpected args: $*" >&2 exit 99 FAKE_IDX @@ -182,6 +194,10 @@ while [[ $# -gt 0 ]]; do done [[ -n "$out" ]] || { echo "stub builder: missing --output-dir" >&2; exit 2; } printf 'history=%s max_bytes=%s\n' "$history" "$max_bytes" >>"${FAKE_BUILDER_CALLS:?}" +if [[ -n "${FAKE_BUILDER_EXIT:-}" ]]; then + echo "stub builder: failed with ${FAKE_BUILDER_EXIT}" >&2 + exit "$FAKE_BUILDER_EXIT" +fi if [[ "${FAKE_BUILDER_TOO_LARGE:-0}" == "1" ]]; then echo "snapshot is 15290368 bytes (6 monthly releases), above --max-snapshot-bytes $max_bytes; lower --history" >&2 exit 1 @@ -625,6 +641,82 @@ else fail "--keep-workdir prints nothing after RESULT (trailing='$after_result')" fi +# =========================================================================== +# 9c. a Cloudflare challenge is "blocked"; every other failure stays FAILED +# =========================================================================== +reset_remote +run_publisher +manifest_before="$(cat "$state/remote/assets/ownership-snapshot-manifest.json")" +calls_before="$(builder_calls)" + +export FAKE_DISCOVER_EXIT="6" +run_publisher +unset FAKE_DISCOVER_EXIT +if [[ "$PUBLISH_STATUS" == "6" ]] && [[ "$PUBLISH_LAST" == "RESULT: blocked stage=discover" ]]; then + pass "challenged discovery is blocked with its own exit status" +else + fail "challenged discovery is blocked (status=$PUBLISH_STATUS last='$PUBLISH_LAST')" +fi +if [[ "$(builder_calls)" == "$calls_before" ]]; then + pass "challenged discovery does not run the builder" +else + fail "challenged discovery does not run the builder" +fi + +export FAKE_DISCOVER_EXIT="4" FAKE_BUILDER_EXIT="4" +run_publisher +unset FAKE_DISCOVER_EXIT FAKE_BUILDER_EXIT +if [[ "$PUBLISH_STATUS" == "4" ]] && [[ "$PUBLISH_LAST" == "RESULT: FAILED stage=build-snapshot (exit 4)" ]]; then + pass "a failed data page is FAILED, never up-to-date" +else + fail "a failed data page is FAILED (status=$PUBLISH_STATUS last='$PUBLISH_LAST')" +fi + +export FAKE_BUILDER_EXIT="6" +run_publisher --force +unset FAKE_BUILDER_EXIT +if [[ "$PUBLISH_STATUS" == "6" ]] && [[ "$PUBLISH_LAST" == "RESULT: blocked stage=build-snapshot" ]]; then + pass "a challenge during the build is blocked" +else + fail "a challenge during the build is blocked (status=$PUBLISH_STATUS last='$PUBLISH_LAST')" +fi + +# A real parser failure from the build (idx exits 1) stays loud. +export FAKE_BUILDER_EXIT="1" +run_publisher --force +unset FAKE_BUILDER_EXIT +if [[ "$PUBLISH_STATUS" == "1" ]] && [[ "$PUBLISH_LAST" == "RESULT: FAILED stage=build-snapshot (exit 1)" ]]; then + pass "a parser failure during the build is FAILED" +else + fail "a parser failure during the build is FAILED (status=$PUBLISH_STATUS last='$PUBLISH_LAST')" +fi + +if [[ "$(cat "$state/remote/assets/ownership-snapshot-manifest.json")" == "$manifest_before" ]]; then + pass "blocked and failed runs leave the published manifest alone" +else + fail "blocked and failed runs leave the published manifest alone" +fi + +# The real builder has to hand idx's exit status back to the publisher. +real_builder_status() { + local status=0 + FAKE_DISCOVER_JSON='[{"family":"above_one_percent","status":"supported","listing_page_url":"https://idx.test/","query_url":"https://idx.test/","pdf_url":"https://idx.test/a.xlsx","title":"t","publish_date":"2026-09-02","original_filename":"a.xlsx","format":"xlsx","as_of_date":"2026-08-31"}]' \ + bash "$repo_root/scripts/build-latest-ownership-snapshot.sh" \ + --idx-bin "$bin_dir/idx" --output-dir "$work/real-build" --history 0 >/dev/null 2>&1 || status=$? + printf '%s' "$status" +} +for case in "FAKE_DISCOVER_EXIT 6" "FAKE_IMPORT_EXIT 6" "FAKE_DISCOVER_EXIT 4" "FAKE_IMPORT_EXIT 1"; do + read -r var code <<<"$case" + export "$var=$code" + got="$(real_builder_status)" + unset "$var" + if [[ "$got" == "$code" ]]; then + pass "real builder keeps exit $code from $var" + else + fail "real builder keeps exit $code from $var (got $got)" + fi +done + # =========================================================================== # 10. auth/network errors never become a trustworthy up-to-date # =========================================================================== diff --git a/scripts/publish-ownership-snapshot.sh b/scripts/publish-ownership-snapshot.sh index e520ed7..8100c5c 100755 --- a/scripts/publish-ownership-snapshot.sh +++ b/scripts/publish-ownership-snapshot.sh @@ -23,8 +23,13 @@ artifact is missing, and a retry repairs the publication. The last line is always one of: RESULT: published RESULT: up-to-date + RESULT: blocked stage= RESULT: FAILED stage= (exit ) +A Cloudflare challenge from IDX is "blocked", exit 6: nothing is wrong with the +run, so retry on the next schedule and alert on stale data instead. Every other +failure is FAILED. + Options: --idx-bin idx binary to use (default: ./target/debug/idx) --output-dir Directory to write the copied snapshot and manifest @@ -66,6 +71,14 @@ fail() { exit "$status" } +# idx's exit status for a Cloudflare challenge. +BLOCKED_STATUS=6 + +blocked() { + printf 'RESULT: blocked stage=%s\n' "$STAGE" >&2 + exit "$BLOCKED_STATUS" +} + # Unexpected command failures (set -e). errtrace is deliberately off: with # it, a failure inside $(...) would run this in the subshell and the parent. on_error() { @@ -291,8 +304,14 @@ remote_asset_sha256() { # asked for more history than the source has does not rebuild forever. # Returns non-zero when discovery is unusable; callers then build. compute_requested_coverage() { - local json latest available wanted - if ! json="$("$IDX_BIN" -o json ownership discover --family above1 --limit 50 2>/dev/null)"; then + local json latest available wanted status=0 + # --require-data-page: the newest reports are only on the data page, so + # without it a failed page would read as "up-to-date" against older results. + json="$("$IDX_BIN" -v -o json ownership discover --family above1 --limit 50 --require-data-page)" || status=$? + if (( status == BLOCKED_STATUS )); then + blocked + fi + if (( status != 0 )); then return 1 fi @@ -410,7 +429,14 @@ if [[ "$KEEP_WORKDIR" == "1" ]]; then fi printf 'Preparing latest ownership snapshot artifacts...\n' -"$BUILDER" "${build_args[@]}" +build_status=0 +"$BUILDER" "${build_args[@]}" || build_status=$? +if (( build_status == BLOCKED_STATUS )); then + blocked +fi +if (( build_status != 0 )); then + fail "$build_status" "snapshot build failed" +fi STAGED_MANIFEST_PATH="$PUBLISH_WORKDIR/ownership-snapshot-manifest.json" if [[ ! -f "$STAGED_MANIFEST_PATH" ]]; then diff --git a/web/src/content/docs/agents/index.md b/web/src/content/docs/agents/index.md index 6e4a460..b44ce2a 100644 --- a/web/src/content/docs/agents/index.md +++ b/web/src/content/docs/agents/index.md @@ -20,7 +20,7 @@ What an agent needs to know: `{"error": true, "code": "...", "message": "..."}`. - Exit codes say what to do next: `2` fix the input, `3` not found (or no cached data offline), `4` provider unavailable, retry later, `5` - unsupported, `1` other. + unsupported, `6` Cloudflare challenge from idx.co.id, retry later, `1` other. - Tickers are IDX codes (`BBCA`); `.JK` is optional. Several tickers can be passed at once to `stocks quote` and `stocks compare`. - Ownership queries need `idx ownership sync` once; after that they run diff --git a/web/src/content/docs/cli/output.md b/web/src/content/docs/cli/output.md index ed72dba..dbb79ec 100644 --- a/web/src/content/docs/cli/output.md +++ b/web/src/content/docs/cli/output.md @@ -34,6 +34,7 @@ stdout is not a terminal, and with `--no-color`. | `3` | not found, no data, or no cached data offline | check the ticker or name; go online | | `4` | upstream unavailable | retry later | | `5` | unsupported | use another command or provider | +| `6` | Cloudflare challenge (`CLOUDFLARECHALLENGE`), on commands that read idx.co.id | retry later, or from another network | | `1` | anything else | read the message | ## Pipes diff --git a/web/src/content/docs/cli/ownership.md b/web/src/content/docs/cli/ownership.md index 07b1037..2896f6f 100644 --- a/web/src/content/docs/cli/ownership.md +++ b/web/src/content/docs/cli/ownership.md @@ -43,6 +43,7 @@ import IDX's reports directly: ```bash idx ownership discover # find the latest report URLs on IDX +idx ownership discover --require-data-page # fail rather than list only older reports idx ownership import --url # needs curl-impersonate idx ownership import --file report.xlsx # or a local XLSX, PDF, ZIP or TXT ``` diff --git a/web/src/content/docs/cli/reference.md b/web/src/content/docs/cli/reference.md index 5a89111..61c5364 100644 --- a/web/src/content/docs/cli/reference.md +++ b/web/src/content/docs/cli/reference.md @@ -924,15 +924,16 @@ Discover the latest IDX-hosted ownership report URLs Usage: idx ownership discover [OPTIONS] Options: - --family Report family to discover: above1 (default), all, above5, or investor-type [default: above1] - -o, --output Output format [default: auto = table on a terminal, json when piped] [possible values: auto, table, json, jsonl] - --limit Maximum number of discovered report URLs to print [default: 6] - --no-color Disable colored output - -q, --quiet Suppress non-essential output - -v, --verbose... Increase verbosity (-v, -vv) - --offline Run without network requests - --no-cache Bypass local cache - -h, --help Print help + --family Report family to discover: above1 (default), all, above5, or investor-type [default: above1] + -o, --output Output format [default: auto = table on a terminal, json when piped] [possible values: auto, table, json, jsonl] + --limit Maximum number of discovered report URLs to print [default: 6] + --no-color Disable colored output + -q, --quiet Suppress non-essential output + --require-data-page Fail if the IDX data page cannot be read, instead of warning and listing older announcements. The newest above-1% reports are only on that page + -v, --verbose... Increase verbosity (-v, -vv) + --offline Run without network requests + --no-cache Bypass local cache + -h, --help Print help Examples: idx ownership discover --limit 1