mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-11-07 09:45:36 +00:00
fix(ownership): report a Cloudflare challenge as its own error and result
When Cloudflare answers idx.co.id with a challenge, curl fails with only `curl: (22)`, which exits 4 like any other HTTP failure, and every profile is tried twice. A challenge needs a browser, so no profile can solve it. The source is monthly, so the publisher only needs one run per month to get through: treat a challenge as "retry tomorrow" and alert on stale data. - Add IdxError::CloudflareChallenge (CLOUDFLARECHALLENGE, exit 6), detected from the cf-mitigated header. curl reports it on stderr through --write-out, because --fail drops the response. - A URL with a challenge gets one round of profiles; other failures keep two. This also covers `stocks flow --source idx`, which now exits 6 on a challenge. - -v logs which profile fetched each URL. - Discovery keeps error types, so a challenge survives an announcement miss. A source answering with something unparseable still is not called a challenge. - `ownership discover --require-data-page` fails when the data page cannot be read. Default discover is unchanged: it warns and lists older announcements. - The publisher and builder use that flag, so a failed data page can no longer end in a false "up-to-date". A challenge ends in `RESULT: blocked stage=<stage>`, exit 6; every other failure stays `RESULT: FAILED`. - The systemd example treats exit 6 as success and runs the freshness check, so the unit fails on FAILED or on stale data. - live-smoke reports a challenge as CHALLENGED, a failure and never a pass. Golden change: completions-bash.golden gains `--require-data-page` in the discover options, one word on one line. It is the direct result of the new flag; no other golden file changes. Live check on 2026-11-05 against release/v0.5.0: 19 passed, 2 failed. Both failures are real challenges (flow-idx, discover), each exit 6 with empty stdout. The import, releases and ticker cases did not run, because discover did not pass. Refs #83
This commit is contained in:
hermes
parent
50d9f92550
commit
b236c7f9d9
23 changed files with 797 additions and 54 deletions
|
|
@ -14,6 +14,7 @@ use idx_core::investor::{
|
|||
ConcentrationRank, HolderSource, ImportSource, Investor, parse_report_family,
|
||||
};
|
||||
use idx_core::ownership::graph;
|
||||
use idx_core::ownership::remote::DataPagePolicy;
|
||||
use idx_core::ownership::snapshot;
|
||||
use idx_core::ownership::types::{ChangeType, FlowSignal, HolderRow, OwnershipSource};
|
||||
|
||||
|
|
@ -88,6 +89,9 @@ pub struct DiscoverArgs {
|
|||
/// Maximum number of discovered report URLs to print.
|
||||
#[arg(long, default_value_t = 6)]
|
||||
pub limit: usize,
|
||||
/// Fail if the IDX data page cannot be read, instead of warning and listing older announcements. The newest above-1% reports are only on that page.
|
||||
#[arg(long)]
|
||||
pub require_data_page: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
|
|
@ -284,7 +288,12 @@ fn handle_discover(
|
|||
}
|
||||
|
||||
let family = parse_report_family(&args.family)?;
|
||||
let reports = investor.discover(family, args.limit)?;
|
||||
let data_page = if args.require_data_page {
|
||||
DataPagePolicy::Required
|
||||
} else {
|
||||
DataPagePolicy::BestEffort
|
||||
};
|
||||
let reports = investor.discover(family, args.limit, data_page)?;
|
||||
|
||||
if config.output.is_json() {
|
||||
return json::print_json(&reports);
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ fn run() -> Result<(), IdxError> {
|
|||
Err(err) => return Err(argument_error(err)),
|
||||
};
|
||||
runtime::set_quiet(cli.quiet);
|
||||
runtime::set_verbose(cli.verbose > 0);
|
||||
let config = match IdxConfig::load_with_cli(&cli) {
|
||||
Ok(config) => config,
|
||||
Err(err) => {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ use std::net::TcpListener;
|
|||
#[cfg(unix)]
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::thread;
|
||||
|
||||
use assert_cmd::Command;
|
||||
|
|
@ -1110,6 +1112,427 @@ fn ownership_releases_uses_xdg_data_home_default_db_path() {
|
|||
);
|
||||
}
|
||||
|
||||
/// Answers every request like Cloudflare's challenge and counts them. Without
|
||||
/// `mitigated` it is a plain 403 from the origin. Returns the URL and the count.
|
||||
fn spawn_403_server(mitigated: bool) -> (String, Arc<AtomicUsize>) {
|
||||
let extra = if mitigated {
|
||||
"cf-mitigated: challenge\r\n"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let body = "<html><title>Just a moment...</title></html>";
|
||||
spawn_counting_server(format!(
|
||||
"HTTP/1.1 403 Forbidden\r\nServer: cloudflare\r\n{extra}Content-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
))
|
||||
}
|
||||
|
||||
/// Answers every request with `response`, counting them.
|
||||
fn spawn_counting_server(response: String) -> (String, Arc<AtomicUsize>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind local test server");
|
||||
let addr = listener.local_addr().expect("local addr");
|
||||
let hits = Arc::new(AtomicUsize::new(0));
|
||||
let counted = Arc::clone(&hits);
|
||||
|
||||
thread::spawn(move || {
|
||||
for stream in listener.incoming() {
|
||||
let Ok(mut stream) = stream else { return };
|
||||
let mut buf = [0u8; 2048];
|
||||
let _ = stream.read(&mut buf);
|
||||
counted.fetch_add(1, Ordering::SeqCst);
|
||||
let _ = stream.write_all(response.as_bytes());
|
||||
}
|
||||
});
|
||||
|
||||
(format!("http://{addr}"), hits)
|
||||
}
|
||||
|
||||
fn discover_with(
|
||||
name: &str,
|
||||
announcements: &str,
|
||||
page: &str,
|
||||
extra: &[&str],
|
||||
) -> std::process::Output {
|
||||
test_bin(name)
|
||||
.env("IDX_CURL_IMPERSONATE_BIN", "curl")
|
||||
.env("IDX_OWNERSHIP_ANNOUNCEMENT_API_URL", announcements)
|
||||
.env("IDX_OWNERSHIP_DATA_PAGE_URL", page)
|
||||
.args(["-o", "json", "ownership", "discover"])
|
||||
.args(extra)
|
||||
.output()
|
||||
.expect("ownership discover output")
|
||||
}
|
||||
|
||||
/// The one JSON error on stderr, after any diagnostic JSON lines.
|
||||
fn stderr_error(output: &std::process::Output) -> Value {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let line = stderr.lines().last().unwrap_or_default();
|
||||
serde_json::from_str(line).unwrap_or_else(|_| panic!("stderr is not JSON: {stderr}"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_discover_names_a_cloudflare_challenge_after_one_round() {
|
||||
let (announcements, announcement_hits) = spawn_403_server(true);
|
||||
let (page, page_hits) = spawn_403_server(true);
|
||||
|
||||
let output = discover_with("ownership-discover-challenge", &announcements, &page, &[]);
|
||||
|
||||
assert_eq!(output.status.code(), Some(6));
|
||||
assert!(output.stdout.is_empty());
|
||||
let error = stderr_error(&output);
|
||||
assert_eq!(error["code"], "CLOUDFLARECHALLENGE");
|
||||
assert!(
|
||||
error["message"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains("cf-mitigated: challenge"),
|
||||
"{error}"
|
||||
);
|
||||
assert!(
|
||||
!error["message"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains("idx-curl-probe"),
|
||||
"{error}"
|
||||
);
|
||||
assert_eq!(announcement_hits.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(page_hits.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_discover_plain_403_stays_http_and_keeps_retrying() {
|
||||
let (announcements, announcement_hits) = spawn_403_server(false);
|
||||
let (page, page_hits) = spawn_403_server(false);
|
||||
|
||||
let output = discover_with("ownership-discover-plain-403", &announcements, &page, &[]);
|
||||
|
||||
assert_eq!(output.status.code(), Some(4));
|
||||
assert!(output.stdout.is_empty());
|
||||
assert_eq!(stderr_error(&output)["code"], "HTTP");
|
||||
assert_eq!(announcement_hits.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(page_hits.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
fn announcement_fixture_url() -> String {
|
||||
let raw = r#"{
|
||||
"Items": [{
|
||||
"PublishDate": "2026-03-10T12:09:09",
|
||||
"Title": "Pemegang Saham di atas 1% (KSEI)",
|
||||
"AnnouncementType": "",
|
||||
"Code": "Semua Emiten Saham",
|
||||
"Attachments": [{
|
||||
"PDFFilename": "above1.pdf",
|
||||
"FullSavePath": "https://www.idx.co.id/StaticData/above1.pdf",
|
||||
"IsAttachment": 1,
|
||||
"OriginalFilename": "above1_lamp1.pdf"
|
||||
}],
|
||||
"PdfPath": ""
|
||||
}],
|
||||
"ItemCount": 1,
|
||||
"PageSize": 10,
|
||||
"PageNumber": 1,
|
||||
"PageCount": 1
|
||||
}"#;
|
||||
spawn_single_response_server("application/json", raw)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_discover_warns_on_a_challenged_data_page_by_default() {
|
||||
let (page, _) = spawn_403_server(true);
|
||||
|
||||
let output = discover_with(
|
||||
"ownership-discover-partial",
|
||||
&announcement_fixture_url(),
|
||||
&page,
|
||||
&[],
|
||||
);
|
||||
|
||||
assert!(output.status.success(), "{output:?}");
|
||||
assert!(
|
||||
String::from_utf8_lossy(&output.stderr).contains("cf-mitigated: challenge"),
|
||||
"{output:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_discover_require_data_page_fails_on_a_challenged_page() {
|
||||
let (page, _) = spawn_403_server(true);
|
||||
|
||||
let output = discover_with(
|
||||
"ownership-discover-require-challenge",
|
||||
&announcement_fixture_url(),
|
||||
&page,
|
||||
&["--require-data-page"],
|
||||
);
|
||||
|
||||
assert_eq!(output.status.code(), Some(6));
|
||||
assert!(output.stdout.is_empty(), "{output:?}");
|
||||
assert_eq!(stderr_error(&output)["code"], "CLOUDFLARECHALLENGE");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_discover_require_data_page_keeps_other_page_failures_distinct() {
|
||||
let unreachable = discover_with(
|
||||
"ownership-discover-require-unreachable",
|
||||
&announcement_fixture_url(),
|
||||
UNREACHABLE_DATA_PAGE_URL,
|
||||
&["--require-data-page"],
|
||||
);
|
||||
assert_eq!(unreachable.status.code(), Some(4));
|
||||
assert!(unreachable.stdout.is_empty(), "{unreachable:?}");
|
||||
assert_eq!(stderr_error(&unreachable)["code"], "HTTP");
|
||||
|
||||
// The page's validator looks for this word, so this loads but has no reports.
|
||||
let empty_page = spawn_single_response_server("text/html", "Prospectus");
|
||||
let unparsed = discover_with(
|
||||
"ownership-discover-require-unparsed",
|
||||
&announcement_fixture_url(),
|
||||
&empty_page,
|
||||
&["--require-data-page"],
|
||||
);
|
||||
assert_eq!(unparsed.status.code(), Some(1));
|
||||
assert!(unparsed.stdout.is_empty(), "{unparsed:?}");
|
||||
assert_eq!(stderr_error(&unparsed)["code"], "PARSEERROR");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_discover_require_data_page_passes_with_the_page() {
|
||||
let page = fs::read_to_string(fixture_path("idx_share_ownership_page_excerpt.html"))
|
||||
.expect("read share ownership page fixture");
|
||||
let page_url = spawn_single_response_server("text/html", page);
|
||||
|
||||
let output = discover_with(
|
||||
"ownership-discover-require-ok",
|
||||
"http://127.0.0.1:9/announcements",
|
||||
&page_url,
|
||||
&["--require-data-page", "--limit", "1"],
|
||||
);
|
||||
|
||||
assert!(output.status.success(), "{output:?}");
|
||||
let reports: Value = serde_json::from_slice(&output.stdout).expect("parse discover json");
|
||||
assert_eq!(reports[0]["as_of_date"], "2026-08-31");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_discover_logs_the_passing_profile_only_when_verbose() {
|
||||
let page = fs::read_to_string(fixture_path("idx_share_ownership_page_excerpt.html"))
|
||||
.expect("read share ownership page fixture");
|
||||
let run = |name: &str, flags: &[&str]| {
|
||||
test_bin(name)
|
||||
.env("IDX_CURL_IMPERSONATE_BIN", "curl")
|
||||
.env(
|
||||
"IDX_OWNERSHIP_ANNOUNCEMENT_API_URL",
|
||||
"http://127.0.0.1:9/announcements",
|
||||
)
|
||||
.env(
|
||||
"IDX_OWNERSHIP_DATA_PAGE_URL",
|
||||
spawn_single_response_server("text/html", page.clone()),
|
||||
)
|
||||
.args(flags)
|
||||
.args(["-o", "json", "ownership", "discover", "--limit", "1"])
|
||||
.output()
|
||||
.expect("ownership discover output")
|
||||
};
|
||||
|
||||
let verbose = run("ownership-discover-verbose", &["-v"]);
|
||||
assert!(verbose.status.success(), "{verbose:?}");
|
||||
assert!(
|
||||
String::from_utf8_lossy(&verbose.stderr).contains("fetched with curl"),
|
||||
"{verbose:?}"
|
||||
);
|
||||
|
||||
let default = run("ownership-discover-not-verbose", &[]);
|
||||
assert!(default.status.success(), "{default:?}");
|
||||
assert!(
|
||||
!String::from_utf8_lossy(&default.stderr).contains("fetched with"),
|
||||
"{default:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A directory holding only curl-impersonate profiles, which are real curl
|
||||
/// routed to a local port, so the code's own fallback list picks them up.
|
||||
#[cfg(unix)]
|
||||
fn fake_profiles(name: &str, profiles: &[(&str, &str)]) -> PathBuf {
|
||||
let curl = std::env::split_paths(&std::env::var_os("PATH").expect("PATH"))
|
||||
.map(|dir| dir.join("curl"))
|
||||
.find(|path| path.is_file())
|
||||
.expect("curl on PATH");
|
||||
let dir = test_env_dir(name).join("profiles");
|
||||
fs::create_dir_all(&dir).expect("create profile dir");
|
||||
for (profile, server) in profiles {
|
||||
let port = server.rsplit(':').next().expect("server port");
|
||||
let path = dir.join(profile);
|
||||
fs::write(
|
||||
&path,
|
||||
format!(
|
||||
"#!/bin/sh\nexec {} --connect-to ::127.0.0.1:{port} \"$@\"\n",
|
||||
curl.display()
|
||||
),
|
||||
)
|
||||
.expect("write fake profile");
|
||||
fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod fake profile");
|
||||
}
|
||||
dir
|
||||
}
|
||||
|
||||
/// Discovery using the profile fallback list, not a pinned binary.
|
||||
#[cfg(unix)]
|
||||
fn discover_with_profiles(name: &str, profiles: &Path, flags: &[&str]) -> std::process::Output {
|
||||
test_bin(name)
|
||||
.env_remove("IDX_CURL_IMPERSONATE_BIN")
|
||||
.env("PATH", profiles)
|
||||
.env("IDX_OWNERSHIP_ANNOUNCEMENT_API_URL", "http://idx.test/api")
|
||||
.env("IDX_OWNERSHIP_DATA_PAGE_URL", "http://idx.test/page")
|
||||
.args(flags)
|
||||
.args(["-o", "json", "ownership", "discover"])
|
||||
.output()
|
||||
.expect("ownership discover output")
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn challenged_profiles_get_one_round_per_url() {
|
||||
let (server, hits) = spawn_403_server(true);
|
||||
let profiles = fake_profiles(
|
||||
"fallback-challenged",
|
||||
&[("curl_chrome142", &server), ("curl_firefox144", &server)],
|
||||
);
|
||||
|
||||
let output = discover_with_profiles("fallback-challenged-run", &profiles, &[]);
|
||||
|
||||
assert_eq!(output.status.code(), Some(6), "{output:?}");
|
||||
assert_eq!(stderr_error(&output)["code"], "CLOUDFLARECHALLENGE");
|
||||
// Two URLs, each tried once by each of two profiles.
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 4);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn plain_403_profiles_keep_both_rounds() {
|
||||
let (server, hits) = spawn_403_server(false);
|
||||
let profiles = fake_profiles(
|
||||
"fallback-plain-403",
|
||||
&[("curl_chrome142", &server), ("curl_firefox144", &server)],
|
||||
);
|
||||
|
||||
let output = discover_with_profiles("fallback-plain-403-run", &profiles, &[]);
|
||||
|
||||
assert_eq!(output.status.code(), Some(4), "{output:?}");
|
||||
assert_eq!(stderr_error(&output)["code"], "HTTP");
|
||||
// Two URLs, two profiles, two rounds.
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 8);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn a_profile_can_pass_after_another_is_challenged_and_is_logged() {
|
||||
let (challenged, _) = spawn_403_server(true);
|
||||
let page = fs::read_to_string(fixture_path("idx_share_ownership_page_excerpt.html"))
|
||||
.expect("read share ownership page fixture");
|
||||
let (passing, _) = spawn_counting_server(format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{page}",
|
||||
page.len()
|
||||
));
|
||||
let profiles = fake_profiles(
|
||||
"fallback-passes",
|
||||
&[
|
||||
("curl_chrome142", &challenged),
|
||||
("curl_firefox144", &passing),
|
||||
],
|
||||
);
|
||||
|
||||
let output = discover_with_profiles("fallback-passes-run", &profiles, &["-v"]);
|
||||
|
||||
assert!(output.status.success(), "{output:?}");
|
||||
let reports: Value = serde_json::from_slice(&output.stdout).expect("parse discover json");
|
||||
assert_eq!(reports[0]["as_of_date"], "2026-08-31");
|
||||
assert!(
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
.contains("IDX share ownership data page discovery: fetched with curl_firefox144"),
|
||||
"{output:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn a_challenge_outranks_a_later_profile_failing() {
|
||||
let (challenged, _) = spawn_403_server(true);
|
||||
let profiles = fake_profiles(
|
||||
"fallback-later-failure",
|
||||
&[
|
||||
("curl_chrome142", &challenged),
|
||||
("curl_firefox144", UNREACHABLE_DATA_PAGE_PORT),
|
||||
],
|
||||
);
|
||||
|
||||
let output = discover_with_profiles("fallback-later-failure-run", &profiles, &[]);
|
||||
|
||||
assert_eq!(output.status.code(), Some(6), "{output:?}");
|
||||
assert_eq!(stderr_error(&output)["code"], "CLOUDFLARECHALLENGE");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_discover_keeps_the_challenge_past_an_announcement_miss() {
|
||||
let announcements = spawn_single_response_server("application/json", r#"{"Items":[]}"#);
|
||||
let (page, _) = spawn_403_server(true);
|
||||
|
||||
let output = discover_with("discover-miss-and-challenge", &announcements, &page, &[]);
|
||||
|
||||
assert_eq!(output.status.code(), Some(6), "{output:?}");
|
||||
assert_eq!(stderr_error(&output)["code"], "CLOUDFLARECHALLENGE");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_discover_does_not_blame_a_challenge_for_a_changed_page() {
|
||||
let (announcements, _) = spawn_403_server(true);
|
||||
// The page's validator looks for this word, so this loads but has no reports.
|
||||
let page = spawn_single_response_server("text/html", "Prospectus");
|
||||
|
||||
let output = discover_with("discover-challenge-and-parse", &announcements, &page, &[]);
|
||||
|
||||
assert_eq!(output.status.code(), Some(4), "{output:?}");
|
||||
assert_eq!(stderr_error(&output)["code"], "HTTP");
|
||||
}
|
||||
|
||||
/// idx.co.id download URLs always go through curl, so a wrapper points real curl
|
||||
/// at the local server, keeping this off the live network.
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn ownership_import_url_names_a_cloudflare_challenge() {
|
||||
let (server, hits) = spawn_403_server(true);
|
||||
let port = server.rsplit(':').next().expect("server port");
|
||||
let root = test_env_dir("ownership-import-challenge");
|
||||
let wrapper = root.join("curl-wrapper");
|
||||
fs::write(
|
||||
&wrapper,
|
||||
format!("#!/bin/sh\nexec curl --connect-to ::127.0.0.1:{port} \"$@\"\n"),
|
||||
)
|
||||
.expect("write curl wrapper");
|
||||
fs::set_permissions(&wrapper, fs::Permissions::from_mode(0o755)).expect("chmod curl wrapper");
|
||||
|
||||
let output = bin_with_root(&root)
|
||||
.env("IDX_CURL_IMPERSONATE_BIN", &wrapper)
|
||||
.args([
|
||||
"-o",
|
||||
"json",
|
||||
"ownership",
|
||||
"import",
|
||||
"--url",
|
||||
"http://www.idx.co.id/Media/fixture/peng-2026-09-00024-satu-persen.xlsx",
|
||||
])
|
||||
.output()
|
||||
.expect("ownership import output");
|
||||
|
||||
assert_eq!(output.status.code(), Some(6), "{output:?}");
|
||||
assert!(output.stdout.is_empty());
|
||||
assert_eq!(stderr_error(&output)["code"], "CLOUDFLARECHALLENGE");
|
||||
assert_eq!(hits.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
/// A fake profile routed here fails to connect (nothing listens on port 9).
|
||||
#[cfg(unix)]
|
||||
const UNREACHABLE_DATA_PAGE_PORT: &str = "http://127.0.0.1:9";
|
||||
|
||||
/// Keeps discovery tests off the live IDX data page (nothing listens on port 9).
|
||||
const UNREACHABLE_DATA_PAGE_URL: &str = "http://127.0.0.1:9/data-kepemilikan-saham/";
|
||||
|
||||
|
|
|
|||
|
|
@ -1610,7 +1610,7 @@ _idx() {
|
|||
return 0
|
||||
;;
|
||||
idx__ownership__discover)
|
||||
opts="-o -q -v -h --family --limit --output --no-color --quiet --verbose --offline --no-cache --help"
|
||||
opts="-o -q -v -h --family --limit --require-data-page --output --no-color --quiet --verbose --offline --no-cache --help"
|
||||
if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then
|
||||
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
|
||||
return 0
|
||||
|
|
|
|||
|
|
@ -44,9 +44,11 @@ pub fn run_owned(stage: &str, args: &[String]) -> Result<Output, IdxError> {
|
|||
run_binary(stage, &binary, args)
|
||||
}
|
||||
|
||||
/// Browser profiles tried, in order, when a site's bot protection rejects
|
||||
/// some TLS fingerprints but not others (IDX's Cloudflare rules block
|
||||
/// `curl_chrome142` on some pages while Firefox/Safari profiles pass).
|
||||
/// Browser profiles tried, in order, because a site's bot protection can reject
|
||||
/// some TLS fingerprints but not others. On IDX which profiles pass changes over
|
||||
/// time and by path, and at times none does (on 2026-11-05 Chrome, Firefox and
|
||||
/// Safari profiles all got a challenge). A profile only changes the fingerprint,
|
||||
/// so it cannot solve a challenge that is active for this client.
|
||||
const FALLBACK_PROFILES: &[&str] = &[
|
||||
"curl_chrome142",
|
||||
"curl_firefox144",
|
||||
|
|
@ -60,9 +62,47 @@ const FALLBACK_PROFILES: &[&str] = &[
|
|||
];
|
||||
const FALLBACK_ROUNDS: usize = 2;
|
||||
|
||||
/// Marks the line curl adds to stderr describing the response. `--fail` drops
|
||||
/// the response itself, and stdout must stay the body alone.
|
||||
const PROBE_PREFIX: &str = "idx-curl-probe";
|
||||
const PROBE_WRITE_OUT: &str =
|
||||
"%{stderr}idx-curl-probe status=%{http_code} cf-mitigated=%header{cf-mitigated}\\n";
|
||||
|
||||
/// What the probe line says about a response.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
struct ResponseProbe {
|
||||
status: String,
|
||||
challenged: bool,
|
||||
}
|
||||
|
||||
/// Separates the probe line from curl's own stderr. Without a probe line (the
|
||||
/// caller did not ask for one) stderr comes back unchanged.
|
||||
fn split_probe(stderr: &[u8]) -> (Option<ResponseProbe>, Vec<u8>) {
|
||||
let mut probe = None;
|
||||
let mut rest = Vec::with_capacity(stderr.len());
|
||||
for line in stderr.split_inclusive(|byte| *byte == b'\n') {
|
||||
let parsed = std::str::from_utf8(line)
|
||||
.ok()
|
||||
.and_then(|text| text.trim_end().strip_prefix(PROBE_PREFIX))
|
||||
.and_then(|fields| fields.strip_prefix(" status="))
|
||||
.and_then(|fields| fields.split_once(" cf-mitigated="));
|
||||
match parsed {
|
||||
Some((status, mitigated)) => {
|
||||
probe = Some(ResponseProbe {
|
||||
status: status.to_string(),
|
||||
challenged: mitigated.trim().eq_ignore_ascii_case("challenge"),
|
||||
});
|
||||
}
|
||||
None => rest.extend_from_slice(line),
|
||||
}
|
||||
}
|
||||
(probe, rest)
|
||||
}
|
||||
|
||||
/// Run curl with each available impersonation profile until `accept` approves
|
||||
/// the response body. `IDX_CURL_IMPERSONATE_BIN`, when set, is the only
|
||||
/// profile used. Returns the first accepted output, or the last error.
|
||||
/// profile used. Returns the first accepted output, or an error: a Cloudflare
|
||||
/// challenge from any profile, otherwise the last error.
|
||||
pub fn run_owned_with_fallback(
|
||||
stage: &str,
|
||||
args: &[String],
|
||||
|
|
@ -83,17 +123,39 @@ pub fn run_owned_with_fallback(
|
|||
)));
|
||||
}
|
||||
|
||||
run_profiles(stage, &profiles, args, accept)
|
||||
}
|
||||
|
||||
/// A challenge can differ by profile, so one round tries them all, but a repeat
|
||||
/// straight away is unlikely to change the answer: it ends after that round.
|
||||
fn run_profiles(
|
||||
stage: &str,
|
||||
profiles: &[String],
|
||||
args: &[String],
|
||||
accept: &dyn Fn(&[u8]) -> Result<(), IdxError>,
|
||||
) -> Result<Output, IdxError> {
|
||||
let mut probed_args = vec!["--write-out".to_string(), PROBE_WRITE_OUT.to_string()];
|
||||
probed_args.extend_from_slice(args);
|
||||
|
||||
let mut last_error = None;
|
||||
let mut challenge = None;
|
||||
for _ in 0..FALLBACK_ROUNDS {
|
||||
for binary in &profiles {
|
||||
match run_binary(stage, binary, args) {
|
||||
for binary in profiles {
|
||||
match run_binary(stage, binary, &probed_args) {
|
||||
Ok(output) => match accept(&output.stdout) {
|
||||
Ok(()) => return Ok(output),
|
||||
Ok(()) => {
|
||||
crate::runtime::verbose(format!("{stage}: fetched with {binary}"));
|
||||
return Ok(output);
|
||||
}
|
||||
Err(err) => last_error = Some(err),
|
||||
},
|
||||
Err(err @ IdxError::CloudflareChallenge(_)) => challenge = Some(err),
|
||||
Err(err) => last_error = Some(err),
|
||||
}
|
||||
}
|
||||
if let Some(err) = challenge {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
Err(last_error.expect("at least one profile was tried"))
|
||||
}
|
||||
|
|
@ -166,12 +228,20 @@ fn run_binary_limited(
|
|||
.wait()
|
||||
.map_err(|e| IdxError::Http(format!("failed to wait for {binary} for {stage}: {e}")))?;
|
||||
let stderr = stderr_reader.join().unwrap_or_default();
|
||||
let (probe, stderr) = split_probe(&stderr);
|
||||
let output = Output {
|
||||
status,
|
||||
stdout,
|
||||
stderr,
|
||||
};
|
||||
|
||||
if let Some(probe) = probe.filter(|probe| probe.challenged) {
|
||||
return Err(IdxError::CloudflareChallenge(format!(
|
||||
"{stage}: Cloudflare answered {binary} with a challenge (HTTP {}, cf-mitigated: challenge), which curl cannot solve; retry later",
|
||||
probe.status
|
||||
)));
|
||||
}
|
||||
|
||||
if output.status.success() {
|
||||
return Ok(output);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,6 +34,10 @@ pub enum IdxError {
|
|||
InvalidInput(String),
|
||||
#[error("no market data: {0}")]
|
||||
NoMarketData(String),
|
||||
/// Cloudflare answered with a challenge only a browser can solve, on every
|
||||
/// client tried. Nothing is wrong with the request, so a later run may pass.
|
||||
#[error("cloudflare challenge: {0}")]
|
||||
CloudflareChallenge(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
|
||||
|
|
@ -53,6 +57,7 @@ pub enum ErrorCode {
|
|||
PdfParseError,
|
||||
InvalidInput,
|
||||
NoMarketData,
|
||||
CloudflareChallenge,
|
||||
}
|
||||
|
||||
/// The stable wire form frontends emit, e.g. `SYMBOLNOTFOUND`.
|
||||
|
|
@ -80,6 +85,7 @@ impl IdxError {
|
|||
Self::PdfParseError(_) => ErrorCode::PdfParseError,
|
||||
Self::InvalidInput(_) => ErrorCode::InvalidInput,
|
||||
Self::NoMarketData(_) => ErrorCode::NoMarketData,
|
||||
Self::CloudflareChallenge(_) => ErrorCode::CloudflareChallenge,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -95,6 +101,7 @@ impl IdxError {
|
|||
| ErrorCode::Offline
|
||||
| ErrorCode::AuthError => 4,
|
||||
ErrorCode::Unsupported => 5,
|
||||
ErrorCode::CloudflareChallenge => 6,
|
||||
ErrorCode::Io
|
||||
| ErrorCode::ParseError
|
||||
| ErrorCode::DatabaseError
|
||||
|
|
|
|||
|
|
@ -14,7 +14,9 @@ use sha2::{Digest, Sha256};
|
|||
|
||||
use crate::config::CoreConfig;
|
||||
use crate::error::IdxError;
|
||||
use crate::ownership::remote::{self, DiscoveredOwnershipPdf, OwnershipReportFamily};
|
||||
use crate::ownership::remote::{
|
||||
self, DataPagePolicy, DiscoveredOwnershipPdf, OwnershipReportFamily,
|
||||
};
|
||||
use crate::ownership::snapshot::{self, OwnershipSyncResult};
|
||||
use crate::ownership::types::{
|
||||
ChangeRow, ConcentrationMetrics, CrossHolderRow, Entity, EntityHoldings, GraphEdge, GraphNode,
|
||||
|
|
@ -264,8 +266,9 @@ impl Investor {
|
|||
&self,
|
||||
family: Option<OwnershipReportFamily>,
|
||||
limit: usize,
|
||||
data_page: DataPagePolicy,
|
||||
) -> Result<Vec<DiscoveredOwnershipPdf>, IdxError> {
|
||||
remote::discover_idx_ownership_reports(family, limit)
|
||||
remote::discover_idx_ownership_reports(family, limit, data_page)
|
||||
}
|
||||
|
||||
/// Installs or refreshes the maintained snapshot. `manifest` overrides
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ pub mod runtime {
|
|||
|
||||
static QUIET: AtomicBool = AtomicBool::new(false);
|
||||
static JSON: AtomicBool = AtomicBool::new(false);
|
||||
static VERBOSE: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
pub fn set_quiet(quiet: bool) {
|
||||
QUIET.store(quiet, Ordering::Relaxed);
|
||||
|
|
@ -36,6 +37,10 @@ pub mod runtime {
|
|||
QUIET.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn set_verbose(verbose: bool) {
|
||||
VERBOSE.store(verbose, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// In JSON mode diagnostics are JSON lines, so stderr stays
|
||||
/// machine-readable alongside a JSON error.
|
||||
pub fn set_json(json: bool) {
|
||||
|
|
@ -61,6 +66,13 @@ pub mod runtime {
|
|||
emit("info", message);
|
||||
}
|
||||
|
||||
/// Detail worth keeping in a job's log but not worth showing by default.
|
||||
pub fn verbose(message: impl Display) {
|
||||
if VERBOSE.load(Ordering::Relaxed) {
|
||||
emit("info", message);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn warn(message: impl Display) {
|
||||
emit("warning", message);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -281,12 +281,24 @@ pub fn select_latest_ownership_reports(
|
|||
Ok(attachments)
|
||||
}
|
||||
|
||||
/// Whether discovery may answer without the data page, the only place the
|
||||
/// newest above-1% reports are listed since June 2026.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum DataPagePolicy {
|
||||
/// Warn and return what the announcements list, which can miss the newest
|
||||
/// reports.
|
||||
BestEffort,
|
||||
/// Fail with the data page's error, so old results cannot pass as current.
|
||||
Required,
|
||||
}
|
||||
|
||||
pub fn discover_idx_ownership_reports(
|
||||
family_filter: Option<OwnershipReportFamily>,
|
||||
limit: usize,
|
||||
data_page: DataPagePolicy,
|
||||
) -> Result<Vec<DiscoveredOwnershipPdf>, IdxError> {
|
||||
let mut discovered = Vec::new();
|
||||
let mut errors = Vec::new();
|
||||
let mut errors: Vec<IdxError> = Vec::new();
|
||||
|
||||
for query in DISCOVERY_QUERIES
|
||||
.iter()
|
||||
|
|
@ -303,7 +315,7 @@ pub fn discover_idx_ownership_reports(
|
|||
.and_then(|page| select_latest_ownership_reports(&page, &query_url, query.family))
|
||||
{
|
||||
Ok(mut reports) => discovered.append(&mut reports),
|
||||
Err(err) => errors.push(err.to_string()),
|
||||
Err(err) => errors.push(err),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -320,10 +332,10 @@ pub fn discover_idx_ownership_reports(
|
|||
// The page loaded (it has its file list) but nothing parsed:
|
||||
// the payload shape changed. Say so instead of silently
|
||||
// falling back to announcement-only results.
|
||||
page_error = Some(format!(
|
||||
page_error = Some(IdxError::ParseError(format!(
|
||||
"IDX share ownership data page {} loaded but no report entries could be parsed; the page layout may have changed",
|
||||
share_ownership_page_url()
|
||||
));
|
||||
)));
|
||||
}
|
||||
page_reports
|
||||
.retain(|report| family_filter.is_none_or(|family| family == report.family));
|
||||
|
|
@ -343,7 +355,13 @@ pub fn discover_idx_ownership_reports(
|
|||
});
|
||||
discovered.extend(page_reports);
|
||||
}
|
||||
Err(err) => page_error = Some(err.to_string()),
|
||||
Err(err) => page_error = Some(err),
|
||||
}
|
||||
|
||||
if data_page == DataPagePolicy::Required
|
||||
&& let Some(err) = page_error
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
discovered.sort_by(|left, right| {
|
||||
|
|
@ -369,21 +387,50 @@ pub fn discover_idx_ownership_reports(
|
|||
errors.extend(page_error);
|
||||
|
||||
if discovered.is_empty() {
|
||||
let detail = if errors.is_empty() {
|
||||
"no matching ownership reports found".to_string()
|
||||
} else {
|
||||
errors.join("; ")
|
||||
};
|
||||
return Err(IdxError::Http(format!(
|
||||
"failed to discover IDX ownership reports from {} or {}: {detail}",
|
||||
announcement_listing_url(),
|
||||
share_ownership_page_url()
|
||||
)));
|
||||
return Err(discovery_failure(
|
||||
&errors,
|
||||
&announcement_listing_url(),
|
||||
&share_ownership_page_url(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(discovered)
|
||||
}
|
||||
|
||||
/// The error for a discovery that found nothing. A challenge names the cause
|
||||
/// unless a source answered with something unparseable: a retry cannot fix that,
|
||||
/// so it stays a plain failure. Announcement misses do not hide a challenge.
|
||||
fn discovery_failure(errors: &[IdxError], listing_url: &str, page_url: &str) -> IdxError {
|
||||
let challenged = errors
|
||||
.iter()
|
||||
.any(|err| matches!(err, IdxError::CloudflareChallenge(_)));
|
||||
let unparseable = errors
|
||||
.iter()
|
||||
.any(|err| matches!(err, IdxError::ParseError(_)));
|
||||
|
||||
let detail = if errors.is_empty() {
|
||||
"no matching ownership reports found".to_string()
|
||||
} else {
|
||||
errors
|
||||
.iter()
|
||||
.map(|err| match err {
|
||||
IdxError::CloudflareChallenge(message) => message.clone(),
|
||||
other => other.to_string(),
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("; ")
|
||||
};
|
||||
let message = format!(
|
||||
"failed to discover IDX ownership reports from {listing_url} or {page_url}: {detail}"
|
||||
);
|
||||
|
||||
if challenged && !unparseable {
|
||||
IdxError::CloudflareChallenge(message)
|
||||
} else {
|
||||
IdxError::Http(message)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn download_idx_pdf(url: &str, target: &Path) -> Result<(), IdxError> {
|
||||
let bytes = fetch_bytes(
|
||||
"IDX ownership PDF download",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue