mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-11-07 09:45:36 +00:00
feat(ownership): discover above-1% XLSX on the IDX data page
IDX stopped posting the monthly above-1% list as an announcement after the 2026-05-29 report; June/July/August are only on the Data Kepemilikan Saham page. `ownership discover` could not see them, and IDX's Cloudflare now rejects curl_chrome142 on that page and on the /Media/ XLSX files. - remote: parse the page's Nuxt payload (Description/Prospectus pairs, / escapes), classify above-1% XLSX as supported and above-5% / investor-type workbooks as unsupported, derive as_of_date from the Indonesian description. Keep all above-1% entries, only the newest of the other series. Merge with the announcement results. - discover JSON gains `format` (xlsx|pdf) and `as_of_date`; page entries use the as-of date as publish_date (the page shows none). - import --url accepts .xlsx and downloads via the XLSX validator. - curl_impersonate: run_owned_with_fallback tries several browser profiles until the body validates (JSON / page payload / %PDF / PK); IDX_CURL_IMPERSONATE_BIN still pins one. - Tests: page fixture parsing, date parsing, payload validation, CLI discover via a local page server; existing discover tests now point IDX_OWNERSHIP_DATA_PAGE_URL at a closed port so no test hits IDX. - Docs: ARCHITECTURE discovery section, AGENTS ownership inputs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E
This commit is contained in:
parent
06d0a7a2ae
commit
29e416109c
7 changed files with 592 additions and 27 deletions
|
|
@ -40,7 +40,65 @@ pub fn run(stage: &str, args: &[&str]) -> Result<Output, IdxError> {
|
|||
|
||||
pub fn run_owned(stage: &str, args: &[String]) -> Result<Output, IdxError> {
|
||||
let binary = chrome_curl_binary()?;
|
||||
let output = Command::new(&binary).args(args).output().map_err(|e| {
|
||||
run_binary(stage, &binary, args)
|
||||
}
|
||||
|
||||
/// Browser profiles tried, in order, when a site's bot protection rejects
|
||||
/// some TLS fingerprints but not others (IDX's Cloudflare rules block
|
||||
/// `curl_chrome142` on some pages while Firefox/Safari profiles pass).
|
||||
const FALLBACK_PROFILES: &[&str] = &[
|
||||
"curl_chrome142",
|
||||
"curl_firefox144",
|
||||
"curl_chrome136",
|
||||
"curl_firefox135",
|
||||
"curl_safari184",
|
||||
"curl_chrome131",
|
||||
"curl_chrome124",
|
||||
"curl_chrome120",
|
||||
"curl_chrome116",
|
||||
];
|
||||
const FALLBACK_ROUNDS: usize = 2;
|
||||
|
||||
/// Run curl with each available impersonation profile until `accept` approves
|
||||
/// the response body. `IDX_CURL_IMPERSONATE_BIN`, when set, is the only
|
||||
/// profile used. Returns the first accepted output, or the last error.
|
||||
pub fn run_owned_with_fallback(
|
||||
stage: &str,
|
||||
args: &[String],
|
||||
accept: &dyn Fn(&[u8]) -> Result<(), IdxError>,
|
||||
) -> Result<Output, IdxError> {
|
||||
let profiles: Vec<String> = match std::env::var(OVERRIDE_ENV) {
|
||||
Ok(value) if !value.trim().is_empty() => vec![value.trim().to_string()],
|
||||
_ => FALLBACK_PROFILES
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|bin| Command::new(bin).arg("--version").output().is_ok())
|
||||
.map(str::to_string)
|
||||
.collect(),
|
||||
};
|
||||
if profiles.is_empty() {
|
||||
return Err(IdxError::Http(format!(
|
||||
"no curl-impersonate binary found; set {OVERRIDE_ENV} or install nixpkgs#curl-impersonate"
|
||||
)));
|
||||
}
|
||||
|
||||
let mut last_error = None;
|
||||
for _ in 0..FALLBACK_ROUNDS {
|
||||
for binary in &profiles {
|
||||
match run_binary(stage, binary, args) {
|
||||
Ok(output) => match accept(&output.stdout) {
|
||||
Ok(()) => return Ok(output),
|
||||
Err(err) => last_error = Some(err),
|
||||
},
|
||||
Err(err) => last_error = Some(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(last_error.expect("at least one profile was tried"))
|
||||
}
|
||||
|
||||
fn run_binary(stage: &str, binary: &str, args: &[String]) -> Result<Output, IdxError> {
|
||||
let output = Command::new(binary).args(args).output().map_err(|e| {
|
||||
if e.kind() == std::io::ErrorKind::NotFound {
|
||||
return IdxError::Http(format!(
|
||||
"curl-impersonate binary '{binary}' not found; set {OVERRIDE_ENV} or install nixpkgs#curl-impersonate-chrome"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue