From 29e416109c7386e0200bb4ae3984353696c7e2a4 Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 24 Sep 2026 20:55:20 +0000 Subject: [PATCH] feat(ownership): discover above-1% XLSX on the IDX data page IDX stopped posting the monthly above-1% list as an announcement after the 2026-05-29 report; June/July/August are only on the Data Kepemilikan Saham page. `ownership discover` could not see them, and IDX's Cloudflare now rejects curl_chrome142 on that page and on the /Media/ XLSX files. - remote: parse the page's Nuxt payload (Description/Prospectus pairs, / escapes), classify above-1% XLSX as supported and above-5% / investor-type workbooks as unsupported, derive as_of_date from the Indonesian description. Keep all above-1% entries, only the newest of the other series. Merge with the announcement results. - discover JSON gains `format` (xlsx|pdf) and `as_of_date`; page entries use the as-of date as publish_date (the page shows none). - import --url accepts .xlsx and downloads via the XLSX validator. - curl_impersonate: run_owned_with_fallback tries several browser profiles until the body validates (JSON / page payload / %PDF / PK); IDX_CURL_IMPERSONATE_BIN still pins one. - Tests: page fixture parsing, date parsing, payload validation, CLI discover via a local page server; existing discover tests now point IDX_OWNERSHIP_DATA_PAGE_URL at a closed port so no test hits IDX. - Docs: ARCHITECTURE discovery section, AGENTS ownership inputs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E --- AGENTS.md | 4 +- docs/ARCHITECTURE.md | 16 + src/cli/ownership.rs | 64 ++- src/curl_impersonate.rs | 60 ++- src/ownership/remote.rs | 427 +++++++++++++++++- tests/cli.rs | 43 +- .../idx_share_ownership_page_excerpt.html | 5 + 7 files changed, 592 insertions(+), 27 deletions(-) create mode 100644 tests/fixtures/idx_share_ownership_page_excerpt.html diff --git a/AGENTS.md b/AGENTS.md index f0d092f..3a4a034 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,11 +57,11 @@ Ownership is local-first after bootstrap. Preferred bootstrap/update path: 1. `idx ownership sync` -2. if no snapshot manifest is available: `idx ownership discover` then `idx ownership import --url ` +2. if no snapshot manifest is available: `idx ownership discover` then `idx ownership import --url ` 3. local `--file` imports remain available for manual/fallback use Ownership input paths: -- primary remote source: discoverable IDX `above1` holder-register PDF +- primary remote source: IDX `above1` holder register — monthly XLSX on the Data Kepemilikan Saham page (since the 2026-05-29 report), legacy PDF announcements before that; `ownership discover` checks both - maintained snapshot path: `ownership sync` - local file imports: IDX above-1% `.xlsx` (current format since June 2026), legacy `.pdf`, plus local archive `.zip` / `.txt` diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 3919f69..446413d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -67,6 +67,22 @@ Unlike the `stocks` module (live-fetch), ownership is **import-then-query**: - **KSEI archive** — monthly ZIP/TXT balance-position matrix, used as a local fallback/backstop import path - **Bing Finance** — global institutional ownership (REST API, quarterly) +### Discovery +`idx ownership discover` merges two IDX sources: +- the announcement API (`GetAllAnnouncement`) for the legacy PDF announcements +- the Data Kepemilikan Saham page (`/id/perusahaan-tercatat/data-kepemilikan-saham/`), + whose server-rendered Nuxt payload lists every XLSX with a description such as + `Pemegang Saham di Atas 1% per 31 Agustus 2026`; the as-of date comes from that text + +Only the above-1% XLSX is `supported`; the daily above-5% and investor-type workbooks +are listed as `unsupported`. Both URLs can be overridden for tests with +`IDX_OWNERSHIP_ANNOUNCEMENT_API_URL` and `IDX_OWNERSHIP_DATA_PAGE_URL`. + +IDX sits behind Cloudflare, which accepts some curl-impersonate TLS profiles and +rejects others per path (for example `curl_chrome142` gets 403 on the data page). +IDX fetches therefore try several profiles until the response validates (JSON, page +payload, `%PDF`, or `PK` zip magic). `IDX_CURL_IMPERSONATE_BIN` pins a single profile. + ### Parser Pipeline ``` KSEI XLSX → zip + quick-xml (sharedStrings + sheet1) → exact 12-column header check diff --git a/src/cli/ownership.rs b/src/cli/ownership.rs index 2540e2c..f8b4bde 100644 --- a/src/cli/ownership.rs +++ b/src/cli/ownership.rs @@ -35,7 +35,7 @@ pub struct OwnershipCmd { pub enum OwnershipCommand { #[command( about = "Discover the latest IDX-hosted ownership report URLs", - after_help = "Examples:\n idx ownership discover --limit 1\n idx ownership discover --family all --limit 6\n\nUse this when no snapshot manifest is available and you need a direct PDF URL for `idx ownership import --url`." + after_help = "Examples:\n idx ownership discover --limit 1\n idx ownership discover --family all --limit 6\n\nUse this when no snapshot manifest is available and you need a direct XLSX or PDF URL for `idx ownership import --url`." )] Discover(DiscoverArgs), #[command( @@ -84,7 +84,7 @@ pub struct DiscoverArgs { #[derive(Debug, Args)] pub struct ImportArgs { - /// Direct URL to a remote ownership PDF, typically from `ownership discover`. + /// Direct URL to a remote above-1% XLSX or PDF, typically from `ownership discover`. #[arg(long)] pub url: Option, /// Path to a local ownership file: above-1% XLSX or PDF (primary), ZIP/TXT archive (fallback). @@ -999,13 +999,16 @@ fn resolve_import_input(args: &ImportArgs) -> Result if let Some(url) = &args.url { let trimmed = url.trim(); - validate_import_url(trimmed)?; - let target = cache_pdf_path(trimmed)?; - download_pdf(trimmed, &target)?; + let format = validate_import_url(trimmed)?; + let target = cache_download_path(trimmed, format)?; + match format { + ImportInputFormat::Xlsx => download_xlsx(trimmed, &target)?, + _ => download_pdf(trimmed, &target)?, + } return Ok(Some(ResolvedImportInput { import_path: target, source_url: Some(trimmed.to_string()), - format: ImportInputFormat::Pdf, + format, })); } @@ -1031,7 +1034,11 @@ fn detect_local_import_format(path: &Path) -> Result Result { +fn cache_download_path(url: &str, format: ImportInputFormat) -> Result { + let extension = match format { + ImportInputFormat::Xlsx => ".xlsx", + _ => ".pdf", + }; let raw_dir = crate::cache::cache_dir()?.join("ownership").join("raw"); fs::create_dir_all(&raw_dir).map_err(|e| IdxError::Io(e.to_string()))?; @@ -1045,20 +1052,20 @@ fn cache_pdf_path(url: &str) -> Result { .to_string(); if file_name.is_empty() || file_name == "/" { - file_name = format!("ksei-{}.pdf", Utc::now().timestamp()); + file_name = format!("ksei-{}{extension}", Utc::now().timestamp()); } - if !file_name.to_ascii_lowercase().ends_with(".pdf") { - file_name.push_str(".pdf"); + if !file_name.to_ascii_lowercase().ends_with(extension) { + file_name.push_str(extension); } Ok(raw_dir.join(file_name)) } -fn validate_import_url(url: &str) -> Result<(), IdxError> { +fn validate_import_url(url: &str) -> Result { let trimmed = url.trim(); if trimmed.is_empty() { return Err(IdxError::InvalidInput( - "ownership import --url accepts direct PDF URLs only".to_string(), + "ownership import --url accepts direct XLSX or PDF URLs only".to_string(), )); } @@ -1068,11 +1075,14 @@ fn validate_import_url(url: &str) -> Result<(), IdxError> { .next() .unwrap_or(normalized.as_str()); if normalized.ends_with(".pdf") { - return Ok(()); + return Ok(ImportInputFormat::Pdf); + } + if normalized.ends_with(".xlsx") { + return Ok(ImportInputFormat::Xlsx); } Err(IdxError::InvalidInput( - "ownership import --url accepts direct PDF URLs only; run `idx ownership discover` first to find the current supported attachment".to_string(), + "ownership import --url accepts direct XLSX or PDF URLs only; run `idx ownership discover` first to find the current supported file".to_string(), )) } @@ -1108,6 +1118,32 @@ fn download_pdf(url: &str, target: &Path) -> Result<(), IdxError> { Ok(()) } +fn download_xlsx(url: &str, target: &Path) -> Result<(), IdxError> { + if is_idx_url(url) { + return remote::download_idx_xlsx(url, target); + } + + let response = ureq::get(url) + .header( + "Accept", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/octet-stream,*/*;q=0.8", + ) + .call() + .map_err(|e| IdxError::Http(format!("failed to download XLSX: {e}")))?; + let bytes = response + .into_body() + .read_to_vec() + .map_err(|e| IdxError::Http(format!("failed reading XLSX body: {e}")))?; + remote::validate_xlsx_payload(&bytes)?; + + fs::write(target, &bytes).map_err(|e| { + IdxError::Io(format!( + "failed writing cached XLSX {}: {e}", + target.display() + )) + }) +} + fn is_idx_url(url: &str) -> bool { let normalized = url.trim().to_ascii_lowercase(); normalized.starts_with("https://www.idx.co.id/") diff --git a/src/curl_impersonate.rs b/src/curl_impersonate.rs index 3735feb..dbabb8c 100644 --- a/src/curl_impersonate.rs +++ b/src/curl_impersonate.rs @@ -40,7 +40,65 @@ pub fn run(stage: &str, args: &[&str]) -> Result { pub fn run_owned(stage: &str, args: &[String]) -> Result { let binary = chrome_curl_binary()?; - let output = Command::new(&binary).args(args).output().map_err(|e| { + run_binary(stage, &binary, args) +} + +/// Browser profiles tried, in order, when a site's bot protection rejects +/// some TLS fingerprints but not others (IDX's Cloudflare rules block +/// `curl_chrome142` on some pages while Firefox/Safari profiles pass). +const FALLBACK_PROFILES: &[&str] = &[ + "curl_chrome142", + "curl_firefox144", + "curl_chrome136", + "curl_firefox135", + "curl_safari184", + "curl_chrome131", + "curl_chrome124", + "curl_chrome120", + "curl_chrome116", +]; +const FALLBACK_ROUNDS: usize = 2; + +/// Run curl with each available impersonation profile until `accept` approves +/// the response body. `IDX_CURL_IMPERSONATE_BIN`, when set, is the only +/// profile used. Returns the first accepted output, or the last error. +pub fn run_owned_with_fallback( + stage: &str, + args: &[String], + accept: &dyn Fn(&[u8]) -> Result<(), IdxError>, +) -> Result { + let profiles: Vec = match std::env::var(OVERRIDE_ENV) { + Ok(value) if !value.trim().is_empty() => vec![value.trim().to_string()], + _ => FALLBACK_PROFILES + .iter() + .copied() + .filter(|bin| Command::new(bin).arg("--version").output().is_ok()) + .map(str::to_string) + .collect(), + }; + if profiles.is_empty() { + return Err(IdxError::Http(format!( + "no curl-impersonate binary found; set {OVERRIDE_ENV} or install nixpkgs#curl-impersonate" + ))); + } + + let mut last_error = None; + for _ in 0..FALLBACK_ROUNDS { + for binary in &profiles { + match run_binary(stage, binary, args) { + Ok(output) => match accept(&output.stdout) { + Ok(()) => return Ok(output), + Err(err) => last_error = Some(err), + }, + Err(err) => last_error = Some(err), + } + } + } + Err(last_error.expect("at least one profile was tried")) +} + +fn run_binary(stage: &str, binary: &str, args: &[String]) -> Result { + let output = Command::new(binary).args(args).output().map_err(|e| { if e.kind() == std::io::ErrorKind::NotFound { return IdxError::Http(format!( "curl-impersonate binary '{binary}' not found; set {OVERRIDE_ENV} or install nixpkgs#curl-impersonate-chrome" diff --git a/src/ownership/remote.rs b/src/ownership/remote.rs index c8b18f6..a26e3f6 100644 --- a/src/ownership/remote.rs +++ b/src/ownership/remote.rs @@ -12,6 +12,10 @@ const DEFAULT_IDX_ANNOUNCEMENT_API_URL: &str = const IDX_ANNOUNCEMENT_API_ENV: &str = "IDX_OWNERSHIP_ANNOUNCEMENT_API_URL"; const IDX_ANNOUNCEMENT_LISTING_ENV: &str = "IDX_OWNERSHIP_ANNOUNCEMENT_PAGE_URL"; const IDX_ANNOUNCEMENT_PAGE_SIZE: usize = 10; +/// Since June 2026 the monthly above-1% list (XLSX) is published only here. +pub const IDX_SHARE_OWNERSHIP_PAGE_URL: &str = + "https://www.idx.co.id/id/perusahaan-tercatat/data-kepemilikan-saham/"; +const IDX_SHARE_OWNERSHIP_PAGE_ENV: &str = "IDX_OWNERSHIP_DATA_PAGE_URL"; const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"; #[derive(Debug, Clone, Deserialize)] @@ -116,6 +120,12 @@ pub struct DiscoveredOwnershipPdf { pub code: Option, pub original_filename: Option, pub is_attachment: bool, + /// File format of `pdf_url`: `pdf` (announcement attachment) or `xlsx` + /// (Data Kepemilikan Saham page). + pub format: String, + /// Report as-of date (`YYYY-MM-DD`) when the source states it. The data + /// page shows no publish date, so its entries use this as `publish_date`. + pub as_of_date: Option, } struct DiscoveryQuery { @@ -152,6 +162,13 @@ pub fn announcement_listing_url() -> String { .unwrap_or_else(|| IDX_ANNOUNCEMENT_LISTING_URL.to_string()) } +pub fn share_ownership_page_url() -> String { + std::env::var(IDX_SHARE_OWNERSHIP_PAGE_ENV) + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| IDX_SHARE_OWNERSHIP_PAGE_URL.to_string()) +} + pub fn announcement_api_url() -> String { std::env::var(IDX_ANNOUNCEMENT_API_ENV) .ok() @@ -240,6 +257,8 @@ pub fn select_latest_ownership_reports( code: clean_option(item.code.as_deref()), original_filename, is_attachment, + format: "pdf".to_string(), + as_of_date: None, } }) .collect::>(); @@ -278,6 +297,7 @@ pub fn discover_idx_ownership_reports( "IDX ownership announcement discovery", &query_url, &json_headers(), + &validate_json_payload, ) .and_then(|raw| parse_announcement_page(&raw)) .and_then(|page| select_latest_ownership_reports(&page, &query_url, query.family)) @@ -287,6 +307,35 @@ pub fn discover_idx_ownership_reports( } } + match fetch_text( + "IDX share ownership data page discovery", + &share_ownership_page_url(), + &page_headers(), + &validate_share_ownership_page, + ) { + Ok(html) => { + let mut page_reports = parse_share_ownership_page(&html, &share_ownership_page_url()); + page_reports + .retain(|report| family_filter.is_none_or(|family| family == report.family)); + // Keep the full above-1% history (snapshot backfill needs it), but + // only the newest entry of the unsupported daily/breakdown series. + page_reports.sort_by(|left, right| right.publish_date.cmp(&left.publish_date)); + let mut seen_unsupported = Vec::new(); + page_reports.retain(|report| { + if report.family == OwnershipReportFamily::AboveOnePercent { + return true; + } + if seen_unsupported.contains(&report.family) { + return false; + } + seen_unsupported.push(report.family); + true + }); + discovered.extend(page_reports); + } + Err(err) => errors.push(err.to_string()), + } + discovered.sort_by(|left, right| { right .publish_date @@ -307,8 +356,9 @@ pub fn discover_idx_ownership_reports( errors.join("; ") }; return Err(IdxError::Http(format!( - "failed to discover IDX ownership reports from {}: {detail}", - announcement_listing_url() + "failed to discover IDX ownership reports from {} or {}: {detail}", + announcement_listing_url(), + share_ownership_page_url() ))); } @@ -316,8 +366,12 @@ pub fn discover_idx_ownership_reports( } pub fn download_idx_pdf(url: &str, target: &Path) -> Result<(), IdxError> { - let bytes = fetch_bytes("IDX ownership PDF download", url, &pdf_headers())?; - validate_pdf_payload(&bytes)?; + let bytes = fetch_bytes( + "IDX ownership PDF download", + url, + &pdf_headers(), + &validate_pdf_payload, + )?; fs::write(target, &bytes).map_err(|e| { IdxError::Io(format!( @@ -329,6 +383,236 @@ pub fn download_idx_pdf(url: &str, target: &Path) -> Result<(), IdxError> { Ok(()) } +pub fn download_idx_xlsx(url: &str, target: &Path) -> Result<(), IdxError> { + let bytes = fetch_bytes( + "IDX ownership XLSX download", + url, + &xlsx_headers(), + &validate_xlsx_payload, + )?; + + fs::write(target, &bytes).map_err(|e| { + IdxError::Io(format!( + "failed writing cached XLSX {}: {e}", + target.display() + )) + })?; + + Ok(()) +} + +/// XLSX is a ZIP container; Cloudflare challenges come back as HTML. +pub fn validate_xlsx_payload(bytes: &[u8]) -> Result<(), IdxError> { + if bytes.starts_with(b"PK\x03\x04") { + return Ok(()); + } + if looks_like_html(bytes) { + return Err(IdxError::Http( + "IDX ownership download returned HTML (likely a Cloudflare challenge) instead of an XLSX workbook".to_string(), + )); + } + Err(IdxError::Http( + "IDX ownership download did not look like an XLSX workbook".to_string(), + )) +} + +fn validate_json_payload(bytes: &[u8]) -> Result<(), IdxError> { + if looks_like_html(bytes) { + return Err(IdxError::Http( + "IDX ownership discovery returned HTML (likely a Cloudflare challenge) instead of announcement JSON".to_string(), + )); + } + Ok(()) +} + +fn validate_share_ownership_page(bytes: &[u8]) -> Result<(), IdxError> { + if contains_bytes(bytes, b"Prospectus") { + return Ok(()); + } + Err(IdxError::Http( + "IDX share ownership data page did not contain its file list (likely a Cloudflare challenge)".to_string(), + )) +} + +fn looks_like_html(bytes: &[u8]) -> bool { + let head: Vec = bytes + .iter() + .skip_while(|byte| byte.is_ascii_whitespace()) + .take(256) + .map(u8::to_ascii_lowercase) + .collect(); + head.starts_with(b" bool { + haystack + .windows(needle.len()) + .any(|window| window == needle) +} + +/// Parse the Data Kepemilikan Saham page. The page is server-rendered by +/// Nuxt and embeds its file list as JS object literals such as +/// `{Description:"Pemegang Saham di Atas 1% per 31 Agustus 2026", +/// Prospectus:"https:\u002F\u002Fwww.idx.co.id\u002FMedia\u002F...xlsx",...}`. +pub fn parse_share_ownership_page(html: &str, page_url: &str) -> Vec { + let mut reports = Vec::new(); + let mut rest = html; + + while let Some(start) = find_js_key(rest, "Description") { + let (description, after_description) = match read_js_string(&rest[start..]) { + Some(value) => value, + None => { + rest = &rest[start..]; + continue; + } + }; + let object_tail = &rest[start + after_description..]; + // The URL belongs to the same object literal, so it must come before + // the next Description key. + let window_end = find_js_key(object_tail, "Description").unwrap_or(object_tail.len()); + let window = &object_tail[..window_end]; + rest = object_tail; + + let Some(url_start) = find_js_key(window, "Prospectus") else { + continue; + }; + let Some((url, _)) = read_js_string(&window[url_start..]) else { + continue; + }; + if let Some(report) = classify_page_entry(description.trim(), url.trim(), page_url) { + reports.push(report); + } + } + + reports +} + +/// Offset just past `Key:` / `"Key":` (the opening quote of the value). +fn find_js_key(haystack: &str, key: &str) -> Option { + let bare = format!("{key}:\""); + let quoted = format!("\"{key}\":\""); + [bare, quoted] + .iter() + .filter_map(|needle| { + haystack + .find(needle.as_str()) + .map(|index| index + needle.len() - 1) + }) + .min() +} + +/// Read a JS/JSON string literal starting at the opening quote. Returns the +/// decoded value and the byte offset just past the closing quote. +fn read_js_string(input: &str) -> Option<(String, usize)> { + let mut chars = input.char_indices(); + if chars.next()?.1 != '"' { + return None; + } + let mut value = String::new(); + while let Some((index, ch)) = chars.next() { + match ch { + '"' => return Some((value, index + 1)), + '\\' => match chars.next()?.1 { + 'u' => { + let hex: String = (0..4) + .filter_map(|_| chars.next().map(|(_, c)| c)) + .collect(); + value.push(char::from_u32(u32::from_str_radix(&hex, 16).ok()?)?); + } + 'n' => value.push('\n'), + 't' => value.push('\t'), + other => value.push(other), + }, + _ => value.push(ch), + } + } + None +} + +fn classify_page_entry( + description: &str, + url: &str, + page_url: &str, +) -> Option { + let upper = description.to_uppercase(); + let family = if upper.contains("DI ATAS 1%") || upper.contains("ABOVE 1%") { + OwnershipReportFamily::AboveOnePercent + } else if upper.contains("DI ATAS 5%") || upper.contains("ABOVE 5%") { + OwnershipReportFamily::AboveFivePercent + } else if upper.contains("TIPE INVESTOR") + || upper.contains("KLASIFIKASI") + || upper.contains("INVESTOR TYPE") + || upper.contains("CLASSIFICATION") + { + OwnershipReportFamily::InvestorTypeBreakdown + } else { + return None; + }; + + let file_name = url.rsplit('/').next().unwrap_or_default().to_string(); + let lower_name = file_name.to_ascii_lowercase(); + let format = if lower_name.ends_with(".xlsx") { + "xlsx" + } else if lower_name.ends_with(".pdf") { + "pdf" + } else { + return None; + }; + let status = if family == OwnershipReportFamily::AboveOnePercent && format == "xlsx" { + OwnershipReportStatus::Supported + } else { + OwnershipReportStatus::Unsupported + }; + let as_of_date = parse_indonesian_as_of_date(description).map(|date| date.to_string()); + + Some(DiscoveredOwnershipPdf { + family, + status, + listing_page_url: page_url.to_string(), + query_url: page_url.to_string(), + pdf_url: url.to_string(), + title: description.to_string(), + publish_date: as_of_date.clone().unwrap_or_default(), + code: None, + original_filename: (!file_name.is_empty()).then_some(file_name), + is_attachment: true, + format: format.to_string(), + as_of_date, + }) +} + +/// `"... per 31 Agustus 2026"` -> 2026-08-31 (Indonesian or English months). +fn parse_indonesian_as_of_date(description: &str) -> Option { + let tokens: Vec = description + .split_whitespace() + .map(|token| { + token + .trim_matches(|c: char| !c.is_alphanumeric()) + .to_lowercase() + }) + .collect(); + tokens.windows(3).rev().find_map(|window| { + let day = window[0].parse::().ok()?; + let month = match window[1].as_str() { + "januari" | "january" | "jan" => 1, + "februari" | "february" | "feb" => 2, + "maret" | "march" | "mar" => 3, + "april" | "apr" => 4, + "mei" | "may" => 5, + "juni" | "june" | "jun" => 6, + "juli" | "july" | "jul" => 7, + "agustus" | "august" | "agu" | "aug" => 8, + "september" | "sep" => 9, + "oktober" | "october" | "okt" | "oct" => 10, + "november" | "nov" => 11, + "desember" | "december" | "des" | "dec" => 12, + _ => return None, + }; + let year = window[2].parse::().ok()?; + chrono::NaiveDate::from_ymd_opt(year, month, day) + }) +} + pub fn validate_pdf_payload(bytes: &[u8]) -> Result<(), IdxError> { if bytes.is_empty() { return Err(IdxError::Http( @@ -357,13 +641,23 @@ pub fn validate_pdf_payload(bytes: &[u8]) -> Result<(), IdxError> { )) } -fn fetch_text(stage: &str, url: &str, headers: &[(String, String)]) -> Result { - let bytes = fetch_bytes(stage, url, headers)?; +fn fetch_text( + stage: &str, + url: &str, + headers: &[(String, String)], + accept: &dyn Fn(&[u8]) -> Result<(), IdxError>, +) -> Result { + let bytes = fetch_bytes(stage, url, headers, accept)?; String::from_utf8(bytes) .map_err(|e| IdxError::Http(format!("failed to decode {stage} response as utf-8: {e}"))) } -fn fetch_bytes(stage: &str, url: &str, headers: &[(String, String)]) -> Result, IdxError> { +fn fetch_bytes( + stage: &str, + url: &str, + headers: &[(String, String)], + accept: &dyn Fn(&[u8]) -> Result<(), IdxError>, +) -> Result, IdxError> { let mut args = vec![ "--silent".to_string(), "--show-error".to_string(), @@ -377,7 +671,7 @@ fn fetch_bytes(stage: &str, url: &str, headers: &[(String, String)]) -> Result Vec<(String, String)> { ] } +fn page_headers() -> Vec<(String, String)> { + vec![ + ("User-Agent".to_string(), USER_AGENT.to_string()), + ( + "Accept".to_string(), + "text/html,application/xhtml+xml,*/*;q=0.8".to_string(), + ), + ( + "Accept-Language".to_string(), + "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7".to_string(), + ), + ] +} + +fn xlsx_headers() -> Vec<(String, String)> { + vec![ + ("User-Agent".to_string(), USER_AGENT.to_string()), + ( + "Accept".to_string(), + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/octet-stream,*/*;q=0.8".to_string(), + ), + ( + "Accept-Language".to_string(), + "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7".to_string(), + ), + ("Referer".to_string(), share_ownership_page_url()), + ] +} + fn pdf_headers() -> Vec<(String, String)> { vec![ ("User-Agent".to_string(), USER_AGENT.to_string()), @@ -502,7 +825,8 @@ mod tests { use super::{ AnnouncementPage, IDX_ANNOUNCEMENT_LISTING_URL, OwnershipReportFamily, OwnershipReportStatus, build_announcement_query_url, parse_announcement_page, - select_latest_ownership_reports, validate_pdf_payload, + parse_indonesian_as_of_date, parse_share_ownership_page, select_latest_ownership_reports, + validate_pdf_payload, validate_share_ownership_page, validate_xlsx_payload, }; use crate::error::IdxError; @@ -762,4 +1086,89 @@ mod tests { assert!(matches!(err, IdxError::Http(_))); assert!(err.to_string().contains("PDF/direct attachment")); } + + #[test] + fn parses_share_ownership_page_entries() { + let page_url = "https://www.idx.co.id/id/perusahaan-tercatat/data-kepemilikan-saham/"; + let reports = parse_share_ownership_page( + include_str!("../../tests/fixtures/idx_share_ownership_page_excerpt.html"), + page_url, + ); + assert_eq!(reports.len(), 7); + + let above1: Vec<_> = reports + .iter() + .filter(|r| r.family == OwnershipReportFamily::AboveOnePercent) + .collect(); + assert_eq!(above1.len(), 3); + assert!( + above1 + .iter() + .all(|r| r.status == OwnershipReportStatus::Supported) + ); + assert!(above1.iter().all(|r| r.format == "xlsx")); + + let august = above1 + .iter() + .find(|r| r.as_of_date.as_deref() == Some("2026-08-31")) + .expect("August above-1% entry"); + assert_eq!( + august.pdf_url, + "https://www.idx.co.id/Media/fahlw1o2/peng-2026-08-00017-satu-persen.xlsx" + ); + assert_eq!(august.publish_date, "2026-08-31"); + assert_eq!(august.listing_page_url, page_url); + assert_eq!( + august.original_filename.as_deref(), + Some("peng-2026-08-00017-satu-persen.xlsx") + ); + + let above5 = reports + .iter() + .filter(|r| r.family == OwnershipReportFamily::AboveFivePercent) + .count(); + let breakdown = reports + .iter() + .filter(|r| r.family == OwnershipReportFamily::InvestorTypeBreakdown) + .count(); + assert_eq!((above5, breakdown), (2, 2)); + assert!( + reports + .iter() + .filter(|r| r.family != OwnershipReportFamily::AboveOnePercent) + .all(|r| r.status == OwnershipReportStatus::Unsupported) + ); + } + + #[test] + fn share_ownership_page_without_payload_yields_nothing() { + assert!(parse_share_ownership_page("Just a moment...", "p").is_empty()); + assert!(validate_share_ownership_page(b"Just a moment...").is_err()); + } + + #[test] + fn parses_indonesian_and_english_as_of_dates() { + let date = |y, m, d| chrono::NaiveDate::from_ymd_opt(y, m, d); + assert_eq!( + parse_indonesian_as_of_date("Pemegang Saham di Atas 1% per 31 Agustus 2026"), + date(2026, 8, 31) + ); + assert_eq!( + parse_indonesian_as_of_date("Shareholders above 1% as of 29 May 2026"), + date(2026, 5, 29) + ); + assert_eq!( + parse_indonesian_as_of_date("Pemegang Saham di Atas 1%"), + None + ); + } + + #[test] + fn validates_xlsx_payloads() { + assert!(validate_xlsx_payload(b"PK\x03\x04rest-of-zip").is_ok()); + let err = validate_xlsx_payload(b"Just a moment...") + .expect_err("html challenge rejected"); + assert!(err.to_string().contains("Cloudflare"), "{err}"); + assert!(validate_xlsx_payload(b"%PDF-1.7").is_err()); + } } diff --git a/tests/cli.rs b/tests/cli.rs index 5ba912b..60d2bd6 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1189,6 +1189,43 @@ fn ownership_releases_uses_xdg_data_home_default_db_path() { ); } +/// Keeps discovery tests off the live IDX data page (nothing listens on port 9). +const UNREACHABLE_DATA_PAGE_URL: &str = "http://127.0.0.1:9/data-kepemilikan-saham/"; + +#[test] +fn ownership_discover_finds_xlsx_on_share_ownership_page() { + let page = fs::read_to_string("tests/fixtures/idx_share_ownership_page_excerpt.html") + .expect("read share ownership page fixture"); + let page_url = spawn_single_response_server("text/html", page); + + let output = test_bin("ownership-discover-data-page") + .env("IDX_CURL_IMPERSONATE_BIN", "curl") + .env( + "IDX_OWNERSHIP_ANNOUNCEMENT_API_URL", + "http://127.0.0.1:9/announcements", + ) + .env("IDX_OWNERSHIP_DATA_PAGE_URL", &page_url) + .args(["-o", "json", "ownership", "discover", "--limit", "1"]) + .output() + .expect("ownership discover json output"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + + let reports: Value = serde_json::from_slice(&output.stdout).expect("parse discover json"); + let latest = &reports[0]; + assert_eq!(latest["family"], "above_one_percent"); + assert_eq!(latest["status"], "supported"); + assert_eq!(latest["format"], "xlsx"); + assert_eq!(latest["as_of_date"], "2026-08-31"); + assert_eq!( + latest["pdf_url"], + "https://www.idx.co.id/Media/fahlw1o2/peng-2026-08-00017-satu-persen.xlsx" + ); +} + #[test] fn ownership_discover_lists_fixture_candidates() { let body = fs::read_to_string("tests/fixtures/idx_announcement_kepemilikan.json") @@ -1198,6 +1235,7 @@ fn ownership_discover_lists_fixture_candidates() { test_bin("ownership-discover") .env("IDX_CURL_IMPERSONATE_BIN", "curl") .env("IDX_OWNERSHIP_ANNOUNCEMENT_API_URL", &json_url) + .env("IDX_OWNERSHIP_DATA_PAGE_URL", UNREACHABLE_DATA_PAGE_URL) .env( "IDX_OWNERSHIP_ANNOUNCEMENT_PAGE_URL", "http://127.0.0.1/pengumuman", @@ -1257,6 +1295,7 @@ fn ownership_discover_supports_above1_family() { test_bin("ownership-discover-above1") .env("IDX_CURL_IMPERSONATE_BIN", "curl") .env("IDX_OWNERSHIP_ANNOUNCEMENT_API_URL", &json_url) + .env("IDX_OWNERSHIP_DATA_PAGE_URL", UNREACHABLE_DATA_PAGE_URL) .env( "IDX_OWNERSHIP_ANNOUNCEMENT_PAGE_URL", "http://127.0.0.1/pengumuman", @@ -1313,6 +1352,7 @@ fn ownership_discover_defaults_to_above1_and_prefers_supported_attachment() { let output = test_bin("ownership-discover-default") .env("IDX_CURL_IMPERSONATE_BIN", "curl") .env("IDX_OWNERSHIP_ANNOUNCEMENT_API_URL", &json_url) + .env("IDX_OWNERSHIP_DATA_PAGE_URL", UNREACHABLE_DATA_PAGE_URL) .env( "IDX_OWNERSHIP_ANNOUNCEMENT_PAGE_URL", "http://127.0.0.1/pengumuman", @@ -1367,6 +1407,7 @@ fn ownership_discover_json_includes_status() { test_bin("ownership-discover-json-status") .env("IDX_CURL_IMPERSONATE_BIN", "curl") .env("IDX_OWNERSHIP_ANNOUNCEMENT_API_URL", &json_url) + .env("IDX_OWNERSHIP_DATA_PAGE_URL", UNREACHABLE_DATA_PAGE_URL) .env( "IDX_OWNERSHIP_ANNOUNCEMENT_PAGE_URL", "http://127.0.0.1/pengumuman", @@ -1406,7 +1447,7 @@ fn ownership_import_url_rejects_listing_page_inputs() { .assert() .failure() .stderr(predicate::str::contains( - "ownership import --url accepts direct PDF URLs only", + "ownership import --url accepts direct XLSX or PDF URLs only", )) .stderr(predicate::str::contains("ownership discover")); } diff --git a/tests/fixtures/idx_share_ownership_page_excerpt.html b/tests/fixtures/idx_share_ownership_page_excerpt.html new file mode 100644 index 0000000..7962b06 --- /dev/null +++ b/tests/fixtures/idx_share_ownership_page_excerpt.html @@ -0,0 +1,5 @@ + +Data Kepemilikan Saham - Bursa Efek Indonesia +
+ +