From 2c85acb8290ce46ce971ce6f776e8c2a8aa610c0 Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 24 Sep 2026 20:48:07 +0000 Subject: [PATCH 1/7] feat(ownership): import IDX above-1% XLSX holder register Since June 2026 IDX publishes the monthly "Pemegang Saham di Atas 1%" list as an XLSX workbook on the Data Kepemilikan Saham page instead of a PDF announcement. Add a parser for it and accept `.xlsx` in `ownership import --file`. - src/ownership/xlsx.rs: zip + quick-xml reader (shared strings incl. rich text, sheet cells), exact 12-column header match so the daily above-5% and investor-type/classification workbooks are rejected, Excel serial dates, numeric shares, percent -> bps with float-noise rounding. Builds KseiHoldingDraft directly and reuses the entity normalizers, so investor types stay uppercase like the PDF path. - Fixtures: trimmed excerpts of the real 2026-08-31 above-1% and 2026-09-23 above-5% workbooks. - Unit + CLI tests: import, ticker, releases, above-5% rejection. - Docs: ARCHITECTURE parser pipeline, AGENTS ownership inputs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E --- AGENTS.md | 2 +- docs/ARCHITECTURE.md | 6 +- src/cli/ownership.rs | 13 +- src/ownership/entities.rs | 4 +- src/ownership/mod.rs | 1 + src/ownership/xlsx.rs | 450 ++++++++++++++++++ tests/cli.rs | 73 ++- .../ksei_above1_20260831_excerpt.xlsx | Bin 0 -> 10999 bytes .../ksei_above5_20260923_excerpt.xlsx | Bin 0 -> 26480 bytes 9 files changed, 539 insertions(+), 10 deletions(-) create mode 100644 src/ownership/xlsx.rs create mode 100644 tests/fixtures/ksei_above1_20260831_excerpt.xlsx create mode 100644 tests/fixtures/ksei_above5_20260923_excerpt.xlsx diff --git a/AGENTS.md b/AGENTS.md index e8bb38a..f0d092f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,7 +63,7 @@ Preferred bootstrap/update path: Ownership input paths: - primary remote source: discoverable IDX `above1` holder-register PDF - maintained snapshot path: `ownership sync` -- local fallback path: PDF, plus local archive `.zip` / `.txt` +- local file imports: IDX above-1% `.xlsx` (current format since June 2026), legacy `.pdf`, plus local archive `.zip` / `.txt` Important scope note: - archive ZIP/TXT ingest is a fallback/backstop path, not the primary product ingest surface diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 1aecd3c..3919f69 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -63,12 +63,16 @@ Unlike the `stocks` module (live-fetch), ownership is **import-then-query**: 4. Fully offline after import/sync ### Data Sources -- **KSEI** — official ≥1% shareholder registry (monthly PDF from IDX) +- **KSEI** — official ≥1% shareholder registry, published monthly by IDX. Through May 2026 it was a PDF announcement; from the 2026-05-29 report onward it is an XLSX workbook on the IDX "Data Kepemilikan Saham" page - **KSEI archive** — monthly ZIP/TXT balance-position matrix, used as a local fallback/backstop import path - **Bing Finance** — global institutional ownership (REST API, quarterly) ### Parser Pipeline ``` +KSEI XLSX → zip + quick-xml (sharedStrings + sheet1) → exact 12-column header check + → KseiHoldingDraft (Excel serial dates, numeric shares, percent → bps) + → SQLite INSERT (within transaction) + KSEI PDF → mutool stext (XML with coordinates) → quick-xml parse → KseiRawRow → normalize (ID locale numbers, dates, entity names) → KseiHolding → SQLite INSERT (within transaction) diff --git a/src/cli/ownership.rs b/src/cli/ownership.rs index 4073649..2540e2c 100644 --- a/src/cli/ownership.rs +++ b/src/cli/ownership.rs @@ -17,13 +17,13 @@ use crate::output::table::format_idr; use crate::ownership::types::{ ChangeType, FlowSignal, HolderRow, KseiHolding, OwnershipRelease, OwnershipSource, }; -use crate::ownership::{archive, db, entities, graph, parser, remote, search, snapshot}; +use crate::ownership::{archive, db, entities, graph, parser, remote, search, snapshot, xlsx}; use crate::runtime; #[derive(Debug, Args)] #[command( about = "Ownership intelligence (KSEI + Bing)", - long_about = "Ownership intelligence (KSEI + Bing).\n\nPreferred bootstrap path:\n 1. Run `idx ownership sync` to install or refresh a maintained SQLite snapshot.\n 2. If no snapshot manifest is available, run `idx ownership discover` and then `idx ownership import --url `.\n 3. Local `.pdf`, `.zip`, and `.txt` imports remain available for manual or fallback workflows.", + long_about = "Ownership intelligence (KSEI + Bing).\n\nPreferred bootstrap path:\n 1. Run `idx ownership sync` to install or refresh a maintained SQLite snapshot.\n 2. If no snapshot manifest is available, run `idx ownership discover` and then `idx ownership import --url `.\n 3. Local `.xlsx`, `.pdf`, `.zip`, and `.txt` imports remain available for manual or fallback workflows.", after_help = "Examples:\n idx ownership sync\n idx ownership sync --manifest /path/to/ownership-snapshot-manifest.json\n idx ownership discover --limit 1\n idx ownership import --url \n idx ownership releases" )] pub struct OwnershipCmd { @@ -41,7 +41,7 @@ pub enum OwnershipCommand { #[command( about = "Import ownership data directly from source files", long_about = "Import ownership data directly from source files.\n\nPrefer `idx ownership sync` for normal bootstrap/update flows. Use `import` when you need a direct PDF import from IDX discovery output, a local PDF, or a local KSEI archive fallback file.", - after_help = "Examples:\n idx ownership import --url \n idx ownership import --file /path/to/ksei.pdf\n idx ownership import --file /path/to/BalanceposEfek20260227.zip" + after_help = "Examples:\n idx ownership import --url \n idx ownership import --file /path/to/peng-2026-08-00017-satu-persen.xlsx\n idx ownership import --file /path/to/ksei.pdf\n idx ownership import --file /path/to/BalanceposEfek20260227.zip" )] Import(ImportArgs), #[command( @@ -87,7 +87,7 @@ pub struct ImportArgs { /// Direct URL to a remote ownership PDF, typically from `ownership discover`. #[arg(long)] pub url: Option, - /// Path to a local ownership file: PDF (primary), ZIP/TXT archive (fallback). + /// Path to a local ownership file: above-1% XLSX or PDF (primary), ZIP/TXT archive (fallback). #[arg(long)] pub file: Option, /// Fetch Bing institutional data for these symbols. @@ -836,6 +836,7 @@ fn handle_import(args: &ImportArgs, config: &IdxConfig) -> Result<(), IdxError> drafts } ImportInputFormat::Archive => archive::parse_balancepos_file(&import_input.import_path)?, + ImportInputFormat::Xlsx => xlsx::parse_above1_xlsx_file(&import_input.import_path)?, }; let mut holdings = Vec::with_capacity(drafts.len()); @@ -977,6 +978,7 @@ struct OwnershipImportResult { enum ImportInputFormat { Pdf, Archive, + Xlsx, } fn resolve_import_input(args: &ImportArgs) -> Result, IdxError> { @@ -1018,11 +1020,12 @@ fn detect_local_import_format(path: &Path) -> Result Ok(ImportInputFormat::Pdf), + Some("xlsx") if xlsx::supports_xlsx_file(path) => Ok(ImportInputFormat::Xlsx), Some("zip") | Some("txt") if archive::supports_local_archive_file(path) => { Ok(ImportInputFormat::Archive) } _ => Err(IdxError::InvalidInput(format!( - "unsupported local ownership file {}; expected a .pdf, .zip, or .txt input", + "unsupported local ownership file {}; expected a .xlsx, .pdf, .zip, or .txt input", path.display() ))), } diff --git a/src/ownership/entities.rs b/src/ownership/entities.rs index 9de47af..21c94a0 100644 --- a/src/ownership/entities.rs +++ b/src/ownership/entities.rs @@ -212,7 +212,7 @@ pub fn resolve_entity( Ok(entity_id) } -fn normalize_investor_type(raw: &str) -> Option { +pub(crate) fn normalize_investor_type(raw: &str) -> Option { let value = raw.trim(); if value.is_empty() { None @@ -221,7 +221,7 @@ fn normalize_investor_type(raw: &str) -> Option { } } -fn normalize_locality(raw: &str) -> Option { +pub(crate) fn normalize_locality(raw: &str) -> Option { match raw.trim().to_uppercase().as_str() { "L" | "D" => Some(Locality::Local), "F" | "A" => Some(Locality::Foreign), diff --git a/src/ownership/mod.rs b/src/ownership/mod.rs index 3735ff5..6635dae 100644 --- a/src/ownership/mod.rs +++ b/src/ownership/mod.rs @@ -7,3 +7,4 @@ pub mod remote; pub mod search; pub mod snapshot; pub mod types; +pub mod xlsx; diff --git a/src/ownership/xlsx.rs b/src/ownership/xlsx.rs new file mode 100644 index 0000000..2669bbd --- /dev/null +++ b/src/ownership/xlsx.rs @@ -0,0 +1,450 @@ +//! Parser for the IDX "Pemegang Saham di Atas 1%" XLSX workbook. +//! +//! Since June 2026 IDX publishes the monthly above-1% holder register as an +//! XLSX file on the "Data Kepemilikan Saham" page instead of a PDF +//! announcement. The workbook has one sheet: a disclaimer block, a `[PUBLIC]` +//! marker, then a fixed 12-column table. Only that exact header is accepted, +//! so the daily above-5% workbook and the investor-type / classification +//! workbooks published on the same page are rejected instead of mis-imported. + +use std::collections::HashMap; +use std::fs; +use std::io::{Cursor, Read}; +use std::path::Path; + +use chrono::{Duration, NaiveDate}; +use quick_xml::Reader; +use quick_xml::events::{BytesStart, Event}; + +use crate::error::IdxError; +use crate::ownership::entities::{normalize_investor_type, normalize_locality, parse_ksei_date}; +use crate::ownership::types::KseiHoldingDraft; + +const EXPECTED_HEADER: [&str; 12] = [ + "DATE", + "SHARE_CODE", + "ISSUER_NAME", + "INVESTOR_NAME", + "INVESTOR_CLASSIFICATION", + "LOCAL_FOREIGN", + "NATIONALITY", + "DOMICILE", + "HOLDINGS_SCRIPLESS", + "HOLDINGS_SCRIP", + "TOTAL_HOLDING_SHARES", + "PERCENTAGE", +]; + +/// Rows above this line are disclaimer text; the header must appear before it. +const MAX_HEADER_SEARCH_ROWS: usize = 50; + +pub fn supports_xlsx_file(path: &Path) -> bool { + path.extension() + .and_then(|value| value.to_str()) + .is_some_and(|value| value.trim().eq_ignore_ascii_case("xlsx")) +} + +pub fn parse_above1_xlsx_file(path: &Path) -> Result, IdxError> { + let bytes = fs::read(path).map_err(|e| { + IdxError::Io(format!( + "failed to read ownership XLSX {}: {e}", + path.display() + )) + })?; + parse_above1_xlsx_bytes(&bytes) +} + +pub fn parse_above1_xlsx_bytes(bytes: &[u8]) -> Result, IdxError> { + let mut zip = zip::ZipArchive::new(Cursor::new(bytes)) + .map_err(|e| IdxError::ParseError(format!("failed to open ownership XLSX: {e}")))?; + + let shared_strings = match read_zip_entry(&mut zip, "xl/sharedStrings.xml")? { + Some(xml) => parse_shared_strings(&xml)?, + None => Vec::new(), + }; + let sheet_name = first_worksheet_name(&mut zip)?; + let sheet_xml = read_zip_entry(&mut zip, &sheet_name)? + .ok_or_else(|| IdxError::ParseError(format!("ownership XLSX is missing {sheet_name}")))?; + + let rows = parse_sheet_rows(&sheet_xml, &shared_strings)?; + drafts_from_rows(&rows) +} + +fn read_zip_entry( + zip: &mut zip::ZipArchive>, + name: &str, +) -> Result, IdxError> { + let mut file = match zip.by_name(name) { + Ok(file) => file, + Err(zip::result::ZipError::FileNotFound) => return Ok(None), + Err(e) => { + return Err(IdxError::ParseError(format!( + "failed to read ownership XLSX entry {name}: {e}" + ))); + } + }; + let mut output = String::new(); + file.read_to_string(&mut output).map_err(|e| { + IdxError::ParseError(format!("failed to decode ownership XLSX entry {name}: {e}")) + })?; + Ok(Some(output)) +} + +fn first_worksheet_name(zip: &mut zip::ZipArchive>) -> Result { + let mut sheets: Vec = zip + .file_names() + .filter(|name| name.starts_with("xl/worksheets/") && name.ends_with(".xml")) + .filter(|name| !name["xl/worksheets/".len()..].contains('/')) + .map(str::to_string) + .collect(); + // sheet1.xml, sheet2.xml, ... sort by their numeric suffix. + sheets.sort_by_key(|name| { + name.trim_start_matches("xl/worksheets/sheet") + .trim_end_matches(".xml") + .parse::() + .unwrap_or(u32::MAX) + }); + sheets + .into_iter() + .next() + .ok_or_else(|| IdxError::ParseError("ownership XLSX has no worksheet".to_string())) +} + +fn xml_error(context: &str, err: impl std::fmt::Display) -> IdxError { + IdxError::ParseError(format!("failed to parse ownership XLSX {context}: {err}")) +} + +/// Shared strings table: each `` is one string, made of one `` or +/// several rich-text runs ``. Phonetic hints (``) are skipped. +fn parse_shared_strings(xml: &str) -> Result, IdxError> { + let mut reader = Reader::from_str(xml); + let mut strings = Vec::new(); + let mut current: Option = None; + let mut in_text = false; + let mut in_phonetic = false; + + loop { + match reader.read_event() { + Ok(Event::Start(e)) => match e.local_name().as_ref() { + b"si" => current = Some(String::new()), + b"t" => in_text = true, + b"rPh" => in_phonetic = true, + _ => {} + }, + Ok(Event::Empty(e)) if e.local_name().as_ref() == b"si" => strings.push(String::new()), + Ok(Event::Text(text)) if in_text && !in_phonetic => { + if let Some(value) = current.as_mut() { + let unescaped = text + .unescape() + .map_err(|e| xml_error("shared strings", e))?; + value.push_str(&unescaped); + } + } + Ok(Event::End(e)) => match e.local_name().as_ref() { + b"si" => strings.push(current.take().unwrap_or_default()), + b"t" => in_text = false, + b"rPh" => in_phonetic = false, + _ => {} + }, + Ok(Event::Eof) => break, + Err(e) => return Err(xml_error("shared strings", e)), + _ => {} + } + } + + Ok(strings) +} + +/// One sheet row: column letters (`A`, `B`, ...) to cell text. +type SheetRow = HashMap; + +fn parse_sheet_rows(xml: &str, shared_strings: &[String]) -> Result, IdxError> { + let mut reader = Reader::from_str(xml); + let mut rows = Vec::new(); + let mut row: Option = None; + let mut cell: Option<(String, String)> = None; + let mut value = String::new(); + let mut in_value = false; + + loop { + match reader.read_event() { + Ok(Event::Start(e)) => match e.local_name().as_ref() { + b"row" => row = Some(SheetRow::new()), + b"c" => { + cell = Some(cell_ref_and_type(&e)?); + value.clear(); + } + b"v" | b"t" if cell.is_some() => in_value = true, + _ => {} + }, + Ok(Event::Empty(e)) if e.local_name().as_ref() == b"row" => rows.push(SheetRow::new()), + Ok(Event::Text(text)) if in_value => { + let unescaped = text.unescape().map_err(|e| xml_error("worksheet", e))?; + value.push_str(&unescaped); + } + Ok(Event::End(e)) => match e.local_name().as_ref() { + b"v" | b"t" => in_value = false, + b"c" => { + if let (Some((column, cell_type)), Some(current_row)) = + (cell.take(), row.as_mut()) + { + let text = if cell_type == "s" { + let index = value.trim().parse::().map_err(|e| { + xml_error("worksheet", format!("bad shared string index: {e}")) + })?; + shared_strings.get(index).cloned().ok_or_else(|| { + xml_error( + "worksheet", + format!("shared string {index} out of range"), + ) + })? + } else { + value.clone() + }; + current_row.insert(column, text); + } + } + b"row" => { + if let Some(current_row) = row.take() { + rows.push(current_row); + } + } + _ => {} + }, + Ok(Event::Eof) => break, + Err(e) => return Err(xml_error("worksheet", e)), + _ => {} + } + } + + Ok(rows) +} + +fn cell_ref_and_type(element: &BytesStart<'_>) -> Result<(String, String), IdxError> { + let mut column = String::new(); + let mut cell_type = String::new(); + for attribute in element.attributes() { + let attribute = attribute.map_err(|e| xml_error("worksheet", e))?; + let raw = String::from_utf8_lossy(&attribute.value).to_string(); + match attribute.key.local_name().as_ref() { + b"r" => column = raw.chars().take_while(char::is_ascii_alphabetic).collect(), + b"t" => cell_type = raw, + _ => {} + } + } + if column.is_empty() { + return Err(xml_error("worksheet", "cell without a reference")); + } + Ok((column, cell_type)) +} + +fn drafts_from_rows(rows: &[SheetRow]) -> Result, IdxError> { + let (header_index, columns) = rows + .iter() + .take(MAX_HEADER_SEARCH_ROWS) + .enumerate() + .find_map(|(index, row)| header_columns(row).map(|columns| (index, columns))) + .ok_or_else(|| { + IdxError::ParseError( + "ownership XLSX header did not match the above-1% holder register layout \ + (DATE, SHARE_CODE, ... PERCENTAGE); the above-5% and investor-type \ + workbooks are not supported" + .to_string(), + ) + })?; + + let mut drafts = Vec::new(); + for (offset, row) in rows[header_index + 1..].iter().enumerate() { + let line = header_index + offset + 2; + let field = |name: &str| -> &str { + columns + .get(name) + .and_then(|column| row.get(column)) + .map(|value| value.trim()) + .unwrap_or("") + }; + + let ticker_code = field("SHARE_CODE"); + let investor_name = field("INVESTOR_NAME"); + if ticker_code.is_empty() && investor_name.is_empty() { + continue; + } + if ticker_code.is_empty() || investor_name.is_empty() { + return Err(IdxError::ParseError(format!( + "ownership XLSX row {line} is missing SHARE_CODE or INVESTOR_NAME" + ))); + } + + drafts.push(KseiHoldingDraft { + ticker_code: ticker_code.to_string(), + issuer_name: optional(field("ISSUER_NAME")), + raw_investor_name: investor_name.to_string(), + investor_type: normalize_investor_type(field("INVESTOR_CLASSIFICATION")), + locality: normalize_locality(field("LOCAL_FOREIGN")), + nationality: optional(field("NATIONALITY")), + domicile: optional(field("DOMICILE")), + holdings_scripless: parse_shares(field("HOLDINGS_SCRIPLESS"), line, false)?, + holdings_scrip: parse_shares(field("HOLDINGS_SCRIP"), line, false)?, + total_shares: parse_shares(field("TOTAL_HOLDING_SHARES"), line, true)?, + percentage_bps: parse_percentage_bps(field("PERCENTAGE"), line)?, + report_date: parse_report_date(field("DATE"), line)?, + }); + } + + if drafts.is_empty() { + return Err(IdxError::ParseError( + "ownership XLSX contained a header but no holder rows".to_string(), + )); + } + + Ok(drafts) +} + +/// Map header labels to column letters if `row` is exactly the expected header. +fn header_columns(row: &SheetRow) -> Option> { + let mut found: HashMap<&'static str, String> = HashMap::new(); + for (column, text) in row { + let label = text.trim().to_ascii_uppercase().replace(' ', "_"); + if label.is_empty() { + continue; + } + let expected = EXPECTED_HEADER.iter().find(|name| **name == label)?; + found.insert(expected, column.clone()); + } + (found.len() == EXPECTED_HEADER.len()).then_some(found) +} + +fn optional(value: &str) -> Option { + (!value.is_empty()).then(|| value.to_string()) +} + +fn parse_number(raw: &str, line: usize, name: &str) -> Result { + raw.replace(',', "") + .parse::() + .ok() + .filter(|value| value.is_finite()) + .ok_or_else(|| { + IdxError::ParseError(format!("ownership XLSX row {line}: invalid {name} '{raw}'")) + }) +} + +fn parse_shares(raw: &str, line: usize, required: bool) -> Result { + if raw.is_empty() { + return if required { + Err(IdxError::ParseError(format!( + "ownership XLSX row {line}: missing TOTAL_HOLDING_SHARES" + ))) + } else { + Ok(0) + }; + } + let value = parse_number(raw, line, "share count")?; + if value < 0.0 || value.fract() != 0.0 || value > i64::MAX as f64 { + return Err(IdxError::ParseError(format!( + "ownership XLSX row {line}: share count '{raw}' is not a whole number" + ))); + } + Ok(value as i64) +} + +/// `54.94` (percent) -> 5494 basis points. Excel stores values such as +/// `1.1499999999999999`, so round to the nearest basis point. +fn parse_percentage_bps(raw: &str, line: usize) -> Result { + let value = parse_number(raw, line, "PERCENTAGE")?; + if !(0.0..=100.0).contains(&value) { + return Err(IdxError::ParseError(format!( + "ownership XLSX row {line}: PERCENTAGE '{raw}' is out of range" + ))); + } + Ok((value * 100.0).round() as i64) +} + +/// DATE is an Excel serial day number (`46265` = 2026-08-31); accept the +/// KSEI text form (`31-Aug-2026`) as well in case the cell is stored as text. +fn parse_report_date(raw: &str, line: usize) -> Result { + if let Ok(serial) = raw.parse::() { + let days = serial.trunc() as i64; + let epoch = NaiveDate::from_ymd_opt(1899, 12, 30).expect("valid Excel epoch"); + if (1..=2_958_465).contains(&days) { + return Ok(epoch + Duration::days(days)); + } + } + parse_ksei_date(raw).map_err(|_| { + IdxError::ParseError(format!("ownership XLSX row {line}: invalid DATE '{raw}'")) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture() -> Vec { + parse_above1_xlsx_bytes(include_bytes!( + "../../tests/fixtures/ksei_above1_20260831_excerpt.xlsx" + )) + .expect("above-1% XLSX excerpt parses") + } + + #[test] + fn parses_above1_xlsx_excerpt() { + let drafts = fixture(); + assert!( + drafts.len() >= 10, + "expected excerpt rows, got {}", + drafts.len() + ); + assert!( + drafts + .iter() + .all(|d| d.report_date == NaiveDate::from_ymd_opt(2026, 8, 31).unwrap()) + ); + + let dwimuria = drafts + .iter() + .find(|d| { + d.ticker_code == "BBCA" && d.raw_investor_name == "PT DWIMURIA INVESTAMA ANDALAN" + }) + .expect("BBCA controlling holder present"); + assert_eq!(dwimuria.percentage_bps, 5494); + assert_eq!( + dwimuria.investor_type.as_ref().map(|t| t.0.as_str()), + Some("CORPORATE") + ); + assert_eq!( + dwimuria.total_shares, + dwimuria.holdings_scripless + dwimuria.holdings_scrip + ); + } + + #[test] + fn rounds_float_noise_in_percentages() { + assert_eq!(parse_percentage_bps("1.1499999999999999", 1).unwrap(), 115); + assert_eq!(parse_percentage_bps("41.1", 1).unwrap(), 4110); + assert!(parse_percentage_bps("100.5", 1).is_err()); + } + + #[test] + fn parses_excel_serial_and_text_dates() { + let expected = NaiveDate::from_ymd_opt(2026, 8, 31).unwrap(); + assert_eq!(parse_report_date("46265", 1).unwrap(), expected); + assert_eq!(parse_report_date("31-Aug-2026", 1).unwrap(), expected); + assert!(parse_report_date("not a date", 1).is_err()); + } + + #[test] + fn rejects_above5_workbook() { + let err = parse_above1_xlsx_bytes(include_bytes!( + "../../tests/fixtures/ksei_above5_20260923_excerpt.xlsx" + )) + .expect_err("above-5% workbook must be rejected"); + assert!( + err.to_string().contains("above-1% holder register layout"), + "{err}" + ); + } + + #[test] + fn rejects_non_xlsx_bytes() { + assert!(parse_above1_xlsx_bytes(b"%PDF-1.7 not a workbook").is_err()); + } +} diff --git a/tests/cli.rs b/tests/cli.rs index 2451d77..5ba912b 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -407,7 +407,7 @@ fn ownership_import_help_mentions_sync_preference_and_fallback_files() { .success() .stdout(predicate::str::contains("Prefer `idx ownership sync`")) .stdout(predicate::str::contains( - "PDF (primary), ZIP/TXT archive (fallback)", + "above-1% XLSX or PDF (primary), ZIP/TXT archive (fallback)", )) .stdout(predicate::str::contains( "idx ownership import --file /path/to/BalanceposEfek20260227.zip", @@ -1524,6 +1524,77 @@ fn ownership_import_url_duplicate_release_is_skipped() { .stdout(predicate::str::contains("Release already imported")); } +#[test] +fn ownership_import_file_xlsx_supports_ticker_and_releases() { + let root = test_env_dir("ownership-import-xlsx"); + let db_path = root.join("ownership.db"); + let xlsx_path = root.join("peng-2026-08-00017-satu-persen.xlsx"); + fs::write( + &xlsx_path, + include_bytes!("fixtures/ksei_above1_20260831_excerpt.xlsx"), + ) + .expect("write xlsx fixture"); + + bin_with_root(&root) + .args([ + "config", + "set", + "ownership.db_path", + db_path.to_str().unwrap(), + ]) + .assert() + .success(); + + bin_with_root(&root) + .args(["ownership", "import", "--file", xlsx_path.to_str().unwrap()]) + .assert() + .success() + .stdout(predicate::str::contains("(as of 2026-08-31)")); + + let output = bin_with_root(&root) + .args(["-o", "json", "ownership", "ticker", "BBCA"]) + .output() + .expect("ownership ticker json output"); + assert!(output.status.success()); + let body = String::from_utf8_lossy(&output.stdout); + assert!(body.contains("DWIMURIA INVESTAMA ANDALAN"), "{body}"); + + let output = bin_with_root(&root) + .args(["-o", "json", "ownership", "releases"]) + .output() + .expect("ownership releases json output"); + let releases: Value = serde_json::from_slice(&output.stdout).expect("parse releases json"); + assert_eq!(releases[0]["as_of_date"], "2026-08-31"); +} + +#[test] +fn ownership_import_file_rejects_above5_xlsx() { + let root = test_env_dir("ownership-import-xlsx-above5"); + let db_path = root.join("ownership.db"); + let xlsx_path = root.join("peng-2026-09-23-00080-lima-persen.xlsx"); + fs::write( + &xlsx_path, + include_bytes!("fixtures/ksei_above5_20260923_excerpt.xlsx"), + ) + .expect("write xlsx fixture"); + + bin_with_root(&root) + .args([ + "config", + "set", + "ownership.db_path", + db_path.to_str().unwrap(), + ]) + .assert() + .success(); + + bin_with_root(&root) + .args(["ownership", "import", "--file", xlsx_path.to_str().unwrap()]) + .assert() + .failure() + .stderr(predicate::str::contains("above-1% holder register layout")); +} + #[test] fn ownership_import_force_reimports_existing_release() { let root = test_env_dir("ownership-import-force"); diff --git a/tests/fixtures/ksei_above1_20260831_excerpt.xlsx b/tests/fixtures/ksei_above1_20260831_excerpt.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..46bc8a15570bb2c1853b9cf6e1c81633cc80eee9 GIT binary patch literal 10999 zcmWIWW@Zs#U|`^2c(lYK_QTR{j$Mol4FA~~7=#%Z7^0o?^GZ_lO5#H*3sQ??^(u06 z&Q86Zb=yIp?LBub`^N1Dqtsi2x}L@?(G{sOP-(r_7p|QBao!Ed<=<|Xt5`0Za)dp{ zD}RfJ@#p&gS!GL3zP+{kk)ZKS39ZuDzR<>lM{c#bF0Hevk=h%~YSTJBTE!f?D%J1(R<+BE*tL=mDNIyLyZ&8pM2y-~2WUH|B28DY=YM)Tf1cxhXDb+L-mjGIcbt|Bit zZ|{la&RXN2I$N~xOZsE>naU4uhkdVAysLj*{M}E#{|)@p3XeUk+SeNQjA25>;|Y4T zhpI0+KM^y!_I=Lo_lJ+Vzu|w&Y{z3!eEpP3+>OJ28bzy7J z8|I#qZ=`&T6g$`Mt6dy&OLte*!&J5Ob2Y_IZj-l~z7cl5@+8Yn$Lrh$&!AgdP5;lZ z<(ISQ=A6Ui%$L3I;0g7|uUB8Rmh#xCskfzHXZMbX&#qqo0y{LsGZe+1Tzopat*LcN z|3Xo@Cqhvop^^8xKU{C_x1N`?DobwJtG3NIk3F`$cf};?k?_f>uG9H?1a|nn5Lwm8 zztdi2?)F4u4&7z5v?t~7lwqxiPs!Hf6ApbYEPFpJ!e@a(;{=fn^V3(qV3vwATD0!b z^t9X~;;m2TWZXPnRNitlOHw0fGMDGKbPu`ssf*V96ZBv2G^XNT&VL&gRJuArXR3Tc+L;U6fO;9_plK*n3swSx)eZ zu)puk`C6u5n!Mwl+k=Xn=QE$rl)N1te?2vEPk^H2`dworR+{kiq~@AlQA;^&Re&F5uUk#haa)Towix|ionbBX)Jx;7#0oU5nE zj7KM@aXWgd6?=t7-dOkeN110tYMy{oAs^fP^O0W3^YpxKFIO z&dUC_?%A3J-)CPkJ(2Wr?i8!w)=UwXRf)$$&hAQMP>a9#mtlp_Y1b_&tP;BHyXR$o zFMDbul;H60ck7}>N?m*vOBY5 zS#rqNdj$MAp`|6Cyl)j3A7iD>3frPnFYH6)J-1d=yJual*O%MTxuNrq2PgBd>mPYt ze)jEgIHS1vW|?=KbHMG(^BIrDEr0r3jL)i;`*>c$R$=kzK!F{Fj~=gRzLmf1o`2%0 zlR-~!+hmos4O)*^ zugg@7UB3SKvsK&VB9;|dC|lpS+grQ%jVAlC6PcE~_O-5_xi?uP|AfkiD`y3je>Vip zo%<{|q}2BQ)moP28{WP6;1u6-t8>S>0N#r=(=S?zcr7zOv~lavS?ju2?U|$I^W0pm z>rBUzs`@2Mc4R+rbUU}H=hf>2Z)$J;`v2*;>;FUi`gwXxPabRkX83d8wwGn~m6Agt zjN6ZO&UVg`iQB`zwg2jY?Jqqfeuc6xY?j%+_Uh%WyC$xxJG?AWpt@yl-Gh=1PkEx} zwceJ!p7nTIHtzzy_Bex{_KDxWY4^Xcdf)b2`G`lc(iN8pUnJjMG@R=$kS^;tSLZ>c zhuNoF6E8D$SvYpK1|HwI=*019;_ABk4j&U5}VeYdpP*f`7m5<-5)2cEkB*@Q(T& z|NriIoVWGgwcJmf=d`mWPG0U&NoZL+B~zmE{=Pdl6OJgGi{`IobgTN15*ru4Hg4fl ziAj4x=K42vf9&x%@@`^~zDmZs;M`x0pztitNKGv%)(2CDpfYjptk~k&9%60pw`cuhK3XgAmyuWNc5tWa&1o~& zUP&u|F6FH|yGvzD!`qZ)&-UkkR%CfD!)debNTJ{Uzh-lmOP9(Qe|_D5_gdBE$azbC zXV11=m;3UsR;>E(=h?sQ|HsvES#@ph|0_HHzCNq}e@B)7=J!9lueGMi zeEYfeZ;(j)%alp$tB;&=e&7) zD{)ruM-TJaDNEB|zA9b*-8Fa8%iNe5ug}L%a;_`6z4q9Rw~yZZe*N&W)?L}2qrsbe zyXH?_nfo^Ea{KezSx5b&i|^ad=n22pf9B7_XX`Eaf{$MG*qt(UY2_|Wna6ixjQ9O5 z5b}Ez`nu(P{S@BGYK-}woF7t-{$Dozdez6tGk?rZtIG~sragO~+UxA!{@Z{4O8K|R z|M>h@`N!Aqe=~b^X%ffuxods$*S$TT^J{j}zAftJ;a4U8`+lju_w(kd_)Fh+>qNSr zjGO&4Bdti#zV>qTUac*XGj3j+Ju&un`;W+a!|A{NW>;4Jy1VV4$K|`*Lt~1*-91sec8X$>PU6&-#=vrmyea0GB0cQ$nT2YYr5A?`*v5j z>!)S=k3GFq>D8@tck0`#leBy?CNHgjQCsSLwRBo}wZo%ZPf}+6Uw^Jc%i8Z2OG&+* zT3X=Quj^-Sy*B$tY4jiM+Wps0e(v5Gb4+%>NyO6Idw%7;om4w@S?r(1zrNc>it9g9 zf2i@S&8c$R*Cn5m0`INKjNAL{nNWScOK)iIA!*@zvkI;!Ui$j8g8yPg$F;-Z^7{mC z&Ny-Tu7~BW?bpA2^PB#_YF-iR?*p|zdOPb+wf@P_3wgZIZ@XZfg}7s~&=$2xmW|In z?SC|Q-`LTzw)+7ezs~IjzQ1-&(DO9e*~NAu3sSTe?x8#H`sa&+?)Vuko|B2=b zpY{Fn_SoK8d!{m*b?I+aTJdff&z(aDc-Wp9Y-nh-me|qW$Sx6a=s>c%f#J=JV5JjN zG>%)(XKmbcz(K69%Z%H=pt;eKM8QvBo!wFwzF91)56RvW^|&IvfVFu?6JyLl13f3E!bDDX<4Qk|13qyA%#JaPM_g|> z#WPXIK~g)8FEJ^YelyGOM@O9Ek>k5}md74F`^qu9KYo$X<8WujdhJZB#s)|>2X@|0s%3){sbQjFUTr?6dAIp$=-DeoejZQy$|@!(WN zMah~?`qfG<#VyWPIm(xG;ZW6MfKlMXdQu|=Lg6lO&< zXI9yYDSfet|1n`%dFL~Z_7#G%nN@#y&OQ|4`P`oIOF2VG%0)Pv<<=7cDVNC-+!s}9 z1bo*VH2M2(`8@GmX{DO{m%ivjovBuS$Jo8vET${bd83?cCh$2lk2;HCvvWGH(Hy2ZMy#AV?d?N2gS*WzDIr?|9LKX+3^N` z`3FBQFwT3tSU<@jLx9@AOAgWk)6&Ht3@gwi1Xb%9jBb^qamljk?bP)Uhc@x zn|~J&>E2=dXuF)4yNxdbRzR z*GGddF(paWGAq0Lc0DstnET;QKDS%Q(JKj&K7yjEK9hwMXHDUdJEtkQ%yOC8vVy{% zSM8Z`S~a@A*Y^BQ<90kKquL#^-szak_cPA|do?DmJSa6&>6AB%ms3likkS|J1%{|mVP;rq^RcyNL z?$zSQJf*|I)8gP~234g8-(Dq5`g!Opb46d|s_v~@3e_euJ>AlH-^<79{qiI0U3YnG zYdtq>`>Q{hA$vo=U;O_0%$55u-|Vc?*4-ZV^7TFq<13xp*1xsgG;ddD_shi#9u-N2 zzW?Ft_w8wQ+4*1FEvo&#?c91*+IY%q4f7ARla*C1_*+8M)Fc10ao!AD(!R=-ZJhz< z=h*u-i#~3e61qRDeA!3W*VC;!8`he5 zf7!RhM#&Qs;- z{r4YPCp6EQWM;kFT9-fJwPVEE)FuB$SeCk^|+!5x9gEj6)$S$_{a0e zEHX(EESPw4;pT;>EbV{)-G04bmGG>frY|$CS{F{vm^**hI<7+LE8Y_?^Ssw>^>KV0 zbSzKhq1?}9cixs+)ta+Tvs7soxmg#X7wK_wPr}PNxikGO>QbXbr4^@d-Y_L2Xv&R@ zr=s>*i;u-jvVP#q{?z4Bo!|bf_JRPZe4e>}KMU_{zW%PS;P!Ctc0?Vu;~{o9ZKGXGg{`Mdru-aY$|v$y~1*E{yOj#>WA$G^;fzFhUZpZ~}^ z;85f?D+lq9cU;rhBf0)vP6>EqSr0`;Ce!n>C}U!G|7vz=jbWA z+>yA{ni9a#5X^OKZkBckdvvSLBBzr@wk7}0Fls09@0>SrcJZHx1^J5IzAt`SW*%Ot zc8>3o?*!2@#l*xz&vce{?OFc)N!XGVJ6jE7Pb?2SBb>m?62~i z1kRnbI&PC{HtlhX*_AyHB~m*w-Apy#b&4((+jPFi^iS1UC(S7lS)XQ36f{`v{VZqU zTbKOEuELu)M2-ZQ*A)Nl>|xoyasQHXUlld`0%`A4(s?z^*}N-5lRb8J*05V`)U>>P zV#C9NrG}RbpEmg@t(wyNtoe?~mAy=FjIV8o|Ll9q#p`UC*%NDlLoRV$S1lS%w!b>F z>By0P3&UnCa>#4hrlmMbG-!<>--NAgzl9h7%1r$`@o1;5W0!SyMB33`; zseXI@QkMRQjY-}=+IhF}ez(tAJ$=KuJdLzZ>(~8Sz4~8I?AEo@`rj`7{q5x}cN!+c$doG{aZNp_`vhNP}w&&(9&HKaxx85~AUutdkR4Z)Rg1CbZA`f z-Qe!^nQ8Zy2Z5RE9{R3fV>-I>?(1^xi!%%lWQP3Mo3w%V^yYRCRe_6EZu71RtS?`3 zo9+I$S&DC$d{4gA$2m#!T)v>Vjq?;M_P`?k(-YIritcn@+m!cUg21PRn~zv>>R*=m zxpEOBWB=xb!mn;k$rtb6)U|5Q*4nod-+q11R`;;|x#59J zy?vJ#nSc1_s(Yt>;nLHpyrrkl)kWSH&h~w()A{aX)B2~n3+~lTjTH?3appyyxRCah z^}_2Tf0l~c)a(rkXa8W+xJ#Y4=SKL4PgTpEcN$$OjxPD{7hH8d!|TK6SwSg!eQWow zu+e)cq0lH|X;sp4rSpK^6m^|BUNSOXkt@UxZqf)*+EWo3^TO->x7yW@0!?>XT->gs zHYsuW<<2Ev+^(0^HP5_mI)8n%Tj~6pD_4JdRqps+W4`#++xlJA2(q;%JY-85_%1uzBE;kS#AHlb-Gtv zg0K0Mz}kWcfe$6W>a>>}l1^I@meCO$uy7)mIqu`-bgdJZ_attm^u)G(O6RXk<3E?4bY6I!_k>e>FD#q0mg`FY_nCox zZ30)McTJgjbGgp%H-+cytJyy{{;PRrZtyu$&$ZpGWowe1_Wp+SGdv*)XM;~JVG2oE zqQq98Q_55vbo-8W*UZqTGlI)jOkKzw+t}0}mYJ_vY?`AKllopqcIxUg zy>};m4E?tC<>{TK+?!K16W^R%RC~(*+uBprcja%~6MuUtr)=|@#H@q%#j{HMn)9s~ zV?^Fe@Q~g%wI@?ucTTDLedArCbH2#RO}(OW_1*CkA3yw#ylVOFl5uRQcinO6XZ+1 zcgcURaSnNuR@eWE^JCvD?b$`8N3u7v+uO{R-(UZt`kcxxttY&yTygprL)9<$xn|^d z&sr8{R?O41UqtWP601+UY&P3Zyc1n>;s1%}YkL2tm3)2n<>ZvStq0;bl73vAHt%ai z&bKLI+n2vx`*X_l&RG*>-p`ENuAh8kZN0kW z-On4gRxLRnWFB_+SM%RO7iHNJVmc$KzZtdRvzkBxX-FdS&H?&=zB&mH><@SX7S0#pTI8AG}*SD3;|E(g! z`dyG~Pl|v-zQv*Bd>bj3IH83Frc!!xi&S0}_UG++$ZqE(`!i#erg{%okyriQ>$8+3 zKTcKoldSsZUHzRuXhTA8zCX%y<78k+lVD&FfoA-S#G=%c;F6-uymWB>pBtW?f7?y$ zuiS$Fi3|M89-heEe%t-QjsC&eGY~mne5Z~ zeePa^x%Thw-;oS9hzWgGtFG7e=kD9JoXHc5E}t!0_#tib%PHbNR&3v{cD>16O>B|J zemtChdy4nf#Kf>wk7}Pkzi+)JlUA4C-}`gp z=_aFC?z2j91!)y$g43LIKf6dAUdJHBf6YnsX&+a=IFMQUfzGH=I&zhS~ zZ(5wDBmy0-I3EL7qwPwDU!Oe=)*XmfAi zuC|`ITlk0dB+mO&zg(ZyAoXd%nso2b&kq@oZMXiHKAm@sOtQ-B11-J}#oxady0DMY zVS@d+vK=`WE7;!UXC=F>@s>EO*&B3cF6*^5ore>@FiJk=GC9d@mAdpl*Z$|JjBig) z(XP9)n%iCLb})me)-9&4l!+QCmz?r6e?B!4w{#3~$#RYPfF3B9Y)t{~~6t^S^+6 zY`Q^QZn^h&#jk(3PQ*v_;NeTDszTCU9nQST+ZNCa}zI1 zSpVieo6Vv$r-iFk9b9^}9?nXQ%3FGfdE3RvQ%r)>qQr#EH~io8)oFLl%&f1K6J2?n zU3FH4wy;*eIuUCo7_;|?k)ZjRIf_-RzTDgkc5p9iU6o~NI)jbJ`f9d7UhuCM_vFf! zMjWtKbl_l@2Rwer(V6_4wEAA@r*@D=?MycK4#pK-25X`^6K( zy{DLn7Ea`j{9zW?GTr;Xu?+3g)#q5ab3FWl=t{hlzmELQ?cZRy9Z+X zT!rpNuw3m-FOT2-+x@-R<@J2Ri;lK%ei#2GliL68dTPmsx+gdHWtH^0O_yG9uE(Y8 zdw@fWx$DG)Ne))~oaDMLh3>oED;2wL2g|GfY$Y0}ss`?o@70a&^c}?rNv}aM(Kw;i zFVik!Wq#aKjx#MW*`5&}TbVC%$Q_&RR5a;YQgdRXq37A%8|GzruCnUyiYwis>nQ%q zj-~Y6-I;YKxy~1ATo>A-?S0_clm_F+WxuqiFP>bI?fCcBy`vLl6rM`mE=XkknYE-W zE>!o*jnlgIX8Kp{o+z*X;>)TQo3th_#@&_Y$9}2M8B7<`Q)YZ}(Ph%j(bc%-vMcjN z!LE&At!qBMeLQpZqFZgd-sEuZ`hUIVs`T%__4DoJD^5TBx%t7tXde5o>CM-q?K#2@ zA6>rEJahl7f32p8-=RppYU}S5{Wg&1|MODghsYQ4e>FBdP5SdK>UYo4ZCq~k`Q)Wbn+)XjejR>!NqXy^ z>U#(BZKW%^B|k4?&VS=O(_Z$>vcm_y-zeIfEg5HYgRh|Z>)xIGDrYXf@ZIxy^IxCK zMll^?i{JW$=tX;e7rR$v_NmV=!a%0N|G>5CoJ8mQiRGa7)7wV^f!<&KG3BoJcADYf zoAdCiyjq<0qlXU`+ovAP*-`O{tzIep?Gr!teeF-!4*c67<#0a7bnaGB>pbmgn~v-e z{A8*Ys~|f4=qtWom!B2{PJgBPvSM!>_dOP`$}cxkEOx|j`Ezh8ZkT7U8v5_0?SanY zc2(RhSE75<7hlh-u-(AM*wZxaa@_j+b#6fmi{cpee6LDudVezOp?UYxRWG^QFTM`z zzW8b}|5o9DSC7no{Nm}Zm%l%&&S==mJ|}VU)g$d!!)oq{NX`B=#lOC&rZU^7@P9n& z$QN6CXUHT*1_mQ$1_nU}28NXUl2TZ1!yfk+%2jfBD_M_vNJP zuJ(wz;gxv7DbmQSY`N$1&al_aFPE(Q|F`e$5l=V0vU0n7pI^L~Dpvk}+sS(hYLD+u zn!33u^}&?u$8x8=`Ta}r|B+2CmojfwuzO!~Oe+6+wY`+t(2bmp>y81qYeM=TwUGQ_hM82C9l0x^BDdqzg(`gXQqkK z`AvH-KQ3EfT;$PHCue2{gt1uEXkJM);wd*@k#vLCnXnroqE{!WPDh8v-UY5$u&NY1- zF3HaYh1=4Jf!WOl0uA@sfAcH8Hw(;O?W!xK{3X#KV*B?67rA_uJYQ)g|F6}I4*#}c z`rD5?cE8`Z+k1O@A&c#e@xB?QWxKH{3cm*?|K|h<3^utZ&s+Be{hEP&a?03 zFBgX?UoU&WX3=6IX%k_cCt!58kvmOHPd9d1aFoGor@P8mOPvpEU!B7;U5r&?eL%5M z7-#;UV+Z~3rERKr-L`7yjhWN`+E~TSt2>x}fzx`6WX)=&pisfSJq>C{K5vY#@|>~8 zRwh|y3ggPjP5Yb;;<}hQRD|Ff`D+=IxdJ%6&^u z_}58OzRsGG=<#U5nnz8)GVXj^Ezv-u6n>!0(!K5KuT|M}wE znLpVBycwB9m_hwD>?;HrprC;f#K5v%5ZwUub!Q-LAl$(Cnh|9c8oEaG)m0!(Al$%c z$BePo3SBe$Di4rG5N=@n%m&tsW6cM;Iq2&eKxTk&1EVJg*c_~@9MDZcpNs&R0KyH7 zw*BdYmiP5ZeTp6hNL?gp34KgS=m4exEZ(^ M4l*$?C}@Cq0Hzi~OaK4? literal 0 HcmV?d00001 diff --git a/tests/fixtures/ksei_above5_20260923_excerpt.xlsx b/tests/fixtures/ksei_above5_20260923_excerpt.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..29781458eedcb3f6fb6cdb846517de710a845da7 GIT binary patch literal 26480 zcmWIWW@Zs#U|`^2c(lYKcGo;pu`)&mhI&>824MyUhG^&fypq(slK7Cyg4E(zy^7qN zwNq|;9d;0Cd(V1b@y%M3=)D3Hm&H{u928!`Z*X_6iTC#-=R$68FP|JHxU%=sy%f*1 zfB%0QCmQXFu!!E1s3csY+IM=P#)Z`jdgq5d`ua4jK#eQugH+Ckh)&f>J0#@gdg7)D zt~`FGcDyKF%6F3S`E8S_f>(H!Y52Dt4qn-m{>A3OHRm4p@QA~A ziVF9h=nwZ4iV`~dbk)YxP^P3UZj8@&)qMWNp1MJ8O61M20m(|TPuUdBg>IdC->~(a zVZWMW?@?iuI)>@HpBy^%eL~t~fwZS(lX*3rW2IZIR_r`IX;PBYg)0i1Hm8f{v>n{IMv(=+AUHISst@hJ5h4Lqv9XgWs zd(Hlin^n&(lXt(D`dWW8%TMOcxgS%{vh8?NH~%p|YBU{^IiIA;$iPs@%)r3Sz`zh+ zl$ukluLmO6UOwx^Y$(9~VfO>;mOHy|u9NR5x!ujd$~y5E!{wARbCZ)TDp$X~_0@I_ zf2DOJIZds;&htddwvBCjlR9R_oIUP(Qqs!PYxlNi`rW@T@}?U$1}B_IJiA@uXHHd} zS*07ZW`5V)Zxf%2h-cqh`Xs-#W0N!YMumR8aw#scpt!cMiV4~ofg!1tx`(dw*u72N zv~yMEjMd!d#G?EEpB57n*6>hJVU_tgp}c*`=g1pOPTeb?bv+5zJO0;Y;^dtce5>jh zB6XQvvw{>)a%D;BmQPtFlKRzGnfJ(w&jFz3dv+S4f{8hbz`66N7i>^+1`f7^U&c4n!`scUKmsEqWIi{C;ix5n@66JOL$ zd1;Lrnuj+C=q_MlV322HVBlw9V5rE^FV8Q^PRh^Eh9vmd{MiN~ZSU3J+AG`-Hc&Ki zR26Os>)zsf@mHy^X3+u8FrKt+nL(j{-&?CGW=@YQxf_sVaii+byyCc;kEzi`uN%FN zx`-;)JX7ipn|UH~YuAOhXQ$daOuMw?6sNdo?*--17pGr*Uf$h1;dQG(HaEl6u7HK5 zvQ~YkxT1tg)v}@vo(VcJDTCwbslcsERKi)8?t0Sqd{<+zP}o9Q#VPL-JDyHTH4%*r zzU1{@VN%f|$9-B6YptSY?qv@b+L@PSEu6~R!TQ+8ep^7w)JTiSP^U?(?OV3*$j?>% z!}KFroNfJW&Sievv?A47g}iIoUhI7_`On7J34d0x+`cTer~iqwpxlg?4+TYQCWy*J z929K3TfZ#vd&g%xkJs`Ua#FE6lFQ4jUq>xjS3luWNtbfB@KXKaWtnmdi#d+A?Wn)O#&>gE*7nM;#`3mL!8+Pgu` z^6D4yXDTyK-mpAz%l7|FyA_iYug_RHP0YssQvQXO>`x53jmMt<)-Pi+l|M7Ts@A?U zS?0FmgSof&*~{EEeE0qBlT)wRKOWe>%SV6y#GiK#`Al#V|7|Cc*7E1G`LcbJbIqIY zZbC*ws#j^pIv^oX|t)!owS9TmTc$S(~>Pu7IwdlkbEdv zaqrHVL)RYqP5=GE95pdF+D)1MhJk@$7ZU@66eKZ2N(ba54Jj#tPG=o95ZELBi$Cf8 z);C+G1Svfi;@Y|G9Xm&6?#vsLF67Do`@M=cSB7ID`}}E=8;uv=jm@_^?zks;kzECA zXke2=e$(69Kr@dwk-$*S~q+{8xB!?~;W)&u#9PB`4Ot>kC-&?Pb)ojM*8UCLXum z9Xs^v{pv?cT|A4AqzApdyFvZV*O04ozk|X z{~w$GfBygD>izpGe#)=^_vg*c*ZkXzYkt4FF8}Z6*Vp_1e0tm;U;FcU`u&=ZuNI5f zSAKh|R$ue+(`@~?{QvJy+t>birLRA~_M<3!?dPw@+wZ>n^wB)t=Kr_D=IKA4p8jrm zyxacY$Dh;B*L{C>QMH9sHC-k-Dl$KlWG=kNcg@AvB=`~8}qr~UtbdD}fb{?EhLy5H;H`N!A%JXpW) z&cEUt|EmA|yT0M?L-X3te@;)=kK6m>@$3F=#`WLL?T_uRulfI@TmSyNmxlkV=cd)) z`p5Y1+a9hD?ZLADW%7_>ve}4YozgPSJeP>_)@6V(6`|E%HSzTZM@9pNbc6p{hzlzuY z*gvoK@0-)t&u@$0^Z!Hhs?GHkA0OFgefQtLDt6oC+CK%`{8uRa{M5XD-rtYi`|JMx z`Yc}me|q})|6iV8zwPNKKX`uuM< z>&us!KewnU?>l_Ec#i!pIlbqnAO1aG_~)I>boqaMm)*bLJ#TsR>+aw?_vR(-|9`Xo z_}|@=@7$mFHX<}fBWD6pLM$avCZ=L=ZmYV=N~jazWVexQ}2Jwiuc)|1Yft}_V@Ik3x)3A`~LaI$Jza6Zx6rUSMm2~e%;qkNAKHL zoV>UG-zV2v#&`9R>hEs1y*y!HL+r2CN@lx({VgFUJG}FaLeVzmop?$~%w0UADEWz4Q3rW!rySe~RD!TQa>q zyR7ol`d{z!%WnSP(evH^U+mxAwR~63-x9i?Yq{_4r^xbo>i>WJESUGTVBY!J_CK#& z-+9?S_PJkv-S2mA^Z&m+JniR(r{(<7ZxT?HetIPXj`xRGwx?Xc>&a;#3pTD2g^S3Mh|LI+< z-QT+D+qbT>eCBEVGe5t~@^8g-_0os;@4qSf_fgOPRnz{PPd_|%d;DREk<0e;lY@S~ zxw~KM{^Rhq-+zS8um8XL-@)k*pUz*mbh`P~$6uTemsFNt{qt_YYku9Uj~(09ORviD z{d)3o!Q;ude*b>9$A9(R-Q_>OeA;r$^tjF1vgzgFyH7jETG;(e`8hk5&wk6oj~kwD zE}yR^&lklXZMT2V?p-mvUlsj)ksdnZwR7vveY+3E$ZMT>?08(=x6fAQi)ZnL$&3E| zeXPeX`t0uRnyT8U#97Jn!e+eQt$pU}#wGLT99_>Z`YiEu`g=QHJDDwV+iL1+%UgH; z{U4B+$1nQq+dHn}>J8_wLoF{{mv=_^s?L zPss88T4MR)e?{n*%d{7-ADKDEC3-u~(nikJ8A z-WB41)&A~*f=TnkX1+K3d)Vsp-$eUWCTA0utH&SxR&hi9ZleGC`Lk=~w9Zw(`TX*B z?Y*$M?^Y}?eZIT)U*glBdGqIs{GNRO-Tk+pAHVv%7QuFM+vLn^|9$iMzk<*9 zZN{&kx##NsM>KzXAK3h@-}J@D4eje!9zSfC$!9%x-n9kCYvr;p?Ee1q%Ue79D+`{j zXqVbj-!HYLUg!Jrzs&dFex5A#WdpjgY_k7;N|jWsacuv-!m<6h)R&4I$d`~ zorPgyNkzX@Nrle$<>EHDOx(KnlKuDUH$NiU``!n(_w|>)__*P?6~xT#%S&Gr&OdJT z4cK+UDpPv;ss_uOGdE2nAHe-RJ$wiS}sh01xF8;irZZ%));?GO!w@SE7ue4aba$@FP*tzUNtKP!qJ%r6${m_ z1d2;#aQK?E=0%p(^p%-U>@d33ZFDQxZLL(6lWgh2#Z``*K}dut_u#%8H+Sa7J9!>crQ??Zq%PSD>;Ht18cQ?&hoq8ko*R;g)os&JLTzu-M6zDeHy3<86_MM7QY6-I7R=+71 zcSgOp!!$x;@#RpJN5v&#sYhhv*CR*#1=W%XIJ#aM<31QRno?q2Ba;@*Xw{M%z zh?+{UXO3y9?x!~hmkaL(CDL4C0=jfZp?RTjxv=@W#Qg93dh+fZ{zf4^CEpm=k8Cp8>7nK%*QjqiH(SJv3~fA1do8dqL(y}CX&oS4+3dwT6W z`Ph7|i+wJ&rvr%6!CmVZeo?$+Z=C0q?+(+52!t&{fv%Ilq_V=o9pL_1{g_XFFE_(L+#ijecG5Xa)o#!v6`YV03-1Wv&#m;@YZRe5aFE*)9 z{2>$jO+{$_;?wh79u=27`@P~US`E9p$3x}U>vdnQ#~$@quKl-kFLJ$wqnzVT-oEbd zp3dCgy)KKc8J4Xkq7+U|d~+?d=kBg~KiX_N^KS^>-#}C~d&}zL@~Wl&U$u_q#l8%F zOH4B2yRxh9-T!TW{gf_VySV(;YNVPTSB-gLcd6*!=x{eHa2P1hts>mYcwMmb`uB^j zU-DwJ#J+voyx(4nhz$5*+S6}pf%mV)zG&QY=lA)Gt#T*g2quh-se!eJF_{D1%iPK}S_?1#xNbI@N*m6Nzb>f;E2GP%pgyt*% zYSK%;XmmmR3NdagFfSA=7Z!haO8?QiKZjx~iD_YMT3^@JJtMW?=I2@!w~JGV$b==b z=f3ZHyt`wkuK2TUG2a$lGu*zEh@?98;?`3ybmr;IpUGJy_S$&POwLQh`00bpH=Sco z=1Wel7u(EpuID6CbwyV0mbdBk=7sC7ZrDD5(Y32tH;8CWPJ3FiyvODB+2GTgcYL$^ zd}R|6o=Ua+WNGy+-0`_^s>(Yx;wq)YWq)?Y->GM9$J!-tq`1}tdvS6rH3~Oigjn92Xw>ph(5mlR) zS{OF{Rhjp3?z4dYb**M zt$gZfv(@G!agmhB^84kj`-kVfxN8M!s0ip?zEjG?NxH5jD3=u_FhU%l=9(ShBk6{0#axL9rBVP(JdG?Uc&&e}1>#YJ_A1S^; zOkjt-e50vcD0*b`r{Mo`hD5c~#onFr{~Nu1+uJ+xo#{`BDYIhRl~+s3yswp%eSg(R z=4i!xVqBIN`6At-tFBGiR^5H$qazdNO(tA)znJzkO80PJ&mP-*d@8T16}*XQ)uitH zV7cqdd9}Xlww=G`O(xE560)^>zI=Cg|0UTce(~DH^B0IIu?)C>-`Ib*+Iri%roN2deou_Qp2mFNz;3wxsl)k`j$h`;5b4An zkF7iE+74H`gJQZI6w|nvYlYXEmp`pVAH7I@DECca-aq*Y^%D=Td}o!dQ1>9J%(#`o zzR~Q~xB96s6r$cUuPrC8*SYp~_qEwG71#}hpB(U+KsZ=3QXdv?5#QRkQx_C82kuNG ztmVb6Rnd2+zP~Cfzqypp`lDsiHa{Xfq%E5!`R1rjv+ZNaZ%Jw==6&QLCaUKur(XHa z`c`t`M)B9z-%2jLNK}S?mZE=b&YPK>o5XF^tJ9jk5Rp$xe#msrRH**d^ez0Ohia@cgcR=VEu7T&2ift!U0io zx@hIKYV~Fv`l@dbl^)!qw+qi#c5{D!Wb#wl#fz&6_wTan*O%|jwZ9#;J@@{) zz1!!%-D*zENWrO1>Vdy4Z*80Jab+71ah2QlY}H*`|CZkVX5D#9JCvv<&ou3=>*Npa zlijKuSiQ81n4Zw-qGi{clzAdYxq1qgjc^V=L{|zqoaOhxUHZNZDNNtFixULx^#gZlLYo z`*q*0^K6$|xKUR7)C#u@4#G(@dYX3UFTK^W`Ya^TmGNzuJXY}fKNdA{O zcYX=JAub3^^)i0n{66Q7Ub#@`ws%G96Ej}pVxIEYQfjK_fjQyxdA8B3_iB0KcJ6!zpQf@|d8W=j6r z7)^|p>;F#eSg=|`yyW)9;!Er8h$;-UWse!h=-0lkUAt5F_v@|t#C3xs`$Au@`^)hy z*Shn_--}a;Y0FHhnn zWNQT%FV^qrdq=oV4U-IfeI()$cXFrxqs-H-8;=lCrRttu`Kw9q(aj@IYt#boS4}6% zZHFXIbW0dkB!)b9?RmAwVAW>A0T-1Ta{hf|V{A=5Xn{=K+# zzWr|Bt%)96N_$?tAsh#c3oH0@fGJj`(CkfD<#HhI@5o(*>>;l z{9Nk3=-;e`#MP_Dy3I3etL(z<5AUphEkiw1+ykb{XqB6Vv-VBynQ01aqa;#1~iW1+|HATEPdY-YjGW<_eUv1(1hgbINW>|NfpEx71H0en$F`bP= zE6z=p@UFQP?3U!aoS5{s?(f8}O?^pw7hGOf)#;z)7D`k(f9c`6KN|B6%f0U1e*64C zEg}~V921gX|99%EiCZ)Jl0@$-XI~_yo7?m1-wV*FPLHZx*Ty66$ApL&L)xx$XR5qw z{DqD80=NDZ(RJRt*b8~x2(F@W&Wcj?z}J3TcoschLR5($c5RVmm-=b`SE z=`QT$1dpBz7I*YiUhPjLZg@5E>}{|$54|K~T_s~R7ZWu%d+{Q>q1CSgv$nHueD=1{ zG@pnrWVF~XNxnt%A3E3{c3ktaWWoG}grjIz>chN83h&hT_FR0aa4p#OgWGb#t$T1c zzkP1T-pI=@72f^eTl12LNx0bGw#C!FC!PD0c>HzRfR&_3h4}yWNHp zs@H`O72WfE-yGfCX}db)?a??ahw%yPefHWWR6NOUyci%^S*- z#J=^-{AB%6?1ZfGQ(~fNTJYVe@6RpjnW4Gg`n&Bmp1U7q%G94teP6+MoA4;^S|g>25zV#7qZP`Nc)!W!HxkjZ&Wi+% z*(v9j_w?QC{2ujALiR2Z7F|-DC{ejPIrQh;iuF(XLWv2y*M8-9{q`)%I$qE=_rweJy-G_uMYIw|?=l{dLP%uRFfua`5%!doNeKF4Ug0_UgXx!T;WWfBWmp zJ$;=Bv+B)8ug{*|vt6V6-X@*u=+xEQ1An^B&!2Yo-kLL2pN-z@o?5hB=k&d6d)@5r z|DJAoH+%1o-+Ob9oJz|NOWRvH^SfB;#pr3N`KxojnrYpC@%ikgo3p1z=EptT>l*oG zTI$W)yTZ>O-~0Z@j_Vy0tMAtmZQr({8}ClNoBploe_Zj+_kTmap_{7}y|?$2`LCMI zIp5M!ug#ubnI9;&H#PFxws%{v-QPQ(r*?helkasJ-Z$TzJ+;PM|MR^?I#s7Py~&-r z_MY2Q>Dqmf=sy4cYUz&ozs@`N*6sh^eE&4j{*!-QwNhT7ZvDi7@5%LhLmn@vA~Hn( z7Th-a;(Y1d)x)Ls->!RK-G3D|iz^{?%He?)8c-pH9>pQYZ5p5mGx zIc@LSO;>NPs9Aj89Ni1A!-^gA|2}vd-MfDO_h1F$WAJLt`DpvQpUdvvS{nU-!#?x? zKNXh0_*1XleI0a%+^+mxx8rNDRsL7MMbZB=wh*uJWz5qH+r7WKe7|OYK^NUv-2TfK z-}Tko;cnmU@;h&@)XSL>uk*Fd-CxcYoXWwOixt!&~BFZ&$`@RXMtv9{);}$== zKCw|4Ehw`}=GOa{*?XB^-m*9AS@n)WH~clLXtOBp>=`&XB~hsM2% zdVDE9dkR`6@B9 zz4TpjI?<80&hGK&wSRr)XTLkJGk(^7q7(J!uRBs-h?m=k@BhB@{dYa0(`L-)fOCIs zrWD`xpa1*!OQO@}ROec`UzHlqU+uR4Ui+Qsygqkz{Np?N|LfOT$~^aupPt3%PGsR? zC9m23&p5W&;@sbE7NR51Dt!O3o$rM$!tbB?d1Jcs*Rx+x>v5dbO7Ys4Rqv*k*k1p! zw0Qr5Y+}<`^z*9Azbn;GE!-#nD?O6$lO8$YkhjV9iIvD?poSt-B2 zWS8tVp6%QG7Z96Va$N%?Z%x0DU;l5V(U$qdN4uXX`=2ZN(<)YdW%0ZIEow2b_N@&* z%Wn1c`n}~<=EqjnUi^bmkK?RxLq0F}|NG}xxeXrxbRAj9;?fV+@<;MElaALz{ zo6?G1SJwL$+JxS&dUg9+@Yc0!iEOz>e7}6WuIJsxYtOB}75^c+60^E;`x4*Z)0g^o z@5*~S-GKN?Gay-r|L@lC3$L|W-?{zz8qp!jXS(jwh4YN|S3d7>{%^C6=!id@y8D{N zm-CH$rpMo3IR8x%HLJYn=bwTWlfCg08V*%NO0)ADF%)tx8j3nR+3BMMhXlx9a1zm6!EzEKc8^Zu*P zr_qEv(-&;{$g|~f`JDaXovGWKM7MW^Zf`T)-dDQ4k$3ylpKI;*_n-<-Tyb2rex9HI z3Gwgc&Kuu^ZFm&6@zu2r&#rBJXSU(tHlzJlpHD{<7W%s5$=4ksD_=+(zMt5+Pw1U_Y>Rq34V~vd?Eeu-Ne>?g74%!U)=lNj?GG?)?4R~tDZZ)wD&>rx#K%U*X#&g zv&(eN&eAozd2@EG)tq$jZ%s^BOo6@pbS2R{=Z`Nv!Selcp{5X6*ajrLVxh|W-;?D& z&ixkm5iJOCg^ZNfCn>K-l~zALn@qdQ6?R7}?5@(XQ}#TjV(#I0 zbGY)0J2KAohIk)x&HXcFV+fb4&Qvc>@1=nUf?iHpAHwG9GSyTmaPMy0g+BLg&+|}w zpX0AE`TJJ&zLUFe2_Jh>TbQx>TN=N0zVi;DNmwmoQh@ASMIGVfFTGy1-7R5v=l z`-X7LlfSPr(qG-}$Vhv&w<9C<)!dGZlvlMK8Og6=J2H}1&Hri_ReNJarq`QNfheJB z(U!I&am)*4H--2d+3ebN^7zkx_8l?* zP4wsg(J@=?zIWq>k}JQtUc2pGyCLJsZZ2)Ny(>4wT&d>Dc5S@=`O~d+`)5u!FZ=s; z&z`NX-#>dgH@4nh|NYxv?LYtR&XSjBU-9%;cV*?m`C9F(p8h&rS-Dc)jD79XU#lxC zMX$^MYv29czdk-z{?*mDU$@7Xi`(%<-`|eqRh3$WC)N@w8C2Vng!N$NKs02Ap ztRhFhBqKF96+(iKSa};2-7Re{Qm6mLzN51ISyI2gqTh$DnVV+o?4CWv=2Y^{#Sz9* zDwXBFoKwE6)Ngf){uT5hTHY&jL1Wv#FUMJgs(uPy%{caW;@Xo-Z+>wq7cbL2`(3uS z_}s;dA8&-Oo9oIlN8y-Z@9t^UCpR*5F z^7}iP^t%OIIXvpl=pSAu$~7@&Tdv%R%oF+-S&n|X)w79%%QB_7`qYD!TcibV7tdKH z>nIbg&^FPey4vh8Bh&Gfp08is`q6R1e&d6eE)F*W7li2;ibLU3#bv*uZv-<)uzd2nE4${k`P^;Olf4(bS$eQ+`O|BqH7Dz8uh%WV`CUqC-@b$9 zW$z2`Nb=V5-oNqSuWZfvkH6;bYWr^f;~C#R!&h7POl%8ZFyTSP#_qgrx8A1p=_wd{ zt&@H@H|=sJZ}AVd)ajeFA0J<&bzP}q{-%vAD;_vCCM$$-&N}GusH$^;43D8dM@*zb zqv6w{TOVV>)#46+?23O{SRr{($a*%{k8>fb*grHpbSYUYeQ8x_lTG!fCoc~l_m}_o z<;&y#`F4L8-#o0;;Mw!qVBYk~{q?_Z%E#BU)cyWA`@a4ELp{$^|9pOLU-|E|zP^#Z zNo2fT)xRGXH?Oz*`*rtu-Vbukla2mJxOX&1zYvkRBToRI^ zhdZWBtPW^Vo9M1BmcjSj$*ro7gLSdyqdP}-o(-FIXLi6eo#Y$ePikG7=<}eJb0@o| zu5x>cAxGjA3Dd_GI?4B+bI)8PAQO-%;`mi|QeUb}LX70Q-4dPGZ(l1Bx? zL1!-cW{VZhcK!(aD|nzw_SnwWYSt@4pYZI`ysI za<1?4Z*toTB;^$<=frF=6JLAw|HiE=Z=DwBJ?rXReDT7cT;8+G+Bcr=ubMJ-#iyO; zd4!8p!dTuGUj19J^ZW8-%av?hAvS4VPCt#}{SKd%3|Y^(_z|^eDR-m=0|KVH9> zSG{EQoR#;trSI75_xO&|+D?t7v-K8DXH8!Ew z$_2BsvX+)H%_vzrv*xu*yj!QqiOWH(CvDzEHJ-5j8}8+OXorN8T$(|tX0byHYf{4l z54CDbJ{7&r*C%{G=ZijH>iA=O=jQE;d(wH8F&Cv{iz*&>dwBAbKC^7rfrUH@tCidu#(Hy2M(Q9RQ>rO5RBgIA@B%S+_F zUVe+csV46C?D1!dvUtB+t7eI+d#v8nvv+Usz9p;j!k$WZ$1Z=n=f{kS&%z5;*5*zN zGxuI?Wpi$$vWD*UH6Lb&OP)2i;oG+Pnx5&jtSz5!n$5j4eWv}Cx34Cp7^_TER<*QD z@cP~27yi@x(B{}lcQ5baneap4EW^BcCnseV`TcuX7$-OD-v@hF{$c;s9=b6GndhY~pJK6nduY7a!SNGkKmZ6(p3)QL}nDlVn z*M*P2OEI69zcOc={VLN}yvwhBtNt2w{`>29C(7q0-mg}8+G$YqSEnIaB5~8H{r?v{ zE8oW~xc2^r7_N+z;IDay!cxAp(W&-g#@DoGpCc$m%C@<1K&tW^A6_WlyOFqwmafi;eSffsTBS#e2aPHHje)Ul^A(fx}p#OmTR|1&$? zI&N|!L2s#7uhz;_dER?%l}mfm;x!+IY&@#UwA^~;#ed&7dNP%938}jl7~Kxke(1G?uCqy-D(?OJQ%0S+~=ldF%3jxCvbI5cz4M(!e9kwN3xC z@WdSD*}Yr0?)pAG*l=zfzpo ztqjiEnf^+i96CMDRB3y>wC=CM-P|@F4w{s z-LpE;@!Rj*z2vHMu+nOw`mJX_GEJAO1ns&P_jt-*zAtqrn?)Nu;uf5ElyNFzPfz9X zzPZOw{@B>^(8Zp^EAg&zgI(XB!tXxE_uV(uNSWW~_+IdLNuFJ?L->N(7gNgnmQU>H zRr);n_3wS`GUq?n+-BMqYxT48vaQ$hiq7R*YD%6h`_3bf`XGAVdk3#0TWqgbGj|IW z2!0ayu+aB^bDcy<|8lEU9+e>zKdrEu`*>yFs>L>|`aZ4L1!k|dS+(xdiYhRBxy`D1 zpH`G@`?2Oji2K!WxzPC75OePA^S9$z_Fuoc&rWaui>;4~r}-Dg#o98t-`{rk?y(BD z6*Y2u_iW0abav*Xo(Vb&*)MBUoV=uvCbdmSXi`*VdHRaU*EMP`m#^!)UUOac{L+P8 z)6W)NE6csSRj?#s`YK_YUa^yMcYAj#t)08s;A-^uwLI5K4+jS=dJ{cMMbn|$Y)9>U z7jd&|vv+SYD-tR)D`J{tRwN{wdeJ1>=6FRozwF0ll~MalnBRSW<~Z~FKO2op4=y$v z#XdR9vFW^F(&;eU23|7{D|efQ<^gIC#Avclz8t=TLX6@N?zII;JV~K9vl3>>#9xr~<_?-IF%O<7T(Z$!& z7YD7f+uy%(=&#VH&B31= zKh^9?d2o$K$CPK!tj4u%jq%0~Hq6!f&HWiHF{Z^8Q(x3*Id-2ij=C9EH+No}^@CSy zPYV~EPcM${3=K`%v7y>VKHq-6&f8Op)5D4zFFT!hudTgv*6YRk4zDyV#3%5qt1_>6rMY5#e?VyMlSz!L^d|&_Ha^jS3QTsa zZERV^rQ+^bTidegmGXR^SKLdM)-h(TT;G3f=Ig}=^JlEv?!R+xF!$Uo#jAAFUpEKq zO}8lgH%ssh>tnNOorQCn!z})N;(k*hu;;tN@9KRC)vNZVRerH%P4ZvVzJ>9*)$iAS zLajBY`)y{}KYTiE@5VWA4G-pjJg&&?_vlR7+x6WaO{;e9x%@Hk)P~$Su^VRpetO5% z==UYhEj^ZA(+-uAZp=IOa!w=Cn|g?zfc%VhVJl}qlQ*|D$t z_cw_TLrzU8&T5}pmb|(3oNbqmT}O;* z_M5ZnI-36$@^1M(!F9P*&Qso3t|F5)boDbew-y9W(2-m#ZT{`u;Z)#DeP5&u3$l*%l8QT-Yu+}H6`u0QnXMX}m`mD=|v#{$oUdK-n$J!a9h zGt2YYs>>4YhT*v6r7$p8LIX zXS|-4vhKZ`@0lWRqtoS&RiACwoq2!vM4{DPx{;w}*~?zUMqk>!IANEofH0fat`wOg z50&_C@V?c$khII;WQpYdM<0BpFJI&Gkl_z7y4HWXQf3L`$xl;{g++?aktk8&{C-|% zb&ZOW!^uM*^<_3(=gBTL6izrW(co%?z+Gk)bm~L?>#wHSC7ZvD3#N= zuiK&M8yz?1c0UF*j!UUOBcCa{Y30YWq0Pe*C|qysZ8~ z@7<@_+g?5u-;m!mJ>fm!10~E&npZ0=51;k0}o?$U@*>-Towz%%$s6 zPH=OoY9~D1qjT1gxyyO~?gvXPI}RURw(IwI^Mc!P{L9O4`*+OzXrphu&FJ4R_u7W^ zi>p>m6h8gdT8v|7=C-MKeAkrzTx-6`AyaSrp9>y_8~=%%`epSubJF0{z%Q}3pJIEci6W=`M!c#S>U2tv1=vs)?NCYGx2OD-^=YKe4cLJrxbp$1V*V$ zx%g;KQY4f4>e80h`42Us929rU+BcXpsP#RtyeDvM+q}NF#y40CW|?1~Sh-nrLY;W^ z;wjf0x+l0<{eTU9Aoxbm<@o%%D zuZ_}*M|T7!n3OJEq#I9F!-~PLQ z>EQ0AGwf^^S+iY@_y5{xUus>>d8o&J-jjFiE^6~zFZtfS7|*$Kr%=J!S5Gk_A_pSDSrZ2eT|pAaf_B2_+OXV3G%FedNkua&Ixizl7^P=4|5 zoT;q3mm|fJf45JW`aLtwXmQEu+iWxPc8cxJ`t0o9t0h_@=(*$Bg4MeB4{y7)$1bdW z@5}G2=e=%mysFw~u5eF5rLVYp?va+q0@uHv$aoN=Uc3Fh)IkgF%^lB*nJVRZzGR$C z2vj?e({@r`E`GX z@V!X~O!XNiJuKoc*Ey6|)Xl$X)sNDvs;4*AxP_lqYF)9QvW9)NQniDe_`Dg1?^d0N z*mkB`zW9{$Y^USu4_6#sWjDpOZ2q|ycW0~CZT@rd5A*6f8|N;*-N4f<_%Y`GjK8~D zRAQ&=?E4+9eBoy4S*hbLg;wIxbxCeNI9k`eX5X=tO?W4$o5H>6G7fRSnz1=j3V*`B?@(|OX(v;O(F zbMLk>A3R=g?&s4xvr{hjuGy<^5@w>M9w;EhFPgn0VD+3mdg=dkKdxaux-d`VSxWab zpYGSIdtOd4WsCP&e9@xu+G~$f%Y2?MDe_xBB`Rzoi*yO!)y~vpl^yRg!a|sTw}@V{ z;871*#LUN+!T;r}hf1|dx^mEZUt0!?#Sx2Lp6c{Ab;#}I>)j&G++6f%%As7-xGi!= zF7Dl!WU?(+ddta#x4+!K)c)He;r6xlRr%2gB9iHv#vJPmk1lEolKAlKbx^^Fd0GMY z?{Z!<)j08@bMyTy-fL%fvum(^`u#Tk;r+&({Y|nWJFkcOmp98z+5Y5Vp{25`%rRm6 zJ)a-XPMxvjq)PAO{(a{o|8Y$D?fuW!hM{ZW_n+T2>ljdjTcvZ)jSfZz24Q9f1_9*Y zPAn(@hj!4xyxRsmd--qJOYAZax}$i5De~yGoEHte+!MA~v#s$t?j&+@f`szT>;7c| z*E7p+z5V#{;mgh?d#vB(UQ6mYw!6t!y(#s@7X9AaVs8rnzI~>ATB%X{U{yP-bz(~S ztH0JedG;N2U`w8+6QdL1d*zH}|E^=UM?AB3Us+jmgL$b@&d&8(iwhrqP3-Th&^yxe zc;bbr5%)}*6|Jq$MgNMweCf4bd`;B+Sx=TfdA@9;8}rimRrB50Q(0T34fQoq48LJ!vmANu>0b(W1bj))ZBkphyVPwH}_ih+5CTXQ2yB4N6DMN zaVzZM`7&$DYAL2^ne2xTushi;NjX^d>$lV$vD4kpau-Mczx?yq)eW7yWo{>#6?V>owbC$A#|h{>3vZzwS9~@bv5VZxVACmiL+# zPl*u}X$;(!cjmE7?XA0ym|V|!2cKSSYTzzj@S^JDzu2D_Zu3>$Z|H7nDJ(Aeuy6L! zjlt|0OLuc+3LD+}w)TtS8~NnbrHd1CudZZ$ob1KXfAwVL?MLtWWqz$SxBk6x($>)P zCGS4JyTrQJ^`}x@XUXf?Reg19HeaopJCWydZ^XkE6>puOiOE3%PSZ%pI8o!=)>Yr7l~*fYdH8?x zeA(Uk=e~2_pTD5z{v)Hh)ZhN6N_3uz{fVlQ2ncerRAUO>ru(v?09&4)OiDf<``{nA()pJ@V^~W_GJAAib&Z-BQ=Q7(qHyNLKWpg-m zUbar~(k7V~A*z#SoH#yfN1&a|#{D@NvzjifIlz?iLS)sUpn^t8|4kp1SUqp5n00M5 z3%mBC;MBC@iRW&){OhZ65!3qB*F}8UtqtQgaHZ~7(t9^%!^9U4LF2$X$D9e z2sbb;XGFO|16?Ef{SY8cAl$(Cl?mgv2z1Tp=kbFyf^Y-lAy%+v9H;Z6n}dF4J;)3Y zZeV=I1~v!l3HIowpwGyFOaS2qMn500DJ5w0Gw24Ok6eK?gKz_5dN9}k#5fkZR`g~g zNE-+@FsjBtw4yg6(M>?F3_-d6kW3 Date: Thu, 24 Sep 2026 20:59:34 +0000 Subject: [PATCH 2/7] fix(ownership): accept Feb/Mar 2026 INVESTOR_TYPE header in XLSX The February and March 2026 above-1% workbooks name the investor column INVESTOR_TYPE (codes such as CP, ID); April onward uses INVESTOR_CLASSIFICATION (words). Treat both as the same field; every other column stays strict. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E --- src/ownership/xlsx.rs | 62 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/src/ownership/xlsx.rs b/src/ownership/xlsx.rs index 2669bbd..f5a0f0f 100644 --- a/src/ownership/xlsx.rs +++ b/src/ownership/xlsx.rs @@ -304,10 +304,15 @@ fn drafts_from_rows(rows: &[SheetRow]) -> Result, IdxError fn header_columns(row: &SheetRow) -> Option> { let mut found: HashMap<&'static str, String> = HashMap::new(); for (column, text) in row { - let label = text.trim().to_ascii_uppercase().replace(' ', "_"); + let mut label = text.trim().to_ascii_uppercase().replace(' ', "_"); if label.is_empty() { continue; } + // Feb/Mar 2026 workbooks name the column INVESTOR_TYPE and hold codes + // (CP, ID, ...); later ones use INVESTOR_CLASSIFICATION with words. + if label == "INVESTOR_TYPE" { + label = "INVESTOR_CLASSIFICATION".to_string(); + } let expected = EXPECTED_HEADER.iter().find(|name| **name == label)?; found.insert(expected, column.clone()); } @@ -416,6 +421,61 @@ mod tests { ); } + fn header_row(labels: &[&str]) -> SheetRow { + labels + .iter() + .enumerate() + .map(|(index, label)| { + ( + ((b'A' + index as u8) as char).to_string(), + label.to_string(), + ) + }) + .collect() + } + + #[test] + fn accepts_legacy_investor_type_header() { + let mut labels = EXPECTED_HEADER.to_vec(); + labels[4] = "INVESTOR_TYPE"; + let columns = header_columns(&header_row(&labels)).expect("legacy header accepted"); + assert_eq!(columns["INVESTOR_CLASSIFICATION"], "E"); + + let mut row = header_row(&[ + "46112", + "AADI", + "ADARO ANDALAN INDONESIA Tbk", + "GARIBALDI THOHIR", + "ID", + "L ", + "INDONESIAN", + "INDONESIA", + "454011607", + "0", + "454011607", + "5.83", + ]); + row.retain(|_, value| !value.is_empty()); + let drafts = drafts_from_rows(&[header_row(&labels), row]).expect("legacy rows parse"); + assert_eq!( + drafts[0].investor_type.as_ref().map(|t| t.0.as_str()), + Some("ID") + ); + assert_eq!( + drafts[0].report_date, + NaiveDate::from_ymd_opt(2026, 3, 31).unwrap() + ); + assert!(drafts[0].locality.is_some()); + } + + #[test] + fn rejects_header_with_unknown_or_missing_columns() { + let mut labels = EXPECTED_HEADER.to_vec(); + labels[11] = "PERUBAHAN"; + assert!(header_columns(&header_row(&labels)).is_none()); + assert!(header_columns(&header_row(&EXPECTED_HEADER[..11])).is_none()); + } + #[test] fn rounds_float_noise_in_percentages() { assert_eq!(parse_percentage_bps("1.1499999999999999", 1).unwrap(), 115); From 06d0a7a2ae0261f75a12d5bed05ded7ddd194a35 Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 24 Sep 2026 21:08:57 +0000 Subject: [PATCH 3/7] fix(ownership): pin XLSX excerpt row count, report Excel row numbers Addresses review on #25: sheet XML omits blank rows, so error messages now use each number instead of the position; the excerpt test pins the exact row count so dropped rows fail loudly. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E --- src/ownership/xlsx.rs | 122 ++++++++++++++++++++++++++++-------------- 1 file changed, 83 insertions(+), 39 deletions(-) diff --git a/src/ownership/xlsx.rs b/src/ownership/xlsx.rs index f5a0f0f..d12be20 100644 --- a/src/ownership/xlsx.rs +++ b/src/ownership/xlsx.rs @@ -155,8 +155,26 @@ fn parse_shared_strings(xml: &str) -> Result, IdxError> { Ok(strings) } -/// One sheet row: column letters (`A`, `B`, ...) to cell text. -type SheetRow = HashMap; +/// One sheet row: its 1-based Excel row number and column letters +/// (`A`, `B`, ...) mapped to cell text. Sheet XML omits blank rows, so the +/// number comes from the row's `r` attribute, not its position. +#[derive(Debug, Default)] +struct SheetRow { + number: usize, + cells: HashMap, +} + +fn row_number(element: &BytesStart<'_>, fallback: usize) -> Result { + for attribute in element.attributes() { + let attribute = attribute.map_err(|e| xml_error("worksheet", e))?; + if attribute.key.local_name().as_ref() == b"r" { + if let Ok(number) = String::from_utf8_lossy(&attribute.value).trim().parse() { + return Ok(number); + } + } + } + Ok(fallback) +} fn parse_sheet_rows(xml: &str, shared_strings: &[String]) -> Result, IdxError> { let mut reader = Reader::from_str(xml); @@ -169,7 +187,12 @@ fn parse_sheet_rows(xml: &str, shared_strings: &[String]) -> Result match e.local_name().as_ref() { - b"row" => row = Some(SheetRow::new()), + b"row" => { + row = Some(SheetRow { + number: row_number(&e, rows.len() + 1)?, + cells: HashMap::new(), + }) + } b"c" => { cell = Some(cell_ref_and_type(&e)?); value.clear(); @@ -177,7 +200,10 @@ fn parse_sheet_rows(xml: &str, shared_strings: &[String]) -> Result in_value = true, _ => {} }, - Ok(Event::Empty(e)) if e.local_name().as_ref() == b"row" => rows.push(SheetRow::new()), + Ok(Event::Empty(e)) if e.local_name().as_ref() == b"row" => rows.push(SheetRow { + number: row_number(&e, rows.len() + 1)?, + cells: HashMap::new(), + }), Ok(Event::Text(text)) if in_value => { let unescaped = text.unescape().map_err(|e| xml_error("worksheet", e))?; value.push_str(&unescaped); @@ -201,7 +227,7 @@ fn parse_sheet_rows(xml: &str, shared_strings: &[String]) -> Result { @@ -254,12 +280,12 @@ fn drafts_from_rows(rows: &[SheetRow]) -> Result, IdxError })?; let mut drafts = Vec::new(); - for (offset, row) in rows[header_index + 1..].iter().enumerate() { - let line = header_index + offset + 2; + for row in &rows[header_index + 1..] { + let line = row.number; let field = |name: &str| -> &str { columns .get(name) - .and_then(|column| row.get(column)) + .and_then(|column| row.cells.get(column)) .map(|value| value.trim()) .unwrap_or("") }; @@ -303,7 +329,7 @@ fn drafts_from_rows(rows: &[SheetRow]) -> Result, IdxError /// Map header labels to column letters if `row` is exactly the expected header. fn header_columns(row: &SheetRow) -> Option> { let mut found: HashMap<&'static str, String> = HashMap::new(); - for (column, text) in row { + for (column, text) in &row.cells { let mut label = text.trim().to_ascii_uppercase().replace(' ', "_"); if label.is_empty() { continue; @@ -393,11 +419,8 @@ mod tests { #[test] fn parses_above1_xlsx_excerpt() { let drafts = fixture(); - assert!( - drafts.len() >= 10, - "expected excerpt rows, got {}", - drafts.len() - ); + // Pinned so a regression that silently drops rows fails loudly. + assert_eq!(drafts.len(), 17); assert!( drafts .iter() @@ -422,16 +445,24 @@ mod tests { } fn header_row(labels: &[&str]) -> SheetRow { - labels - .iter() - .enumerate() - .map(|(index, label)| { - ( - ((b'A' + index as u8) as char).to_string(), - label.to_string(), - ) - }) - .collect() + sheet_row(1, labels) + } + + fn sheet_row(number: usize, values: &[&str]) -> SheetRow { + SheetRow { + number, + cells: values + .iter() + .enumerate() + .filter(|(_, value)| !value.is_empty()) + .map(|(index, value)| { + ( + ((b'A' + index as u8) as char).to_string(), + value.to_string(), + ) + }) + .collect(), + } } #[test] @@ -441,21 +472,23 @@ mod tests { let columns = header_columns(&header_row(&labels)).expect("legacy header accepted"); assert_eq!(columns["INVESTOR_CLASSIFICATION"], "E"); - let mut row = header_row(&[ - "46112", - "AADI", - "ADARO ANDALAN INDONESIA Tbk", - "GARIBALDI THOHIR", - "ID", - "L ", - "INDONESIAN", - "INDONESIA", - "454011607", - "0", - "454011607", - "5.83", - ]); - row.retain(|_, value| !value.is_empty()); + let row = sheet_row( + 2, + &[ + "46112", + "AADI", + "ADARO ANDALAN INDONESIA Tbk", + "GARIBALDI THOHIR", + "ID", + "L ", + "INDONESIAN", + "INDONESIA", + "454011607", + "0", + "454011607", + "5.83", + ], + ); let drafts = drafts_from_rows(&[header_row(&labels), row]).expect("legacy rows parse"); assert_eq!( drafts[0].investor_type.as_ref().map(|t| t.0.as_str()), @@ -468,6 +501,17 @@ mod tests { assert!(drafts[0].locality.is_some()); } + #[test] + fn row_errors_report_excel_row_numbers() { + let missing_investor = sheet_row( + 709, + &["46265", "BBCA", "BANK CENTRAL ASIA Tbk", "", "Corporate"], + ); + let err = drafts_from_rows(&[header_row(&EXPECTED_HEADER), missing_investor]) + .expect_err("row without investor rejected"); + assert!(err.to_string().contains("row 709"), "{err}"); + } + #[test] fn rejects_header_with_unknown_or_missing_columns() { let mut labels = EXPECTED_HEADER.to_vec(); From 29e416109c7386e0200bb4ae3984353696c7e2a4 Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 24 Sep 2026 20:55:20 +0000 Subject: [PATCH 4/7] feat(ownership): discover above-1% XLSX on the IDX data page IDX stopped posting the monthly above-1% list as an announcement after the 2026-05-29 report; June/July/August are only on the Data Kepemilikan Saham page. `ownership discover` could not see them, and IDX's Cloudflare now rejects curl_chrome142 on that page and on the /Media/ XLSX files. - remote: parse the page's Nuxt payload (Description/Prospectus pairs, / escapes), classify above-1% XLSX as supported and above-5% / investor-type workbooks as unsupported, derive as_of_date from the Indonesian description. Keep all above-1% entries, only the newest of the other series. Merge with the announcement results. - discover JSON gains `format` (xlsx|pdf) and `as_of_date`; page entries use the as-of date as publish_date (the page shows none). - import --url accepts .xlsx and downloads via the XLSX validator. - curl_impersonate: run_owned_with_fallback tries several browser profiles until the body validates (JSON / page payload / %PDF / PK); IDX_CURL_IMPERSONATE_BIN still pins one. - Tests: page fixture parsing, date parsing, payload validation, CLI discover via a local page server; existing discover tests now point IDX_OWNERSHIP_DATA_PAGE_URL at a closed port so no test hits IDX. - Docs: ARCHITECTURE discovery section, AGENTS ownership inputs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E --- AGENTS.md | 4 +- docs/ARCHITECTURE.md | 16 + src/cli/ownership.rs | 64 ++- src/curl_impersonate.rs | 60 ++- src/ownership/remote.rs | 427 +++++++++++++++++- tests/cli.rs | 43 +- .../idx_share_ownership_page_excerpt.html | 5 + 7 files changed, 592 insertions(+), 27 deletions(-) create mode 100644 tests/fixtures/idx_share_ownership_page_excerpt.html diff --git a/AGENTS.md b/AGENTS.md index f0d092f..3a4a034 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,11 +57,11 @@ Ownership is local-first after bootstrap. Preferred bootstrap/update path: 1. `idx ownership sync` -2. if no snapshot manifest is available: `idx ownership discover` then `idx ownership import --url ` +2. if no snapshot manifest is available: `idx ownership discover` then `idx ownership import --url ` 3. local `--file` imports remain available for manual/fallback use Ownership input paths: -- primary remote source: discoverable IDX `above1` holder-register PDF +- primary remote source: IDX `above1` holder register — monthly XLSX on the Data Kepemilikan Saham page (since the 2026-05-29 report), legacy PDF announcements before that; `ownership discover` checks both - maintained snapshot path: `ownership sync` - local file imports: IDX above-1% `.xlsx` (current format since June 2026), legacy `.pdf`, plus local archive `.zip` / `.txt` diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 3919f69..446413d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -67,6 +67,22 @@ Unlike the `stocks` module (live-fetch), ownership is **import-then-query**: - **KSEI archive** — monthly ZIP/TXT balance-position matrix, used as a local fallback/backstop import path - **Bing Finance** — global institutional ownership (REST API, quarterly) +### Discovery +`idx ownership discover` merges two IDX sources: +- the announcement API (`GetAllAnnouncement`) for the legacy PDF announcements +- the Data Kepemilikan Saham page (`/id/perusahaan-tercatat/data-kepemilikan-saham/`), + whose server-rendered Nuxt payload lists every XLSX with a description such as + `Pemegang Saham di Atas 1% per 31 Agustus 2026`; the as-of date comes from that text + +Only the above-1% XLSX is `supported`; the daily above-5% and investor-type workbooks +are listed as `unsupported`. Both URLs can be overridden for tests with +`IDX_OWNERSHIP_ANNOUNCEMENT_API_URL` and `IDX_OWNERSHIP_DATA_PAGE_URL`. + +IDX sits behind Cloudflare, which accepts some curl-impersonate TLS profiles and +rejects others per path (for example `curl_chrome142` gets 403 on the data page). +IDX fetches therefore try several profiles until the response validates (JSON, page +payload, `%PDF`, or `PK` zip magic). `IDX_CURL_IMPERSONATE_BIN` pins a single profile. + ### Parser Pipeline ``` KSEI XLSX → zip + quick-xml (sharedStrings + sheet1) → exact 12-column header check diff --git a/src/cli/ownership.rs b/src/cli/ownership.rs index 2540e2c..f8b4bde 100644 --- a/src/cli/ownership.rs +++ b/src/cli/ownership.rs @@ -35,7 +35,7 @@ pub struct OwnershipCmd { pub enum OwnershipCommand { #[command( about = "Discover the latest IDX-hosted ownership report URLs", - after_help = "Examples:\n idx ownership discover --limit 1\n idx ownership discover --family all --limit 6\n\nUse this when no snapshot manifest is available and you need a direct PDF URL for `idx ownership import --url`." + after_help = "Examples:\n idx ownership discover --limit 1\n idx ownership discover --family all --limit 6\n\nUse this when no snapshot manifest is available and you need a direct XLSX or PDF URL for `idx ownership import --url`." )] Discover(DiscoverArgs), #[command( @@ -84,7 +84,7 @@ pub struct DiscoverArgs { #[derive(Debug, Args)] pub struct ImportArgs { - /// Direct URL to a remote ownership PDF, typically from `ownership discover`. + /// Direct URL to a remote above-1% XLSX or PDF, typically from `ownership discover`. #[arg(long)] pub url: Option, /// Path to a local ownership file: above-1% XLSX or PDF (primary), ZIP/TXT archive (fallback). @@ -999,13 +999,16 @@ fn resolve_import_input(args: &ImportArgs) -> Result if let Some(url) = &args.url { let trimmed = url.trim(); - validate_import_url(trimmed)?; - let target = cache_pdf_path(trimmed)?; - download_pdf(trimmed, &target)?; + let format = validate_import_url(trimmed)?; + let target = cache_download_path(trimmed, format)?; + match format { + ImportInputFormat::Xlsx => download_xlsx(trimmed, &target)?, + _ => download_pdf(trimmed, &target)?, + } return Ok(Some(ResolvedImportInput { import_path: target, source_url: Some(trimmed.to_string()), - format: ImportInputFormat::Pdf, + format, })); } @@ -1031,7 +1034,11 @@ fn detect_local_import_format(path: &Path) -> Result Result { +fn cache_download_path(url: &str, format: ImportInputFormat) -> Result { + let extension = match format { + ImportInputFormat::Xlsx => ".xlsx", + _ => ".pdf", + }; let raw_dir = crate::cache::cache_dir()?.join("ownership").join("raw"); fs::create_dir_all(&raw_dir).map_err(|e| IdxError::Io(e.to_string()))?; @@ -1045,20 +1052,20 @@ fn cache_pdf_path(url: &str) -> Result { .to_string(); if file_name.is_empty() || file_name == "/" { - file_name = format!("ksei-{}.pdf", Utc::now().timestamp()); + file_name = format!("ksei-{}{extension}", Utc::now().timestamp()); } - if !file_name.to_ascii_lowercase().ends_with(".pdf") { - file_name.push_str(".pdf"); + if !file_name.to_ascii_lowercase().ends_with(extension) { + file_name.push_str(extension); } Ok(raw_dir.join(file_name)) } -fn validate_import_url(url: &str) -> Result<(), IdxError> { +fn validate_import_url(url: &str) -> Result { let trimmed = url.trim(); if trimmed.is_empty() { return Err(IdxError::InvalidInput( - "ownership import --url accepts direct PDF URLs only".to_string(), + "ownership import --url accepts direct XLSX or PDF URLs only".to_string(), )); } @@ -1068,11 +1075,14 @@ fn validate_import_url(url: &str) -> Result<(), IdxError> { .next() .unwrap_or(normalized.as_str()); if normalized.ends_with(".pdf") { - return Ok(()); + return Ok(ImportInputFormat::Pdf); + } + if normalized.ends_with(".xlsx") { + return Ok(ImportInputFormat::Xlsx); } Err(IdxError::InvalidInput( - "ownership import --url accepts direct PDF URLs only; run `idx ownership discover` first to find the current supported attachment".to_string(), + "ownership import --url accepts direct XLSX or PDF URLs only; run `idx ownership discover` first to find the current supported file".to_string(), )) } @@ -1108,6 +1118,32 @@ fn download_pdf(url: &str, target: &Path) -> Result<(), IdxError> { Ok(()) } +fn download_xlsx(url: &str, target: &Path) -> Result<(), IdxError> { + if is_idx_url(url) { + return remote::download_idx_xlsx(url, target); + } + + let response = ureq::get(url) + .header( + "Accept", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/octet-stream,*/*;q=0.8", + ) + .call() + .map_err(|e| IdxError::Http(format!("failed to download XLSX: {e}")))?; + let bytes = response + .into_body() + .read_to_vec() + .map_err(|e| IdxError::Http(format!("failed reading XLSX body: {e}")))?; + remote::validate_xlsx_payload(&bytes)?; + + fs::write(target, &bytes).map_err(|e| { + IdxError::Io(format!( + "failed writing cached XLSX {}: {e}", + target.display() + )) + }) +} + fn is_idx_url(url: &str) -> bool { let normalized = url.trim().to_ascii_lowercase(); normalized.starts_with("https://www.idx.co.id/") diff --git a/src/curl_impersonate.rs b/src/curl_impersonate.rs index 3735feb..dbabb8c 100644 --- a/src/curl_impersonate.rs +++ b/src/curl_impersonate.rs @@ -40,7 +40,65 @@ pub fn run(stage: &str, args: &[&str]) -> Result { pub fn run_owned(stage: &str, args: &[String]) -> Result { let binary = chrome_curl_binary()?; - let output = Command::new(&binary).args(args).output().map_err(|e| { + run_binary(stage, &binary, args) +} + +/// Browser profiles tried, in order, when a site's bot protection rejects +/// some TLS fingerprints but not others (IDX's Cloudflare rules block +/// `curl_chrome142` on some pages while Firefox/Safari profiles pass). +const FALLBACK_PROFILES: &[&str] = &[ + "curl_chrome142", + "curl_firefox144", + "curl_chrome136", + "curl_firefox135", + "curl_safari184", + "curl_chrome131", + "curl_chrome124", + "curl_chrome120", + "curl_chrome116", +]; +const FALLBACK_ROUNDS: usize = 2; + +/// Run curl with each available impersonation profile until `accept` approves +/// the response body. `IDX_CURL_IMPERSONATE_BIN`, when set, is the only +/// profile used. Returns the first accepted output, or the last error. +pub fn run_owned_with_fallback( + stage: &str, + args: &[String], + accept: &dyn Fn(&[u8]) -> Result<(), IdxError>, +) -> Result { + let profiles: Vec = match std::env::var(OVERRIDE_ENV) { + Ok(value) if !value.trim().is_empty() => vec![value.trim().to_string()], + _ => FALLBACK_PROFILES + .iter() + .copied() + .filter(|bin| Command::new(bin).arg("--version").output().is_ok()) + .map(str::to_string) + .collect(), + }; + if profiles.is_empty() { + return Err(IdxError::Http(format!( + "no curl-impersonate binary found; set {OVERRIDE_ENV} or install nixpkgs#curl-impersonate" + ))); + } + + let mut last_error = None; + for _ in 0..FALLBACK_ROUNDS { + for binary in &profiles { + match run_binary(stage, binary, args) { + Ok(output) => match accept(&output.stdout) { + Ok(()) => return Ok(output), + Err(err) => last_error = Some(err), + }, + Err(err) => last_error = Some(err), + } + } + } + Err(last_error.expect("at least one profile was tried")) +} + +fn run_binary(stage: &str, binary: &str, args: &[String]) -> Result { + let output = Command::new(binary).args(args).output().map_err(|e| { if e.kind() == std::io::ErrorKind::NotFound { return IdxError::Http(format!( "curl-impersonate binary '{binary}' not found; set {OVERRIDE_ENV} or install nixpkgs#curl-impersonate-chrome" diff --git a/src/ownership/remote.rs b/src/ownership/remote.rs index c8b18f6..a26e3f6 100644 --- a/src/ownership/remote.rs +++ b/src/ownership/remote.rs @@ -12,6 +12,10 @@ const DEFAULT_IDX_ANNOUNCEMENT_API_URL: &str = const IDX_ANNOUNCEMENT_API_ENV: &str = "IDX_OWNERSHIP_ANNOUNCEMENT_API_URL"; const IDX_ANNOUNCEMENT_LISTING_ENV: &str = "IDX_OWNERSHIP_ANNOUNCEMENT_PAGE_URL"; const IDX_ANNOUNCEMENT_PAGE_SIZE: usize = 10; +/// Since June 2026 the monthly above-1% list (XLSX) is published only here. +pub const IDX_SHARE_OWNERSHIP_PAGE_URL: &str = + "https://www.idx.co.id/id/perusahaan-tercatat/data-kepemilikan-saham/"; +const IDX_SHARE_OWNERSHIP_PAGE_ENV: &str = "IDX_OWNERSHIP_DATA_PAGE_URL"; const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"; #[derive(Debug, Clone, Deserialize)] @@ -116,6 +120,12 @@ pub struct DiscoveredOwnershipPdf { pub code: Option, pub original_filename: Option, pub is_attachment: bool, + /// File format of `pdf_url`: `pdf` (announcement attachment) or `xlsx` + /// (Data Kepemilikan Saham page). + pub format: String, + /// Report as-of date (`YYYY-MM-DD`) when the source states it. The data + /// page shows no publish date, so its entries use this as `publish_date`. + pub as_of_date: Option, } struct DiscoveryQuery { @@ -152,6 +162,13 @@ pub fn announcement_listing_url() -> String { .unwrap_or_else(|| IDX_ANNOUNCEMENT_LISTING_URL.to_string()) } +pub fn share_ownership_page_url() -> String { + std::env::var(IDX_SHARE_OWNERSHIP_PAGE_ENV) + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| IDX_SHARE_OWNERSHIP_PAGE_URL.to_string()) +} + pub fn announcement_api_url() -> String { std::env::var(IDX_ANNOUNCEMENT_API_ENV) .ok() @@ -240,6 +257,8 @@ pub fn select_latest_ownership_reports( code: clean_option(item.code.as_deref()), original_filename, is_attachment, + format: "pdf".to_string(), + as_of_date: None, } }) .collect::>(); @@ -278,6 +297,7 @@ pub fn discover_idx_ownership_reports( "IDX ownership announcement discovery", &query_url, &json_headers(), + &validate_json_payload, ) .and_then(|raw| parse_announcement_page(&raw)) .and_then(|page| select_latest_ownership_reports(&page, &query_url, query.family)) @@ -287,6 +307,35 @@ pub fn discover_idx_ownership_reports( } } + match fetch_text( + "IDX share ownership data page discovery", + &share_ownership_page_url(), + &page_headers(), + &validate_share_ownership_page, + ) { + Ok(html) => { + let mut page_reports = parse_share_ownership_page(&html, &share_ownership_page_url()); + page_reports + .retain(|report| family_filter.is_none_or(|family| family == report.family)); + // Keep the full above-1% history (snapshot backfill needs it), but + // only the newest entry of the unsupported daily/breakdown series. + page_reports.sort_by(|left, right| right.publish_date.cmp(&left.publish_date)); + let mut seen_unsupported = Vec::new(); + page_reports.retain(|report| { + if report.family == OwnershipReportFamily::AboveOnePercent { + return true; + } + if seen_unsupported.contains(&report.family) { + return false; + } + seen_unsupported.push(report.family); + true + }); + discovered.extend(page_reports); + } + Err(err) => errors.push(err.to_string()), + } + discovered.sort_by(|left, right| { right .publish_date @@ -307,8 +356,9 @@ pub fn discover_idx_ownership_reports( errors.join("; ") }; return Err(IdxError::Http(format!( - "failed to discover IDX ownership reports from {}: {detail}", - announcement_listing_url() + "failed to discover IDX ownership reports from {} or {}: {detail}", + announcement_listing_url(), + share_ownership_page_url() ))); } @@ -316,8 +366,12 @@ pub fn discover_idx_ownership_reports( } pub fn download_idx_pdf(url: &str, target: &Path) -> Result<(), IdxError> { - let bytes = fetch_bytes("IDX ownership PDF download", url, &pdf_headers())?; - validate_pdf_payload(&bytes)?; + let bytes = fetch_bytes( + "IDX ownership PDF download", + url, + &pdf_headers(), + &validate_pdf_payload, + )?; fs::write(target, &bytes).map_err(|e| { IdxError::Io(format!( @@ -329,6 +383,236 @@ pub fn download_idx_pdf(url: &str, target: &Path) -> Result<(), IdxError> { Ok(()) } +pub fn download_idx_xlsx(url: &str, target: &Path) -> Result<(), IdxError> { + let bytes = fetch_bytes( + "IDX ownership XLSX download", + url, + &xlsx_headers(), + &validate_xlsx_payload, + )?; + + fs::write(target, &bytes).map_err(|e| { + IdxError::Io(format!( + "failed writing cached XLSX {}: {e}", + target.display() + )) + })?; + + Ok(()) +} + +/// XLSX is a ZIP container; Cloudflare challenges come back as HTML. +pub fn validate_xlsx_payload(bytes: &[u8]) -> Result<(), IdxError> { + if bytes.starts_with(b"PK\x03\x04") { + return Ok(()); + } + if looks_like_html(bytes) { + return Err(IdxError::Http( + "IDX ownership download returned HTML (likely a Cloudflare challenge) instead of an XLSX workbook".to_string(), + )); + } + Err(IdxError::Http( + "IDX ownership download did not look like an XLSX workbook".to_string(), + )) +} + +fn validate_json_payload(bytes: &[u8]) -> Result<(), IdxError> { + if looks_like_html(bytes) { + return Err(IdxError::Http( + "IDX ownership discovery returned HTML (likely a Cloudflare challenge) instead of announcement JSON".to_string(), + )); + } + Ok(()) +} + +fn validate_share_ownership_page(bytes: &[u8]) -> Result<(), IdxError> { + if contains_bytes(bytes, b"Prospectus") { + return Ok(()); + } + Err(IdxError::Http( + "IDX share ownership data page did not contain its file list (likely a Cloudflare challenge)".to_string(), + )) +} + +fn looks_like_html(bytes: &[u8]) -> bool { + let head: Vec = bytes + .iter() + .skip_while(|byte| byte.is_ascii_whitespace()) + .take(256) + .map(u8::to_ascii_lowercase) + .collect(); + head.starts_with(b" bool { + haystack + .windows(needle.len()) + .any(|window| window == needle) +} + +/// Parse the Data Kepemilikan Saham page. The page is server-rendered by +/// Nuxt and embeds its file list as JS object literals such as +/// `{Description:"Pemegang Saham di Atas 1% per 31 Agustus 2026", +/// Prospectus:"https:\u002F\u002Fwww.idx.co.id\u002FMedia\u002F...xlsx",...}`. +pub fn parse_share_ownership_page(html: &str, page_url: &str) -> Vec { + let mut reports = Vec::new(); + let mut rest = html; + + while let Some(start) = find_js_key(rest, "Description") { + let (description, after_description) = match read_js_string(&rest[start..]) { + Some(value) => value, + None => { + rest = &rest[start..]; + continue; + } + }; + let object_tail = &rest[start + after_description..]; + // The URL belongs to the same object literal, so it must come before + // the next Description key. + let window_end = find_js_key(object_tail, "Description").unwrap_or(object_tail.len()); + let window = &object_tail[..window_end]; + rest = object_tail; + + let Some(url_start) = find_js_key(window, "Prospectus") else { + continue; + }; + let Some((url, _)) = read_js_string(&window[url_start..]) else { + continue; + }; + if let Some(report) = classify_page_entry(description.trim(), url.trim(), page_url) { + reports.push(report); + } + } + + reports +} + +/// Offset just past `Key:` / `"Key":` (the opening quote of the value). +fn find_js_key(haystack: &str, key: &str) -> Option { + let bare = format!("{key}:\""); + let quoted = format!("\"{key}\":\""); + [bare, quoted] + .iter() + .filter_map(|needle| { + haystack + .find(needle.as_str()) + .map(|index| index + needle.len() - 1) + }) + .min() +} + +/// Read a JS/JSON string literal starting at the opening quote. Returns the +/// decoded value and the byte offset just past the closing quote. +fn read_js_string(input: &str) -> Option<(String, usize)> { + let mut chars = input.char_indices(); + if chars.next()?.1 != '"' { + return None; + } + let mut value = String::new(); + while let Some((index, ch)) = chars.next() { + match ch { + '"' => return Some((value, index + 1)), + '\\' => match chars.next()?.1 { + 'u' => { + let hex: String = (0..4) + .filter_map(|_| chars.next().map(|(_, c)| c)) + .collect(); + value.push(char::from_u32(u32::from_str_radix(&hex, 16).ok()?)?); + } + 'n' => value.push('\n'), + 't' => value.push('\t'), + other => value.push(other), + }, + _ => value.push(ch), + } + } + None +} + +fn classify_page_entry( + description: &str, + url: &str, + page_url: &str, +) -> Option { + let upper = description.to_uppercase(); + let family = if upper.contains("DI ATAS 1%") || upper.contains("ABOVE 1%") { + OwnershipReportFamily::AboveOnePercent + } else if upper.contains("DI ATAS 5%") || upper.contains("ABOVE 5%") { + OwnershipReportFamily::AboveFivePercent + } else if upper.contains("TIPE INVESTOR") + || upper.contains("KLASIFIKASI") + || upper.contains("INVESTOR TYPE") + || upper.contains("CLASSIFICATION") + { + OwnershipReportFamily::InvestorTypeBreakdown + } else { + return None; + }; + + let file_name = url.rsplit('/').next().unwrap_or_default().to_string(); + let lower_name = file_name.to_ascii_lowercase(); + let format = if lower_name.ends_with(".xlsx") { + "xlsx" + } else if lower_name.ends_with(".pdf") { + "pdf" + } else { + return None; + }; + let status = if family == OwnershipReportFamily::AboveOnePercent && format == "xlsx" { + OwnershipReportStatus::Supported + } else { + OwnershipReportStatus::Unsupported + }; + let as_of_date = parse_indonesian_as_of_date(description).map(|date| date.to_string()); + + Some(DiscoveredOwnershipPdf { + family, + status, + listing_page_url: page_url.to_string(), + query_url: page_url.to_string(), + pdf_url: url.to_string(), + title: description.to_string(), + publish_date: as_of_date.clone().unwrap_or_default(), + code: None, + original_filename: (!file_name.is_empty()).then_some(file_name), + is_attachment: true, + format: format.to_string(), + as_of_date, + }) +} + +/// `"... per 31 Agustus 2026"` -> 2026-08-31 (Indonesian or English months). +fn parse_indonesian_as_of_date(description: &str) -> Option { + let tokens: Vec = description + .split_whitespace() + .map(|token| { + token + .trim_matches(|c: char| !c.is_alphanumeric()) + .to_lowercase() + }) + .collect(); + tokens.windows(3).rev().find_map(|window| { + let day = window[0].parse::().ok()?; + let month = match window[1].as_str() { + "januari" | "january" | "jan" => 1, + "februari" | "february" | "feb" => 2, + "maret" | "march" | "mar" => 3, + "april" | "apr" => 4, + "mei" | "may" => 5, + "juni" | "june" | "jun" => 6, + "juli" | "july" | "jul" => 7, + "agustus" | "august" | "agu" | "aug" => 8, + "september" | "sep" => 9, + "oktober" | "october" | "okt" | "oct" => 10, + "november" | "nov" => 11, + "desember" | "december" | "des" | "dec" => 12, + _ => return None, + }; + let year = window[2].parse::().ok()?; + chrono::NaiveDate::from_ymd_opt(year, month, day) + }) +} + pub fn validate_pdf_payload(bytes: &[u8]) -> Result<(), IdxError> { if bytes.is_empty() { return Err(IdxError::Http( @@ -357,13 +641,23 @@ pub fn validate_pdf_payload(bytes: &[u8]) -> Result<(), IdxError> { )) } -fn fetch_text(stage: &str, url: &str, headers: &[(String, String)]) -> Result { - let bytes = fetch_bytes(stage, url, headers)?; +fn fetch_text( + stage: &str, + url: &str, + headers: &[(String, String)], + accept: &dyn Fn(&[u8]) -> Result<(), IdxError>, +) -> Result { + let bytes = fetch_bytes(stage, url, headers, accept)?; String::from_utf8(bytes) .map_err(|e| IdxError::Http(format!("failed to decode {stage} response as utf-8: {e}"))) } -fn fetch_bytes(stage: &str, url: &str, headers: &[(String, String)]) -> Result, IdxError> { +fn fetch_bytes( + stage: &str, + url: &str, + headers: &[(String, String)], + accept: &dyn Fn(&[u8]) -> Result<(), IdxError>, +) -> Result, IdxError> { let mut args = vec![ "--silent".to_string(), "--show-error".to_string(), @@ -377,7 +671,7 @@ fn fetch_bytes(stage: &str, url: &str, headers: &[(String, String)]) -> Result Vec<(String, String)> { ] } +fn page_headers() -> Vec<(String, String)> { + vec![ + ("User-Agent".to_string(), USER_AGENT.to_string()), + ( + "Accept".to_string(), + "text/html,application/xhtml+xml,*/*;q=0.8".to_string(), + ), + ( + "Accept-Language".to_string(), + "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7".to_string(), + ), + ] +} + +fn xlsx_headers() -> Vec<(String, String)> { + vec![ + ("User-Agent".to_string(), USER_AGENT.to_string()), + ( + "Accept".to_string(), + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/octet-stream,*/*;q=0.8".to_string(), + ), + ( + "Accept-Language".to_string(), + "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7".to_string(), + ), + ("Referer".to_string(), share_ownership_page_url()), + ] +} + fn pdf_headers() -> Vec<(String, String)> { vec![ ("User-Agent".to_string(), USER_AGENT.to_string()), @@ -502,7 +825,8 @@ mod tests { use super::{ AnnouncementPage, IDX_ANNOUNCEMENT_LISTING_URL, OwnershipReportFamily, OwnershipReportStatus, build_announcement_query_url, parse_announcement_page, - select_latest_ownership_reports, validate_pdf_payload, + parse_indonesian_as_of_date, parse_share_ownership_page, select_latest_ownership_reports, + validate_pdf_payload, validate_share_ownership_page, validate_xlsx_payload, }; use crate::error::IdxError; @@ -762,4 +1086,89 @@ mod tests { assert!(matches!(err, IdxError::Http(_))); assert!(err.to_string().contains("PDF/direct attachment")); } + + #[test] + fn parses_share_ownership_page_entries() { + let page_url = "https://www.idx.co.id/id/perusahaan-tercatat/data-kepemilikan-saham/"; + let reports = parse_share_ownership_page( + include_str!("../../tests/fixtures/idx_share_ownership_page_excerpt.html"), + page_url, + ); + assert_eq!(reports.len(), 7); + + let above1: Vec<_> = reports + .iter() + .filter(|r| r.family == OwnershipReportFamily::AboveOnePercent) + .collect(); + assert_eq!(above1.len(), 3); + assert!( + above1 + .iter() + .all(|r| r.status == OwnershipReportStatus::Supported) + ); + assert!(above1.iter().all(|r| r.format == "xlsx")); + + let august = above1 + .iter() + .find(|r| r.as_of_date.as_deref() == Some("2026-08-31")) + .expect("August above-1% entry"); + assert_eq!( + august.pdf_url, + "https://www.idx.co.id/Media/fahlw1o2/peng-2026-08-00017-satu-persen.xlsx" + ); + assert_eq!(august.publish_date, "2026-08-31"); + assert_eq!(august.listing_page_url, page_url); + assert_eq!( + august.original_filename.as_deref(), + Some("peng-2026-08-00017-satu-persen.xlsx") + ); + + let above5 = reports + .iter() + .filter(|r| r.family == OwnershipReportFamily::AboveFivePercent) + .count(); + let breakdown = reports + .iter() + .filter(|r| r.family == OwnershipReportFamily::InvestorTypeBreakdown) + .count(); + assert_eq!((above5, breakdown), (2, 2)); + assert!( + reports + .iter() + .filter(|r| r.family != OwnershipReportFamily::AboveOnePercent) + .all(|r| r.status == OwnershipReportStatus::Unsupported) + ); + } + + #[test] + fn share_ownership_page_without_payload_yields_nothing() { + assert!(parse_share_ownership_page("Just a moment...", "p").is_empty()); + assert!(validate_share_ownership_page(b"Just a moment...").is_err()); + } + + #[test] + fn parses_indonesian_and_english_as_of_dates() { + let date = |y, m, d| chrono::NaiveDate::from_ymd_opt(y, m, d); + assert_eq!( + parse_indonesian_as_of_date("Pemegang Saham di Atas 1% per 31 Agustus 2026"), + date(2026, 8, 31) + ); + assert_eq!( + parse_indonesian_as_of_date("Shareholders above 1% as of 29 May 2026"), + date(2026, 5, 29) + ); + assert_eq!( + parse_indonesian_as_of_date("Pemegang Saham di Atas 1%"), + None + ); + } + + #[test] + fn validates_xlsx_payloads() { + assert!(validate_xlsx_payload(b"PK\x03\x04rest-of-zip").is_ok()); + let err = validate_xlsx_payload(b"Just a moment...") + .expect_err("html challenge rejected"); + assert!(err.to_string().contains("Cloudflare"), "{err}"); + assert!(validate_xlsx_payload(b"%PDF-1.7").is_err()); + } } diff --git a/tests/cli.rs b/tests/cli.rs index 5ba912b..60d2bd6 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1189,6 +1189,43 @@ fn ownership_releases_uses_xdg_data_home_default_db_path() { ); } +/// Keeps discovery tests off the live IDX data page (nothing listens on port 9). +const UNREACHABLE_DATA_PAGE_URL: &str = "http://127.0.0.1:9/data-kepemilikan-saham/"; + +#[test] +fn ownership_discover_finds_xlsx_on_share_ownership_page() { + let page = fs::read_to_string("tests/fixtures/idx_share_ownership_page_excerpt.html") + .expect("read share ownership page fixture"); + let page_url = spawn_single_response_server("text/html", page); + + let output = test_bin("ownership-discover-data-page") + .env("IDX_CURL_IMPERSONATE_BIN", "curl") + .env( + "IDX_OWNERSHIP_ANNOUNCEMENT_API_URL", + "http://127.0.0.1:9/announcements", + ) + .env("IDX_OWNERSHIP_DATA_PAGE_URL", &page_url) + .args(["-o", "json", "ownership", "discover", "--limit", "1"]) + .output() + .expect("ownership discover json output"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + + let reports: Value = serde_json::from_slice(&output.stdout).expect("parse discover json"); + let latest = &reports[0]; + assert_eq!(latest["family"], "above_one_percent"); + assert_eq!(latest["status"], "supported"); + assert_eq!(latest["format"], "xlsx"); + assert_eq!(latest["as_of_date"], "2026-08-31"); + assert_eq!( + latest["pdf_url"], + "https://www.idx.co.id/Media/fahlw1o2/peng-2026-08-00017-satu-persen.xlsx" + ); +} + #[test] fn ownership_discover_lists_fixture_candidates() { let body = fs::read_to_string("tests/fixtures/idx_announcement_kepemilikan.json") @@ -1198,6 +1235,7 @@ fn ownership_discover_lists_fixture_candidates() { test_bin("ownership-discover") .env("IDX_CURL_IMPERSONATE_BIN", "curl") .env("IDX_OWNERSHIP_ANNOUNCEMENT_API_URL", &json_url) + .env("IDX_OWNERSHIP_DATA_PAGE_URL", UNREACHABLE_DATA_PAGE_URL) .env( "IDX_OWNERSHIP_ANNOUNCEMENT_PAGE_URL", "http://127.0.0.1/pengumuman", @@ -1257,6 +1295,7 @@ fn ownership_discover_supports_above1_family() { test_bin("ownership-discover-above1") .env("IDX_CURL_IMPERSONATE_BIN", "curl") .env("IDX_OWNERSHIP_ANNOUNCEMENT_API_URL", &json_url) + .env("IDX_OWNERSHIP_DATA_PAGE_URL", UNREACHABLE_DATA_PAGE_URL) .env( "IDX_OWNERSHIP_ANNOUNCEMENT_PAGE_URL", "http://127.0.0.1/pengumuman", @@ -1313,6 +1352,7 @@ fn ownership_discover_defaults_to_above1_and_prefers_supported_attachment() { let output = test_bin("ownership-discover-default") .env("IDX_CURL_IMPERSONATE_BIN", "curl") .env("IDX_OWNERSHIP_ANNOUNCEMENT_API_URL", &json_url) + .env("IDX_OWNERSHIP_DATA_PAGE_URL", UNREACHABLE_DATA_PAGE_URL) .env( "IDX_OWNERSHIP_ANNOUNCEMENT_PAGE_URL", "http://127.0.0.1/pengumuman", @@ -1367,6 +1407,7 @@ fn ownership_discover_json_includes_status() { test_bin("ownership-discover-json-status") .env("IDX_CURL_IMPERSONATE_BIN", "curl") .env("IDX_OWNERSHIP_ANNOUNCEMENT_API_URL", &json_url) + .env("IDX_OWNERSHIP_DATA_PAGE_URL", UNREACHABLE_DATA_PAGE_URL) .env( "IDX_OWNERSHIP_ANNOUNCEMENT_PAGE_URL", "http://127.0.0.1/pengumuman", @@ -1406,7 +1447,7 @@ fn ownership_import_url_rejects_listing_page_inputs() { .assert() .failure() .stderr(predicate::str::contains( - "ownership import --url accepts direct PDF URLs only", + "ownership import --url accepts direct XLSX or PDF URLs only", )) .stderr(predicate::str::contains("ownership discover")); } diff --git a/tests/fixtures/idx_share_ownership_page_excerpt.html b/tests/fixtures/idx_share_ownership_page_excerpt.html new file mode 100644 index 0000000..7962b06 --- /dev/null +++ b/tests/fixtures/idx_share_ownership_page_excerpt.html @@ -0,0 +1,5 @@ + +Data Kepemilikan Saham - Bursa Efek Indonesia +
+ + From bb39e98fab41d6b1f08a0d59980ae6c2da49aa1d Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 24 Sep 2026 21:11:56 +0000 Subject: [PATCH 5/7] fix(ownership): harden data-page discovery per review Addresses review on #26: - strip ?query/#fragment before the .xlsx/.pdf extension check - skip page entries without a parseable as-of date instead of emitting an empty publish_date - warn on stderr when the data page fails or parses to zero entries while announcement results still exist, instead of silently degrading to announcement-only results Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E --- src/ownership/remote.rs | 50 ++++++++++++++++++++++++++++++++++++----- tests/cli.rs | 4 ++++ 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/ownership/remote.rs b/src/ownership/remote.rs index a26e3f6..8960dde 100644 --- a/src/ownership/remote.rs +++ b/src/ownership/remote.rs @@ -307,6 +307,7 @@ pub fn discover_idx_ownership_reports( } } + let mut page_error = None; match fetch_text( "IDX share ownership data page discovery", &share_ownership_page_url(), @@ -315,6 +316,15 @@ pub fn discover_idx_ownership_reports( ) { Ok(html) => { let mut page_reports = parse_share_ownership_page(&html, &share_ownership_page_url()); + if page_reports.is_empty() { + // The page loaded (it has its file list) but nothing parsed: + // the payload shape changed. Say so instead of silently + // falling back to announcement-only results. + page_error = Some(format!( + "IDX share ownership data page {} loaded but no report entries could be parsed; the page layout may have changed", + share_ownership_page_url() + )); + } page_reports .retain(|report| family_filter.is_none_or(|family| family == report.family)); // Keep the full above-1% history (snapshot backfill needs it), but @@ -333,7 +343,7 @@ pub fn discover_idx_ownership_reports( }); discovered.extend(page_reports); } - Err(err) => errors.push(err.to_string()), + Err(err) => page_error = Some(err.to_string()), } discovered.sort_by(|left, right| { @@ -349,6 +359,15 @@ pub fn discover_idx_ownership_reports( discovered.truncate(limit); } + if !discovered.is_empty() + && let Some(err) = &page_error + { + // Announcement results still exist, but since June 2026 the newest + // above-1% reports are only on the data page: don't hide its failure. + crate::runtime::warn(err); + } + errors.extend(page_error); + if discovered.is_empty() { let detail = if errors.is_empty() { "no matching ownership reports found".to_string() @@ -549,7 +568,8 @@ fn classify_page_entry( return None; }; - let file_name = url.rsplit('/').next().unwrap_or_default().to_string(); + let path = url.split(['?', '#']).next().unwrap_or_default(); + let file_name = path.rsplit('/').next().unwrap_or_default().to_string(); let lower_name = file_name.to_ascii_lowercase(); let format = if lower_name.ends_with(".xlsx") { "xlsx" @@ -563,7 +583,9 @@ fn classify_page_entry( } else { OwnershipReportStatus::Unsupported }; - let as_of_date = parse_indonesian_as_of_date(description).map(|date| date.to_string()); + // Without an as-of date the entry cannot be ordered against other + // months, and the snapshot manifest needs a date, so skip it. + let as_of_date = parse_indonesian_as_of_date(description)?.to_string(); Some(DiscoveredOwnershipPdf { family, @@ -572,12 +594,12 @@ fn classify_page_entry( query_url: page_url.to_string(), pdf_url: url.to_string(), title: description.to_string(), - publish_date: as_of_date.clone().unwrap_or_default(), + publish_date: as_of_date.clone(), code: None, original_filename: (!file_name.is_empty()).then_some(file_name), is_attachment: true, format: format.to_string(), - as_of_date, + as_of_date: Some(as_of_date), }) } @@ -1140,6 +1162,24 @@ mod tests { ); } + #[test] + fn share_ownership_page_handles_query_strings_and_dateless_entries() { + let html = r#"{Description:"Pemegang Saham di Atas 1% per 31 Agustus 2026",Prospectus:"https://www.idx.co.id/Media/x/peng-satu-persen.xlsx?download=1",ListingDate:r},{Description:"Pemegang Saham di Atas 1%",Prospectus:"https://www.idx.co.id/Media/y/no-date.xlsx",ListingDate:r}"#; + let reports = parse_share_ownership_page(html, "p"); + assert_eq!( + reports.len(), + 1, + "dateless entry skipped, query-string entry kept" + ); + assert_eq!(reports[0].format, "xlsx"); + assert_eq!(reports[0].status, OwnershipReportStatus::Supported); + assert_eq!( + reports[0].original_filename.as_deref(), + Some("peng-satu-persen.xlsx") + ); + assert_eq!(reports[0].publish_date, "2026-08-31"); + } + #[test] fn share_ownership_page_without_payload_yields_nothing() { assert!(parse_share_ownership_page("Just a moment...", "p").is_empty()); diff --git a/tests/cli.rs b/tests/cli.rs index 60d2bd6..89a58a7 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1313,6 +1313,10 @@ fn ownership_discover_supports_above1_family() { .stdout(predicate::str::contains("Above 1%")) .stdout(predicate::str::contains( "20260310_Semua Emiten Saham_Pengumuman Bursa_32052554_lamp1.pdf", + )) + .stderr(predicate::str::contains("warning:")) + .stderr(predicate::str::contains( + "IDX share ownership data page discovery", )); } From c2d2734739005a101baa93e580d9fbca4cdf7edc Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 24 Sep 2026 20:59:34 +0000 Subject: [PATCH 6/7] feat(ownership): build snapshots from XLSX with monthly history - build-latest-ownership-snapshot.sh: discover with a wider limit and add --history to import the n previous supported above-1% XLSX months (one per as-of date, oldest first) before the latest, so `idx ownership changes` works straight from a synced snapshot. - publish-ownership-snapshot.sh: pass --history through (for the clan-private systemd timer via extraArgs). - publish-ownership-snapshot.yml: `history` input (default 5). - Docs: publish/self-hosted flow now describes the XLSX source, observed upload days (2nd-3rd of the month), and --history. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E --- .../workflows/publish-ownership-snapshot.yml | 7 +++- docs/OWNERSHIP_PUBLISH.md | 6 ++- docs/OWNERSHIP_SELF_HOSTED.md | 12 +++++- scripts/build-latest-ownership-snapshot.sh | 42 +++++++++++++++++-- scripts/publish-ownership-snapshot.sh | 8 ++++ 5 files changed, 66 insertions(+), 9 deletions(-) diff --git a/.github/workflows/publish-ownership-snapshot.yml b/.github/workflows/publish-ownership-snapshot.yml index cb1baa4..e392355 100644 --- a/.github/workflows/publish-ownership-snapshot.yml +++ b/.github/workflows/publish-ownership-snapshot.yml @@ -7,6 +7,10 @@ on: description: Stable GitHub release tag to publish the snapshot assets to required: true default: ownership-snapshot-current + history: + description: Number of earlier monthly above-1% reports to include for `ownership changes` + required: false + default: "5" permissions: contents: write @@ -36,7 +40,8 @@ jobs: --idx-bin ./target/debug/idx \ --output-dir dist/ownership-snapshot \ --repo "${{ github.repository }}" \ - --release-tag "${{ inputs.release_tag }}" + --release-tag "${{ inputs.release_tag }}" \ + --history "${{ inputs.history }}" - name: Upload workflow artifact copy uses: actions/upload-artifact@v4 diff --git a/docs/OWNERSHIP_PUBLISH.md b/docs/OWNERSHIP_PUBLISH.md index 278a16e..05821fe 100644 --- a/docs/OWNERSHIP_PUBLISH.md +++ b/docs/OWNERSHIP_PUBLISH.md @@ -45,9 +45,11 @@ nix develop --command scripts/build-latest-ownership-snapshot.sh \ The script performs these steps: -1. `idx -o json ownership discover --family above1 --limit 1` +1. `idx -o json ownership discover --family above1 --limit 50` (newest supported + report first; XLSX from the Data Kepemilikan Saham page since June 2026) 2. verifies the discovered report is the current supported import path -3. imports that PDF into an isolated temp ownership DB +3. imports that report (plus `--history ` earlier months, oldest first) into + an isolated temp ownership DB 4. checks that the imported release metadata is non-empty and tied to the same source URL 5. runs `scripts/build-ownership-snapshot.sh` to emit the SQLite artifact and diff --git a/docs/OWNERSHIP_SELF_HOSTED.md b/docs/OWNERSHIP_SELF_HOSTED.md index 7457fbb..57be012 100644 --- a/docs/OWNERSHIP_SELF_HOSTED.md +++ b/docs/OWNERSHIP_SELF_HOSTED.md @@ -38,7 +38,9 @@ nix develop --command scripts/publish-ownership-snapshot.sh \ That helper: 1. optionally builds `idx` -2. discovers the latest supported IDX/KSEI PDF +2. discovers the latest supported IDX/KSEI above-1% report (XLSX on the IDX + Data Kepemilikan Saham page since the 2026-05-29 report; PDF announcements + before that) 3. imports it into an isolated temp DB 4. emits the SQLite snapshot and manifest 5. ensures the stable GitHub release exists @@ -66,7 +68,13 @@ The sample timer uses: - `RandomizedDelaySec=30m` That is intentionally conservative. The ownership source is monthly, but the -exact publish day can drift. Start with an early-month schedule and adjust after +exact publish day can drift. Observed XLSX upload times (HTTP Last-Modified): +2026-06-03, 2026-07-02 03:55Z, 2026-08-02 01:50Z, 2026-09-02 03:51Z, so a run +on the 2nd can occasionally land before the file exists; the 3rd is safer. + +Pass `--history ` (for example via `services.idxOwnershipPublish.extraArgs = +[ "--history" "5" ];` in `clan-private`) to include earlier months so +`idx ownership changes` works straight from the synced snapshot. Start with an early-month schedule and adjust after observing a few real runs. ## Clan Integration Later diff --git a/scripts/build-latest-ownership-snapshot.sh b/scripts/build-latest-ownership-snapshot.sh index d69b54f..72da16d 100755 --- a/scripts/build-latest-ownership-snapshot.sh +++ b/scripts/build-latest-ownership-snapshot.sh @@ -6,7 +6,8 @@ usage() { cat <<'EOF' Usage: scripts/build-latest-ownership-snapshot.sh --output-dir [options] -Discover the latest supported IDX/KSEI ownership PDF, import it into an isolated +Discover the latest supported IDX/KSEI above-1% report (XLSX from the IDX Data +Kepemilikan Saham page, or a legacy PDF announcement), import it into an isolated ownership database, and emit GitHub-release-ready snapshot artifacts plus manifest. Options: @@ -17,6 +18,9 @@ Options: (default: 0xrsydn/idx-cli) --release-tag Stable GitHub release tag used for the default base URL (default: ownership-snapshot-current) + --history Also import the previous monthly above-1% XLSX + reports (oldest first) so `ownership changes` works + from the snapshot (default: 0) --keep-workdir Keep the temp workdir instead of deleting it --help Show this help EOF @@ -28,6 +32,7 @@ BASE_URL="" REPO_FULL_NAME="0xrsydn/idx-cli" RELEASE_TAG="ownership-snapshot-current" KEEP_WORKDIR="0" +HISTORY="0" while [[ $# -gt 0 ]]; do case "$1" in @@ -51,6 +56,10 @@ while [[ $# -gt 0 ]]; do RELEASE_TAG="${2:-}" shift 2 ;; + --history) + HISTORY="${2:-}" + shift 2 + ;; --keep-workdir) KEEP_WORKDIR="1" shift @@ -73,6 +82,11 @@ if [[ -z "$OUTPUT_DIR" ]]; then exit 2 fi +if ! [[ "$HISTORY" =~ ^[0-9]+$ ]]; then + echo "--history must be a non-negative integer" >&2 + exit 2 +fi + if [[ -z "$BASE_URL" ]]; then BASE_URL="https://github.com/${REPO_FULL_NAME}/releases/download/${RELEASE_TAG}" fi @@ -115,8 +129,8 @@ RELEASES_JSON="$WORKDIR/releases.json" MANIFEST_PATH="$OUTPUT_DIR/ownership-snapshot-manifest.json" DB_PATH="$XDG_DATA_HOME/idx/ownership.db" -printf 'Discovering latest supported IDX/KSEI ownership PDF...\n' -"$IDX_BIN" -o json ownership discover --family above1 --limit 1 > "$DISCOVERY_JSON" +printf 'Discovering latest supported IDX/KSEI above-1%% report...\n' +"$IDX_BIN" -o json ownership discover --family above1 --limit 50 > "$DISCOVERY_JSON" discovery_payload="$( jq -r ' @@ -151,7 +165,27 @@ DISCOVERED_TITLE="${discovery_fields[4]:-}" DISCOVERED_PUBLISH_DATE="${discovery_fields[5]:-}" DISCOVERED_ORIGINAL_FILENAME="${discovery_fields[6]:-}" -printf 'Importing discovered PDF into isolated DB...\n' +if (( HISTORY > 0 )); then + # Earlier supported XLSX months, one per as-of date, newest first; import oldest first. + mapfile -t history_urls < <( + jq -r --arg latest "$DISCOVERED_PDF_URL" --argjson n "$HISTORY" ' + (map(select(.pdf_url == $latest)) | .[0].as_of_date // "") as $latest_as_of + | [ .[] + | select(.status == "supported" and .format == "xlsx" and .as_of_date != null) + | select($latest_as_of == "" or .as_of_date < $latest_as_of) ] + | unique_by(.as_of_date) + | sort_by(.as_of_date) | reverse + | .[:$n] | reverse + | .[].pdf_url + ' "$DISCOVERY_JSON" + ) + printf 'Importing %d earlier monthly report(s) for history...\n' "${#history_urls[@]}" + for url in "${history_urls[@]}"; do + "$IDX_BIN" ownership import --url "$url" + done +fi + +printf 'Importing discovered report into isolated DB...\n' "$IDX_BIN" ownership import --url "$DISCOVERED_PDF_URL" printf 'Inspecting imported release metadata...\n' diff --git a/scripts/publish-ownership-snapshot.sh b/scripts/publish-ownership-snapshot.sh index 22a4d36..6bbc60d 100755 --- a/scripts/publish-ownership-snapshot.sh +++ b/scripts/publish-ownership-snapshot.sh @@ -16,6 +16,8 @@ Options: (default: 0xrsydn/idx-cli) --release-tag Stable GitHub release tag used for snapshot assets (default: ownership-snapshot-current) + --history Also include the previous monthly above-1% reports + in the snapshot (passed to the builder; default: 0) --build Run `cargo build` before publishing --keep-workdir Keep the temp workdir created by the builder helper --help Show this help @@ -28,6 +30,7 @@ REPO_FULL_NAME="0xrsydn/idx-cli" RELEASE_TAG="ownership-snapshot-current" BUILD_FIRST="0" KEEP_WORKDIR="0" +HISTORY="0" PUBLISH_WORKDIR="" while [[ $# -gt 0 ]]; do @@ -48,6 +51,10 @@ while [[ $# -gt 0 ]]; do RELEASE_TAG="${2:-}" shift 2 ;; + --history) + HISTORY="${2:-}" + shift 2 + ;; --build) BUILD_FIRST="1" shift @@ -120,6 +127,7 @@ build_args=( --output-dir "$PUBLISH_WORKDIR" --repo "$REPO_FULL_NAME" --release-tag "$RELEASE_TAG" + --history "$HISTORY" ) if [[ "$KEEP_WORKDIR" == "1" ]]; then From c866dd7a65b33c4c1e6845eb7f7aa5ff9bd0aeba Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 24 Sep 2026 21:12:34 +0000 Subject: [PATCH 7/7] fix(ownership): harden --history parsing per review Addresses review on #27: canonicalize --history ("08" no longer hits bash octal parsing), run the history jq in a plain command substitution so a jq failure aborts under set -e, and restore the schedule-guidance paragraph in OWNERSHIP_SELF_HOSTED.md. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E --- docs/OWNERSHIP_SELF_HOSTED.md | 4 ++-- scripts/build-latest-ownership-snapshot.sh | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/OWNERSHIP_SELF_HOSTED.md b/docs/OWNERSHIP_SELF_HOSTED.md index 57be012..b51962a 100644 --- a/docs/OWNERSHIP_SELF_HOSTED.md +++ b/docs/OWNERSHIP_SELF_HOSTED.md @@ -71,11 +71,11 @@ That is intentionally conservative. The ownership source is monthly, but the exact publish day can drift. Observed XLSX upload times (HTTP Last-Modified): 2026-06-03, 2026-07-02 03:55Z, 2026-08-02 01:50Z, 2026-09-02 03:51Z, so a run on the 2nd can occasionally land before the file exists; the 3rd is safer. +Start with an early-month schedule and adjust after observing a few real runs. Pass `--history ` (for example via `services.idxOwnershipPublish.extraArgs = [ "--history" "5" ];` in `clan-private`) to include earlier months so -`idx ownership changes` works straight from the synced snapshot. Start with an early-month schedule and adjust after -observing a few real runs. +`idx ownership changes` works straight from the synced snapshot. ## Clan Integration Later diff --git a/scripts/build-latest-ownership-snapshot.sh b/scripts/build-latest-ownership-snapshot.sh index 72da16d..9b787b4 100755 --- a/scripts/build-latest-ownership-snapshot.sh +++ b/scripts/build-latest-ownership-snapshot.sh @@ -86,6 +86,8 @@ if ! [[ "$HISTORY" =~ ^[0-9]+$ ]]; then echo "--history must be a non-negative integer" >&2 exit 2 fi +# Canonicalize so "08" is 8, not an invalid octal literal in (( )). +HISTORY=$((10#$HISTORY)) if [[ -z "$BASE_URL" ]]; then BASE_URL="https://github.com/${REPO_FULL_NAME}/releases/download/${RELEASE_TAG}" @@ -167,7 +169,9 @@ DISCOVERED_ORIGINAL_FILENAME="${discovery_fields[6]:-}" if (( HISTORY > 0 )); then # Earlier supported XLSX months, one per as-of date, newest first; import oldest first. - mapfile -t history_urls < <( + # Run jq in a plain command substitution (not <(...)) so a jq failure + # stops the script under set -e instead of looking like zero months. + history_payload="$( jq -r --arg latest "$DISCOVERED_PDF_URL" --argjson n "$HISTORY" ' (map(select(.pdf_url == $latest)) | .[0].as_of_date // "") as $latest_as_of | [ .[] @@ -178,7 +182,11 @@ if (( HISTORY > 0 )); then | .[:$n] | reverse | .[].pdf_url ' "$DISCOVERY_JSON" - ) + )" + history_urls=() + if [[ -n "$history_payload" ]]; then + mapfile -t history_urls <<< "$history_payload" + fi printf 'Importing %d earlier monthly report(s) for history...\n' "${#history_urls[@]}" for url in "${history_urls[@]}"; do "$IDX_BIN" ownership import --url "$url"