From a5cfdd79b40c84a4d684e15696985afa19063b42 Mon Sep 17 00:00:00 2001 From: hermes Date: Wed, 29 Jul 2026 04:21:54 +0000 Subject: [PATCH] fix: handle spaced KSEI PDF headers (SHARE CODE vs SHARE_CODE) Newer KSEI above1 PDFs use spaces in column headers (SHARE CODE, INVESTOR NAME, INVESTOR CLASSIFICATION) instead of the underscored form (SHARE_CODE, INVESTOR_NAME, INVESTOR_TYPE) that the parser schema markers expected. This caused valid holder-register PDFs to be misclassified as LegacyAboveFivePercent (false positive on REKENING TAMPUNGAN KSEI text in the data), blocking import. Changes: - Add normalize_stext_for_classification() that replaces spaces with underscores inside text="..." attributes before schema marker matching, so both old and new header layouts are recognized - Add INVESTOR_CLASSIFICATION to HOLDER_REGISTER_SCHEMA_MARKERS and HEADER_LABELS to match the renamed column - Update ANNOUNCEMENT_WRAPPER and ABOVE_FIVE marker constants to use underscores (matching the normalized text) - Add test fixture and test case for the spaced-header layout - Bump version to 0.2.3 Tested against the live June 2026 KSEI PDF (as_of 2026-05-29): idx ownership import --url -> Imported 7124 rows for 956 tickers (as of 2026-05-29) --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/ownership/parser.rs | 49 +++++++++++++++++-- .../ksei_above1_spaced_stext_excerpt.xml | 27 ++++++++++ 4 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 tests/fixtures/ksei_above1_spaced_stext_excerpt.xml diff --git a/Cargo.lock b/Cargo.lock index 8aa291f..9c0c2af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,7 +711,7 @@ dependencies = [ [[package]] name = "idx-cli" -version = "0.2.2" +version = "0.2.3" dependencies = [ "assert_cmd", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 26bdb22..976eb68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "idx-cli" -version = "0.2.2" +version = "0.2.3" edition = "2024" rust-version = "1.85" description = "CLI tool for Indonesian stock market (IDX) analysis" diff --git a/src/ownership/parser.rs b/src/ownership/parser.rs index 4239247..6e117f9 100644 --- a/src/ownership/parser.rs +++ b/src/ownership/parser.rs @@ -17,6 +17,7 @@ const HEADER_LABELS: &[&str] = &[ "ISSUERNAME", "INVESTORNAME", "INVESTORTYPE", + "INVESTORCLASSIFICATION", "LOCALFOREIGN", "NATIONALITY", "DOMICILE", @@ -31,14 +32,15 @@ const HOLDER_REGISTER_SCHEMA_MARKERS: &[&str] = &[ "TEXT=\"SHARE_CODE\"", "TEXT=\"INVESTOR_NAME\"", "TEXT=\"INVESTOR_TYPE\"", + "TEXT=\"INVESTOR_CLASSIFICATION\"", "TEXT=\"LOCAL_FOREIGN\"", "TEXT=\"TOTAL_HOLDING_SHARES\"", "TEXT=\"PERCENTAGE\"", ]; const ANNOUNCEMENT_WRAPPER_SCHEMA_MARKERS: &[&str] = - &["TEXT=\"PENGUMUMAN\"", "PT BURSA EFEK INDONESIA (BEI)"]; + &["TEXT=\"PENGUMUMAN\"", "PT_BURSA_EFEK_INDONESIA_(BEI)"]; const ABOVE_FIVE_SCHEMA_MARKERS: &[&str] = - &["TEXT=\"INVS\"", "REKENING TAMPUNGAN KSEI", "CLOSED MEMBER-"]; + &["TEXT=\"INVS\"", "REKENING_TAMPUNGAN_KSEI", "CLOSED_MEMBER-"]; const INVESTOR_TYPE_SCHEMA_MARKERS: &[&str] = &[ "TEXT=\"STOCK_CODE\"", "TEXT=\"NUMBER_OF_SHARES\"", @@ -103,7 +105,7 @@ pub fn extract_pdf_stext(path: &Path) -> Result { /// Classify a PDF schema from mutool stext XML before the row parser runs. pub fn classify_stext_xml(xml: &str) -> OwnershipPdfSchema { - let normalized = xml.to_ascii_uppercase(); + let normalized = normalize_stext_for_classification(xml); if count_schema_markers(&normalized, HOLDER_REGISTER_SCHEMA_MARKERS) >= 5 { return OwnershipPdfSchema::HolderRegister; @@ -121,6 +123,41 @@ pub fn classify_stext_xml(xml: &str) -> OwnershipPdfSchema { OwnershipPdfSchema::Unknown } +/// Normalize stext XML for schema classification. +/// +/// Newer KSEI PDFs use spaces in header text (e.g. `SHARE CODE` instead of +/// `SHARE_CODE`, `INVESTOR CLASSIFICATION` instead of `INVESTOR_TYPE`). The +/// schema markers use the underscored form, so we replace spaces with +/// underscores inside `text="..."` attribute values to match both layouts. +fn normalize_stext_for_classification(xml: &str) -> String { + let upper = xml.to_ascii_uppercase(); + let mut result = String::with_capacity(upper.len()); + let bytes = upper.as_bytes(); + let mut i = 0; + + while i < bytes.len() { + if bytes[i..].starts_with(b"TEXT=\"") { + result.push_str("TEXT=\""); + i += 6; // skip past TEXT=" + // Collect everything until the closing quote. + while i < bytes.len() && bytes[i] != b'"' { + let ch = bytes[i] as char; + result.push(if ch == ' ' { '_' } else { ch }); + i += 1; + } + if i < bytes.len() { + result.push('"'); + i += 1; // skip closing quote + } + } else { + result.push(bytes[i] as char); + i += 1; + } + } + + result +} + /// Parse a KSEI ownership PDF into raw rows. /// Shells out to `mutool` for XML extraction, classifies the schema, /// and only parses the supported holder-register layout. @@ -684,6 +721,12 @@ mod tests { ); } + #[test] + fn classify_stext_xml_detects_spaced_holder_register_schema() { + let xml = include_str!("../../tests/fixtures/ksei_above1_spaced_stext_excerpt.xml"); + assert_eq!(classify_stext_xml(xml), OwnershipPdfSchema::HolderRegister); + } + #[test] fn classify_stext_xml_detects_legacy_above5_schema() { let xml = include_str!("../../tests/fixtures/ksei_above5_stext_excerpt.xml"); diff --git a/tests/fixtures/ksei_above1_spaced_stext_excerpt.xml b/tests/fixtures/ksei_above1_spaced_stext_excerpt.xml new file mode 100644 index 0000000..e75c5f7 --- /dev/null +++ b/tests/fixtures/ksei_above1_spaced_stext_excerpt.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + +