fix: handle spaced KSEI PDF headers (SHARE CODE vs SHARE_CODE)

Newer KSEI above1 PDFs use spaces in column headers (SHARE CODE,
INVESTOR NAME, INVESTOR CLASSIFICATION) instead of the underscored
form (SHARE_CODE, INVESTOR_NAME, INVESTOR_TYPE) that the parser
schema markers expected.  This caused valid holder-register PDFs to
be misclassified as LegacyAboveFivePercent (false positive on
REKENING TAMPUNGAN KSEI text in the data), blocking import.

Changes:
- Add normalize_stext_for_classification() that replaces spaces with
  underscores inside text="..." attributes before schema marker
  matching, so both old and new header layouts are recognized
- Add INVESTOR_CLASSIFICATION to HOLDER_REGISTER_SCHEMA_MARKERS and
  HEADER_LABELS to match the renamed column
- Update ANNOUNCEMENT_WRAPPER and ABOVE_FIVE marker constants to use
  underscores (matching the normalized text)
- Add test fixture and test case for the spaced-header layout
- Bump version to 0.2.3

Tested against the live June 2026 KSEI PDF (as_of 2026-05-29):
  idx ownership import --url <latest-pdf-url>
  -> Imported 7124 rows for 956 tickers (as of 2026-05-29)
This commit is contained in:
hermes 2026-07-29 04:21:54 +00:00
commit a5cfdd79b4
4 changed files with 75 additions and 5 deletions

2
Cargo.lock generated
View file

@ -711,7 +711,7 @@ dependencies = [
[[package]] [[package]]
name = "idx-cli" name = "idx-cli"
version = "0.2.2" version = "0.2.3"
dependencies = [ dependencies = [
"assert_cmd", "assert_cmd",
"chrono", "chrono",

View file

@ -1,6 +1,6 @@
[package] [package]
name = "idx-cli" name = "idx-cli"
version = "0.2.2" version = "0.2.3"
edition = "2024" edition = "2024"
rust-version = "1.85" rust-version = "1.85"
description = "CLI tool for Indonesian stock market (IDX) analysis" description = "CLI tool for Indonesian stock market (IDX) analysis"

View file

@ -17,6 +17,7 @@ const HEADER_LABELS: &[&str] = &[
"ISSUERNAME", "ISSUERNAME",
"INVESTORNAME", "INVESTORNAME",
"INVESTORTYPE", "INVESTORTYPE",
"INVESTORCLASSIFICATION",
"LOCALFOREIGN", "LOCALFOREIGN",
"NATIONALITY", "NATIONALITY",
"DOMICILE", "DOMICILE",
@ -31,14 +32,15 @@ const HOLDER_REGISTER_SCHEMA_MARKERS: &[&str] = &[
"TEXT=\"SHARE_CODE\"", "TEXT=\"SHARE_CODE\"",
"TEXT=\"INVESTOR_NAME\"", "TEXT=\"INVESTOR_NAME\"",
"TEXT=\"INVESTOR_TYPE\"", "TEXT=\"INVESTOR_TYPE\"",
"TEXT=\"INVESTOR_CLASSIFICATION\"",
"TEXT=\"LOCAL_FOREIGN\"", "TEXT=\"LOCAL_FOREIGN\"",
"TEXT=\"TOTAL_HOLDING_SHARES\"", "TEXT=\"TOTAL_HOLDING_SHARES\"",
"TEXT=\"PERCENTAGE\"", "TEXT=\"PERCENTAGE\"",
]; ];
const ANNOUNCEMENT_WRAPPER_SCHEMA_MARKERS: &[&str] = const ANNOUNCEMENT_WRAPPER_SCHEMA_MARKERS: &[&str] =
&["TEXT=\"PENGUMUMAN\"", "PT BURSA EFEK INDONESIA (BEI)"]; &["TEXT=\"PENGUMUMAN\"", "PT_BURSA_EFEK_INDONESIA_(BEI)"];
const ABOVE_FIVE_SCHEMA_MARKERS: &[&str] = const ABOVE_FIVE_SCHEMA_MARKERS: &[&str] =
&["TEXT=\"INVS\"", "REKENING TAMPUNGAN KSEI", "CLOSED MEMBER-"]; &["TEXT=\"INVS\"", "REKENING_TAMPUNGAN_KSEI", "CLOSED_MEMBER-"];
const INVESTOR_TYPE_SCHEMA_MARKERS: &[&str] = &[ const INVESTOR_TYPE_SCHEMA_MARKERS: &[&str] = &[
"TEXT=\"STOCK_CODE\"", "TEXT=\"STOCK_CODE\"",
"TEXT=\"NUMBER_OF_SHARES\"", "TEXT=\"NUMBER_OF_SHARES\"",
@ -103,7 +105,7 @@ pub fn extract_pdf_stext(path: &Path) -> Result<String, IdxError> {
/// Classify a PDF schema from mutool stext XML before the row parser runs. /// Classify a PDF schema from mutool stext XML before the row parser runs.
pub fn classify_stext_xml(xml: &str) -> OwnershipPdfSchema { pub fn classify_stext_xml(xml: &str) -> OwnershipPdfSchema {
let normalized = xml.to_ascii_uppercase(); let normalized = normalize_stext_for_classification(xml);
if count_schema_markers(&normalized, HOLDER_REGISTER_SCHEMA_MARKERS) >= 5 { if count_schema_markers(&normalized, HOLDER_REGISTER_SCHEMA_MARKERS) >= 5 {
return OwnershipPdfSchema::HolderRegister; return OwnershipPdfSchema::HolderRegister;
@ -121,6 +123,41 @@ pub fn classify_stext_xml(xml: &str) -> OwnershipPdfSchema {
OwnershipPdfSchema::Unknown OwnershipPdfSchema::Unknown
} }
/// Normalize stext XML for schema classification.
///
/// Newer KSEI PDFs use spaces in header text (e.g. `SHARE CODE` instead of
/// `SHARE_CODE`, `INVESTOR CLASSIFICATION` instead of `INVESTOR_TYPE`). The
/// schema markers use the underscored form, so we replace spaces with
/// underscores inside `text="..."` attribute values to match both layouts.
fn normalize_stext_for_classification(xml: &str) -> String {
let upper = xml.to_ascii_uppercase();
let mut result = String::with_capacity(upper.len());
let bytes = upper.as_bytes();
let mut i = 0;
while i < bytes.len() {
if bytes[i..].starts_with(b"TEXT=\"") {
result.push_str("TEXT=\"");
i += 6; // skip past TEXT="
// Collect everything until the closing quote.
while i < bytes.len() && bytes[i] != b'"' {
let ch = bytes[i] as char;
result.push(if ch == ' ' { '_' } else { ch });
i += 1;
}
if i < bytes.len() {
result.push('"');
i += 1; // skip closing quote
}
} else {
result.push(bytes[i] as char);
i += 1;
}
}
result
}
/// Parse a KSEI ownership PDF into raw rows. /// Parse a KSEI ownership PDF into raw rows.
/// Shells out to `mutool` for XML extraction, classifies the schema, /// Shells out to `mutool` for XML extraction, classifies the schema,
/// and only parses the supported holder-register layout. /// and only parses the supported holder-register layout.
@ -684,6 +721,12 @@ mod tests {
); );
} }
#[test]
fn classify_stext_xml_detects_spaced_holder_register_schema() {
let xml = include_str!("../../tests/fixtures/ksei_above1_spaced_stext_excerpt.xml");
assert_eq!(classify_stext_xml(xml), OwnershipPdfSchema::HolderRegister);
}
#[test] #[test]
fn classify_stext_xml_detects_legacy_above5_schema() { fn classify_stext_xml_detects_legacy_above5_schema() {
let xml = include_str!("../../tests/fixtures/ksei_above5_stext_excerpt.xml"); let xml = include_str!("../../tests/fixtures/ksei_above5_stext_excerpt.xml");

View file

@ -0,0 +1,27 @@
<?xml version="1.0"?>
<document>
<page id="page1" width="612" height="792">
<line bbox="58.2 135.37999 63.566404 137.78" wmode="0" dir="1 0" flags="0" text="DATE"></line>
<line bbox="73.944 135.37999 87.46321 137.78" wmode="0" dir="1 0" flags="0" text="SHARE CODE"></line>
<line bbox="111.26 135.37999 125.50881 137.78" wmode="0" dir="1 0" flags="0" text="ISSUER NAME"></line>
<line bbox="199.73 135.37999 217.21158 137.78" wmode="0" dir="1 0" flags="0" text="INVESTOR NAME"></line>
<line bbox="288.53 135.37999 315.77238 137.78" wmode="0" dir="1 0" flags="0" text="INVESTOR CLASSIFICATION"></line>
<line bbox="337.63 135.37999 354.19963 137.78" wmode="0" dir="1 0" flags="0" text="LOCAL FOREIGN"></line>
<line bbox="363.31 135.37999 376.7908 137.78" wmode="0" dir="1 0" flags="0" text="NATIONALITY"></line>
<line bbox="398.11 135.37999 408.034 137.78" wmode="0" dir="1 0" flags="0" text="DOMICILE"></line>
<line bbox="432.55 135.37999 454.30836 137.78" wmode="0" dir="1 0" flags="0" text="HOLDINGS SCRIPLESS"></line>
<line bbox="466.78 135.37999 484.1248 137.78" wmode="0" dir="1 0" flags="0" text="HOLDINGS SCRIP"></line>
<line bbox="496.9 135.37999 522.97366 137.78" wmode="0" dir="1 0" flags="0" text="TOTAL HOLDING SHARES"></line>
<line bbox="534.34 135.37999 547.6168 137.78" wmode="0" dir="1 0" flags="0" text="PERCENTAGE"></line>
<line bbox="55.68 138.49999 75.26399 140.9" wmode="0" dir="1 0" flags="0" text="29-May-2026 AADI"></line>
<line bbox="91.944 138.49999 125.39758 140.9" wmode="0" dir="1 0" flags="0" text="ADARO ANDALAN INDONESIA Tbk"></line>
<line bbox="145.7 138.49999 175.97843 140.9" wmode="0" dir="1 0" flags="0" text="PERSADA CAPITAL INVESTAMA"></line>
<line bbox="272.09 138.49999 282.27076 140.9" wmode="0" dir="1 0" flags="0" text="Corporate"></line>
<line bbox="330.0 138.49999 332.0 140.9" wmode="0" dir="1 0" flags="0" text="D"></line>
<line bbox="360.0 138.49999 380.0 140.9" wmode="0" dir="1 0" flags="0" text="INDONESIA"></line>
<line bbox="430.0 138.49999 460.0 140.9" wmode="0" dir="1 0" flags="0" text="3.200.142.830"></line>
<line bbox="465.0 138.49999 470.0 140.9" wmode="0" dir="1 0" flags="0" text="0"></line>
<line bbox="495.0 138.49999 525.0 140.9" wmode="0" dir="1 0" flags="0" text="3.200.142.830"></line>
<line bbox="533.0 138.49999 548.0 140.9" wmode="0" dir="1 0" flags="0" text="41,10"></line>
</page>
</document>