fix: handle spaced KSEI PDF headers (SHARE CODE vs SHARE_CODE)

Newer KSEI above1 PDFs use spaces in column headers (SHARE CODE,
INVESTOR NAME, INVESTOR CLASSIFICATION) instead of the underscored
form (SHARE_CODE, INVESTOR_NAME, INVESTOR_TYPE) that the parser
schema markers expected.  This caused valid holder-register PDFs to
be misclassified as LegacyAboveFivePercent (false positive on
REKENING TAMPUNGAN KSEI text in the data), blocking import.

Changes:
- Add normalize_stext_for_classification() that replaces spaces with
  underscores inside text="..." attributes before schema marker
  matching, so both old and new header layouts are recognized
- Add INVESTOR_CLASSIFICATION to HOLDER_REGISTER_SCHEMA_MARKERS and
  HEADER_LABELS to match the renamed column
- Update ANNOUNCEMENT_WRAPPER and ABOVE_FIVE marker constants to use
  underscores (matching the normalized text)
- Add test fixture and test case for the spaced-header layout
- Bump version to 0.2.3

Tested against the live June 2026 KSEI PDF (as_of 2026-05-29):
  idx ownership import --url <latest-pdf-url>
  -> Imported 7124 rows for 956 tickers (as of 2026-05-29)
This commit is contained in:
hermes 2026-07-29 04:21:54 +00:00
commit a5cfdd79b4
4 changed files with 75 additions and 5 deletions

View file

@ -17,6 +17,7 @@ const HEADER_LABELS: &[&str] = &[
"ISSUERNAME",
"INVESTORNAME",
"INVESTORTYPE",
"INVESTORCLASSIFICATION",
"LOCALFOREIGN",
"NATIONALITY",
"DOMICILE",
@ -31,14 +32,15 @@ const HOLDER_REGISTER_SCHEMA_MARKERS: &[&str] = &[
"TEXT=\"SHARE_CODE\"",
"TEXT=\"INVESTOR_NAME\"",
"TEXT=\"INVESTOR_TYPE\"",
"TEXT=\"INVESTOR_CLASSIFICATION\"",
"TEXT=\"LOCAL_FOREIGN\"",
"TEXT=\"TOTAL_HOLDING_SHARES\"",
"TEXT=\"PERCENTAGE\"",
];
const ANNOUNCEMENT_WRAPPER_SCHEMA_MARKERS: &[&str] =
&["TEXT=\"PENGUMUMAN\"", "PT BURSA EFEK INDONESIA (BEI)"];
&["TEXT=\"PENGUMUMAN\"", "PT_BURSA_EFEK_INDONESIA_(BEI)"];
const ABOVE_FIVE_SCHEMA_MARKERS: &[&str] =
&["TEXT=\"INVS\"", "REKENING TAMPUNGAN KSEI", "CLOSED MEMBER-"];
&["TEXT=\"INVS\"", "REKENING_TAMPUNGAN_KSEI", "CLOSED_MEMBER-"];
const INVESTOR_TYPE_SCHEMA_MARKERS: &[&str] = &[
"TEXT=\"STOCK_CODE\"",
"TEXT=\"NUMBER_OF_SHARES\"",
@ -103,7 +105,7 @@ pub fn extract_pdf_stext(path: &Path) -> Result<String, IdxError> {
/// Classify a PDF schema from mutool stext XML before the row parser runs.
pub fn classify_stext_xml(xml: &str) -> OwnershipPdfSchema {
let normalized = xml.to_ascii_uppercase();
let normalized = normalize_stext_for_classification(xml);
if count_schema_markers(&normalized, HOLDER_REGISTER_SCHEMA_MARKERS) >= 5 {
return OwnershipPdfSchema::HolderRegister;
@ -121,6 +123,41 @@ pub fn classify_stext_xml(xml: &str) -> OwnershipPdfSchema {
OwnershipPdfSchema::Unknown
}
/// Normalize stext XML for schema classification.
///
/// Newer KSEI PDFs use spaces in header text (e.g. `SHARE CODE` instead of
/// `SHARE_CODE`, `INVESTOR CLASSIFICATION` instead of `INVESTOR_TYPE`). The
/// schema markers use the underscored form, so we replace spaces with
/// underscores inside `text="..."` attribute values to match both layouts.
fn normalize_stext_for_classification(xml: &str) -> String {
let upper = xml.to_ascii_uppercase();
let mut result = String::with_capacity(upper.len());
let bytes = upper.as_bytes();
let mut i = 0;
while i < bytes.len() {
if bytes[i..].starts_with(b"TEXT=\"") {
result.push_str("TEXT=\"");
i += 6; // skip past TEXT="
// Collect everything until the closing quote.
while i < bytes.len() && bytes[i] != b'"' {
let ch = bytes[i] as char;
result.push(if ch == ' ' { '_' } else { ch });
i += 1;
}
if i < bytes.len() {
result.push('"');
i += 1; // skip closing quote
}
} else {
result.push(bytes[i] as char);
i += 1;
}
}
result
}
/// Parse a KSEI ownership PDF into raw rows.
/// Shells out to `mutool` for XML extraction, classifies the schema,
/// and only parses the supported holder-register layout.
@ -684,6 +721,12 @@ mod tests {
);
}
#[test]
fn classify_stext_xml_detects_spaced_holder_register_schema() {
let xml = include_str!("../../tests/fixtures/ksei_above1_spaced_stext_excerpt.xml");
assert_eq!(classify_stext_xml(xml), OwnershipPdfSchema::HolderRegister);
}
#[test]
fn classify_stext_xml_detects_legacy_above5_schema() {
let xml = include_str!("../../tests/fixtures/ksei_above5_stext_excerpt.xml");