mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-08-07 01:33:52 +00:00
fix: handle spaced KSEI PDF headers (SHARE CODE vs SHARE_CODE)
Newer KSEI above1 PDFs use spaces in column headers (SHARE CODE, INVESTOR NAME, INVESTOR CLASSIFICATION) instead of the underscored form (SHARE_CODE, INVESTOR_NAME, INVESTOR_TYPE) that the parser schema markers expected. This caused valid holder-register PDFs to be misclassified as LegacyAboveFivePercent (false positive on REKENING TAMPUNGAN KSEI text in the data), blocking import. Changes: - Add normalize_stext_for_classification() that replaces spaces with underscores inside text="..." attributes before schema marker matching, so both old and new header layouts are recognized - Add INVESTOR_CLASSIFICATION to HOLDER_REGISTER_SCHEMA_MARKERS and HEADER_LABELS to match the renamed column - Update ANNOUNCEMENT_WRAPPER and ABOVE_FIVE marker constants to use underscores (matching the normalized text) - Add test fixture and test case for the spaced-header layout - Bump version to 0.2.3 Tested against the live June 2026 KSEI PDF (as_of 2026-05-29): idx ownership import --url <latest-pdf-url> -> Imported 7124 rows for 956 tickers (as of 2026-05-29)
This commit is contained in:
parent
e2c0425a3b
commit
a5cfdd79b4
4 changed files with 75 additions and 5 deletions
2
Cargo.lock
generated
2
Cargo.lock
generated
|
|
@ -711,7 +711,7 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "idx-cli"
|
name = "idx-cli"
|
||||||
version = "0.2.2"
|
version = "0.2.3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"assert_cmd",
|
"assert_cmd",
|
||||||
"chrono",
|
"chrono",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "idx-cli"
|
name = "idx-cli"
|
||||||
version = "0.2.2"
|
version = "0.2.3"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
rust-version = "1.85"
|
rust-version = "1.85"
|
||||||
description = "CLI tool for Indonesian stock market (IDX) analysis"
|
description = "CLI tool for Indonesian stock market (IDX) analysis"
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,7 @@ const HEADER_LABELS: &[&str] = &[
|
||||||
"ISSUERNAME",
|
"ISSUERNAME",
|
||||||
"INVESTORNAME",
|
"INVESTORNAME",
|
||||||
"INVESTORTYPE",
|
"INVESTORTYPE",
|
||||||
|
"INVESTORCLASSIFICATION",
|
||||||
"LOCALFOREIGN",
|
"LOCALFOREIGN",
|
||||||
"NATIONALITY",
|
"NATIONALITY",
|
||||||
"DOMICILE",
|
"DOMICILE",
|
||||||
|
|
@ -31,14 +32,15 @@ const HOLDER_REGISTER_SCHEMA_MARKERS: &[&str] = &[
|
||||||
"TEXT=\"SHARE_CODE\"",
|
"TEXT=\"SHARE_CODE\"",
|
||||||
"TEXT=\"INVESTOR_NAME\"",
|
"TEXT=\"INVESTOR_NAME\"",
|
||||||
"TEXT=\"INVESTOR_TYPE\"",
|
"TEXT=\"INVESTOR_TYPE\"",
|
||||||
|
"TEXT=\"INVESTOR_CLASSIFICATION\"",
|
||||||
"TEXT=\"LOCAL_FOREIGN\"",
|
"TEXT=\"LOCAL_FOREIGN\"",
|
||||||
"TEXT=\"TOTAL_HOLDING_SHARES\"",
|
"TEXT=\"TOTAL_HOLDING_SHARES\"",
|
||||||
"TEXT=\"PERCENTAGE\"",
|
"TEXT=\"PERCENTAGE\"",
|
||||||
];
|
];
|
||||||
const ANNOUNCEMENT_WRAPPER_SCHEMA_MARKERS: &[&str] =
|
const ANNOUNCEMENT_WRAPPER_SCHEMA_MARKERS: &[&str] =
|
||||||
&["TEXT=\"PENGUMUMAN\"", "PT BURSA EFEK INDONESIA (BEI)"];
|
&["TEXT=\"PENGUMUMAN\"", "PT_BURSA_EFEK_INDONESIA_(BEI)"];
|
||||||
const ABOVE_FIVE_SCHEMA_MARKERS: &[&str] =
|
const ABOVE_FIVE_SCHEMA_MARKERS: &[&str] =
|
||||||
&["TEXT=\"INVS\"", "REKENING TAMPUNGAN KSEI", "CLOSED MEMBER-"];
|
&["TEXT=\"INVS\"", "REKENING_TAMPUNGAN_KSEI", "CLOSED_MEMBER-"];
|
||||||
const INVESTOR_TYPE_SCHEMA_MARKERS: &[&str] = &[
|
const INVESTOR_TYPE_SCHEMA_MARKERS: &[&str] = &[
|
||||||
"TEXT=\"STOCK_CODE\"",
|
"TEXT=\"STOCK_CODE\"",
|
||||||
"TEXT=\"NUMBER_OF_SHARES\"",
|
"TEXT=\"NUMBER_OF_SHARES\"",
|
||||||
|
|
@ -103,7 +105,7 @@ pub fn extract_pdf_stext(path: &Path) -> Result<String, IdxError> {
|
||||||
|
|
||||||
/// Classify a PDF schema from mutool stext XML before the row parser runs.
|
/// Classify a PDF schema from mutool stext XML before the row parser runs.
|
||||||
pub fn classify_stext_xml(xml: &str) -> OwnershipPdfSchema {
|
pub fn classify_stext_xml(xml: &str) -> OwnershipPdfSchema {
|
||||||
let normalized = xml.to_ascii_uppercase();
|
let normalized = normalize_stext_for_classification(xml);
|
||||||
|
|
||||||
if count_schema_markers(&normalized, HOLDER_REGISTER_SCHEMA_MARKERS) >= 5 {
|
if count_schema_markers(&normalized, HOLDER_REGISTER_SCHEMA_MARKERS) >= 5 {
|
||||||
return OwnershipPdfSchema::HolderRegister;
|
return OwnershipPdfSchema::HolderRegister;
|
||||||
|
|
@ -121,6 +123,41 @@ pub fn classify_stext_xml(xml: &str) -> OwnershipPdfSchema {
|
||||||
OwnershipPdfSchema::Unknown
|
OwnershipPdfSchema::Unknown
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Normalize stext XML for schema classification.
|
||||||
|
///
|
||||||
|
/// Newer KSEI PDFs use spaces in header text (e.g. `SHARE CODE` instead of
|
||||||
|
/// `SHARE_CODE`, `INVESTOR CLASSIFICATION` instead of `INVESTOR_TYPE`). The
|
||||||
|
/// schema markers use the underscored form, so we replace spaces with
|
||||||
|
/// underscores inside `text="..."` attribute values to match both layouts.
|
||||||
|
fn normalize_stext_for_classification(xml: &str) -> String {
|
||||||
|
let upper = xml.to_ascii_uppercase();
|
||||||
|
let mut result = String::with_capacity(upper.len());
|
||||||
|
let bytes = upper.as_bytes();
|
||||||
|
let mut i = 0;
|
||||||
|
|
||||||
|
while i < bytes.len() {
|
||||||
|
if bytes[i..].starts_with(b"TEXT=\"") {
|
||||||
|
result.push_str("TEXT=\"");
|
||||||
|
i += 6; // skip past TEXT="
|
||||||
|
// Collect everything until the closing quote.
|
||||||
|
while i < bytes.len() && bytes[i] != b'"' {
|
||||||
|
let ch = bytes[i] as char;
|
||||||
|
result.push(if ch == ' ' { '_' } else { ch });
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
if i < bytes.len() {
|
||||||
|
result.push('"');
|
||||||
|
i += 1; // skip closing quote
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
result.push(bytes[i] as char);
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
/// Parse a KSEI ownership PDF into raw rows.
|
/// Parse a KSEI ownership PDF into raw rows.
|
||||||
/// Shells out to `mutool` for XML extraction, classifies the schema,
|
/// Shells out to `mutool` for XML extraction, classifies the schema,
|
||||||
/// and only parses the supported holder-register layout.
|
/// and only parses the supported holder-register layout.
|
||||||
|
|
@ -684,6 +721,12 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn classify_stext_xml_detects_spaced_holder_register_schema() {
|
||||||
|
let xml = include_str!("../../tests/fixtures/ksei_above1_spaced_stext_excerpt.xml");
|
||||||
|
assert_eq!(classify_stext_xml(xml), OwnershipPdfSchema::HolderRegister);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn classify_stext_xml_detects_legacy_above5_schema() {
|
fn classify_stext_xml_detects_legacy_above5_schema() {
|
||||||
let xml = include_str!("../../tests/fixtures/ksei_above5_stext_excerpt.xml");
|
let xml = include_str!("../../tests/fixtures/ksei_above5_stext_excerpt.xml");
|
||||||
|
|
|
||||||
27
tests/fixtures/ksei_above1_spaced_stext_excerpt.xml
vendored
Normal file
27
tests/fixtures/ksei_above1_spaced_stext_excerpt.xml
vendored
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
<?xml version="1.0"?>
|
||||||
|
<document>
|
||||||
|
<page id="page1" width="612" height="792">
|
||||||
|
<line bbox="58.2 135.37999 63.566404 137.78" wmode="0" dir="1 0" flags="0" text="DATE"></line>
|
||||||
|
<line bbox="73.944 135.37999 87.46321 137.78" wmode="0" dir="1 0" flags="0" text="SHARE CODE"></line>
|
||||||
|
<line bbox="111.26 135.37999 125.50881 137.78" wmode="0" dir="1 0" flags="0" text="ISSUER NAME"></line>
|
||||||
|
<line bbox="199.73 135.37999 217.21158 137.78" wmode="0" dir="1 0" flags="0" text="INVESTOR NAME"></line>
|
||||||
|
<line bbox="288.53 135.37999 315.77238 137.78" wmode="0" dir="1 0" flags="0" text="INVESTOR CLASSIFICATION"></line>
|
||||||
|
<line bbox="337.63 135.37999 354.19963 137.78" wmode="0" dir="1 0" flags="0" text="LOCAL FOREIGN"></line>
|
||||||
|
<line bbox="363.31 135.37999 376.7908 137.78" wmode="0" dir="1 0" flags="0" text="NATIONALITY"></line>
|
||||||
|
<line bbox="398.11 135.37999 408.034 137.78" wmode="0" dir="1 0" flags="0" text="DOMICILE"></line>
|
||||||
|
<line bbox="432.55 135.37999 454.30836 137.78" wmode="0" dir="1 0" flags="0" text="HOLDINGS SCRIPLESS"></line>
|
||||||
|
<line bbox="466.78 135.37999 484.1248 137.78" wmode="0" dir="1 0" flags="0" text="HOLDINGS SCRIP"></line>
|
||||||
|
<line bbox="496.9 135.37999 522.97366 137.78" wmode="0" dir="1 0" flags="0" text="TOTAL HOLDING SHARES"></line>
|
||||||
|
<line bbox="534.34 135.37999 547.6168 137.78" wmode="0" dir="1 0" flags="0" text="PERCENTAGE"></line>
|
||||||
|
<line bbox="55.68 138.49999 75.26399 140.9" wmode="0" dir="1 0" flags="0" text="29-May-2026 AADI"></line>
|
||||||
|
<line bbox="91.944 138.49999 125.39758 140.9" wmode="0" dir="1 0" flags="0" text="ADARO ANDALAN INDONESIA Tbk"></line>
|
||||||
|
<line bbox="145.7 138.49999 175.97843 140.9" wmode="0" dir="1 0" flags="0" text="PERSADA CAPITAL INVESTAMA"></line>
|
||||||
|
<line bbox="272.09 138.49999 282.27076 140.9" wmode="0" dir="1 0" flags="0" text="Corporate"></line>
|
||||||
|
<line bbox="330.0 138.49999 332.0 140.9" wmode="0" dir="1 0" flags="0" text="D"></line>
|
||||||
|
<line bbox="360.0 138.49999 380.0 140.9" wmode="0" dir="1 0" flags="0" text="INDONESIA"></line>
|
||||||
|
<line bbox="430.0 138.49999 460.0 140.9" wmode="0" dir="1 0" flags="0" text="3.200.142.830"></line>
|
||||||
|
<line bbox="465.0 138.49999 470.0 140.9" wmode="0" dir="1 0" flags="0" text="0"></line>
|
||||||
|
<line bbox="495.0 138.49999 525.0 140.9" wmode="0" dir="1 0" flags="0" text="3.200.142.830"></line>
|
||||||
|
<line bbox="533.0 138.49999 548.0 140.9" wmode="0" dir="1 0" flags="0" text="41,10"></line>
|
||||||
|
</page>
|
||||||
|
</document>
|
||||||
Loading…
Add table
Add a link
Reference in a new issue