mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-08-07 01:33:52 +00:00
fix: handle missing scrip column in April 2026 KSEI PDF
The April 2026 IDX ownership PDF omits the zero-valued scrip column for some holders, causing the parser to leave holdings_scripless/scrip as empty strings. This broke normalize_ksei_row which called parse_id_number on empty input. Two-layer fix: - Parser: pop_numeric_tail handles 2-column (missing scrip) rows by defaulting scrip to "0" - Normalizer: parse_id_number_or_zero backstop treats empty component share fields as 0 while still requiring total_shares Also includes AGENTS.md refresh, formatting cleanup (rustfmt), and version bump to 0.2.2. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
c94c826307
commit
04eaa75882
10 changed files with 286 additions and 144 deletions
|
|
@ -14,11 +14,11 @@ use crate::error::IdxError;
|
|||
use crate::output::OutputFormat;
|
||||
use crate::output::json;
|
||||
use crate::output::table::format_idr;
|
||||
use crate::runtime;
|
||||
use crate::ownership::types::{
|
||||
ChangeType, FlowSignal, HolderRow, KseiHolding, OwnershipRelease, OwnershipSource,
|
||||
};
|
||||
use crate::ownership::{archive, db, entities, graph, parser, remote, search, snapshot};
|
||||
use crate::runtime;
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
#[command(
|
||||
|
|
|
|||
18
src/main.rs
18
src/main.rs
|
|
@ -78,16 +78,14 @@ fn run() -> Result<(), IdxError> {
|
|||
}
|
||||
Commands::Stocks(stocks) => {
|
||||
let provider = default_provider(config.provider, cli.verbose > 0);
|
||||
if let Err(err) =
|
||||
cli::stocks::handle(
|
||||
stocks,
|
||||
&config,
|
||||
&provider,
|
||||
cli.offline,
|
||||
cli.no_cache,
|
||||
cli.verbose > 0,
|
||||
)
|
||||
{
|
||||
if let Err(err) = cli::stocks::handle(
|
||||
stocks,
|
||||
&config,
|
||||
&provider,
|
||||
cli.offline,
|
||||
cli.no_cache,
|
||||
cli.verbose > 0,
|
||||
) {
|
||||
emit_error(&err, &config.output);
|
||||
return Err(err);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -159,7 +159,10 @@ pub fn upsert_ticker(conn: &Connection, code: &str, name: Option<&str>) -> Resul
|
|||
.map_err(|e| IdxError::DatabaseError(e.to_string()))
|
||||
}
|
||||
|
||||
fn insert_ksei_holdings_rows(conn: &Connection, holdings: &[KseiHolding]) -> Result<usize, IdxError> {
|
||||
fn insert_ksei_holdings_rows(
|
||||
conn: &Connection,
|
||||
holdings: &[KseiHolding],
|
||||
) -> Result<usize, IdxError> {
|
||||
if holdings.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
|
@ -591,8 +594,10 @@ pub fn query_cross_holders(
|
|||
min_tickers: usize,
|
||||
limit: usize,
|
||||
) -> Result<Vec<CrossHolderRow>, IdxError> {
|
||||
let mut aggregates: std::collections::BTreeMap<i64, (Entity, std::collections::BTreeSet<i64>, i64)> =
|
||||
std::collections::BTreeMap::new();
|
||||
let mut aggregates: std::collections::BTreeMap<
|
||||
i64,
|
||||
(Entity, std::collections::BTreeSet<i64>, i64),
|
||||
> = std::collections::BTreeMap::new();
|
||||
|
||||
if let Some(latest_ksei) = latest_ksei_release(conn)? {
|
||||
let mut stmt = conn
|
||||
|
|
@ -1437,7 +1442,10 @@ mod tests {
|
|||
row_count: holdings.len(),
|
||||
imported_at: 1,
|
||||
};
|
||||
assert_eq!(write_ksei_release(&conn, &release, &holdings, false).unwrap(), 2);
|
||||
assert_eq!(
|
||||
write_ksei_release(&conn, &release, &holdings, false).unwrap(),
|
||||
2
|
||||
);
|
||||
|
||||
let data = query_ticker_holdings(&conn, "BBCA").unwrap();
|
||||
assert_eq!(data.ticker.code, "BBCA");
|
||||
|
|
@ -1512,8 +1520,13 @@ mod tests {
|
|||
};
|
||||
|
||||
assert_eq!(
|
||||
write_ksei_release(&conn, &initial_release, std::slice::from_ref(&initial), false)
|
||||
.unwrap(),
|
||||
write_ksei_release(
|
||||
&conn,
|
||||
&initial_release,
|
||||
std::slice::from_ref(&initial),
|
||||
false
|
||||
)
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
|
||||
|
|
@ -1794,21 +1807,21 @@ mod tests {
|
|||
let holdings = rows
|
||||
.into_iter()
|
||||
.map(|(tid, bps, name)| KseiHolding {
|
||||
id: 0,
|
||||
ticker_id: tid,
|
||||
entity_id: None,
|
||||
raw_investor_name: name.to_string(),
|
||||
investor_type: None,
|
||||
locality: None,
|
||||
nationality: None,
|
||||
domicile: None,
|
||||
holdings_scripless: 1,
|
||||
holdings_scrip: 0,
|
||||
total_shares: 1,
|
||||
percentage_bps: bps,
|
||||
report_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
||||
release_sha256: "current-release".to_string(),
|
||||
})
|
||||
id: 0,
|
||||
ticker_id: tid,
|
||||
entity_id: None,
|
||||
raw_investor_name: name.to_string(),
|
||||
investor_type: None,
|
||||
locality: None,
|
||||
nationality: None,
|
||||
domicile: None,
|
||||
holdings_scripless: 1,
|
||||
holdings_scrip: 0,
|
||||
total_shares: 1,
|
||||
percentage_bps: bps,
|
||||
report_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
||||
release_sha256: "current-release".to_string(),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let release = OwnershipRelease {
|
||||
id: 0,
|
||||
|
|
|
|||
|
|
@ -28,6 +28,14 @@ pub fn parse_id_number(s: &str) -> Result<i64, IdxError> {
|
|||
.map_err(|e| IdxError::ParseError(format!("failed to parse Indonesian number '{s}': {e}")))
|
||||
}
|
||||
|
||||
fn parse_id_number_or_zero(s: &str) -> Result<i64, IdxError> {
|
||||
if s.trim().is_empty() {
|
||||
Ok(0)
|
||||
} else {
|
||||
parse_id_number(s)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse Indonesian locale percentage to basis points (i64).
|
||||
/// "54,94" → 5494, "0,00" → 0, "100,00" → 10000
|
||||
pub fn parse_id_percentage(s: &str) -> Result<i64, IdxError> {
|
||||
|
|
@ -141,8 +149,8 @@ pub fn normalize_ksei_row(raw: &KseiRawRow) -> Result<KseiHoldingDraft, IdxError
|
|||
locality,
|
||||
nationality: optional_string(&raw.nationality),
|
||||
domicile: optional_string(&raw.domicile),
|
||||
holdings_scripless: parse_id_number(&raw.holdings_scripless)?,
|
||||
holdings_scrip: parse_id_number(&raw.holdings_scrip)?,
|
||||
holdings_scripless: parse_id_number_or_zero(&raw.holdings_scripless)?,
|
||||
holdings_scrip: parse_id_number_or_zero(&raw.holdings_scrip)?,
|
||||
total_shares: parse_id_number(&raw.total_holding_shares)?,
|
||||
percentage_bps: parse_id_percentage(&raw.percentage)?,
|
||||
report_date: parse_ksei_date(&raw.date)?,
|
||||
|
|
@ -297,9 +305,10 @@ mod tests {
|
|||
use rusqlite::Connection;
|
||||
|
||||
use crate::ownership::entities::{
|
||||
normalize_name, parse_id_number, parse_id_percentage, parse_ksei_date, resolve_entity,
|
||||
normalize_ksei_row, normalize_name, parse_id_number, parse_id_percentage, parse_ksei_date,
|
||||
resolve_entity,
|
||||
};
|
||||
use crate::ownership::types::OwnershipSource;
|
||||
use crate::ownership::types::{KseiRawRow, OwnershipSource};
|
||||
|
||||
#[test]
|
||||
fn test_parse_id_number() {
|
||||
|
|
@ -401,4 +410,47 @@ mod tests {
|
|||
|
||||
assert_eq!(id1, id2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_normalize_ksei_row_defaults_empty_component_shares_to_zero() {
|
||||
let raw = KseiRawRow {
|
||||
date: "27-Apr-2026".to_string(),
|
||||
share_code: "AADI".to_string(),
|
||||
issuer_name: "ADARO ANDALAN INDONESIA Tbk".to_string(),
|
||||
investor_name: "ADARO STRATEGIC INVESTMENTS".to_string(),
|
||||
investor_type: "CP".to_string(),
|
||||
local_foreign: "D".to_string(),
|
||||
nationality: "INDONESIA".to_string(),
|
||||
domicile: String::new(),
|
||||
holdings_scripless: String::new(),
|
||||
holdings_scrip: String::new(),
|
||||
total_holding_shares: "3.200.142.830".to_string(),
|
||||
percentage: "66,18".to_string(),
|
||||
};
|
||||
|
||||
let draft = normalize_ksei_row(&raw).expect("empty component shares should normalize");
|
||||
assert_eq!(draft.holdings_scripless, 0);
|
||||
assert_eq!(draft.holdings_scrip, 0);
|
||||
assert_eq!(draft.total_shares, 3_200_142_830);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_normalize_ksei_row_still_requires_total_shares() {
|
||||
let raw = KseiRawRow {
|
||||
date: "27-Apr-2026".to_string(),
|
||||
share_code: "AADI".to_string(),
|
||||
issuer_name: "ADARO ANDALAN INDONESIA Tbk".to_string(),
|
||||
investor_name: "ADARO STRATEGIC INVESTMENTS".to_string(),
|
||||
investor_type: "CP".to_string(),
|
||||
local_foreign: "D".to_string(),
|
||||
nationality: "INDONESIA".to_string(),
|
||||
domicile: String::new(),
|
||||
holdings_scripless: String::new(),
|
||||
holdings_scrip: String::new(),
|
||||
total_holding_shares: String::new(),
|
||||
percentage: "66,18".to_string(),
|
||||
};
|
||||
|
||||
assert!(normalize_ksei_row(&raw).is_err());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -223,16 +223,14 @@ fn detect_root_node(conn: &Connection, root: &str) -> Result<String, IdxError> {
|
|||
fn query_all_edges(conn: &Connection) -> Result<Vec<GraphEdge>, IdxError> {
|
||||
let mut out = Vec::new();
|
||||
|
||||
if let Ok(release_sha) = conn
|
||||
.query_row(
|
||||
"SELECT sha256
|
||||
if let Ok(release_sha) = conn.query_row(
|
||||
"SELECT sha256
|
||||
FROM ownership_releases
|
||||
ORDER BY as_of_date DESC, imported_at DESC
|
||||
LIMIT 1",
|
||||
[],
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
{
|
||||
[],
|
||||
|row| row.get::<_, String>(0),
|
||||
) {
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT 'entity:' || k.entity_id,
|
||||
|
|
@ -371,7 +369,7 @@ mod tests {
|
|||
use chrono::NaiveDate;
|
||||
use rusqlite::Connection;
|
||||
|
||||
use crate::ownership::db::{ensure_schema, write_ksei_release, upsert_ticker};
|
||||
use crate::ownership::db::{ensure_schema, upsert_ticker, write_ksei_release};
|
||||
use crate::ownership::types::{KseiHolding, OwnershipRelease};
|
||||
|
||||
use super::query_ownership_graph;
|
||||
|
|
|
|||
|
|
@ -355,23 +355,23 @@ fn parse_row_segments(texts: &[String]) -> KseiRawRow {
|
|||
{
|
||||
row.percentage = remaining.pop().unwrap_or_default();
|
||||
}
|
||||
if remaining
|
||||
.last()
|
||||
.is_some_and(|segment| is_id_number_like(segment))
|
||||
{
|
||||
row.total_holding_shares = remaining.pop().unwrap_or_default();
|
||||
}
|
||||
if remaining
|
||||
.last()
|
||||
.is_some_and(|segment| is_id_number_like(segment))
|
||||
{
|
||||
row.holdings_scrip = remaining.pop().unwrap_or_default();
|
||||
}
|
||||
if remaining
|
||||
.last()
|
||||
.is_some_and(|segment| is_id_number_like(segment))
|
||||
{
|
||||
row.holdings_scripless = remaining.pop().unwrap_or_default();
|
||||
let numeric_tail = pop_numeric_tail(&mut remaining);
|
||||
match numeric_tail.as_slice() {
|
||||
[scripless, scrip, total] => {
|
||||
row.holdings_scripless = scripless.clone();
|
||||
row.holdings_scrip = scrip.clone();
|
||||
row.total_holding_shares = total.clone();
|
||||
}
|
||||
// Some PDFs omit the zero-valued scrip column entirely.
|
||||
[scripless, total] => {
|
||||
row.holdings_scripless = scripless.clone();
|
||||
row.holdings_scrip = "0".to_string();
|
||||
row.total_holding_shares = total.clone();
|
||||
}
|
||||
[total] => {
|
||||
row.total_holding_shares = total.clone();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
let mut geo_fields = Vec::new();
|
||||
|
|
@ -439,6 +439,19 @@ fn is_share_code_like(value: &str) -> bool {
|
|||
.all(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit())
|
||||
}
|
||||
|
||||
fn pop_numeric_tail(remaining: &mut Vec<String>) -> Vec<String> {
|
||||
let mut numeric_tail = Vec::new();
|
||||
while numeric_tail.len() < 3
|
||||
&& remaining
|
||||
.last()
|
||||
.is_some_and(|segment| is_id_number_like(segment))
|
||||
{
|
||||
numeric_tail.push(remaining.pop().unwrap_or_default());
|
||||
}
|
||||
numeric_tail.reverse();
|
||||
numeric_tail
|
||||
}
|
||||
|
||||
fn is_id_number_like(value: &str) -> bool {
|
||||
let trimmed = value.trim();
|
||||
!trimmed.is_empty()
|
||||
|
|
@ -554,8 +567,11 @@ fn is_percentage_like(s: &str) -> bool {
|
|||
mod tests {
|
||||
use std::path::Path;
|
||||
|
||||
use crate::ownership::entities::normalize_ksei_row;
|
||||
|
||||
use super::{
|
||||
OwnershipPdfSchema, check_mutool, classify_stext_xml, parse_ksei_pdf, parse_stext_xml,
|
||||
OwnershipPdfSchema, check_mutool, classify_stext_xml, parse_ksei_pdf, parse_row_segments,
|
||||
parse_stext_xml,
|
||||
};
|
||||
|
||||
#[test]
|
||||
|
|
@ -608,6 +624,51 @@ mod tests {
|
|||
assert_eq!(row.percentage, "41,10");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_row_segments_missing_scrip_defaults_to_zero() {
|
||||
let texts = vec![
|
||||
"27-Apr-2026 AADI".to_string(),
|
||||
"ADARO ANDALAN INDONESIA Tbk".to_string(),
|
||||
"ADARO STRATEGIC INVESTMENTS".to_string(),
|
||||
"CP".to_string(),
|
||||
"D".to_string(),
|
||||
"INDONESIA".to_string(),
|
||||
"3.200.142.830".to_string(),
|
||||
"3.200.142.830".to_string(),
|
||||
"66,18".to_string(),
|
||||
];
|
||||
|
||||
let row = parse_row_segments(&texts);
|
||||
assert_eq!(row.date, "27-Apr-2026");
|
||||
assert_eq!(row.share_code, "AADI");
|
||||
assert_eq!(row.holdings_scripless, "3.200.142.830");
|
||||
assert_eq!(row.holdings_scrip, "0");
|
||||
assert_eq!(row.total_holding_shares, "3.200.142.830");
|
||||
assert_eq!(row.percentage, "66,18");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_row_segments_missing_scrip_normalizes_without_error() {
|
||||
let texts = vec![
|
||||
"27-Apr-2026 AADI".to_string(),
|
||||
"ADARO ANDALAN INDONESIA Tbk".to_string(),
|
||||
"ADARO STRATEGIC INVESTMENTS".to_string(),
|
||||
"CP".to_string(),
|
||||
"D".to_string(),
|
||||
"INDONESIA".to_string(),
|
||||
"3.200.142.830".to_string(),
|
||||
"3.200.142.830".to_string(),
|
||||
"66,18".to_string(),
|
||||
];
|
||||
|
||||
let row = parse_row_segments(&texts);
|
||||
let draft = normalize_ksei_row(&row).expect("missing scrip column should normalize");
|
||||
|
||||
assert_eq!(draft.holdings_scripless, 3_200_142_830);
|
||||
assert_eq!(draft.holdings_scrip, 0);
|
||||
assert_eq!(draft.total_shares, 3_200_142_830);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_stext_xml_detects_supported_holder_register_schema() {
|
||||
let xml = include_str!("../../tests/fixtures/ksei_above1_stext_excerpt.xml");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue