mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-11-07 09:45:36 +00:00
feat(ownership): import IDX above-1% XLSX holder register
Since June 2026 IDX publishes the monthly "Pemegang Saham di Atas 1%" list as an XLSX workbook on the Data Kepemilikan Saham page instead of a PDF announcement. Add a parser for it and accept `.xlsx` in `ownership import --file`. - src/ownership/xlsx.rs: zip + quick-xml reader (shared strings incl. rich text, sheet cells), exact 12-column header match so the daily above-5% and investor-type/classification workbooks are rejected, Excel serial dates, numeric shares, percent -> bps with float-noise rounding. Builds KseiHoldingDraft directly and reuses the entity normalizers, so investor types stay uppercase like the PDF path. - Fixtures: trimmed excerpts of the real 2026-08-31 above-1% and 2026-09-23 above-5% workbooks. - Unit + CLI tests: import, ticker, releases, above-5% rejection. - Docs: ARCHITECTURE parser pipeline, AGENTS ownership inputs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E
This commit is contained in:
parent
5bf3e5d5e3
commit
2c85acb829
9 changed files with 539 additions and 10 deletions
|
|
@ -63,7 +63,7 @@ Preferred bootstrap/update path:
|
||||||
Ownership input paths:
|
Ownership input paths:
|
||||||
- primary remote source: discoverable IDX `above1` holder-register PDF
|
- primary remote source: discoverable IDX `above1` holder-register PDF
|
||||||
- maintained snapshot path: `ownership sync`
|
- maintained snapshot path: `ownership sync`
|
||||||
- local fallback path: PDF, plus local archive `.zip` / `.txt`
|
- local file imports: IDX above-1% `.xlsx` (current format since June 2026), legacy `.pdf`, plus local archive `.zip` / `.txt`
|
||||||
|
|
||||||
Important scope note:
|
Important scope note:
|
||||||
- archive ZIP/TXT ingest is a fallback/backstop path, not the primary product ingest surface
|
- archive ZIP/TXT ingest is a fallback/backstop path, not the primary product ingest surface
|
||||||
|
|
|
||||||
|
|
@ -63,12 +63,16 @@ Unlike the `stocks` module (live-fetch), ownership is **import-then-query**:
|
||||||
4. Fully offline after import/sync
|
4. Fully offline after import/sync
|
||||||
|
|
||||||
### Data Sources
|
### Data Sources
|
||||||
- **KSEI** — official ≥1% shareholder registry (monthly PDF from IDX)
|
- **KSEI** — official ≥1% shareholder registry, published monthly by IDX. Through May 2026 it was a PDF announcement; from the 2026-05-29 report onward it is an XLSX workbook on the IDX "Data Kepemilikan Saham" page
|
||||||
- **KSEI archive** — monthly ZIP/TXT balance-position matrix, used as a local fallback/backstop import path
|
- **KSEI archive** — monthly ZIP/TXT balance-position matrix, used as a local fallback/backstop import path
|
||||||
- **Bing Finance** — global institutional ownership (REST API, quarterly)
|
- **Bing Finance** — global institutional ownership (REST API, quarterly)
|
||||||
|
|
||||||
### Parser Pipeline
|
### Parser Pipeline
|
||||||
```
|
```
|
||||||
|
KSEI XLSX → zip + quick-xml (sharedStrings + sheet1) → exact 12-column header check
|
||||||
|
→ KseiHoldingDraft (Excel serial dates, numeric shares, percent → bps)
|
||||||
|
→ SQLite INSERT (within transaction)
|
||||||
|
|
||||||
KSEI PDF → mutool stext (XML with coordinates) → quick-xml parse → KseiRawRow
|
KSEI PDF → mutool stext (XML with coordinates) → quick-xml parse → KseiRawRow
|
||||||
→ normalize (ID locale numbers, dates, entity names) → KseiHolding
|
→ normalize (ID locale numbers, dates, entity names) → KseiHolding
|
||||||
→ SQLite INSERT (within transaction)
|
→ SQLite INSERT (within transaction)
|
||||||
|
|
|
||||||
|
|
@ -17,13 +17,13 @@ use crate::output::table::format_idr;
|
||||||
use crate::ownership::types::{
|
use crate::ownership::types::{
|
||||||
ChangeType, FlowSignal, HolderRow, KseiHolding, OwnershipRelease, OwnershipSource,
|
ChangeType, FlowSignal, HolderRow, KseiHolding, OwnershipRelease, OwnershipSource,
|
||||||
};
|
};
|
||||||
use crate::ownership::{archive, db, entities, graph, parser, remote, search, snapshot};
|
use crate::ownership::{archive, db, entities, graph, parser, remote, search, snapshot, xlsx};
|
||||||
use crate::runtime;
|
use crate::runtime;
|
||||||
|
|
||||||
#[derive(Debug, Args)]
|
#[derive(Debug, Args)]
|
||||||
#[command(
|
#[command(
|
||||||
about = "Ownership intelligence (KSEI + Bing)",
|
about = "Ownership intelligence (KSEI + Bing)",
|
||||||
long_about = "Ownership intelligence (KSEI + Bing).\n\nPreferred bootstrap path:\n 1. Run `idx ownership sync` to install or refresh a maintained SQLite snapshot.\n 2. If no snapshot manifest is available, run `idx ownership discover` and then `idx ownership import --url <pdf-url>`.\n 3. Local `.pdf`, `.zip`, and `.txt` imports remain available for manual or fallback workflows.",
|
long_about = "Ownership intelligence (KSEI + Bing).\n\nPreferred bootstrap path:\n 1. Run `idx ownership sync` to install or refresh a maintained SQLite snapshot.\n 2. If no snapshot manifest is available, run `idx ownership discover` and then `idx ownership import --url <pdf-url>`.\n 3. Local `.xlsx`, `.pdf`, `.zip`, and `.txt` imports remain available for manual or fallback workflows.",
|
||||||
after_help = "Examples:\n idx ownership sync\n idx ownership sync --manifest /path/to/ownership-snapshot-manifest.json\n idx ownership discover --limit 1\n idx ownership import --url <pdf-url>\n idx ownership releases"
|
after_help = "Examples:\n idx ownership sync\n idx ownership sync --manifest /path/to/ownership-snapshot-manifest.json\n idx ownership discover --limit 1\n idx ownership import --url <pdf-url>\n idx ownership releases"
|
||||||
)]
|
)]
|
||||||
pub struct OwnershipCmd {
|
pub struct OwnershipCmd {
|
||||||
|
|
@ -41,7 +41,7 @@ pub enum OwnershipCommand {
|
||||||
#[command(
|
#[command(
|
||||||
about = "Import ownership data directly from source files",
|
about = "Import ownership data directly from source files",
|
||||||
long_about = "Import ownership data directly from source files.\n\nPrefer `idx ownership sync` for normal bootstrap/update flows. Use `import` when you need a direct PDF import from IDX discovery output, a local PDF, or a local KSEI archive fallback file.",
|
long_about = "Import ownership data directly from source files.\n\nPrefer `idx ownership sync` for normal bootstrap/update flows. Use `import` when you need a direct PDF import from IDX discovery output, a local PDF, or a local KSEI archive fallback file.",
|
||||||
after_help = "Examples:\n idx ownership import --url <pdf-url-from-discover>\n idx ownership import --file /path/to/ksei.pdf\n idx ownership import --file /path/to/BalanceposEfek20260227.zip"
|
after_help = "Examples:\n idx ownership import --url <pdf-url-from-discover>\n idx ownership import --file /path/to/peng-2026-08-00017-satu-persen.xlsx\n idx ownership import --file /path/to/ksei.pdf\n idx ownership import --file /path/to/BalanceposEfek20260227.zip"
|
||||||
)]
|
)]
|
||||||
Import(ImportArgs),
|
Import(ImportArgs),
|
||||||
#[command(
|
#[command(
|
||||||
|
|
@ -87,7 +87,7 @@ pub struct ImportArgs {
|
||||||
/// Direct URL to a remote ownership PDF, typically from `ownership discover`.
|
/// Direct URL to a remote ownership PDF, typically from `ownership discover`.
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
pub url: Option<String>,
|
pub url: Option<String>,
|
||||||
/// Path to a local ownership file: PDF (primary), ZIP/TXT archive (fallback).
|
/// Path to a local ownership file: above-1% XLSX or PDF (primary), ZIP/TXT archive (fallback).
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
pub file: Option<PathBuf>,
|
pub file: Option<PathBuf>,
|
||||||
/// Fetch Bing institutional data for these symbols.
|
/// Fetch Bing institutional data for these symbols.
|
||||||
|
|
@ -836,6 +836,7 @@ fn handle_import(args: &ImportArgs, config: &IdxConfig) -> Result<(), IdxError>
|
||||||
drafts
|
drafts
|
||||||
}
|
}
|
||||||
ImportInputFormat::Archive => archive::parse_balancepos_file(&import_input.import_path)?,
|
ImportInputFormat::Archive => archive::parse_balancepos_file(&import_input.import_path)?,
|
||||||
|
ImportInputFormat::Xlsx => xlsx::parse_above1_xlsx_file(&import_input.import_path)?,
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut holdings = Vec::with_capacity(drafts.len());
|
let mut holdings = Vec::with_capacity(drafts.len());
|
||||||
|
|
@ -977,6 +978,7 @@ struct OwnershipImportResult {
|
||||||
enum ImportInputFormat {
|
enum ImportInputFormat {
|
||||||
Pdf,
|
Pdf,
|
||||||
Archive,
|
Archive,
|
||||||
|
Xlsx,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn resolve_import_input(args: &ImportArgs) -> Result<Option<ResolvedImportInput>, IdxError> {
|
fn resolve_import_input(args: &ImportArgs) -> Result<Option<ResolvedImportInput>, IdxError> {
|
||||||
|
|
@ -1018,11 +1020,12 @@ fn detect_local_import_format(path: &Path) -> Result<ImportInputFormat, IdxError
|
||||||
.as_deref()
|
.as_deref()
|
||||||
{
|
{
|
||||||
Some("pdf") => Ok(ImportInputFormat::Pdf),
|
Some("pdf") => Ok(ImportInputFormat::Pdf),
|
||||||
|
Some("xlsx") if xlsx::supports_xlsx_file(path) => Ok(ImportInputFormat::Xlsx),
|
||||||
Some("zip") | Some("txt") if archive::supports_local_archive_file(path) => {
|
Some("zip") | Some("txt") if archive::supports_local_archive_file(path) => {
|
||||||
Ok(ImportInputFormat::Archive)
|
Ok(ImportInputFormat::Archive)
|
||||||
}
|
}
|
||||||
_ => Err(IdxError::InvalidInput(format!(
|
_ => Err(IdxError::InvalidInput(format!(
|
||||||
"unsupported local ownership file {}; expected a .pdf, .zip, or .txt input",
|
"unsupported local ownership file {}; expected a .xlsx, .pdf, .zip, or .txt input",
|
||||||
path.display()
|
path.display()
|
||||||
))),
|
))),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -212,7 +212,7 @@ pub fn resolve_entity(
|
||||||
Ok(entity_id)
|
Ok(entity_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn normalize_investor_type(raw: &str) -> Option<InvestorTypeCode> {
|
pub(crate) fn normalize_investor_type(raw: &str) -> Option<InvestorTypeCode> {
|
||||||
let value = raw.trim();
|
let value = raw.trim();
|
||||||
if value.is_empty() {
|
if value.is_empty() {
|
||||||
None
|
None
|
||||||
|
|
@ -221,7 +221,7 @@ fn normalize_investor_type(raw: &str) -> Option<InvestorTypeCode> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn normalize_locality(raw: &str) -> Option<Locality> {
|
pub(crate) fn normalize_locality(raw: &str) -> Option<Locality> {
|
||||||
match raw.trim().to_uppercase().as_str() {
|
match raw.trim().to_uppercase().as_str() {
|
||||||
"L" | "D" => Some(Locality::Local),
|
"L" | "D" => Some(Locality::Local),
|
||||||
"F" | "A" => Some(Locality::Foreign),
|
"F" | "A" => Some(Locality::Foreign),
|
||||||
|
|
|
||||||
|
|
@ -7,3 +7,4 @@ pub mod remote;
|
||||||
pub mod search;
|
pub mod search;
|
||||||
pub mod snapshot;
|
pub mod snapshot;
|
||||||
pub mod types;
|
pub mod types;
|
||||||
|
pub mod xlsx;
|
||||||
|
|
|
||||||
450
src/ownership/xlsx.rs
Normal file
450
src/ownership/xlsx.rs
Normal file
|
|
@ -0,0 +1,450 @@
|
||||||
|
//! Parser for the IDX "Pemegang Saham di Atas 1%" XLSX workbook.
|
||||||
|
//!
|
||||||
|
//! Since June 2026 IDX publishes the monthly above-1% holder register as an
|
||||||
|
//! XLSX file on the "Data Kepemilikan Saham" page instead of a PDF
|
||||||
|
//! announcement. The workbook has one sheet: a disclaimer block, a `[PUBLIC]`
|
||||||
|
//! marker, then a fixed 12-column table. Only that exact header is accepted,
|
||||||
|
//! so the daily above-5% workbook and the investor-type / classification
|
||||||
|
//! workbooks published on the same page are rejected instead of mis-imported.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::fs;
|
||||||
|
use std::io::{Cursor, Read};
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use chrono::{Duration, NaiveDate};
|
||||||
|
use quick_xml::Reader;
|
||||||
|
use quick_xml::events::{BytesStart, Event};
|
||||||
|
|
||||||
|
use crate::error::IdxError;
|
||||||
|
use crate::ownership::entities::{normalize_investor_type, normalize_locality, parse_ksei_date};
|
||||||
|
use crate::ownership::types::KseiHoldingDraft;
|
||||||
|
|
||||||
|
const EXPECTED_HEADER: [&str; 12] = [
|
||||||
|
"DATE",
|
||||||
|
"SHARE_CODE",
|
||||||
|
"ISSUER_NAME",
|
||||||
|
"INVESTOR_NAME",
|
||||||
|
"INVESTOR_CLASSIFICATION",
|
||||||
|
"LOCAL_FOREIGN",
|
||||||
|
"NATIONALITY",
|
||||||
|
"DOMICILE",
|
||||||
|
"HOLDINGS_SCRIPLESS",
|
||||||
|
"HOLDINGS_SCRIP",
|
||||||
|
"TOTAL_HOLDING_SHARES",
|
||||||
|
"PERCENTAGE",
|
||||||
|
];
|
||||||
|
|
||||||
|
/// Rows above this line are disclaimer text; the header must appear before it.
|
||||||
|
const MAX_HEADER_SEARCH_ROWS: usize = 50;
|
||||||
|
|
||||||
|
pub fn supports_xlsx_file(path: &Path) -> bool {
|
||||||
|
path.extension()
|
||||||
|
.and_then(|value| value.to_str())
|
||||||
|
.is_some_and(|value| value.trim().eq_ignore_ascii_case("xlsx"))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn parse_above1_xlsx_file(path: &Path) -> Result<Vec<KseiHoldingDraft>, IdxError> {
|
||||||
|
let bytes = fs::read(path).map_err(|e| {
|
||||||
|
IdxError::Io(format!(
|
||||||
|
"failed to read ownership XLSX {}: {e}",
|
||||||
|
path.display()
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
parse_above1_xlsx_bytes(&bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn parse_above1_xlsx_bytes(bytes: &[u8]) -> Result<Vec<KseiHoldingDraft>, IdxError> {
|
||||||
|
let mut zip = zip::ZipArchive::new(Cursor::new(bytes))
|
||||||
|
.map_err(|e| IdxError::ParseError(format!("failed to open ownership XLSX: {e}")))?;
|
||||||
|
|
||||||
|
let shared_strings = match read_zip_entry(&mut zip, "xl/sharedStrings.xml")? {
|
||||||
|
Some(xml) => parse_shared_strings(&xml)?,
|
||||||
|
None => Vec::new(),
|
||||||
|
};
|
||||||
|
let sheet_name = first_worksheet_name(&mut zip)?;
|
||||||
|
let sheet_xml = read_zip_entry(&mut zip, &sheet_name)?
|
||||||
|
.ok_or_else(|| IdxError::ParseError(format!("ownership XLSX is missing {sheet_name}")))?;
|
||||||
|
|
||||||
|
let rows = parse_sheet_rows(&sheet_xml, &shared_strings)?;
|
||||||
|
drafts_from_rows(&rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_zip_entry(
|
||||||
|
zip: &mut zip::ZipArchive<Cursor<&[u8]>>,
|
||||||
|
name: &str,
|
||||||
|
) -> Result<Option<String>, IdxError> {
|
||||||
|
let mut file = match zip.by_name(name) {
|
||||||
|
Ok(file) => file,
|
||||||
|
Err(zip::result::ZipError::FileNotFound) => return Ok(None),
|
||||||
|
Err(e) => {
|
||||||
|
return Err(IdxError::ParseError(format!(
|
||||||
|
"failed to read ownership XLSX entry {name}: {e}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let mut output = String::new();
|
||||||
|
file.read_to_string(&mut output).map_err(|e| {
|
||||||
|
IdxError::ParseError(format!("failed to decode ownership XLSX entry {name}: {e}"))
|
||||||
|
})?;
|
||||||
|
Ok(Some(output))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn first_worksheet_name(zip: &mut zip::ZipArchive<Cursor<&[u8]>>) -> Result<String, IdxError> {
|
||||||
|
let mut sheets: Vec<String> = zip
|
||||||
|
.file_names()
|
||||||
|
.filter(|name| name.starts_with("xl/worksheets/") && name.ends_with(".xml"))
|
||||||
|
.filter(|name| !name["xl/worksheets/".len()..].contains('/'))
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect();
|
||||||
|
// sheet1.xml, sheet2.xml, ... sort by their numeric suffix.
|
||||||
|
sheets.sort_by_key(|name| {
|
||||||
|
name.trim_start_matches("xl/worksheets/sheet")
|
||||||
|
.trim_end_matches(".xml")
|
||||||
|
.parse::<u32>()
|
||||||
|
.unwrap_or(u32::MAX)
|
||||||
|
});
|
||||||
|
sheets
|
||||||
|
.into_iter()
|
||||||
|
.next()
|
||||||
|
.ok_or_else(|| IdxError::ParseError("ownership XLSX has no worksheet".to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn xml_error(context: &str, err: impl std::fmt::Display) -> IdxError {
|
||||||
|
IdxError::ParseError(format!("failed to parse ownership XLSX {context}: {err}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shared strings table: each `<si>` is one string, made of one `<t>` or
|
||||||
|
/// several rich-text runs `<r><t>`. Phonetic hints (`<rPh>`) are skipped.
|
||||||
|
fn parse_shared_strings(xml: &str) -> Result<Vec<String>, IdxError> {
|
||||||
|
let mut reader = Reader::from_str(xml);
|
||||||
|
let mut strings = Vec::new();
|
||||||
|
let mut current: Option<String> = None;
|
||||||
|
let mut in_text = false;
|
||||||
|
let mut in_phonetic = false;
|
||||||
|
|
||||||
|
loop {
|
||||||
|
match reader.read_event() {
|
||||||
|
Ok(Event::Start(e)) => match e.local_name().as_ref() {
|
||||||
|
b"si" => current = Some(String::new()),
|
||||||
|
b"t" => in_text = true,
|
||||||
|
b"rPh" => in_phonetic = true,
|
||||||
|
_ => {}
|
||||||
|
},
|
||||||
|
Ok(Event::Empty(e)) if e.local_name().as_ref() == b"si" => strings.push(String::new()),
|
||||||
|
Ok(Event::Text(text)) if in_text && !in_phonetic => {
|
||||||
|
if let Some(value) = current.as_mut() {
|
||||||
|
let unescaped = text
|
||||||
|
.unescape()
|
||||||
|
.map_err(|e| xml_error("shared strings", e))?;
|
||||||
|
value.push_str(&unescaped);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(Event::End(e)) => match e.local_name().as_ref() {
|
||||||
|
b"si" => strings.push(current.take().unwrap_or_default()),
|
||||||
|
b"t" => in_text = false,
|
||||||
|
b"rPh" => in_phonetic = false,
|
||||||
|
_ => {}
|
||||||
|
},
|
||||||
|
Ok(Event::Eof) => break,
|
||||||
|
Err(e) => return Err(xml_error("shared strings", e)),
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(strings)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One sheet row: column letters (`A`, `B`, ...) to cell text.
|
||||||
|
type SheetRow = HashMap<String, String>;
|
||||||
|
|
||||||
|
fn parse_sheet_rows(xml: &str, shared_strings: &[String]) -> Result<Vec<SheetRow>, IdxError> {
|
||||||
|
let mut reader = Reader::from_str(xml);
|
||||||
|
let mut rows = Vec::new();
|
||||||
|
let mut row: Option<SheetRow> = None;
|
||||||
|
let mut cell: Option<(String, String)> = None;
|
||||||
|
let mut value = String::new();
|
||||||
|
let mut in_value = false;
|
||||||
|
|
||||||
|
loop {
|
||||||
|
match reader.read_event() {
|
||||||
|
Ok(Event::Start(e)) => match e.local_name().as_ref() {
|
||||||
|
b"row" => row = Some(SheetRow::new()),
|
||||||
|
b"c" => {
|
||||||
|
cell = Some(cell_ref_and_type(&e)?);
|
||||||
|
value.clear();
|
||||||
|
}
|
||||||
|
b"v" | b"t" if cell.is_some() => in_value = true,
|
||||||
|
_ => {}
|
||||||
|
},
|
||||||
|
Ok(Event::Empty(e)) if e.local_name().as_ref() == b"row" => rows.push(SheetRow::new()),
|
||||||
|
Ok(Event::Text(text)) if in_value => {
|
||||||
|
let unescaped = text.unescape().map_err(|e| xml_error("worksheet", e))?;
|
||||||
|
value.push_str(&unescaped);
|
||||||
|
}
|
||||||
|
Ok(Event::End(e)) => match e.local_name().as_ref() {
|
||||||
|
b"v" | b"t" => in_value = false,
|
||||||
|
b"c" => {
|
||||||
|
if let (Some((column, cell_type)), Some(current_row)) =
|
||||||
|
(cell.take(), row.as_mut())
|
||||||
|
{
|
||||||
|
let text = if cell_type == "s" {
|
||||||
|
let index = value.trim().parse::<usize>().map_err(|e| {
|
||||||
|
xml_error("worksheet", format!("bad shared string index: {e}"))
|
||||||
|
})?;
|
||||||
|
shared_strings.get(index).cloned().ok_or_else(|| {
|
||||||
|
xml_error(
|
||||||
|
"worksheet",
|
||||||
|
format!("shared string {index} out of range"),
|
||||||
|
)
|
||||||
|
})?
|
||||||
|
} else {
|
||||||
|
value.clone()
|
||||||
|
};
|
||||||
|
current_row.insert(column, text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b"row" => {
|
||||||
|
if let Some(current_row) = row.take() {
|
||||||
|
rows.push(current_row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
},
|
||||||
|
Ok(Event::Eof) => break,
|
||||||
|
Err(e) => return Err(xml_error("worksheet", e)),
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cell_ref_and_type(element: &BytesStart<'_>) -> Result<(String, String), IdxError> {
|
||||||
|
let mut column = String::new();
|
||||||
|
let mut cell_type = String::new();
|
||||||
|
for attribute in element.attributes() {
|
||||||
|
let attribute = attribute.map_err(|e| xml_error("worksheet", e))?;
|
||||||
|
let raw = String::from_utf8_lossy(&attribute.value).to_string();
|
||||||
|
match attribute.key.local_name().as_ref() {
|
||||||
|
b"r" => column = raw.chars().take_while(char::is_ascii_alphabetic).collect(),
|
||||||
|
b"t" => cell_type = raw,
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if column.is_empty() {
|
||||||
|
return Err(xml_error("worksheet", "cell without a reference"));
|
||||||
|
}
|
||||||
|
Ok((column, cell_type))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drafts_from_rows(rows: &[SheetRow]) -> Result<Vec<KseiHoldingDraft>, IdxError> {
|
||||||
|
let (header_index, columns) = rows
|
||||||
|
.iter()
|
||||||
|
.take(MAX_HEADER_SEARCH_ROWS)
|
||||||
|
.enumerate()
|
||||||
|
.find_map(|(index, row)| header_columns(row).map(|columns| (index, columns)))
|
||||||
|
.ok_or_else(|| {
|
||||||
|
IdxError::ParseError(
|
||||||
|
"ownership XLSX header did not match the above-1% holder register layout \
|
||||||
|
(DATE, SHARE_CODE, ... PERCENTAGE); the above-5% and investor-type \
|
||||||
|
workbooks are not supported"
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut drafts = Vec::new();
|
||||||
|
for (offset, row) in rows[header_index + 1..].iter().enumerate() {
|
||||||
|
let line = header_index + offset + 2;
|
||||||
|
let field = |name: &str| -> &str {
|
||||||
|
columns
|
||||||
|
.get(name)
|
||||||
|
.and_then(|column| row.get(column))
|
||||||
|
.map(|value| value.trim())
|
||||||
|
.unwrap_or("")
|
||||||
|
};
|
||||||
|
|
||||||
|
let ticker_code = field("SHARE_CODE");
|
||||||
|
let investor_name = field("INVESTOR_NAME");
|
||||||
|
if ticker_code.is_empty() && investor_name.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if ticker_code.is_empty() || investor_name.is_empty() {
|
||||||
|
return Err(IdxError::ParseError(format!(
|
||||||
|
"ownership XLSX row {line} is missing SHARE_CODE or INVESTOR_NAME"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
drafts.push(KseiHoldingDraft {
|
||||||
|
ticker_code: ticker_code.to_string(),
|
||||||
|
issuer_name: optional(field("ISSUER_NAME")),
|
||||||
|
raw_investor_name: investor_name.to_string(),
|
||||||
|
investor_type: normalize_investor_type(field("INVESTOR_CLASSIFICATION")),
|
||||||
|
locality: normalize_locality(field("LOCAL_FOREIGN")),
|
||||||
|
nationality: optional(field("NATIONALITY")),
|
||||||
|
domicile: optional(field("DOMICILE")),
|
||||||
|
holdings_scripless: parse_shares(field("HOLDINGS_SCRIPLESS"), line, false)?,
|
||||||
|
holdings_scrip: parse_shares(field("HOLDINGS_SCRIP"), line, false)?,
|
||||||
|
total_shares: parse_shares(field("TOTAL_HOLDING_SHARES"), line, true)?,
|
||||||
|
percentage_bps: parse_percentage_bps(field("PERCENTAGE"), line)?,
|
||||||
|
report_date: parse_report_date(field("DATE"), line)?,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if drafts.is_empty() {
|
||||||
|
return Err(IdxError::ParseError(
|
||||||
|
"ownership XLSX contained a header but no holder rows".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(drafts)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Map header labels to column letters if `row` is exactly the expected header.
|
||||||
|
fn header_columns(row: &SheetRow) -> Option<HashMap<&'static str, String>> {
|
||||||
|
let mut found: HashMap<&'static str, String> = HashMap::new();
|
||||||
|
for (column, text) in row {
|
||||||
|
let label = text.trim().to_ascii_uppercase().replace(' ', "_");
|
||||||
|
if label.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let expected = EXPECTED_HEADER.iter().find(|name| **name == label)?;
|
||||||
|
found.insert(expected, column.clone());
|
||||||
|
}
|
||||||
|
(found.len() == EXPECTED_HEADER.len()).then_some(found)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn optional(value: &str) -> Option<String> {
|
||||||
|
(!value.is_empty()).then(|| value.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_number(raw: &str, line: usize, name: &str) -> Result<f64, IdxError> {
|
||||||
|
raw.replace(',', "")
|
||||||
|
.parse::<f64>()
|
||||||
|
.ok()
|
||||||
|
.filter(|value| value.is_finite())
|
||||||
|
.ok_or_else(|| {
|
||||||
|
IdxError::ParseError(format!("ownership XLSX row {line}: invalid {name} '{raw}'"))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_shares(raw: &str, line: usize, required: bool) -> Result<i64, IdxError> {
|
||||||
|
if raw.is_empty() {
|
||||||
|
return if required {
|
||||||
|
Err(IdxError::ParseError(format!(
|
||||||
|
"ownership XLSX row {line}: missing TOTAL_HOLDING_SHARES"
|
||||||
|
)))
|
||||||
|
} else {
|
||||||
|
Ok(0)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
let value = parse_number(raw, line, "share count")?;
|
||||||
|
if value < 0.0 || value.fract() != 0.0 || value > i64::MAX as f64 {
|
||||||
|
return Err(IdxError::ParseError(format!(
|
||||||
|
"ownership XLSX row {line}: share count '{raw}' is not a whole number"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(value as i64)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `54.94` (percent) -> 5494 basis points. Excel stores values such as
|
||||||
|
/// `1.1499999999999999`, so round to the nearest basis point.
|
||||||
|
fn parse_percentage_bps(raw: &str, line: usize) -> Result<i64, IdxError> {
|
||||||
|
let value = parse_number(raw, line, "PERCENTAGE")?;
|
||||||
|
if !(0.0..=100.0).contains(&value) {
|
||||||
|
return Err(IdxError::ParseError(format!(
|
||||||
|
"ownership XLSX row {line}: PERCENTAGE '{raw}' is out of range"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok((value * 100.0).round() as i64)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// DATE is an Excel serial day number (`46265` = 2026-08-31); accept the
|
||||||
|
/// KSEI text form (`31-Aug-2026`) as well in case the cell is stored as text.
|
||||||
|
fn parse_report_date(raw: &str, line: usize) -> Result<NaiveDate, IdxError> {
|
||||||
|
if let Ok(serial) = raw.parse::<f64>() {
|
||||||
|
let days = serial.trunc() as i64;
|
||||||
|
let epoch = NaiveDate::from_ymd_opt(1899, 12, 30).expect("valid Excel epoch");
|
||||||
|
if (1..=2_958_465).contains(&days) {
|
||||||
|
return Ok(epoch + Duration::days(days));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
parse_ksei_date(raw).map_err(|_| {
|
||||||
|
IdxError::ParseError(format!("ownership XLSX row {line}: invalid DATE '{raw}'"))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn fixture() -> Vec<KseiHoldingDraft> {
|
||||||
|
parse_above1_xlsx_bytes(include_bytes!(
|
||||||
|
"../../tests/fixtures/ksei_above1_20260831_excerpt.xlsx"
|
||||||
|
))
|
||||||
|
.expect("above-1% XLSX excerpt parses")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_above1_xlsx_excerpt() {
|
||||||
|
let drafts = fixture();
|
||||||
|
assert!(
|
||||||
|
drafts.len() >= 10,
|
||||||
|
"expected excerpt rows, got {}",
|
||||||
|
drafts.len()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
drafts
|
||||||
|
.iter()
|
||||||
|
.all(|d| d.report_date == NaiveDate::from_ymd_opt(2026, 8, 31).unwrap())
|
||||||
|
);
|
||||||
|
|
||||||
|
let dwimuria = drafts
|
||||||
|
.iter()
|
||||||
|
.find(|d| {
|
||||||
|
d.ticker_code == "BBCA" && d.raw_investor_name == "PT DWIMURIA INVESTAMA ANDALAN"
|
||||||
|
})
|
||||||
|
.expect("BBCA controlling holder present");
|
||||||
|
assert_eq!(dwimuria.percentage_bps, 5494);
|
||||||
|
assert_eq!(
|
||||||
|
dwimuria.investor_type.as_ref().map(|t| t.0.as_str()),
|
||||||
|
Some("CORPORATE")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
dwimuria.total_shares,
|
||||||
|
dwimuria.holdings_scripless + dwimuria.holdings_scrip
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rounds_float_noise_in_percentages() {
|
||||||
|
assert_eq!(parse_percentage_bps("1.1499999999999999", 1).unwrap(), 115);
|
||||||
|
assert_eq!(parse_percentage_bps("41.1", 1).unwrap(), 4110);
|
||||||
|
assert!(parse_percentage_bps("100.5", 1).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_excel_serial_and_text_dates() {
|
||||||
|
let expected = NaiveDate::from_ymd_opt(2026, 8, 31).unwrap();
|
||||||
|
assert_eq!(parse_report_date("46265", 1).unwrap(), expected);
|
||||||
|
assert_eq!(parse_report_date("31-Aug-2026", 1).unwrap(), expected);
|
||||||
|
assert!(parse_report_date("not a date", 1).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_above5_workbook() {
|
||||||
|
let err = parse_above1_xlsx_bytes(include_bytes!(
|
||||||
|
"../../tests/fixtures/ksei_above5_20260923_excerpt.xlsx"
|
||||||
|
))
|
||||||
|
.expect_err("above-5% workbook must be rejected");
|
||||||
|
assert!(
|
||||||
|
err.to_string().contains("above-1% holder register layout"),
|
||||||
|
"{err}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_non_xlsx_bytes() {
|
||||||
|
assert!(parse_above1_xlsx_bytes(b"%PDF-1.7 not a workbook").is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
73
tests/cli.rs
73
tests/cli.rs
|
|
@ -407,7 +407,7 @@ fn ownership_import_help_mentions_sync_preference_and_fallback_files() {
|
||||||
.success()
|
.success()
|
||||||
.stdout(predicate::str::contains("Prefer `idx ownership sync`"))
|
.stdout(predicate::str::contains("Prefer `idx ownership sync`"))
|
||||||
.stdout(predicate::str::contains(
|
.stdout(predicate::str::contains(
|
||||||
"PDF (primary), ZIP/TXT archive (fallback)",
|
"above-1% XLSX or PDF (primary), ZIP/TXT archive (fallback)",
|
||||||
))
|
))
|
||||||
.stdout(predicate::str::contains(
|
.stdout(predicate::str::contains(
|
||||||
"idx ownership import --file /path/to/BalanceposEfek20260227.zip",
|
"idx ownership import --file /path/to/BalanceposEfek20260227.zip",
|
||||||
|
|
@ -1524,6 +1524,77 @@ fn ownership_import_url_duplicate_release_is_skipped() {
|
||||||
.stdout(predicate::str::contains("Release already imported"));
|
.stdout(predicate::str::contains("Release already imported"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ownership_import_file_xlsx_supports_ticker_and_releases() {
|
||||||
|
let root = test_env_dir("ownership-import-xlsx");
|
||||||
|
let db_path = root.join("ownership.db");
|
||||||
|
let xlsx_path = root.join("peng-2026-08-00017-satu-persen.xlsx");
|
||||||
|
fs::write(
|
||||||
|
&xlsx_path,
|
||||||
|
include_bytes!("fixtures/ksei_above1_20260831_excerpt.xlsx"),
|
||||||
|
)
|
||||||
|
.expect("write xlsx fixture");
|
||||||
|
|
||||||
|
bin_with_root(&root)
|
||||||
|
.args([
|
||||||
|
"config",
|
||||||
|
"set",
|
||||||
|
"ownership.db_path",
|
||||||
|
db_path.to_str().unwrap(),
|
||||||
|
])
|
||||||
|
.assert()
|
||||||
|
.success();
|
||||||
|
|
||||||
|
bin_with_root(&root)
|
||||||
|
.args(["ownership", "import", "--file", xlsx_path.to_str().unwrap()])
|
||||||
|
.assert()
|
||||||
|
.success()
|
||||||
|
.stdout(predicate::str::contains("(as of 2026-08-31)"));
|
||||||
|
|
||||||
|
let output = bin_with_root(&root)
|
||||||
|
.args(["-o", "json", "ownership", "ticker", "BBCA"])
|
||||||
|
.output()
|
||||||
|
.expect("ownership ticker json output");
|
||||||
|
assert!(output.status.success());
|
||||||
|
let body = String::from_utf8_lossy(&output.stdout);
|
||||||
|
assert!(body.contains("DWIMURIA INVESTAMA ANDALAN"), "{body}");
|
||||||
|
|
||||||
|
let output = bin_with_root(&root)
|
||||||
|
.args(["-o", "json", "ownership", "releases"])
|
||||||
|
.output()
|
||||||
|
.expect("ownership releases json output");
|
||||||
|
let releases: Value = serde_json::from_slice(&output.stdout).expect("parse releases json");
|
||||||
|
assert_eq!(releases[0]["as_of_date"], "2026-08-31");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ownership_import_file_rejects_above5_xlsx() {
|
||||||
|
let root = test_env_dir("ownership-import-xlsx-above5");
|
||||||
|
let db_path = root.join("ownership.db");
|
||||||
|
let xlsx_path = root.join("peng-2026-09-23-00080-lima-persen.xlsx");
|
||||||
|
fs::write(
|
||||||
|
&xlsx_path,
|
||||||
|
include_bytes!("fixtures/ksei_above5_20260923_excerpt.xlsx"),
|
||||||
|
)
|
||||||
|
.expect("write xlsx fixture");
|
||||||
|
|
||||||
|
bin_with_root(&root)
|
||||||
|
.args([
|
||||||
|
"config",
|
||||||
|
"set",
|
||||||
|
"ownership.db_path",
|
||||||
|
db_path.to_str().unwrap(),
|
||||||
|
])
|
||||||
|
.assert()
|
||||||
|
.success();
|
||||||
|
|
||||||
|
bin_with_root(&root)
|
||||||
|
.args(["ownership", "import", "--file", xlsx_path.to_str().unwrap()])
|
||||||
|
.assert()
|
||||||
|
.failure()
|
||||||
|
.stderr(predicate::str::contains("above-1% holder register layout"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn ownership_import_force_reimports_existing_release() {
|
fn ownership_import_force_reimports_existing_release() {
|
||||||
let root = test_env_dir("ownership-import-force");
|
let root = test_env_dir("ownership-import-force");
|
||||||
|
|
|
||||||
BIN
tests/fixtures/ksei_above1_20260831_excerpt.xlsx
vendored
Normal file
BIN
tests/fixtures/ksei_above1_20260831_excerpt.xlsx
vendored
Normal file
Binary file not shown.
BIN
tests/fixtures/ksei_above5_20260923_excerpt.xlsx
vendored
Normal file
BIN
tests/fixtures/ksei_above5_20260923_excerpt.xlsx
vendored
Normal file
Binary file not shown.
Loading…
Add table
Add a link
Reference in a new issue