diff --git a/Cargo.lock b/Cargo.lock index 8d9b8a5..0367ad2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,6 +88,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "assert_cmd" version = "2.1.2" @@ -302,6 +311,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crossterm" version = "0.29.0" @@ -365,6 +380,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "difflib" version = "0.4.0" @@ -704,6 +730,7 @@ dependencies = [ "thiserror 2.0.18", "toml", "ureq", + "zip", ] [[package]] @@ -1894,8 +1921,37 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror 2.0.18", + "zopfli", +] + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index b210e00..1491e0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ path = "src/main.rs" [features] default = ["ownership"] -ownership = ["dep:rusqlite", "dep:quick-xml", "dep:sha2"] +ownership = ["dep:rusqlite", "dep:quick-xml", "dep:sha2", "dep:zip"] [dependencies] clap = { version = "4", features = ["derive"] } @@ -33,6 +33,7 @@ fastrand = "2" rusqlite = { version = "0.32", features = ["bundled-full"], optional = true } quick-xml = { version = "0.37", optional = true } sha2 = { version = "0.10", optional = true } +zip = { version = "2", default-features = false, features = ["deflate"], optional = true } [dev-dependencies] assert_cmd = "2" diff --git a/FEATURE_SPEC.md b/FEATURE_SPEC.md index 1059f1c..4149612 100644 --- a/FEATURE_SPEC.md +++ b/FEATURE_SPEC.md @@ -26,7 +26,7 @@ Use `TODO.md` as the execution log and smoke-history record. ## Verified Current State -- Current automated coverage is `176` tests: `115` unit and `61` integration. +- Current automated coverage is `180` tests: `117` unit and `63` integration. - Reusable smoke coverage exists via `scripts/live-smoke.sh`; command groups are documented in `docs/SMOKE.md`. - The latest smoke notes in `TODO.md` report passing live table and JSON checks for all shipped `stocks` commands. - Cache/offline parity, JSON startup-error handling, screener input validation, and the recent MSN output cleanups have already been completed. @@ -37,6 +37,7 @@ Use `TODO.md` as the execution log and smoke-history record. - The currently discoverable `above 5%` and `investor-type` BEI families remain different schemas and are now classified and rejected explicitly during import instead of falling through to a generic zero-row parse failure. - Ownership smoke coverage now includes a dedicated `ownership-import` group that verifies live supported import plus expected unsupported-family failures. - `idx ownership sync` now installs maintained SQLite snapshots via a manifest-driven contract with checksum validation, conservative local replacement rules, and fixture-backed regression coverage for install/no-op/force-refresh behavior. +- The KSEI archive ZIP/TXT path is now implemented as a local-file maintainer fallback/backstop, with cross-check coverage against the `above1` PDF contract for a shared monthly release. This means the main gap is no longer endpoint coverage. The remaining work is architecture cleanup, a few correctness edge cases, and selective UX expansion on top of already-shipped commands. @@ -64,6 +65,7 @@ The remaining work is architecture cleanup, a few correctness edge cases, and se | Screener | `idx stocks screen` | Implemented with gaps | Validation landed; expression/preset workflow is still future work | | MSN charts | `idx stocks history --history-provider msn` | Missing | Explicit MSN history still returns unsupported for IDX | | KSEI ownership import/query | `idx ownership import --file`, `idx ownership import --url`, `idx ownership releases`, `idx ownership ticker` | Implemented | Local PDF import and SQLite-backed query flow are verified against the March 2026 KSEI release; remote IDX import now works for the discovered `above 1%` `lamp1` BEI attachment, and legacy `above 5%` / `investor-type` BEI report families are rejected explicitly | +| KSEI archive fallback import | `idx ownership import --file <.zip|.txt>` | Implemented as fallback | Local archive ZIP/TXT ingest maps investor-type/locality buckets into synthetic aggregate holders for validation/backstop use, not the primary ingest surface | | Ownership snapshot sync | `idx ownership sync` | Implemented | Manifest-driven SQLite snapshot install with checksum validation, conservative replacement/no-op rules, and publisher helper script | | Bing ownership CLI | `idx ownership import --fetch-bing` | Not implemented | Client groundwork exists, CLI import path is still deferred | @@ -87,6 +89,7 @@ The following items should no longer be treated as active backlog in this spec: - Real KSEI ownership CLI verification from local file import into SQLite (`7261` rows across `955` tickers on `2026-03-28`) - Ownership remote-import hardening for the `above1` contract, including direct-PDF-only `--url` input, discovery status output, explicit legacy-schema rejection, and live ownership-import smoke coverage - Ownership snapshot publishing/sync contract, including `idx ownership sync`, manifest/checksum validation, conservative local replacement rules, and the `scripts/build-ownership-snapshot.sh` publisher helper +- KSEI archive ZIP/TXT fallback ingest for local `.zip`/`.txt` files, including cross-check coverage against the `above1` PDF holder-register fixture and compatible `releases`/`ticker`/`changes` verification If any of the above regress, capture that in `TODO.md` as a new finding rather than reopening the old section here wholesale. @@ -148,26 +151,6 @@ Done when: - The retry/fallback story for Yahoo failures is deliberate and documented. - SMA200 behavior is either improved or clearly documented as expected. -#### 5. Ownership fallback ingest and cross-check path - -Current state: -- Remote IDX discovery/fetch is already in place for the supported `above1` holder-register family. -- `idx ownership sync` now covers the maintained-snapshot distribution path on top of that import flow. -- The KSEI ZIP/TXT archive remains the main unresolved ownership follow-up and should stay a fallback/cross-check input, not the primary ingest path. - -Why it matters: -- The core ownership product path is now stable: local import, remote IDX discovery/import, and maintained snapshot sync all exist. -- The remaining ownership work is about fallback resilience and validation against a secondary upstream. - -Done when: -- The KSEI ZIP/TXT archive ingest shape is defined. -- At least one monthly archive release is cross-checked against the IDX-PDF-derived SQLite state. -- Fallback ingest produces query-compatible data for `ownership releases`, `ticker`, and `changes`. - -Roadmap: -1. Add KSEI ZIP/TXT ingest later as fallback or cross-check input. -2. Decide whether that fallback stays maintainer-only or becomes a user-facing alternative path. - ### P1 - UX and output contract cleanup Tasks: @@ -191,14 +174,10 @@ Priority order: - Reuse the existing client groundwork in `src/api/msn/bing.rs`. - Define the import shape and output contract for `idx ownership import --fetch-bing`. -3. Ownership fallback ingest and cross-check work - - Treat the KSEI archive (`https://web.ksei.co.id/archive_download/holding_composition`) as fallback/backstop input, not the primary product ingest path. - - Define the ZIP/TXT ingest shape and verify compatibility with the IDX-PDF-derived SQLite contract. - -4. Richer financial statements +3. Richer financial statements - Decide whether to stay with the current single-period model or add multi-period fetch support. -5. New user-facing surfaces from `TODO.md` +4. New user-facing surfaces from `TODO.md` - `market summary` - `market movers` - `market sectors` diff --git a/TODO.md b/TODO.md index ab69b26..ee37310 100644 --- a/TODO.md +++ b/TODO.md @@ -106,13 +106,13 @@ - [x] Batch 3 verification: sync installs into an empty temp data dir, preserves query behavior, and no-ops when already current ### Batch 4 — KSEI ZIP/TXT fallback and cross-check path -- [ ] Keep KSEI ZIP/TXT ingest as fallback and validation/backstop work, not the first milestone -- [ ] Define whether the KSEI archive is only a maintainer fallback or a user-facing alternative import source -- [ ] Add cross-check coverage between IDX-PDF-derived output and KSEI-archive-derived output for at least one monthly release -- [ ] Batch 4 verification: `cargo build` -- [ ] Batch 4 verification: `cargo clippy -- -D warnings` -- [ ] Batch 4 verification: `cargo test` -- [ ] Batch 4 verification: fallback ingest produces a compatible SQLite state for `ownership releases`, `ticker`, and `changes` +- [x] Keep KSEI ZIP/TXT ingest as fallback and validation/backstop work, not the first milestone +- [x] Define whether the KSEI archive is only a maintainer fallback or a user-facing alternative import source +- [x] Add cross-check coverage between IDX-PDF-derived output and KSEI-archive-derived output for at least one monthly release +- [x] Batch 4 verification: `cargo build` +- [x] Batch 4 verification: `cargo clippy -- -D warnings` +- [x] Batch 4 verification: `cargo test` +- [x] Batch 4 verification: fallback ingest produces a compatible SQLite state for `ownership releases`, `ticker`, and `changes` ## 📋 Backlog (per SPEC.md) - [ ] `market summary` — IHSG index, market breadth @@ -122,7 +122,7 @@ - [ ] `screen presets` / `screen run ` — built-in screener presets - [ ] `watchlist` commands — create, manage, live watch - [ ] `alerts` system (v0.2+) — price alerts with daemon -- [ ] `completions ` — shell completion generation +- [x] `completions ` — shell completion generation - [ ] CSV/TSV output formats - [ ] Additional providers (Alpha Vantage, Twelve Data, IDX official) @@ -168,6 +168,8 @@ - [x] Regression coverage now covers default `ownership discover` behavior, status visibility, listing-page rejection, duplicate SHA imports, and explicit unsupported-schema detection with compact `stext` fixtures plus fake-`mutool` CLI tests - [x] New `ownership-import` smoke coverage now discovers the current live `above1`/`above5`/`investor-type` URLs, imports the supported `above1` attachment successfully, and confirms the legacy families fail with explicit unsupported-schema UX (`tmp/live-smoke/20260330-160201`) - [x] New ownership snapshot sync coverage now verifies manifest parsing, checksum validation, install into an empty temp data dir, preserved query behavior for `releases`/`ticker`/`changes`, no-op sync when current, and `--force` refresh via fixture-backed local manifests on `2026-03-31` +- [x] KSEI archive ZIP/TXT fallback import now works through `ownership import --file` for local `.zip` and `.txt` inputs, using synthetic investor-type/locality aggregate holders as a maintainer backstop rather than the primary product ingest +- [x] Batch 4 coverage now cross-checks the live-like `2026-02-27` `above1` PDF fixture against the matching KSEI archive bucket excerpt and verifies fallback import/query behavior for `releases`, `ticker`, and `changes` ## 🐛 Known Issues - [ ] Yahoo Finance returns 429 from datacenter IPs occasionally diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c0c240c..114e7e2 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -61,6 +61,7 @@ Unlike the `stocks` module (live-fetch), ownership is **import-then-query**: ### Data Sources - **KSEI** — official ≥1% shareholder registry (monthly PDF from IDX) +- **KSEI archive** — monthly ZIP/TXT balance-position matrix, used as a local fallback/backstop import path - **Bing Finance** — global institutional ownership (REST API, quarterly) ### Parser Pipeline @@ -68,6 +69,10 @@ Unlike the `stocks` module (live-fetch), ownership is **import-then-query**: KSEI PDF → mutool stext (XML with coordinates) → quick-xml parse → KseiRawRow → normalize (ID locale numbers, dates, entity names) → KseiHolding → SQLite INSERT (within transaction) + +KSEI archive ZIP/TXT → pipe-delimited balance-position rows + → map investor-type/locality buckets into synthetic aggregate holders + → SQLite INSERT (within transaction) ``` ## Data Flow Patterns diff --git a/src/cli/ownership.rs b/src/cli/ownership.rs index 6289b6a..6c3329a 100644 --- a/src/cli/ownership.rs +++ b/src/cli/ownership.rs @@ -18,7 +18,7 @@ use crate::output::table::format_idr; use crate::ownership::types::{ ChangeType, FlowSignal, HolderRow, KseiHolding, OwnershipRelease, OwnershipSource, }; -use crate::ownership::{db, entities, graph, parser, remote, search, snapshot}; +use crate::ownership::{archive, db, entities, graph, parser, remote, search, snapshot}; #[derive(Debug, Args)] pub struct OwnershipCmd { @@ -30,7 +30,7 @@ pub struct OwnershipCmd { pub enum OwnershipCommand { /// Discover the latest IDX-hosted ownership report URLs. Discover(DiscoverArgs), - /// Import ownership data from KSEI PDF or Bing API. + /// Import ownership data from KSEI PDF or archive fallback files. Import(ImportArgs), /// Install or refresh a maintained ownership SQLite snapshot. Sync(SyncArgs), @@ -71,7 +71,7 @@ pub struct ImportArgs { /// URL to a remote ownership PDF. #[arg(long)] pub url: Option, - /// Path to local KSEI PDF file. + /// Path to local KSEI ownership PDF, ZIP, or TXT file. #[arg(long)] pub file: Option, /// Fetch Bing institutional data for these symbols. @@ -773,30 +773,40 @@ fn handle_import(args: &ImportArgs, config: &IdxConfig) -> Result<(), IdxError> } } - let Some(pdf_input) = resolve_pdf_input(args)? else { + let Some(import_input) = resolve_import_input(args)? else { return Ok(()); }; let conn = db::open_db(config)?; - let sha256 = sha256_file(&pdf_input.pdf_path)?; + let sha256 = sha256_file(&import_input.import_path)?; if !args.force && db::release_exists(&conn, &sha256)? { println!("Release already imported (sha256: {sha256}). Use --force to re-import."); return Ok(()); } - let raw_rows = parser::parse_ksei_pdf(&pdf_input.pdf_path)?; - if raw_rows.is_empty() { - return Err(IdxError::ParseError( - "no KSEI rows parsed from PDF".to_string(), - )); - } + let drafts = match import_input.format { + ImportInputFormat::Pdf => { + let raw_rows = parser::parse_ksei_pdf(&import_input.import_path)?; + if raw_rows.is_empty() { + return Err(IdxError::ParseError( + "no KSEI rows parsed from PDF".to_string(), + )); + } - let mut holdings = Vec::with_capacity(raw_rows.len()); + let mut drafts = Vec::with_capacity(raw_rows.len()); + for raw in &raw_rows { + drafts.push(entities::normalize_ksei_row(raw)?); + } + drafts + } + ImportInputFormat::Archive => archive::parse_balancepos_file(&import_input.import_path)?, + }; + + let mut holdings = Vec::with_capacity(drafts.len()); let mut ticker_ids = HashSet::new(); - for raw in &raw_rows { - let draft = entities::normalize_ksei_row(raw)?; + for draft in drafts { let ticker_id = db::upsert_ticker(&conn, &draft.ticker_code, draft.issuer_name.as_deref())?; let entity_id = entities::resolve_entity(&conn, &draft.raw_investor_name, OwnershipSource::Ksei)?; @@ -831,7 +841,7 @@ fn handle_import(args: &ImportArgs, config: &IdxConfig) -> Result<(), IdxError> let release = OwnershipRelease { id: 0, - source_url: pdf_input.source_url, + source_url: import_input.source_url, sha256, as_of_date, row_count: inserted_rows, @@ -900,23 +910,31 @@ fn parse_discovery_family(raw: &str) -> Result, + format: ImportInputFormat, } -fn resolve_pdf_input(args: &ImportArgs) -> Result, IdxError> { +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ImportInputFormat { + Pdf, + Archive, +} + +fn resolve_import_input(args: &ImportArgs) -> Result, IdxError> { if let Some(path) = &args.file { if !path.exists() { return Err(IdxError::Io(format!( - "input PDF not found: {}", + "input ownership file not found: {}", path.display() ))); } - return Ok(Some(ResolvedPdfInput { - pdf_path: path.clone(), + return Ok(Some(ResolvedImportInput { + import_path: path.clone(), source_url: None, + format: detect_local_import_format(path)?, })); } @@ -925,15 +943,34 @@ fn resolve_pdf_input(args: &ImportArgs) -> Result, IdxE validate_import_url(trimmed)?; let target = cache_pdf_path(trimmed)?; download_pdf(trimmed, &target)?; - return Ok(Some(ResolvedPdfInput { - pdf_path: target, + return Ok(Some(ResolvedImportInput { + import_path: target, source_url: Some(trimmed.to_string()), + format: ImportInputFormat::Pdf, })); } Ok(None) } +fn detect_local_import_format(path: &Path) -> Result { + match path + .extension() + .and_then(|value| value.to_str()) + .map(|value| value.trim().to_ascii_lowercase()) + .as_deref() + { + Some("pdf") => Ok(ImportInputFormat::Pdf), + Some("zip") | Some("txt") if archive::supports_local_archive_file(path) => { + Ok(ImportInputFormat::Archive) + } + _ => Err(IdxError::InvalidInput(format!( + "unsupported local ownership file {}; expected a .pdf, .zip, or .txt input", + path.display() + ))), + } +} + fn cache_pdf_path(url: &str) -> Result { let dirs = ProjectDirs::from("", "", "idx") .ok_or_else(|| IdxError::Io("unable to resolve cache directory".to_string()))?; diff --git a/src/ownership/archive.rs b/src/ownership/archive.rs new file mode 100644 index 0000000..1c7f3a9 --- /dev/null +++ b/src/ownership/archive.rs @@ -0,0 +1,360 @@ +use std::fs; +use std::io::{Cursor, Read}; +use std::path::Path; + +use chrono::NaiveDate; + +use crate::error::IdxError; +use crate::ownership::types::{InvestorTypeCode, KseiHoldingDraft, Locality}; + +const EXPECTED_HEADER: &[&str] = &[ + "Date", + "Code", + "Type", + "Sec. Num", + "Price", + "Local IS", + "Local CP", + "Local PF", + "Local IB", + "Local ID", + "Local MF", + "Local SC", + "Local FD", + "Local OT", + "Total", + "Foreign IS", + "Foreign CP", + "Foreign PF", + "Foreign IB", + "Foreign ID", + "Foreign MF", + "Foreign SC", + "Foreign FD", + "Foreign OT", + "Total", +]; + +const LOCAL_BUCKETS: &[(&str, usize)] = &[ + ("IS", 5), + ("CP", 6), + ("PF", 7), + ("IB", 8), + ("ID", 9), + ("MF", 10), + ("SC", 11), + ("FD", 12), + ("OT", 13), +]; + +const FOREIGN_BUCKETS: &[(&str, usize)] = &[ + ("IS", 15), + ("CP", 16), + ("PF", 17), + ("IB", 18), + ("ID", 19), + ("MF", 20), + ("SC", 21), + ("FD", 22), + ("OT", 23), +]; + +pub fn supports_local_archive_file(path: &Path) -> bool { + path.extension() + .and_then(|value| value.to_str()) + .map(|value| { + let ext = value.trim().to_ascii_lowercase(); + ext == "txt" || ext == "zip" + }) + .unwrap_or(false) +} + +pub fn parse_balancepos_file(path: &Path) -> Result, IdxError> { + let raw = match path + .extension() + .and_then(|value| value.to_str()) + .map(|value| value.trim().to_ascii_lowercase()) + .as_deref() + { + Some("txt") => fs::read_to_string(path).map_err(|e| { + IdxError::Io(format!( + "failed to read KSEI archive TXT {}: {e}", + path.display() + )) + })?, + Some("zip") => extract_txt_from_zip(path)?, + _ => { + return Err(IdxError::InvalidInput(format!( + "unsupported local ownership archive file {}; expected .txt or .zip", + path.display() + ))); + } + }; + + parse_balancepos_text(&raw) +} + +pub fn parse_balancepos_text(raw: &str) -> Result, IdxError> { + let mut lines = raw.lines().filter(|line| !line.trim().is_empty()); + let header = lines + .next() + .ok_or_else(|| IdxError::ParseError("empty KSEI archive TXT input".to_string()))?; + validate_header(header)?; + + let mut drafts = Vec::new(); + for (line_number, line) in lines.enumerate() { + let columns: Vec<&str> = line.split('|').map(str::trim).collect(); + if columns.len() != EXPECTED_HEADER.len() { + return Err(IdxError::ParseError(format!( + "invalid KSEI archive TXT row {}: expected {} columns, got {}", + line_number + 2, + EXPECTED_HEADER.len(), + columns.len() + ))); + } + + if !columns[2].eq_ignore_ascii_case("EQUITY") { + continue; + } + + let report_date = parse_archive_date(columns[0])?; + let ticker_code = columns[1].trim().to_uppercase(); + let sec_num = parse_archive_number(columns[3], line_number + 2, "Sec. Num")?; + if sec_num <= 0 { + continue; + } + + append_bucket_drafts( + &mut drafts, + &ticker_code, + report_date, + sec_num, + columns.as_slice(), + Locality::Local, + LOCAL_BUCKETS, + )?; + append_bucket_drafts( + &mut drafts, + &ticker_code, + report_date, + sec_num, + columns.as_slice(), + Locality::Foreign, + FOREIGN_BUCKETS, + )?; + } + + if drafts.is_empty() { + return Err(IdxError::ParseError( + "no importable EQUITY rows found in KSEI archive TXT".to_string(), + )); + } + + Ok(drafts) +} + +fn extract_txt_from_zip(path: &Path) -> Result { + let bytes = fs::read(path).map_err(|e| { + IdxError::Io(format!( + "failed to read KSEI archive ZIP {}: {e}", + path.display() + )) + })?; + let cursor = Cursor::new(bytes); + let mut zip = zip::ZipArchive::new(cursor) + .map_err(|e| IdxError::ParseError(format!("failed to open KSEI archive ZIP: {e}")))?; + + for index in 0..zip.len() { + let mut file = zip.by_index(index).map_err(|e| { + IdxError::ParseError(format!("failed to read KSEI archive ZIP entry: {e}")) + })?; + if file.is_dir() { + continue; + } + + let name = file.name().to_ascii_lowercase(); + if !name.ends_with(".txt") { + continue; + } + + let mut output = String::new(); + file.read_to_string(&mut output).map_err(|e| { + IdxError::ParseError(format!("failed to decode KSEI archive TXT entry: {e}")) + })?; + return Ok(output); + } + + Err(IdxError::ParseError( + "KSEI archive ZIP did not contain a TXT payload".to_string(), + )) +} + +fn validate_header(header: &str) -> Result<(), IdxError> { + let columns: Vec<&str> = header.split('|').map(str::trim).collect(); + if columns != EXPECTED_HEADER { + return Err(IdxError::ParseError( + "KSEI archive TXT header did not match the expected balancepos layout".to_string(), + )); + } + Ok(()) +} + +fn parse_archive_date(raw: &str) -> Result { + let trimmed = raw.trim(); + if trimmed.len() != 11 { + return Err(IdxError::ParseError(format!( + "invalid KSEI archive date `{trimmed}`" + ))); + } + + let canonical = format!( + "{}-{}-{}", + &trimmed[..2], + titlecase_month(&trimmed[3..6]), + &trimmed[7..11] + ); + + NaiveDate::parse_from_str(&canonical, "%d-%b-%Y") + .map_err(|e| IdxError::ParseError(format!("invalid KSEI archive date `{trimmed}`: {e}"))) +} + +fn titlecase_month(raw: &str) -> String { + let upper = raw.trim().to_ascii_uppercase(); + let mut chars = upper.chars(); + match chars.next() { + Some(first) => { + let mut output = String::new(); + output.push(first.to_ascii_uppercase()); + output.push_str(&chars.as_str().to_ascii_lowercase()); + output + } + None => String::new(), + } +} + +fn parse_archive_number(raw: &str, line_number: usize, field: &str) -> Result { + raw.trim().parse::().map_err(|e| { + IdxError::ParseError(format!( + "invalid KSEI archive TXT value in row {line_number} field `{field}`: {e}" + )) + }) +} + +fn append_bucket_drafts( + drafts: &mut Vec, + ticker_code: &str, + report_date: NaiveDate, + sec_num: i64, + columns: &[&str], + locality: Locality, + buckets: &[(&str, usize)], +) -> Result<(), IdxError> { + for (investor_type, column_index) in buckets { + let shares = + parse_archive_number(columns[*column_index], 0, investor_type).map_err(|_| { + IdxError::ParseError(format!( + "invalid KSEI archive TXT share count for {ticker_code} {investor_type}" + )) + })?; + if shares <= 0 { + continue; + } + + drafts.push(KseiHoldingDraft { + ticker_code: ticker_code.to_string(), + issuer_name: None, + raw_investor_name: synthetic_holder_name(locality, investor_type), + investor_type: Some(InvestorTypeCode((*investor_type).to_string())), + locality: Some(locality), + nationality: None, + domicile: None, + holdings_scripless: shares, + holdings_scrip: 0, + total_shares: shares, + percentage_bps: compute_percentage_bps(shares, sec_num), + report_date, + }); + } + + Ok(()) +} + +fn synthetic_holder_name(locality: Locality, investor_type: &str) -> String { + let prefix = match locality { + Locality::Local => "LOCAL", + Locality::Foreign => "FOREIGN", + }; + format!("KSEI AGGREGATE {prefix} {investor_type}") +} + +fn compute_percentage_bps(shares: i64, sec_num: i64) -> i64 { + let shares_i128 = i128::from(shares); + let sec_num_i128 = i128::from(sec_num); + let rounded = ((shares_i128 * 10_000) + (sec_num_i128 / 2)) / sec_num_i128; + i64::try_from(rounded).unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::parse_balancepos_text; + use crate::ownership::entities::normalize_ksei_row; + use crate::ownership::parser::parse_stext_xml; + use crate::ownership::types::Locality; + + #[test] + fn parses_balancepos_excerpt_into_bucket_holders() { + let raw = include_str!("../../tests/fixtures/ksei_balancepos_20260227_excerpt.txt"); + let drafts = parse_balancepos_text(raw).expect("balancepos excerpt parses"); + + let local_cp = drafts + .iter() + .find(|draft| { + draft.ticker_code == "AADI" + && draft.investor_type.as_ref().map(|code| code.0.as_str()) == Some("CP") + && draft.locality == Some(Locality::Local) + }) + .expect("AADI local CP bucket"); + + assert_eq!(local_cp.raw_investor_name, "KSEI AGGREGATE LOCAL CP"); + assert_eq!(local_cp.total_shares, 5_035_745_466); + assert_eq!(local_cp.percentage_bps, 6467); + } + + #[test] + fn balancepos_cross_check_contains_pdf_holder_bucket() { + let pdf_rows = parse_stext_xml(include_str!( + "../../tests/fixtures/ksei_above1_stext_excerpt.xml" + )) + .expect("pdf fixture rows"); + let pdf_drafts: Vec<_> = pdf_rows + .iter() + .map(normalize_ksei_row) + .collect::>() + .expect("normalized pdf drafts"); + let archive_drafts = parse_balancepos_text(include_str!( + "../../tests/fixtures/ksei_balancepos_20260227_excerpt.txt" + )) + .expect("archive drafts"); + + let pdf_local_cp = pdf_drafts + .iter() + .find(|draft| { + draft.ticker_code == "AADI" + && draft.investor_type.as_ref().map(|code| code.0.as_str()) == Some("CP") + && draft.locality == Some(Locality::Local) + }) + .expect("pdf local cp"); + let archive_local_cp = archive_drafts + .iter() + .find(|draft| { + draft.ticker_code == "AADI" + && draft.investor_type.as_ref().map(|code| code.0.as_str()) == Some("CP") + && draft.locality == Some(Locality::Local) + }) + .expect("archive local cp"); + + assert_eq!(pdf_local_cp.report_date, archive_local_cp.report_date); + assert!(pdf_local_cp.total_shares <= archive_local_cp.total_shares); + assert!(pdf_local_cp.percentage_bps <= archive_local_cp.percentage_bps); + } +} diff --git a/src/ownership/mod.rs b/src/ownership/mod.rs index feaf860..3735ff5 100644 --- a/src/ownership/mod.rs +++ b/src/ownership/mod.rs @@ -1,3 +1,4 @@ +pub mod archive; pub mod db; pub mod entities; pub mod graph; diff --git a/tests/cli.rs b/tests/cli.rs index d61cad0..bd0e5d9 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -10,6 +10,7 @@ use assert_cmd::Command; use predicates::prelude::*; use rusqlite::Connection; use sha2::{Digest, Sha256}; +use zip::write::SimpleFileOptions; fn bin() -> Command { let current = std::env::current_exe().expect("current test executable path"); @@ -167,6 +168,18 @@ fn pdf_url(base: &str, name: &str) -> String { format!("{base}/{name}.pdf") } +fn write_zip_with_text(zip_path: &Path, entry_name: &str, text: &str) { + let file = fs::File::create(zip_path).expect("create zip fixture"); + let mut writer = zip::ZipWriter::new(file); + writer + .start_file(entry_name, SimpleFileOptions::default()) + .expect("start zip file"); + writer + .write_all(text.as_bytes()) + .expect("write zip fixture text"); + writer.finish().expect("finish zip fixture"); +} + fn sha256_hex(bytes: &[u8]) -> String { let mut hasher = Sha256::new(); hasher.update(bytes); @@ -1175,6 +1188,111 @@ fn ownership_import_url_rejects_legacy_investor_type_pdf_schema() { )); } +#[test] +fn ownership_import_file_txt_archive_succeeds() { + let root = test_env_dir("ownership-import-txt-archive"); + let db_path = root.join("ownership.db"); + let txt_path = root.join("Balancepos20260227.txt"); + fs::write( + &txt_path, + include_str!("fixtures/ksei_balancepos_20260227_excerpt.txt"), + ) + .expect("write txt archive fixture"); + + bin_with_root(&root) + .args([ + "config", + "set", + "ownership.db_path", + db_path.to_str().unwrap(), + ]) + .assert() + .success(); + + bin_with_root(&root) + .args(["ownership", "import", "--file", txt_path.to_str().unwrap()]) + .assert() + .success() + .stdout(predicate::str::contains("Imported 18 rows for 1 tickers")); + + bin_with_root(&root) + .args(["ownership", "ticker", "AADI", "--source", "ksei"]) + .assert() + .success() + .stdout(predicate::str::contains("KSEI AGGREGATE LOCAL CP")) + .stdout(predicate::str::contains("64.67%")); +} + +#[test] +fn ownership_import_file_zip_archive_supports_releases_ticker_and_changes() { + let root = test_env_dir("ownership-import-zip-archive"); + let db_path = root.join("ownership.db"); + let jan_zip = root.join("BalanceposEfek20260130.zip"); + let feb_zip = root.join("BalanceposEfek20260227.zip"); + + write_zip_with_text( + &jan_zip, + "Balancepos20260130.txt", + include_str!("fixtures/ksei_balancepos_20260130_excerpt.txt"), + ); + write_zip_with_text( + &feb_zip, + "Balancepos20260227.txt", + include_str!("fixtures/ksei_balancepos_20260227_excerpt.txt"), + ); + + bin_with_root(&root) + .args([ + "config", + "set", + "ownership.db_path", + db_path.to_str().unwrap(), + ]) + .assert() + .success(); + + bin_with_root(&root) + .args(["ownership", "import", "--file", jan_zip.to_str().unwrap()]) + .assert() + .success() + .stdout(predicate::str::contains("Imported 18 rows for 1 tickers")); + + bin_with_root(&root) + .args(["ownership", "import", "--file", feb_zip.to_str().unwrap()]) + .assert() + .success() + .stdout(predicate::str::contains("Imported 18 rows for 1 tickers")); + + bin_with_root(&root) + .args(["ownership", "releases"]) + .assert() + .success() + .stdout(predicate::str::contains("2026-02-27")) + .stdout(predicate::str::contains("2026-01-30")); + + bin_with_root(&root) + .args(["ownership", "ticker", "AADI", "--source", "ksei"]) + .assert() + .success() + .stdout(predicate::str::contains("KSEI AGGREGATE LOCAL CP")) + .stdout(predicate::str::contains("KSEI AGGREGATE FOREIGN MF")); + + bin_with_root(&root) + .args([ + "ownership", + "changes", + "--from", + "2026-01-30", + "--to", + "2026-02-27", + ]) + .assert() + .success() + .stdout(predicate::str::contains("AADI")) + .stdout(predicate::str::contains("KSEI AGGREGATE LOCAL CP")) + .stdout(predicate::str::contains("DECREASED")); +} + #[test] fn ownership_sync_installs_snapshot_and_preserves_query_behavior() { let publisher_root = test_env_dir("ownership-sync-publisher"); diff --git a/tests/fixtures/ksei_balancepos_20260130_excerpt.txt b/tests/fixtures/ksei_balancepos_20260130_excerpt.txt new file mode 100644 index 0000000..f27590d --- /dev/null +++ b/tests/fixtures/ksei_balancepos_20260130_excerpt.txt @@ -0,0 +1,2 @@ +Date|Code|Type|Sec. Num|Price|Local IS|Local CP|Local PF|Local IB|Local ID|Local MF|Local SC|Local FD|Local OT|Total|Foreign IS|Foreign CP|Foreign PF|Foreign IB|Foreign ID|Foreign MF|Foreign SC|Foreign FD|Foreign OT|Total +30-JAN-2026|AADI|EQUITY|7786891760|7600|58704495|5046580437|13135269|41|1868661103|91475881|27984191|1966845|85275|7108593537|1150700|55820749|58897143|207783974|2127405|239139003|39765137|2493100|71121012|678298223 diff --git a/tests/fixtures/ksei_balancepos_20260227_excerpt.txt b/tests/fixtures/ksei_balancepos_20260227_excerpt.txt new file mode 100644 index 0000000..66d1d25 --- /dev/null +++ b/tests/fixtures/ksei_balancepos_20260227_excerpt.txt @@ -0,0 +1,2 @@ +Date|Code|Type|Sec. Num|Price|Local IS|Local CP|Local PF|Local IB|Local ID|Local MF|Local SC|Local FD|Local OT|Total|Foreign IS|Foreign CP|Foreign PF|Foreign IB|Foreign ID|Foreign MF|Foreign SC|Foreign FD|Foreign OT|Total +27-FEB-2026|AADI|EQUITY|7786891760|9250|100730965|5035745466|15622269|41|1817092689|117982099|22932698|2060140|105275|7112271642|1150700|66705786|43311343|204936195|1261705|262226503|42723690|2493100|49811096|674620118