Add KSEI archive fallback import

This commit is contained in:
Rasyidan Akbar F. 2026-03-31 19:29:00 +07:00
commit 1f5c4b2798
11 changed files with 622 additions and 59 deletions

56
Cargo.lock generated
View file

@ -88,6 +88,15 @@ dependencies = [
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
[[package]]
name = "arbitrary"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"
dependencies = [
"derive_arbitrary",
]
[[package]] [[package]]
name = "assert_cmd" name = "assert_cmd"
version = "2.1.2" version = "2.1.2"
@ -302,6 +311,12 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]] [[package]]
name = "crossterm" name = "crossterm"
version = "0.29.0" version = "0.29.0"
@ -365,6 +380,17 @@ dependencies = [
"powerfmt", "powerfmt",
] ]
[[package]]
name = "derive_arbitrary"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "difflib" name = "difflib"
version = "0.4.0" version = "0.4.0"
@ -704,6 +730,7 @@ dependencies = [
"thiserror 2.0.18", "thiserror 2.0.18",
"toml", "toml",
"ureq", "ureq",
"zip",
] ]
[[package]] [[package]]
@ -1894,8 +1921,37 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "zip"
version = "2.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50"
dependencies = [
"arbitrary",
"crc32fast",
"crossbeam-utils",
"displaydoc",
"flate2",
"indexmap",
"memchr",
"thiserror 2.0.18",
"zopfli",
]
[[package]] [[package]]
name = "zmij" name = "zmij"
version = "1.0.21" version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
[[package]]
name = "zopfli"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
dependencies = [
"bumpalo",
"crc32fast",
"log",
"simd-adler32",
]

View file

@ -15,7 +15,7 @@ path = "src/main.rs"
[features] [features]
default = ["ownership"] default = ["ownership"]
ownership = ["dep:rusqlite", "dep:quick-xml", "dep:sha2"] ownership = ["dep:rusqlite", "dep:quick-xml", "dep:sha2", "dep:zip"]
[dependencies] [dependencies]
clap = { version = "4", features = ["derive"] } clap = { version = "4", features = ["derive"] }
@ -33,6 +33,7 @@ fastrand = "2"
rusqlite = { version = "0.32", features = ["bundled-full"], optional = true } rusqlite = { version = "0.32", features = ["bundled-full"], optional = true }
quick-xml = { version = "0.37", optional = true } quick-xml = { version = "0.37", optional = true }
sha2 = { version = "0.10", optional = true } sha2 = { version = "0.10", optional = true }
zip = { version = "2", default-features = false, features = ["deflate"], optional = true }
[dev-dependencies] [dev-dependencies]
assert_cmd = "2" assert_cmd = "2"

View file

@ -26,7 +26,7 @@ Use `TODO.md` as the execution log and smoke-history record.
## Verified Current State ## Verified Current State
- Current automated coverage is `176` tests: `115` unit and `61` integration. - Current automated coverage is `180` tests: `117` unit and `63` integration.
- Reusable smoke coverage exists via `scripts/live-smoke.sh`; command groups are documented in `docs/SMOKE.md`. - Reusable smoke coverage exists via `scripts/live-smoke.sh`; command groups are documented in `docs/SMOKE.md`.
- The latest smoke notes in `TODO.md` report passing live table and JSON checks for all shipped `stocks` commands. - The latest smoke notes in `TODO.md` report passing live table and JSON checks for all shipped `stocks` commands.
- Cache/offline parity, JSON startup-error handling, screener input validation, and the recent MSN output cleanups have already been completed. - Cache/offline parity, JSON startup-error handling, screener input validation, and the recent MSN output cleanups have already been completed.
@ -37,6 +37,7 @@ Use `TODO.md` as the execution log and smoke-history record.
- The currently discoverable `above 5%` and `investor-type` BEI families remain different schemas and are now classified and rejected explicitly during import instead of falling through to a generic zero-row parse failure. - The currently discoverable `above 5%` and `investor-type` BEI families remain different schemas and are now classified and rejected explicitly during import instead of falling through to a generic zero-row parse failure.
- Ownership smoke coverage now includes a dedicated `ownership-import` group that verifies live supported import plus expected unsupported-family failures. - Ownership smoke coverage now includes a dedicated `ownership-import` group that verifies live supported import plus expected unsupported-family failures.
- `idx ownership sync` now installs maintained SQLite snapshots via a manifest-driven contract with checksum validation, conservative local replacement rules, and fixture-backed regression coverage for install/no-op/force-refresh behavior. - `idx ownership sync` now installs maintained SQLite snapshots via a manifest-driven contract with checksum validation, conservative local replacement rules, and fixture-backed regression coverage for install/no-op/force-refresh behavior.
- The KSEI archive ZIP/TXT path is now implemented as a local-file maintainer fallback/backstop, with cross-check coverage against the `above1` PDF contract for a shared monthly release.
This means the main gap is no longer endpoint coverage. This means the main gap is no longer endpoint coverage.
The remaining work is architecture cleanup, a few correctness edge cases, and selective UX expansion on top of already-shipped commands. The remaining work is architecture cleanup, a few correctness edge cases, and selective UX expansion on top of already-shipped commands.
@ -64,6 +65,7 @@ The remaining work is architecture cleanup, a few correctness edge cases, and se
| Screener | `idx stocks screen` | Implemented with gaps | Validation landed; expression/preset workflow is still future work | | Screener | `idx stocks screen` | Implemented with gaps | Validation landed; expression/preset workflow is still future work |
| MSN charts | `idx stocks history --history-provider msn` | Missing | Explicit MSN history still returns unsupported for IDX | | MSN charts | `idx stocks history --history-provider msn` | Missing | Explicit MSN history still returns unsupported for IDX |
| KSEI ownership import/query | `idx ownership import --file`, `idx ownership import --url`, `idx ownership releases`, `idx ownership ticker` | Implemented | Local PDF import and SQLite-backed query flow are verified against the March 2026 KSEI release; remote IDX import now works for the discovered `above 1%` `lamp1` BEI attachment, and legacy `above 5%` / `investor-type` BEI report families are rejected explicitly | | KSEI ownership import/query | `idx ownership import --file`, `idx ownership import --url`, `idx ownership releases`, `idx ownership ticker` | Implemented | Local PDF import and SQLite-backed query flow are verified against the March 2026 KSEI release; remote IDX import now works for the discovered `above 1%` `lamp1` BEI attachment, and legacy `above 5%` / `investor-type` BEI report families are rejected explicitly |
| KSEI archive fallback import | `idx ownership import --file <.zip|.txt>` | Implemented as fallback | Local archive ZIP/TXT ingest maps investor-type/locality buckets into synthetic aggregate holders for validation/backstop use, not the primary ingest surface |
| Ownership snapshot sync | `idx ownership sync` | Implemented | Manifest-driven SQLite snapshot install with checksum validation, conservative replacement/no-op rules, and publisher helper script | | Ownership snapshot sync | `idx ownership sync` | Implemented | Manifest-driven SQLite snapshot install with checksum validation, conservative replacement/no-op rules, and publisher helper script |
| Bing ownership CLI | `idx ownership import --fetch-bing` | Not implemented | Client groundwork exists, CLI import path is still deferred | | Bing ownership CLI | `idx ownership import --fetch-bing` | Not implemented | Client groundwork exists, CLI import path is still deferred |
@ -87,6 +89,7 @@ The following items should no longer be treated as active backlog in this spec:
- Real KSEI ownership CLI verification from local file import into SQLite (`7261` rows across `955` tickers on `2026-03-28`) - Real KSEI ownership CLI verification from local file import into SQLite (`7261` rows across `955` tickers on `2026-03-28`)
- Ownership remote-import hardening for the `above1` contract, including direct-PDF-only `--url` input, discovery status output, explicit legacy-schema rejection, and live ownership-import smoke coverage - Ownership remote-import hardening for the `above1` contract, including direct-PDF-only `--url` input, discovery status output, explicit legacy-schema rejection, and live ownership-import smoke coverage
- Ownership snapshot publishing/sync contract, including `idx ownership sync`, manifest/checksum validation, conservative local replacement rules, and the `scripts/build-ownership-snapshot.sh` publisher helper - Ownership snapshot publishing/sync contract, including `idx ownership sync`, manifest/checksum validation, conservative local replacement rules, and the `scripts/build-ownership-snapshot.sh` publisher helper
- KSEI archive ZIP/TXT fallback ingest for local `.zip`/`.txt` files, including cross-check coverage against the `above1` PDF holder-register fixture and compatible `releases`/`ticker`/`changes` verification
If any of the above regress, capture that in `TODO.md` as a new finding rather than reopening the old section here wholesale. If any of the above regress, capture that in `TODO.md` as a new finding rather than reopening the old section here wholesale.
@ -148,26 +151,6 @@ Done when:
- The retry/fallback story for Yahoo failures is deliberate and documented. - The retry/fallback story for Yahoo failures is deliberate and documented.
- SMA200 behavior is either improved or clearly documented as expected. - SMA200 behavior is either improved or clearly documented as expected.
#### 5. Ownership fallback ingest and cross-check path
Current state:
- Remote IDX discovery/fetch is already in place for the supported `above1` holder-register family.
- `idx ownership sync` now covers the maintained-snapshot distribution path on top of that import flow.
- The KSEI ZIP/TXT archive remains the main unresolved ownership follow-up and should stay a fallback/cross-check input, not the primary ingest path.
Why it matters:
- The core ownership product path is now stable: local import, remote IDX discovery/import, and maintained snapshot sync all exist.
- The remaining ownership work is about fallback resilience and validation against a secondary upstream.
Done when:
- The KSEI ZIP/TXT archive ingest shape is defined.
- At least one monthly archive release is cross-checked against the IDX-PDF-derived SQLite state.
- Fallback ingest produces query-compatible data for `ownership releases`, `ticker`, and `changes`.
Roadmap:
1. Add KSEI ZIP/TXT ingest later as fallback or cross-check input.
2. Decide whether that fallback stays maintainer-only or becomes a user-facing alternative path.
### P1 - UX and output contract cleanup ### P1 - UX and output contract cleanup
Tasks: Tasks:
@ -191,14 +174,10 @@ Priority order:
- Reuse the existing client groundwork in `src/api/msn/bing.rs`. - Reuse the existing client groundwork in `src/api/msn/bing.rs`.
- Define the import shape and output contract for `idx ownership import --fetch-bing`. - Define the import shape and output contract for `idx ownership import --fetch-bing`.
3. Ownership fallback ingest and cross-check work 3. Richer financial statements
- Treat the KSEI archive (`https://web.ksei.co.id/archive_download/holding_composition`) as fallback/backstop input, not the primary product ingest path.
- Define the ZIP/TXT ingest shape and verify compatibility with the IDX-PDF-derived SQLite contract.
4. Richer financial statements
- Decide whether to stay with the current single-period model or add multi-period fetch support. - Decide whether to stay with the current single-period model or add multi-period fetch support.
5. New user-facing surfaces from `TODO.md` 4. New user-facing surfaces from `TODO.md`
- `market summary` - `market summary`
- `market movers` - `market movers`
- `market sectors` - `market sectors`

18
TODO.md
View file

@ -106,13 +106,13 @@
- [x] Batch 3 verification: sync installs into an empty temp data dir, preserves query behavior, and no-ops when already current - [x] Batch 3 verification: sync installs into an empty temp data dir, preserves query behavior, and no-ops when already current
### Batch 4 — KSEI ZIP/TXT fallback and cross-check path ### Batch 4 — KSEI ZIP/TXT fallback and cross-check path
- [ ] Keep KSEI ZIP/TXT ingest as fallback and validation/backstop work, not the first milestone - [x] Keep KSEI ZIP/TXT ingest as fallback and validation/backstop work, not the first milestone
- [ ] Define whether the KSEI archive is only a maintainer fallback or a user-facing alternative import source - [x] Define whether the KSEI archive is only a maintainer fallback or a user-facing alternative import source
- [ ] Add cross-check coverage between IDX-PDF-derived output and KSEI-archive-derived output for at least one monthly release - [x] Add cross-check coverage between IDX-PDF-derived output and KSEI-archive-derived output for at least one monthly release
- [ ] Batch 4 verification: `cargo build` - [x] Batch 4 verification: `cargo build`
- [ ] Batch 4 verification: `cargo clippy -- -D warnings` - [x] Batch 4 verification: `cargo clippy -- -D warnings`
- [ ] Batch 4 verification: `cargo test` - [x] Batch 4 verification: `cargo test`
- [ ] Batch 4 verification: fallback ingest produces a compatible SQLite state for `ownership releases`, `ticker`, and `changes` - [x] Batch 4 verification: fallback ingest produces a compatible SQLite state for `ownership releases`, `ticker`, and `changes`
## 📋 Backlog (per SPEC.md) ## 📋 Backlog (per SPEC.md)
- [ ] `market summary` — IHSG index, market breadth - [ ] `market summary` — IHSG index, market breadth
@ -122,7 +122,7 @@
- [ ] `screen presets` / `screen run <PRESET>` — built-in screener presets - [ ] `screen presets` / `screen run <PRESET>` — built-in screener presets
- [ ] `watchlist` commands — create, manage, live watch - [ ] `watchlist` commands — create, manage, live watch
- [ ] `alerts` system (v0.2+) — price alerts with daemon - [ ] `alerts` system (v0.2+) — price alerts with daemon
- [ ] `completions <SHELL>` — shell completion generation - [x] `completions <SHELL>` — shell completion generation
- [ ] CSV/TSV output formats - [ ] CSV/TSV output formats
- [ ] Additional providers (Alpha Vantage, Twelve Data, IDX official) - [ ] Additional providers (Alpha Vantage, Twelve Data, IDX official)
@ -168,6 +168,8 @@
- [x] Regression coverage now covers default `ownership discover` behavior, status visibility, listing-page rejection, duplicate SHA imports, and explicit unsupported-schema detection with compact `stext` fixtures plus fake-`mutool` CLI tests - [x] Regression coverage now covers default `ownership discover` behavior, status visibility, listing-page rejection, duplicate SHA imports, and explicit unsupported-schema detection with compact `stext` fixtures plus fake-`mutool` CLI tests
- [x] New `ownership-import` smoke coverage now discovers the current live `above1`/`above5`/`investor-type` URLs, imports the supported `above1` attachment successfully, and confirms the legacy families fail with explicit unsupported-schema UX (`tmp/live-smoke/20260330-160201`) - [x] New `ownership-import` smoke coverage now discovers the current live `above1`/`above5`/`investor-type` URLs, imports the supported `above1` attachment successfully, and confirms the legacy families fail with explicit unsupported-schema UX (`tmp/live-smoke/20260330-160201`)
- [x] New ownership snapshot sync coverage now verifies manifest parsing, checksum validation, install into an empty temp data dir, preserved query behavior for `releases`/`ticker`/`changes`, no-op sync when current, and `--force` refresh via fixture-backed local manifests on `2026-03-31` - [x] New ownership snapshot sync coverage now verifies manifest parsing, checksum validation, install into an empty temp data dir, preserved query behavior for `releases`/`ticker`/`changes`, no-op sync when current, and `--force` refresh via fixture-backed local manifests on `2026-03-31`
- [x] KSEI archive ZIP/TXT fallback import now works through `ownership import --file` for local `.zip` and `.txt` inputs, using synthetic investor-type/locality aggregate holders as a maintainer backstop rather than the primary product ingest
- [x] Batch 4 coverage now cross-checks the live-like `2026-02-27` `above1` PDF fixture against the matching KSEI archive bucket excerpt and verifies fallback import/query behavior for `releases`, `ticker`, and `changes`
## 🐛 Known Issues ## 🐛 Known Issues
- [ ] Yahoo Finance returns 429 from datacenter IPs occasionally - [ ] Yahoo Finance returns 429 from datacenter IPs occasionally

View file

@ -61,6 +61,7 @@ Unlike the `stocks` module (live-fetch), ownership is **import-then-query**:
### Data Sources ### Data Sources
- **KSEI** — official ≥1% shareholder registry (monthly PDF from IDX) - **KSEI** — official ≥1% shareholder registry (monthly PDF from IDX)
- **KSEI archive** — monthly ZIP/TXT balance-position matrix, used as a local fallback/backstop import path
- **Bing Finance** — global institutional ownership (REST API, quarterly) - **Bing Finance** — global institutional ownership (REST API, quarterly)
### Parser Pipeline ### Parser Pipeline
@ -68,6 +69,10 @@ Unlike the `stocks` module (live-fetch), ownership is **import-then-query**:
KSEI PDF → mutool stext (XML with coordinates) → quick-xml parse → KseiRawRow KSEI PDF → mutool stext (XML with coordinates) → quick-xml parse → KseiRawRow
→ normalize (ID locale numbers, dates, entity names) → KseiHolding → normalize (ID locale numbers, dates, entity names) → KseiHolding
→ SQLite INSERT (within transaction) → SQLite INSERT (within transaction)
KSEI archive ZIP/TXT → pipe-delimited balance-position rows
→ map investor-type/locality buckets into synthetic aggregate holders
→ SQLite INSERT (within transaction)
``` ```
## Data Flow Patterns ## Data Flow Patterns

View file

@ -18,7 +18,7 @@ use crate::output::table::format_idr;
use crate::ownership::types::{ use crate::ownership::types::{
ChangeType, FlowSignal, HolderRow, KseiHolding, OwnershipRelease, OwnershipSource, ChangeType, FlowSignal, HolderRow, KseiHolding, OwnershipRelease, OwnershipSource,
}; };
use crate::ownership::{db, entities, graph, parser, remote, search, snapshot}; use crate::ownership::{archive, db, entities, graph, parser, remote, search, snapshot};
#[derive(Debug, Args)] #[derive(Debug, Args)]
pub struct OwnershipCmd { pub struct OwnershipCmd {
@ -30,7 +30,7 @@ pub struct OwnershipCmd {
pub enum OwnershipCommand { pub enum OwnershipCommand {
/// Discover the latest IDX-hosted ownership report URLs. /// Discover the latest IDX-hosted ownership report URLs.
Discover(DiscoverArgs), Discover(DiscoverArgs),
/// Import ownership data from KSEI PDF or Bing API. /// Import ownership data from KSEI PDF or archive fallback files.
Import(ImportArgs), Import(ImportArgs),
/// Install or refresh a maintained ownership SQLite snapshot. /// Install or refresh a maintained ownership SQLite snapshot.
Sync(SyncArgs), Sync(SyncArgs),
@ -71,7 +71,7 @@ pub struct ImportArgs {
/// URL to a remote ownership PDF. /// URL to a remote ownership PDF.
#[arg(long)] #[arg(long)]
pub url: Option<String>, pub url: Option<String>,
/// Path to local KSEI PDF file. /// Path to local KSEI ownership PDF, ZIP, or TXT file.
#[arg(long)] #[arg(long)]
pub file: Option<PathBuf>, pub file: Option<PathBuf>,
/// Fetch Bing institutional data for these symbols. /// Fetch Bing institutional data for these symbols.
@ -773,30 +773,40 @@ fn handle_import(args: &ImportArgs, config: &IdxConfig) -> Result<(), IdxError>
} }
} }
let Some(pdf_input) = resolve_pdf_input(args)? else { let Some(import_input) = resolve_import_input(args)? else {
return Ok(()); return Ok(());
}; };
let conn = db::open_db(config)?; let conn = db::open_db(config)?;
let sha256 = sha256_file(&pdf_input.pdf_path)?; let sha256 = sha256_file(&import_input.import_path)?;
if !args.force && db::release_exists(&conn, &sha256)? { if !args.force && db::release_exists(&conn, &sha256)? {
println!("Release already imported (sha256: {sha256}). Use --force to re-import."); println!("Release already imported (sha256: {sha256}). Use --force to re-import.");
return Ok(()); return Ok(());
} }
let raw_rows = parser::parse_ksei_pdf(&pdf_input.pdf_path)?; let drafts = match import_input.format {
ImportInputFormat::Pdf => {
let raw_rows = parser::parse_ksei_pdf(&import_input.import_path)?;
if raw_rows.is_empty() { if raw_rows.is_empty() {
return Err(IdxError::ParseError( return Err(IdxError::ParseError(
"no KSEI rows parsed from PDF".to_string(), "no KSEI rows parsed from PDF".to_string(),
)); ));
} }
let mut holdings = Vec::with_capacity(raw_rows.len()); let mut drafts = Vec::with_capacity(raw_rows.len());
for raw in &raw_rows {
drafts.push(entities::normalize_ksei_row(raw)?);
}
drafts
}
ImportInputFormat::Archive => archive::parse_balancepos_file(&import_input.import_path)?,
};
let mut holdings = Vec::with_capacity(drafts.len());
let mut ticker_ids = HashSet::new(); let mut ticker_ids = HashSet::new();
for raw in &raw_rows { for draft in drafts {
let draft = entities::normalize_ksei_row(raw)?;
let ticker_id = db::upsert_ticker(&conn, &draft.ticker_code, draft.issuer_name.as_deref())?; let ticker_id = db::upsert_ticker(&conn, &draft.ticker_code, draft.issuer_name.as_deref())?;
let entity_id = let entity_id =
entities::resolve_entity(&conn, &draft.raw_investor_name, OwnershipSource::Ksei)?; entities::resolve_entity(&conn, &draft.raw_investor_name, OwnershipSource::Ksei)?;
@ -831,7 +841,7 @@ fn handle_import(args: &ImportArgs, config: &IdxConfig) -> Result<(), IdxError>
let release = OwnershipRelease { let release = OwnershipRelease {
id: 0, id: 0,
source_url: pdf_input.source_url, source_url: import_input.source_url,
sha256, sha256,
as_of_date, as_of_date,
row_count: inserted_rows, row_count: inserted_rows,
@ -900,23 +910,31 @@ fn parse_discovery_family(raw: &str) -> Result<Option<remote::OwnershipReportFam
} }
} }
struct ResolvedPdfInput { struct ResolvedImportInput {
pdf_path: PathBuf, import_path: PathBuf,
source_url: Option<String>, source_url: Option<String>,
format: ImportInputFormat,
} }
fn resolve_pdf_input(args: &ImportArgs) -> Result<Option<ResolvedPdfInput>, IdxError> { #[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ImportInputFormat {
Pdf,
Archive,
}
fn resolve_import_input(args: &ImportArgs) -> Result<Option<ResolvedImportInput>, IdxError> {
if let Some(path) = &args.file { if let Some(path) = &args.file {
if !path.exists() { if !path.exists() {
return Err(IdxError::Io(format!( return Err(IdxError::Io(format!(
"input PDF not found: {}", "input ownership file not found: {}",
path.display() path.display()
))); )));
} }
return Ok(Some(ResolvedPdfInput { return Ok(Some(ResolvedImportInput {
pdf_path: path.clone(), import_path: path.clone(),
source_url: None, source_url: None,
format: detect_local_import_format(path)?,
})); }));
} }
@ -925,15 +943,34 @@ fn resolve_pdf_input(args: &ImportArgs) -> Result<Option<ResolvedPdfInput>, IdxE
validate_import_url(trimmed)?; validate_import_url(trimmed)?;
let target = cache_pdf_path(trimmed)?; let target = cache_pdf_path(trimmed)?;
download_pdf(trimmed, &target)?; download_pdf(trimmed, &target)?;
return Ok(Some(ResolvedPdfInput { return Ok(Some(ResolvedImportInput {
pdf_path: target, import_path: target,
source_url: Some(trimmed.to_string()), source_url: Some(trimmed.to_string()),
format: ImportInputFormat::Pdf,
})); }));
} }
Ok(None) Ok(None)
} }
fn detect_local_import_format(path: &Path) -> Result<ImportInputFormat, IdxError> {
match path
.extension()
.and_then(|value| value.to_str())
.map(|value| value.trim().to_ascii_lowercase())
.as_deref()
{
Some("pdf") => Ok(ImportInputFormat::Pdf),
Some("zip") | Some("txt") if archive::supports_local_archive_file(path) => {
Ok(ImportInputFormat::Archive)
}
_ => Err(IdxError::InvalidInput(format!(
"unsupported local ownership file {}; expected a .pdf, .zip, or .txt input",
path.display()
))),
}
}
fn cache_pdf_path(url: &str) -> Result<PathBuf, IdxError> { fn cache_pdf_path(url: &str) -> Result<PathBuf, IdxError> {
let dirs = ProjectDirs::from("", "", "idx") let dirs = ProjectDirs::from("", "", "idx")
.ok_or_else(|| IdxError::Io("unable to resolve cache directory".to_string()))?; .ok_or_else(|| IdxError::Io("unable to resolve cache directory".to_string()))?;

360
src/ownership/archive.rs Normal file
View file

@ -0,0 +1,360 @@
use std::fs;
use std::io::{Cursor, Read};
use std::path::Path;
use chrono::NaiveDate;
use crate::error::IdxError;
use crate::ownership::types::{InvestorTypeCode, KseiHoldingDraft, Locality};
const EXPECTED_HEADER: &[&str] = &[
"Date",
"Code",
"Type",
"Sec. Num",
"Price",
"Local IS",
"Local CP",
"Local PF",
"Local IB",
"Local ID",
"Local MF",
"Local SC",
"Local FD",
"Local OT",
"Total",
"Foreign IS",
"Foreign CP",
"Foreign PF",
"Foreign IB",
"Foreign ID",
"Foreign MF",
"Foreign SC",
"Foreign FD",
"Foreign OT",
"Total",
];
const LOCAL_BUCKETS: &[(&str, usize)] = &[
("IS", 5),
("CP", 6),
("PF", 7),
("IB", 8),
("ID", 9),
("MF", 10),
("SC", 11),
("FD", 12),
("OT", 13),
];
const FOREIGN_BUCKETS: &[(&str, usize)] = &[
("IS", 15),
("CP", 16),
("PF", 17),
("IB", 18),
("ID", 19),
("MF", 20),
("SC", 21),
("FD", 22),
("OT", 23),
];
pub fn supports_local_archive_file(path: &Path) -> bool {
path.extension()
.and_then(|value| value.to_str())
.map(|value| {
let ext = value.trim().to_ascii_lowercase();
ext == "txt" || ext == "zip"
})
.unwrap_or(false)
}
pub fn parse_balancepos_file(path: &Path) -> Result<Vec<KseiHoldingDraft>, IdxError> {
let raw = match path
.extension()
.and_then(|value| value.to_str())
.map(|value| value.trim().to_ascii_lowercase())
.as_deref()
{
Some("txt") => fs::read_to_string(path).map_err(|e| {
IdxError::Io(format!(
"failed to read KSEI archive TXT {}: {e}",
path.display()
))
})?,
Some("zip") => extract_txt_from_zip(path)?,
_ => {
return Err(IdxError::InvalidInput(format!(
"unsupported local ownership archive file {}; expected .txt or .zip",
path.display()
)));
}
};
parse_balancepos_text(&raw)
}
pub fn parse_balancepos_text(raw: &str) -> Result<Vec<KseiHoldingDraft>, IdxError> {
let mut lines = raw.lines().filter(|line| !line.trim().is_empty());
let header = lines
.next()
.ok_or_else(|| IdxError::ParseError("empty KSEI archive TXT input".to_string()))?;
validate_header(header)?;
let mut drafts = Vec::new();
for (line_number, line) in lines.enumerate() {
let columns: Vec<&str> = line.split('|').map(str::trim).collect();
if columns.len() != EXPECTED_HEADER.len() {
return Err(IdxError::ParseError(format!(
"invalid KSEI archive TXT row {}: expected {} columns, got {}",
line_number + 2,
EXPECTED_HEADER.len(),
columns.len()
)));
}
if !columns[2].eq_ignore_ascii_case("EQUITY") {
continue;
}
let report_date = parse_archive_date(columns[0])?;
let ticker_code = columns[1].trim().to_uppercase();
let sec_num = parse_archive_number(columns[3], line_number + 2, "Sec. Num")?;
if sec_num <= 0 {
continue;
}
append_bucket_drafts(
&mut drafts,
&ticker_code,
report_date,
sec_num,
columns.as_slice(),
Locality::Local,
LOCAL_BUCKETS,
)?;
append_bucket_drafts(
&mut drafts,
&ticker_code,
report_date,
sec_num,
columns.as_slice(),
Locality::Foreign,
FOREIGN_BUCKETS,
)?;
}
if drafts.is_empty() {
return Err(IdxError::ParseError(
"no importable EQUITY rows found in KSEI archive TXT".to_string(),
));
}
Ok(drafts)
}
fn extract_txt_from_zip(path: &Path) -> Result<String, IdxError> {
let bytes = fs::read(path).map_err(|e| {
IdxError::Io(format!(
"failed to read KSEI archive ZIP {}: {e}",
path.display()
))
})?;
let cursor = Cursor::new(bytes);
let mut zip = zip::ZipArchive::new(cursor)
.map_err(|e| IdxError::ParseError(format!("failed to open KSEI archive ZIP: {e}")))?;
for index in 0..zip.len() {
let mut file = zip.by_index(index).map_err(|e| {
IdxError::ParseError(format!("failed to read KSEI archive ZIP entry: {e}"))
})?;
if file.is_dir() {
continue;
}
let name = file.name().to_ascii_lowercase();
if !name.ends_with(".txt") {
continue;
}
let mut output = String::new();
file.read_to_string(&mut output).map_err(|e| {
IdxError::ParseError(format!("failed to decode KSEI archive TXT entry: {e}"))
})?;
return Ok(output);
}
Err(IdxError::ParseError(
"KSEI archive ZIP did not contain a TXT payload".to_string(),
))
}
fn validate_header(header: &str) -> Result<(), IdxError> {
let columns: Vec<&str> = header.split('|').map(str::trim).collect();
if columns != EXPECTED_HEADER {
return Err(IdxError::ParseError(
"KSEI archive TXT header did not match the expected balancepos layout".to_string(),
));
}
Ok(())
}
fn parse_archive_date(raw: &str) -> Result<NaiveDate, IdxError> {
let trimmed = raw.trim();
if trimmed.len() != 11 {
return Err(IdxError::ParseError(format!(
"invalid KSEI archive date `{trimmed}`"
)));
}
let canonical = format!(
"{}-{}-{}",
&trimmed[..2],
titlecase_month(&trimmed[3..6]),
&trimmed[7..11]
);
NaiveDate::parse_from_str(&canonical, "%d-%b-%Y")
.map_err(|e| IdxError::ParseError(format!("invalid KSEI archive date `{trimmed}`: {e}")))
}
fn titlecase_month(raw: &str) -> String {
let upper = raw.trim().to_ascii_uppercase();
let mut chars = upper.chars();
match chars.next() {
Some(first) => {
let mut output = String::new();
output.push(first.to_ascii_uppercase());
output.push_str(&chars.as_str().to_ascii_lowercase());
output
}
None => String::new(),
}
}
fn parse_archive_number(raw: &str, line_number: usize, field: &str) -> Result<i64, IdxError> {
raw.trim().parse::<i64>().map_err(|e| {
IdxError::ParseError(format!(
"invalid KSEI archive TXT value in row {line_number} field `{field}`: {e}"
))
})
}
fn append_bucket_drafts(
drafts: &mut Vec<KseiHoldingDraft>,
ticker_code: &str,
report_date: NaiveDate,
sec_num: i64,
columns: &[&str],
locality: Locality,
buckets: &[(&str, usize)],
) -> Result<(), IdxError> {
for (investor_type, column_index) in buckets {
let shares =
parse_archive_number(columns[*column_index], 0, investor_type).map_err(|_| {
IdxError::ParseError(format!(
"invalid KSEI archive TXT share count for {ticker_code} {investor_type}"
))
})?;
if shares <= 0 {
continue;
}
drafts.push(KseiHoldingDraft {
ticker_code: ticker_code.to_string(),
issuer_name: None,
raw_investor_name: synthetic_holder_name(locality, investor_type),
investor_type: Some(InvestorTypeCode((*investor_type).to_string())),
locality: Some(locality),
nationality: None,
domicile: None,
holdings_scripless: shares,
holdings_scrip: 0,
total_shares: shares,
percentage_bps: compute_percentage_bps(shares, sec_num),
report_date,
});
}
Ok(())
}
fn synthetic_holder_name(locality: Locality, investor_type: &str) -> String {
let prefix = match locality {
Locality::Local => "LOCAL",
Locality::Foreign => "FOREIGN",
};
format!("KSEI AGGREGATE {prefix} {investor_type}")
}
fn compute_percentage_bps(shares: i64, sec_num: i64) -> i64 {
let shares_i128 = i128::from(shares);
let sec_num_i128 = i128::from(sec_num);
let rounded = ((shares_i128 * 10_000) + (sec_num_i128 / 2)) / sec_num_i128;
i64::try_from(rounded).unwrap_or(0)
}
#[cfg(test)]
mod tests {
use super::parse_balancepos_text;
use crate::ownership::entities::normalize_ksei_row;
use crate::ownership::parser::parse_stext_xml;
use crate::ownership::types::Locality;
#[test]
fn parses_balancepos_excerpt_into_bucket_holders() {
let raw = include_str!("../../tests/fixtures/ksei_balancepos_20260227_excerpt.txt");
let drafts = parse_balancepos_text(raw).expect("balancepos excerpt parses");
let local_cp = drafts
.iter()
.find(|draft| {
draft.ticker_code == "AADI"
&& draft.investor_type.as_ref().map(|code| code.0.as_str()) == Some("CP")
&& draft.locality == Some(Locality::Local)
})
.expect("AADI local CP bucket");
assert_eq!(local_cp.raw_investor_name, "KSEI AGGREGATE LOCAL CP");
assert_eq!(local_cp.total_shares, 5_035_745_466);
assert_eq!(local_cp.percentage_bps, 6467);
}
#[test]
fn balancepos_cross_check_contains_pdf_holder_bucket() {
let pdf_rows = parse_stext_xml(include_str!(
"../../tests/fixtures/ksei_above1_stext_excerpt.xml"
))
.expect("pdf fixture rows");
let pdf_drafts: Vec<_> = pdf_rows
.iter()
.map(normalize_ksei_row)
.collect::<Result<_, _>>()
.expect("normalized pdf drafts");
let archive_drafts = parse_balancepos_text(include_str!(
"../../tests/fixtures/ksei_balancepos_20260227_excerpt.txt"
))
.expect("archive drafts");
let pdf_local_cp = pdf_drafts
.iter()
.find(|draft| {
draft.ticker_code == "AADI"
&& draft.investor_type.as_ref().map(|code| code.0.as_str()) == Some("CP")
&& draft.locality == Some(Locality::Local)
})
.expect("pdf local cp");
let archive_local_cp = archive_drafts
.iter()
.find(|draft| {
draft.ticker_code == "AADI"
&& draft.investor_type.as_ref().map(|code| code.0.as_str()) == Some("CP")
&& draft.locality == Some(Locality::Local)
})
.expect("archive local cp");
assert_eq!(pdf_local_cp.report_date, archive_local_cp.report_date);
assert!(pdf_local_cp.total_shares <= archive_local_cp.total_shares);
assert!(pdf_local_cp.percentage_bps <= archive_local_cp.percentage_bps);
}
}

View file

@ -1,3 +1,4 @@
pub mod archive;
pub mod db; pub mod db;
pub mod entities; pub mod entities;
pub mod graph; pub mod graph;

View file

@ -10,6 +10,7 @@ use assert_cmd::Command;
use predicates::prelude::*; use predicates::prelude::*;
use rusqlite::Connection; use rusqlite::Connection;
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use zip::write::SimpleFileOptions;
fn bin() -> Command { fn bin() -> Command {
let current = std::env::current_exe().expect("current test executable path"); let current = std::env::current_exe().expect("current test executable path");
@ -167,6 +168,18 @@ fn pdf_url(base: &str, name: &str) -> String {
format!("{base}/{name}.pdf") format!("{base}/{name}.pdf")
} }
fn write_zip_with_text(zip_path: &Path, entry_name: &str, text: &str) {
let file = fs::File::create(zip_path).expect("create zip fixture");
let mut writer = zip::ZipWriter::new(file);
writer
.start_file(entry_name, SimpleFileOptions::default())
.expect("start zip file");
writer
.write_all(text.as_bytes())
.expect("write zip fixture text");
writer.finish().expect("finish zip fixture");
}
fn sha256_hex(bytes: &[u8]) -> String { fn sha256_hex(bytes: &[u8]) -> String {
let mut hasher = Sha256::new(); let mut hasher = Sha256::new();
hasher.update(bytes); hasher.update(bytes);
@ -1175,6 +1188,111 @@ fn ownership_import_url_rejects_legacy_investor_type_pdf_schema() {
)); ));
} }
#[test]
fn ownership_import_file_txt_archive_succeeds() {
let root = test_env_dir("ownership-import-txt-archive");
let db_path = root.join("ownership.db");
let txt_path = root.join("Balancepos20260227.txt");
fs::write(
&txt_path,
include_str!("fixtures/ksei_balancepos_20260227_excerpt.txt"),
)
.expect("write txt archive fixture");
bin_with_root(&root)
.args([
"config",
"set",
"ownership.db_path",
db_path.to_str().unwrap(),
])
.assert()
.success();
bin_with_root(&root)
.args(["ownership", "import", "--file", txt_path.to_str().unwrap()])
.assert()
.success()
.stdout(predicate::str::contains("Imported 18 rows for 1 tickers"));
bin_with_root(&root)
.args(["ownership", "ticker", "AADI", "--source", "ksei"])
.assert()
.success()
.stdout(predicate::str::contains("KSEI AGGREGATE LOCAL CP"))
.stdout(predicate::str::contains("64.67%"));
}
#[test]
fn ownership_import_file_zip_archive_supports_releases_ticker_and_changes() {
let root = test_env_dir("ownership-import-zip-archive");
let db_path = root.join("ownership.db");
let jan_zip = root.join("BalanceposEfek20260130.zip");
let feb_zip = root.join("BalanceposEfek20260227.zip");
write_zip_with_text(
&jan_zip,
"Balancepos20260130.txt",
include_str!("fixtures/ksei_balancepos_20260130_excerpt.txt"),
);
write_zip_with_text(
&feb_zip,
"Balancepos20260227.txt",
include_str!("fixtures/ksei_balancepos_20260227_excerpt.txt"),
);
bin_with_root(&root)
.args([
"config",
"set",
"ownership.db_path",
db_path.to_str().unwrap(),
])
.assert()
.success();
bin_with_root(&root)
.args(["ownership", "import", "--file", jan_zip.to_str().unwrap()])
.assert()
.success()
.stdout(predicate::str::contains("Imported 18 rows for 1 tickers"));
bin_with_root(&root)
.args(["ownership", "import", "--file", feb_zip.to_str().unwrap()])
.assert()
.success()
.stdout(predicate::str::contains("Imported 18 rows for 1 tickers"));
bin_with_root(&root)
.args(["ownership", "releases"])
.assert()
.success()
.stdout(predicate::str::contains("2026-02-27"))
.stdout(predicate::str::contains("2026-01-30"));
bin_with_root(&root)
.args(["ownership", "ticker", "AADI", "--source", "ksei"])
.assert()
.success()
.stdout(predicate::str::contains("KSEI AGGREGATE LOCAL CP"))
.stdout(predicate::str::contains("KSEI AGGREGATE FOREIGN MF"));
bin_with_root(&root)
.args([
"ownership",
"changes",
"--from",
"2026-01-30",
"--to",
"2026-02-27",
])
.assert()
.success()
.stdout(predicate::str::contains("AADI"))
.stdout(predicate::str::contains("KSEI AGGREGATE LOCAL CP"))
.stdout(predicate::str::contains("DECREASED"));
}
#[test] #[test]
fn ownership_sync_installs_snapshot_and_preserves_query_behavior() { fn ownership_sync_installs_snapshot_and_preserves_query_behavior() {
let publisher_root = test_env_dir("ownership-sync-publisher"); let publisher_root = test_env_dir("ownership-sync-publisher");

View file

@ -0,0 +1,2 @@
Date|Code|Type|Sec. Num|Price|Local IS|Local CP|Local PF|Local IB|Local ID|Local MF|Local SC|Local FD|Local OT|Total|Foreign IS|Foreign CP|Foreign PF|Foreign IB|Foreign ID|Foreign MF|Foreign SC|Foreign FD|Foreign OT|Total
30-JAN-2026|AADI|EQUITY|7786891760|7600|58704495|5046580437|13135269|41|1868661103|91475881|27984191|1966845|85275|7108593537|1150700|55820749|58897143|207783974|2127405|239139003|39765137|2493100|71121012|678298223

View file

@ -0,0 +1,2 @@
Date|Code|Type|Sec. Num|Price|Local IS|Local CP|Local PF|Local IB|Local ID|Local MF|Local SC|Local FD|Local OT|Total|Foreign IS|Foreign CP|Foreign PF|Foreign IB|Foreign ID|Foreign MF|Foreign SC|Foreign FD|Foreign OT|Total
27-FEB-2026|AADI|EQUITY|7786891760|9250|100730965|5035745466|15622269|41|1817092689|117982099|22932698|2060140|105275|7112271642|1150700|66705786|43311343|204936195|1261705|262226503|42723690|2493100|49811096|674620118