mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-08-07 01:33:52 +00:00
Add KSEI archive fallback import
This commit is contained in:
parent
8404b27628
commit
1f5c4b2798
11 changed files with 622 additions and 59 deletions
56
Cargo.lock
generated
56
Cargo.lock
generated
|
|
@ -88,6 +88,15 @@ dependencies = [
|
|||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arbitrary"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"
|
||||
dependencies = [
|
||||
"derive_arbitrary",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "assert_cmd"
|
||||
version = "2.1.2"
|
||||
|
|
@ -302,6 +311,12 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
|
||||
|
||||
[[package]]
|
||||
name = "crossterm"
|
||||
version = "0.29.0"
|
||||
|
|
@ -365,6 +380,17 @@ dependencies = [
|
|||
"powerfmt",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_arbitrary"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "difflib"
|
||||
version = "0.4.0"
|
||||
|
|
@ -704,6 +730,7 @@ dependencies = [
|
|||
"thiserror 2.0.18",
|
||||
"toml",
|
||||
"ureq",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1894,8 +1921,37 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "2.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50"
|
||||
dependencies = [
|
||||
"arbitrary",
|
||||
"crc32fast",
|
||||
"crossbeam-utils",
|
||||
"displaydoc",
|
||||
"flate2",
|
||||
"indexmap",
|
||||
"memchr",
|
||||
"thiserror 2.0.18",
|
||||
"zopfli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
|
||||
[[package]]
|
||||
name = "zopfli"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"crc32fast",
|
||||
"log",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ path = "src/main.rs"
|
|||
|
||||
[features]
|
||||
default = ["ownership"]
|
||||
ownership = ["dep:rusqlite", "dep:quick-xml", "dep:sha2"]
|
||||
ownership = ["dep:rusqlite", "dep:quick-xml", "dep:sha2", "dep:zip"]
|
||||
|
||||
[dependencies]
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
|
|
@ -33,6 +33,7 @@ fastrand = "2"
|
|||
rusqlite = { version = "0.32", features = ["bundled-full"], optional = true }
|
||||
quick-xml = { version = "0.37", optional = true }
|
||||
sha2 = { version = "0.10", optional = true }
|
||||
zip = { version = "2", default-features = false, features = ["deflate"], optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
assert_cmd = "2"
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ Use `TODO.md` as the execution log and smoke-history record.
|
|||
|
||||
## Verified Current State
|
||||
|
||||
- Current automated coverage is `176` tests: `115` unit and `61` integration.
|
||||
- Current automated coverage is `180` tests: `117` unit and `63` integration.
|
||||
- Reusable smoke coverage exists via `scripts/live-smoke.sh`; command groups are documented in `docs/SMOKE.md`.
|
||||
- The latest smoke notes in `TODO.md` report passing live table and JSON checks for all shipped `stocks` commands.
|
||||
- Cache/offline parity, JSON startup-error handling, screener input validation, and the recent MSN output cleanups have already been completed.
|
||||
|
|
@ -37,6 +37,7 @@ Use `TODO.md` as the execution log and smoke-history record.
|
|||
- The currently discoverable `above 5%` and `investor-type` BEI families remain different schemas and are now classified and rejected explicitly during import instead of falling through to a generic zero-row parse failure.
|
||||
- Ownership smoke coverage now includes a dedicated `ownership-import` group that verifies live supported import plus expected unsupported-family failures.
|
||||
- `idx ownership sync` now installs maintained SQLite snapshots via a manifest-driven contract with checksum validation, conservative local replacement rules, and fixture-backed regression coverage for install/no-op/force-refresh behavior.
|
||||
- The KSEI archive ZIP/TXT path is now implemented as a local-file maintainer fallback/backstop, with cross-check coverage against the `above1` PDF contract for a shared monthly release.
|
||||
|
||||
This means the main gap is no longer endpoint coverage.
|
||||
The remaining work is architecture cleanup, a few correctness edge cases, and selective UX expansion on top of already-shipped commands.
|
||||
|
|
@ -64,6 +65,7 @@ The remaining work is architecture cleanup, a few correctness edge cases, and se
|
|||
| Screener | `idx stocks screen` | Implemented with gaps | Validation landed; expression/preset workflow is still future work |
|
||||
| MSN charts | `idx stocks history --history-provider msn` | Missing | Explicit MSN history still returns unsupported for IDX |
|
||||
| KSEI ownership import/query | `idx ownership import --file`, `idx ownership import --url`, `idx ownership releases`, `idx ownership ticker` | Implemented | Local PDF import and SQLite-backed query flow are verified against the March 2026 KSEI release; remote IDX import now works for the discovered `above 1%` `lamp1` BEI attachment, and legacy `above 5%` / `investor-type` BEI report families are rejected explicitly |
|
||||
| KSEI archive fallback import | `idx ownership import --file <.zip|.txt>` | Implemented as fallback | Local archive ZIP/TXT ingest maps investor-type/locality buckets into synthetic aggregate holders for validation/backstop use, not the primary ingest surface |
|
||||
| Ownership snapshot sync | `idx ownership sync` | Implemented | Manifest-driven SQLite snapshot install with checksum validation, conservative replacement/no-op rules, and publisher helper script |
|
||||
| Bing ownership CLI | `idx ownership import --fetch-bing` | Not implemented | Client groundwork exists, CLI import path is still deferred |
|
||||
|
||||
|
|
@ -87,6 +89,7 @@ The following items should no longer be treated as active backlog in this spec:
|
|||
- Real KSEI ownership CLI verification from local file import into SQLite (`7261` rows across `955` tickers on `2026-03-28`)
|
||||
- Ownership remote-import hardening for the `above1` contract, including direct-PDF-only `--url` input, discovery status output, explicit legacy-schema rejection, and live ownership-import smoke coverage
|
||||
- Ownership snapshot publishing/sync contract, including `idx ownership sync`, manifest/checksum validation, conservative local replacement rules, and the `scripts/build-ownership-snapshot.sh` publisher helper
|
||||
- KSEI archive ZIP/TXT fallback ingest for local `.zip`/`.txt` files, including cross-check coverage against the `above1` PDF holder-register fixture and compatible `releases`/`ticker`/`changes` verification
|
||||
|
||||
If any of the above regress, capture that in `TODO.md` as a new finding rather than reopening the old section here wholesale.
|
||||
|
||||
|
|
@ -148,26 +151,6 @@ Done when:
|
|||
- The retry/fallback story for Yahoo failures is deliberate and documented.
|
||||
- SMA200 behavior is either improved or clearly documented as expected.
|
||||
|
||||
#### 5. Ownership fallback ingest and cross-check path
|
||||
|
||||
Current state:
|
||||
- Remote IDX discovery/fetch is already in place for the supported `above1` holder-register family.
|
||||
- `idx ownership sync` now covers the maintained-snapshot distribution path on top of that import flow.
|
||||
- The KSEI ZIP/TXT archive remains the main unresolved ownership follow-up and should stay a fallback/cross-check input, not the primary ingest path.
|
||||
|
||||
Why it matters:
|
||||
- The core ownership product path is now stable: local import, remote IDX discovery/import, and maintained snapshot sync all exist.
|
||||
- The remaining ownership work is about fallback resilience and validation against a secondary upstream.
|
||||
|
||||
Done when:
|
||||
- The KSEI ZIP/TXT archive ingest shape is defined.
|
||||
- At least one monthly archive release is cross-checked against the IDX-PDF-derived SQLite state.
|
||||
- Fallback ingest produces query-compatible data for `ownership releases`, `ticker`, and `changes`.
|
||||
|
||||
Roadmap:
|
||||
1. Add KSEI ZIP/TXT ingest later as fallback or cross-check input.
|
||||
2. Decide whether that fallback stays maintainer-only or becomes a user-facing alternative path.
|
||||
|
||||
### P1 - UX and output contract cleanup
|
||||
|
||||
Tasks:
|
||||
|
|
@ -191,14 +174,10 @@ Priority order:
|
|||
- Reuse the existing client groundwork in `src/api/msn/bing.rs`.
|
||||
- Define the import shape and output contract for `idx ownership import --fetch-bing`.
|
||||
|
||||
3. Ownership fallback ingest and cross-check work
|
||||
- Treat the KSEI archive (`https://web.ksei.co.id/archive_download/holding_composition`) as fallback/backstop input, not the primary product ingest path.
|
||||
- Define the ZIP/TXT ingest shape and verify compatibility with the IDX-PDF-derived SQLite contract.
|
||||
|
||||
4. Richer financial statements
|
||||
3. Richer financial statements
|
||||
- Decide whether to stay with the current single-period model or add multi-period fetch support.
|
||||
|
||||
5. New user-facing surfaces from `TODO.md`
|
||||
4. New user-facing surfaces from `TODO.md`
|
||||
- `market summary`
|
||||
- `market movers`
|
||||
- `market sectors`
|
||||
|
|
|
|||
18
TODO.md
18
TODO.md
|
|
@ -106,13 +106,13 @@
|
|||
- [x] Batch 3 verification: sync installs into an empty temp data dir, preserves query behavior, and no-ops when already current
|
||||
|
||||
### Batch 4 — KSEI ZIP/TXT fallback and cross-check path
|
||||
- [ ] Keep KSEI ZIP/TXT ingest as fallback and validation/backstop work, not the first milestone
|
||||
- [ ] Define whether the KSEI archive is only a maintainer fallback or a user-facing alternative import source
|
||||
- [ ] Add cross-check coverage between IDX-PDF-derived output and KSEI-archive-derived output for at least one monthly release
|
||||
- [ ] Batch 4 verification: `cargo build`
|
||||
- [ ] Batch 4 verification: `cargo clippy -- -D warnings`
|
||||
- [ ] Batch 4 verification: `cargo test`
|
||||
- [ ] Batch 4 verification: fallback ingest produces a compatible SQLite state for `ownership releases`, `ticker`, and `changes`
|
||||
- [x] Keep KSEI ZIP/TXT ingest as fallback and validation/backstop work, not the first milestone
|
||||
- [x] Define whether the KSEI archive is only a maintainer fallback or a user-facing alternative import source
|
||||
- [x] Add cross-check coverage between IDX-PDF-derived output and KSEI-archive-derived output for at least one monthly release
|
||||
- [x] Batch 4 verification: `cargo build`
|
||||
- [x] Batch 4 verification: `cargo clippy -- -D warnings`
|
||||
- [x] Batch 4 verification: `cargo test`
|
||||
- [x] Batch 4 verification: fallback ingest produces a compatible SQLite state for `ownership releases`, `ticker`, and `changes`
|
||||
|
||||
## 📋 Backlog (per SPEC.md)
|
||||
- [ ] `market summary` — IHSG index, market breadth
|
||||
|
|
@ -122,7 +122,7 @@
|
|||
- [ ] `screen presets` / `screen run <PRESET>` — built-in screener presets
|
||||
- [ ] `watchlist` commands — create, manage, live watch
|
||||
- [ ] `alerts` system (v0.2+) — price alerts with daemon
|
||||
- [ ] `completions <SHELL>` — shell completion generation
|
||||
- [x] `completions <SHELL>` — shell completion generation
|
||||
- [ ] CSV/TSV output formats
|
||||
- [ ] Additional providers (Alpha Vantage, Twelve Data, IDX official)
|
||||
|
||||
|
|
@ -168,6 +168,8 @@
|
|||
- [x] Regression coverage now covers default `ownership discover` behavior, status visibility, listing-page rejection, duplicate SHA imports, and explicit unsupported-schema detection with compact `stext` fixtures plus fake-`mutool` CLI tests
|
||||
- [x] New `ownership-import` smoke coverage now discovers the current live `above1`/`above5`/`investor-type` URLs, imports the supported `above1` attachment successfully, and confirms the legacy families fail with explicit unsupported-schema UX (`tmp/live-smoke/20260330-160201`)
|
||||
- [x] New ownership snapshot sync coverage now verifies manifest parsing, checksum validation, install into an empty temp data dir, preserved query behavior for `releases`/`ticker`/`changes`, no-op sync when current, and `--force` refresh via fixture-backed local manifests on `2026-03-31`
|
||||
- [x] KSEI archive ZIP/TXT fallback import now works through `ownership import --file` for local `.zip` and `.txt` inputs, using synthetic investor-type/locality aggregate holders as a maintainer backstop rather than the primary product ingest
|
||||
- [x] Batch 4 coverage now cross-checks the live-like `2026-02-27` `above1` PDF fixture against the matching KSEI archive bucket excerpt and verifies fallback import/query behavior for `releases`, `ticker`, and `changes`
|
||||
|
||||
## 🐛 Known Issues
|
||||
- [ ] Yahoo Finance returns 429 from datacenter IPs occasionally
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ Unlike the `stocks` module (live-fetch), ownership is **import-then-query**:
|
|||
|
||||
### Data Sources
|
||||
- **KSEI** — official ≥1% shareholder registry (monthly PDF from IDX)
|
||||
- **KSEI archive** — monthly ZIP/TXT balance-position matrix, used as a local fallback/backstop import path
|
||||
- **Bing Finance** — global institutional ownership (REST API, quarterly)
|
||||
|
||||
### Parser Pipeline
|
||||
|
|
@ -68,6 +69,10 @@ Unlike the `stocks` module (live-fetch), ownership is **import-then-query**:
|
|||
KSEI PDF → mutool stext (XML with coordinates) → quick-xml parse → KseiRawRow
|
||||
→ normalize (ID locale numbers, dates, entity names) → KseiHolding
|
||||
→ SQLite INSERT (within transaction)
|
||||
|
||||
KSEI archive ZIP/TXT → pipe-delimited balance-position rows
|
||||
→ map investor-type/locality buckets into synthetic aggregate holders
|
||||
→ SQLite INSERT (within transaction)
|
||||
```
|
||||
|
||||
## Data Flow Patterns
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ use crate::output::table::format_idr;
|
|||
use crate::ownership::types::{
|
||||
ChangeType, FlowSignal, HolderRow, KseiHolding, OwnershipRelease, OwnershipSource,
|
||||
};
|
||||
use crate::ownership::{db, entities, graph, parser, remote, search, snapshot};
|
||||
use crate::ownership::{archive, db, entities, graph, parser, remote, search, snapshot};
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
pub struct OwnershipCmd {
|
||||
|
|
@ -30,7 +30,7 @@ pub struct OwnershipCmd {
|
|||
pub enum OwnershipCommand {
|
||||
/// Discover the latest IDX-hosted ownership report URLs.
|
||||
Discover(DiscoverArgs),
|
||||
/// Import ownership data from KSEI PDF or Bing API.
|
||||
/// Import ownership data from KSEI PDF or archive fallback files.
|
||||
Import(ImportArgs),
|
||||
/// Install or refresh a maintained ownership SQLite snapshot.
|
||||
Sync(SyncArgs),
|
||||
|
|
@ -71,7 +71,7 @@ pub struct ImportArgs {
|
|||
/// URL to a remote ownership PDF.
|
||||
#[arg(long)]
|
||||
pub url: Option<String>,
|
||||
/// Path to local KSEI PDF file.
|
||||
/// Path to local KSEI ownership PDF, ZIP, or TXT file.
|
||||
#[arg(long)]
|
||||
pub file: Option<PathBuf>,
|
||||
/// Fetch Bing institutional data for these symbols.
|
||||
|
|
@ -773,30 +773,40 @@ fn handle_import(args: &ImportArgs, config: &IdxConfig) -> Result<(), IdxError>
|
|||
}
|
||||
}
|
||||
|
||||
let Some(pdf_input) = resolve_pdf_input(args)? else {
|
||||
let Some(import_input) = resolve_import_input(args)? else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let conn = db::open_db(config)?;
|
||||
|
||||
let sha256 = sha256_file(&pdf_input.pdf_path)?;
|
||||
let sha256 = sha256_file(&import_input.import_path)?;
|
||||
if !args.force && db::release_exists(&conn, &sha256)? {
|
||||
println!("Release already imported (sha256: {sha256}). Use --force to re-import.");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let raw_rows = parser::parse_ksei_pdf(&pdf_input.pdf_path)?;
|
||||
let drafts = match import_input.format {
|
||||
ImportInputFormat::Pdf => {
|
||||
let raw_rows = parser::parse_ksei_pdf(&import_input.import_path)?;
|
||||
if raw_rows.is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"no KSEI rows parsed from PDF".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut holdings = Vec::with_capacity(raw_rows.len());
|
||||
let mut drafts = Vec::with_capacity(raw_rows.len());
|
||||
for raw in &raw_rows {
|
||||
drafts.push(entities::normalize_ksei_row(raw)?);
|
||||
}
|
||||
drafts
|
||||
}
|
||||
ImportInputFormat::Archive => archive::parse_balancepos_file(&import_input.import_path)?,
|
||||
};
|
||||
|
||||
let mut holdings = Vec::with_capacity(drafts.len());
|
||||
let mut ticker_ids = HashSet::new();
|
||||
|
||||
for raw in &raw_rows {
|
||||
let draft = entities::normalize_ksei_row(raw)?;
|
||||
for draft in drafts {
|
||||
let ticker_id = db::upsert_ticker(&conn, &draft.ticker_code, draft.issuer_name.as_deref())?;
|
||||
let entity_id =
|
||||
entities::resolve_entity(&conn, &draft.raw_investor_name, OwnershipSource::Ksei)?;
|
||||
|
|
@ -831,7 +841,7 @@ fn handle_import(args: &ImportArgs, config: &IdxConfig) -> Result<(), IdxError>
|
|||
|
||||
let release = OwnershipRelease {
|
||||
id: 0,
|
||||
source_url: pdf_input.source_url,
|
||||
source_url: import_input.source_url,
|
||||
sha256,
|
||||
as_of_date,
|
||||
row_count: inserted_rows,
|
||||
|
|
@ -900,23 +910,31 @@ fn parse_discovery_family(raw: &str) -> Result<Option<remote::OwnershipReportFam
|
|||
}
|
||||
}
|
||||
|
||||
struct ResolvedPdfInput {
|
||||
pdf_path: PathBuf,
|
||||
struct ResolvedImportInput {
|
||||
import_path: PathBuf,
|
||||
source_url: Option<String>,
|
||||
format: ImportInputFormat,
|
||||
}
|
||||
|
||||
fn resolve_pdf_input(args: &ImportArgs) -> Result<Option<ResolvedPdfInput>, IdxError> {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ImportInputFormat {
|
||||
Pdf,
|
||||
Archive,
|
||||
}
|
||||
|
||||
fn resolve_import_input(args: &ImportArgs) -> Result<Option<ResolvedImportInput>, IdxError> {
|
||||
if let Some(path) = &args.file {
|
||||
if !path.exists() {
|
||||
return Err(IdxError::Io(format!(
|
||||
"input PDF not found: {}",
|
||||
"input ownership file not found: {}",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
|
||||
return Ok(Some(ResolvedPdfInput {
|
||||
pdf_path: path.clone(),
|
||||
return Ok(Some(ResolvedImportInput {
|
||||
import_path: path.clone(),
|
||||
source_url: None,
|
||||
format: detect_local_import_format(path)?,
|
||||
}));
|
||||
}
|
||||
|
||||
|
|
@ -925,15 +943,34 @@ fn resolve_pdf_input(args: &ImportArgs) -> Result<Option<ResolvedPdfInput>, IdxE
|
|||
validate_import_url(trimmed)?;
|
||||
let target = cache_pdf_path(trimmed)?;
|
||||
download_pdf(trimmed, &target)?;
|
||||
return Ok(Some(ResolvedPdfInput {
|
||||
pdf_path: target,
|
||||
return Ok(Some(ResolvedImportInput {
|
||||
import_path: target,
|
||||
source_url: Some(trimmed.to_string()),
|
||||
format: ImportInputFormat::Pdf,
|
||||
}));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn detect_local_import_format(path: &Path) -> Result<ImportInputFormat, IdxError> {
|
||||
match path
|
||||
.extension()
|
||||
.and_then(|value| value.to_str())
|
||||
.map(|value| value.trim().to_ascii_lowercase())
|
||||
.as_deref()
|
||||
{
|
||||
Some("pdf") => Ok(ImportInputFormat::Pdf),
|
||||
Some("zip") | Some("txt") if archive::supports_local_archive_file(path) => {
|
||||
Ok(ImportInputFormat::Archive)
|
||||
}
|
||||
_ => Err(IdxError::InvalidInput(format!(
|
||||
"unsupported local ownership file {}; expected a .pdf, .zip, or .txt input",
|
||||
path.display()
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn cache_pdf_path(url: &str) -> Result<PathBuf, IdxError> {
|
||||
let dirs = ProjectDirs::from("", "", "idx")
|
||||
.ok_or_else(|| IdxError::Io("unable to resolve cache directory".to_string()))?;
|
||||
|
|
|
|||
360
src/ownership/archive.rs
Normal file
360
src/ownership/archive.rs
Normal file
|
|
@ -0,0 +1,360 @@
|
|||
use std::fs;
|
||||
use std::io::{Cursor, Read};
|
||||
use std::path::Path;
|
||||
|
||||
use chrono::NaiveDate;
|
||||
|
||||
use crate::error::IdxError;
|
||||
use crate::ownership::types::{InvestorTypeCode, KseiHoldingDraft, Locality};
|
||||
|
||||
const EXPECTED_HEADER: &[&str] = &[
|
||||
"Date",
|
||||
"Code",
|
||||
"Type",
|
||||
"Sec. Num",
|
||||
"Price",
|
||||
"Local IS",
|
||||
"Local CP",
|
||||
"Local PF",
|
||||
"Local IB",
|
||||
"Local ID",
|
||||
"Local MF",
|
||||
"Local SC",
|
||||
"Local FD",
|
||||
"Local OT",
|
||||
"Total",
|
||||
"Foreign IS",
|
||||
"Foreign CP",
|
||||
"Foreign PF",
|
||||
"Foreign IB",
|
||||
"Foreign ID",
|
||||
"Foreign MF",
|
||||
"Foreign SC",
|
||||
"Foreign FD",
|
||||
"Foreign OT",
|
||||
"Total",
|
||||
];
|
||||
|
||||
const LOCAL_BUCKETS: &[(&str, usize)] = &[
|
||||
("IS", 5),
|
||||
("CP", 6),
|
||||
("PF", 7),
|
||||
("IB", 8),
|
||||
("ID", 9),
|
||||
("MF", 10),
|
||||
("SC", 11),
|
||||
("FD", 12),
|
||||
("OT", 13),
|
||||
];
|
||||
|
||||
const FOREIGN_BUCKETS: &[(&str, usize)] = &[
|
||||
("IS", 15),
|
||||
("CP", 16),
|
||||
("PF", 17),
|
||||
("IB", 18),
|
||||
("ID", 19),
|
||||
("MF", 20),
|
||||
("SC", 21),
|
||||
("FD", 22),
|
||||
("OT", 23),
|
||||
];
|
||||
|
||||
pub fn supports_local_archive_file(path: &Path) -> bool {
|
||||
path.extension()
|
||||
.and_then(|value| value.to_str())
|
||||
.map(|value| {
|
||||
let ext = value.trim().to_ascii_lowercase();
|
||||
ext == "txt" || ext == "zip"
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
pub fn parse_balancepos_file(path: &Path) -> Result<Vec<KseiHoldingDraft>, IdxError> {
|
||||
let raw = match path
|
||||
.extension()
|
||||
.and_then(|value| value.to_str())
|
||||
.map(|value| value.trim().to_ascii_lowercase())
|
||||
.as_deref()
|
||||
{
|
||||
Some("txt") => fs::read_to_string(path).map_err(|e| {
|
||||
IdxError::Io(format!(
|
||||
"failed to read KSEI archive TXT {}: {e}",
|
||||
path.display()
|
||||
))
|
||||
})?,
|
||||
Some("zip") => extract_txt_from_zip(path)?,
|
||||
_ => {
|
||||
return Err(IdxError::InvalidInput(format!(
|
||||
"unsupported local ownership archive file {}; expected .txt or .zip",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
||||
parse_balancepos_text(&raw)
|
||||
}
|
||||
|
||||
pub fn parse_balancepos_text(raw: &str) -> Result<Vec<KseiHoldingDraft>, IdxError> {
|
||||
let mut lines = raw.lines().filter(|line| !line.trim().is_empty());
|
||||
let header = lines
|
||||
.next()
|
||||
.ok_or_else(|| IdxError::ParseError("empty KSEI archive TXT input".to_string()))?;
|
||||
validate_header(header)?;
|
||||
|
||||
let mut drafts = Vec::new();
|
||||
for (line_number, line) in lines.enumerate() {
|
||||
let columns: Vec<&str> = line.split('|').map(str::trim).collect();
|
||||
if columns.len() != EXPECTED_HEADER.len() {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"invalid KSEI archive TXT row {}: expected {} columns, got {}",
|
||||
line_number + 2,
|
||||
EXPECTED_HEADER.len(),
|
||||
columns.len()
|
||||
)));
|
||||
}
|
||||
|
||||
if !columns[2].eq_ignore_ascii_case("EQUITY") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let report_date = parse_archive_date(columns[0])?;
|
||||
let ticker_code = columns[1].trim().to_uppercase();
|
||||
let sec_num = parse_archive_number(columns[3], line_number + 2, "Sec. Num")?;
|
||||
if sec_num <= 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
append_bucket_drafts(
|
||||
&mut drafts,
|
||||
&ticker_code,
|
||||
report_date,
|
||||
sec_num,
|
||||
columns.as_slice(),
|
||||
Locality::Local,
|
||||
LOCAL_BUCKETS,
|
||||
)?;
|
||||
append_bucket_drafts(
|
||||
&mut drafts,
|
||||
&ticker_code,
|
||||
report_date,
|
||||
sec_num,
|
||||
columns.as_slice(),
|
||||
Locality::Foreign,
|
||||
FOREIGN_BUCKETS,
|
||||
)?;
|
||||
}
|
||||
|
||||
if drafts.is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"no importable EQUITY rows found in KSEI archive TXT".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(drafts)
|
||||
}
|
||||
|
||||
fn extract_txt_from_zip(path: &Path) -> Result<String, IdxError> {
|
||||
let bytes = fs::read(path).map_err(|e| {
|
||||
IdxError::Io(format!(
|
||||
"failed to read KSEI archive ZIP {}: {e}",
|
||||
path.display()
|
||||
))
|
||||
})?;
|
||||
let cursor = Cursor::new(bytes);
|
||||
let mut zip = zip::ZipArchive::new(cursor)
|
||||
.map_err(|e| IdxError::ParseError(format!("failed to open KSEI archive ZIP: {e}")))?;
|
||||
|
||||
for index in 0..zip.len() {
|
||||
let mut file = zip.by_index(index).map_err(|e| {
|
||||
IdxError::ParseError(format!("failed to read KSEI archive ZIP entry: {e}"))
|
||||
})?;
|
||||
if file.is_dir() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = file.name().to_ascii_lowercase();
|
||||
if !name.ends_with(".txt") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut output = String::new();
|
||||
file.read_to_string(&mut output).map_err(|e| {
|
||||
IdxError::ParseError(format!("failed to decode KSEI archive TXT entry: {e}"))
|
||||
})?;
|
||||
return Ok(output);
|
||||
}
|
||||
|
||||
Err(IdxError::ParseError(
|
||||
"KSEI archive ZIP did not contain a TXT payload".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
fn validate_header(header: &str) -> Result<(), IdxError> {
|
||||
let columns: Vec<&str> = header.split('|').map(str::trim).collect();
|
||||
if columns != EXPECTED_HEADER {
|
||||
return Err(IdxError::ParseError(
|
||||
"KSEI archive TXT header did not match the expected balancepos layout".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_archive_date(raw: &str) -> Result<NaiveDate, IdxError> {
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.len() != 11 {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"invalid KSEI archive date `{trimmed}`"
|
||||
)));
|
||||
}
|
||||
|
||||
let canonical = format!(
|
||||
"{}-{}-{}",
|
||||
&trimmed[..2],
|
||||
titlecase_month(&trimmed[3..6]),
|
||||
&trimmed[7..11]
|
||||
);
|
||||
|
||||
NaiveDate::parse_from_str(&canonical, "%d-%b-%Y")
|
||||
.map_err(|e| IdxError::ParseError(format!("invalid KSEI archive date `{trimmed}`: {e}")))
|
||||
}
|
||||
|
||||
fn titlecase_month(raw: &str) -> String {
|
||||
let upper = raw.trim().to_ascii_uppercase();
|
||||
let mut chars = upper.chars();
|
||||
match chars.next() {
|
||||
Some(first) => {
|
||||
let mut output = String::new();
|
||||
output.push(first.to_ascii_uppercase());
|
||||
output.push_str(&chars.as_str().to_ascii_lowercase());
|
||||
output
|
||||
}
|
||||
None => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_archive_number(raw: &str, line_number: usize, field: &str) -> Result<i64, IdxError> {
|
||||
raw.trim().parse::<i64>().map_err(|e| {
|
||||
IdxError::ParseError(format!(
|
||||
"invalid KSEI archive TXT value in row {line_number} field `{field}`: {e}"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn append_bucket_drafts(
|
||||
drafts: &mut Vec<KseiHoldingDraft>,
|
||||
ticker_code: &str,
|
||||
report_date: NaiveDate,
|
||||
sec_num: i64,
|
||||
columns: &[&str],
|
||||
locality: Locality,
|
||||
buckets: &[(&str, usize)],
|
||||
) -> Result<(), IdxError> {
|
||||
for (investor_type, column_index) in buckets {
|
||||
let shares =
|
||||
parse_archive_number(columns[*column_index], 0, investor_type).map_err(|_| {
|
||||
IdxError::ParseError(format!(
|
||||
"invalid KSEI archive TXT share count for {ticker_code} {investor_type}"
|
||||
))
|
||||
})?;
|
||||
if shares <= 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
drafts.push(KseiHoldingDraft {
|
||||
ticker_code: ticker_code.to_string(),
|
||||
issuer_name: None,
|
||||
raw_investor_name: synthetic_holder_name(locality, investor_type),
|
||||
investor_type: Some(InvestorTypeCode((*investor_type).to_string())),
|
||||
locality: Some(locality),
|
||||
nationality: None,
|
||||
domicile: None,
|
||||
holdings_scripless: shares,
|
||||
holdings_scrip: 0,
|
||||
total_shares: shares,
|
||||
percentage_bps: compute_percentage_bps(shares, sec_num),
|
||||
report_date,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn synthetic_holder_name(locality: Locality, investor_type: &str) -> String {
|
||||
let prefix = match locality {
|
||||
Locality::Local => "LOCAL",
|
||||
Locality::Foreign => "FOREIGN",
|
||||
};
|
||||
format!("KSEI AGGREGATE {prefix} {investor_type}")
|
||||
}
|
||||
|
||||
fn compute_percentage_bps(shares: i64, sec_num: i64) -> i64 {
|
||||
let shares_i128 = i128::from(shares);
|
||||
let sec_num_i128 = i128::from(sec_num);
|
||||
let rounded = ((shares_i128 * 10_000) + (sec_num_i128 / 2)) / sec_num_i128;
|
||||
i64::try_from(rounded).unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::parse_balancepos_text;
|
||||
use crate::ownership::entities::normalize_ksei_row;
|
||||
use crate::ownership::parser::parse_stext_xml;
|
||||
use crate::ownership::types::Locality;
|
||||
|
||||
#[test]
|
||||
fn parses_balancepos_excerpt_into_bucket_holders() {
|
||||
let raw = include_str!("../../tests/fixtures/ksei_balancepos_20260227_excerpt.txt");
|
||||
let drafts = parse_balancepos_text(raw).expect("balancepos excerpt parses");
|
||||
|
||||
let local_cp = drafts
|
||||
.iter()
|
||||
.find(|draft| {
|
||||
draft.ticker_code == "AADI"
|
||||
&& draft.investor_type.as_ref().map(|code| code.0.as_str()) == Some("CP")
|
||||
&& draft.locality == Some(Locality::Local)
|
||||
})
|
||||
.expect("AADI local CP bucket");
|
||||
|
||||
assert_eq!(local_cp.raw_investor_name, "KSEI AGGREGATE LOCAL CP");
|
||||
assert_eq!(local_cp.total_shares, 5_035_745_466);
|
||||
assert_eq!(local_cp.percentage_bps, 6467);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn balancepos_cross_check_contains_pdf_holder_bucket() {
|
||||
let pdf_rows = parse_stext_xml(include_str!(
|
||||
"../../tests/fixtures/ksei_above1_stext_excerpt.xml"
|
||||
))
|
||||
.expect("pdf fixture rows");
|
||||
let pdf_drafts: Vec<_> = pdf_rows
|
||||
.iter()
|
||||
.map(normalize_ksei_row)
|
||||
.collect::<Result<_, _>>()
|
||||
.expect("normalized pdf drafts");
|
||||
let archive_drafts = parse_balancepos_text(include_str!(
|
||||
"../../tests/fixtures/ksei_balancepos_20260227_excerpt.txt"
|
||||
))
|
||||
.expect("archive drafts");
|
||||
|
||||
let pdf_local_cp = pdf_drafts
|
||||
.iter()
|
||||
.find(|draft| {
|
||||
draft.ticker_code == "AADI"
|
||||
&& draft.investor_type.as_ref().map(|code| code.0.as_str()) == Some("CP")
|
||||
&& draft.locality == Some(Locality::Local)
|
||||
})
|
||||
.expect("pdf local cp");
|
||||
let archive_local_cp = archive_drafts
|
||||
.iter()
|
||||
.find(|draft| {
|
||||
draft.ticker_code == "AADI"
|
||||
&& draft.investor_type.as_ref().map(|code| code.0.as_str()) == Some("CP")
|
||||
&& draft.locality == Some(Locality::Local)
|
||||
})
|
||||
.expect("archive local cp");
|
||||
|
||||
assert_eq!(pdf_local_cp.report_date, archive_local_cp.report_date);
|
||||
assert!(pdf_local_cp.total_shares <= archive_local_cp.total_shares);
|
||||
assert!(pdf_local_cp.percentage_bps <= archive_local_cp.percentage_bps);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
pub mod archive;
|
||||
pub mod db;
|
||||
pub mod entities;
|
||||
pub mod graph;
|
||||
|
|
|
|||
118
tests/cli.rs
118
tests/cli.rs
|
|
@ -10,6 +10,7 @@ use assert_cmd::Command;
|
|||
use predicates::prelude::*;
|
||||
use rusqlite::Connection;
|
||||
use sha2::{Digest, Sha256};
|
||||
use zip::write::SimpleFileOptions;
|
||||
|
||||
fn bin() -> Command {
|
||||
let current = std::env::current_exe().expect("current test executable path");
|
||||
|
|
@ -167,6 +168,18 @@ fn pdf_url(base: &str, name: &str) -> String {
|
|||
format!("{base}/{name}.pdf")
|
||||
}
|
||||
|
||||
fn write_zip_with_text(zip_path: &Path, entry_name: &str, text: &str) {
|
||||
let file = fs::File::create(zip_path).expect("create zip fixture");
|
||||
let mut writer = zip::ZipWriter::new(file);
|
||||
writer
|
||||
.start_file(entry_name, SimpleFileOptions::default())
|
||||
.expect("start zip file");
|
||||
writer
|
||||
.write_all(text.as_bytes())
|
||||
.expect("write zip fixture text");
|
||||
writer.finish().expect("finish zip fixture");
|
||||
}
|
||||
|
||||
fn sha256_hex(bytes: &[u8]) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(bytes);
|
||||
|
|
@ -1175,6 +1188,111 @@ fn ownership_import_url_rejects_legacy_investor_type_pdf_schema() {
|
|||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_import_file_txt_archive_succeeds() {
|
||||
let root = test_env_dir("ownership-import-txt-archive");
|
||||
let db_path = root.join("ownership.db");
|
||||
let txt_path = root.join("Balancepos20260227.txt");
|
||||
fs::write(
|
||||
&txt_path,
|
||||
include_str!("fixtures/ksei_balancepos_20260227_excerpt.txt"),
|
||||
)
|
||||
.expect("write txt archive fixture");
|
||||
|
||||
bin_with_root(&root)
|
||||
.args([
|
||||
"config",
|
||||
"set",
|
||||
"ownership.db_path",
|
||||
db_path.to_str().unwrap(),
|
||||
])
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
bin_with_root(&root)
|
||||
.args(["ownership", "import", "--file", txt_path.to_str().unwrap()])
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Imported 18 rows for 1 tickers"));
|
||||
|
||||
bin_with_root(&root)
|
||||
.args(["ownership", "ticker", "AADI", "--source", "ksei"])
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("KSEI AGGREGATE LOCAL CP"))
|
||||
.stdout(predicate::str::contains("64.67%"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_import_file_zip_archive_supports_releases_ticker_and_changes() {
|
||||
let root = test_env_dir("ownership-import-zip-archive");
|
||||
let db_path = root.join("ownership.db");
|
||||
let jan_zip = root.join("BalanceposEfek20260130.zip");
|
||||
let feb_zip = root.join("BalanceposEfek20260227.zip");
|
||||
|
||||
write_zip_with_text(
|
||||
&jan_zip,
|
||||
"Balancepos20260130.txt",
|
||||
include_str!("fixtures/ksei_balancepos_20260130_excerpt.txt"),
|
||||
);
|
||||
write_zip_with_text(
|
||||
&feb_zip,
|
||||
"Balancepos20260227.txt",
|
||||
include_str!("fixtures/ksei_balancepos_20260227_excerpt.txt"),
|
||||
);
|
||||
|
||||
bin_with_root(&root)
|
||||
.args([
|
||||
"config",
|
||||
"set",
|
||||
"ownership.db_path",
|
||||
db_path.to_str().unwrap(),
|
||||
])
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
bin_with_root(&root)
|
||||
.args(["ownership", "import", "--file", jan_zip.to_str().unwrap()])
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Imported 18 rows for 1 tickers"));
|
||||
|
||||
bin_with_root(&root)
|
||||
.args(["ownership", "import", "--file", feb_zip.to_str().unwrap()])
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Imported 18 rows for 1 tickers"));
|
||||
|
||||
bin_with_root(&root)
|
||||
.args(["ownership", "releases"])
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("2026-02-27"))
|
||||
.stdout(predicate::str::contains("2026-01-30"));
|
||||
|
||||
bin_with_root(&root)
|
||||
.args(["ownership", "ticker", "AADI", "--source", "ksei"])
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("KSEI AGGREGATE LOCAL CP"))
|
||||
.stdout(predicate::str::contains("KSEI AGGREGATE FOREIGN MF"));
|
||||
|
||||
bin_with_root(&root)
|
||||
.args([
|
||||
"ownership",
|
||||
"changes",
|
||||
"--from",
|
||||
"2026-01-30",
|
||||
"--to",
|
||||
"2026-02-27",
|
||||
])
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("AADI"))
|
||||
.stdout(predicate::str::contains("KSEI AGGREGATE LOCAL CP"))
|
||||
.stdout(predicate::str::contains("DECREASED"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_sync_installs_snapshot_and_preserves_query_behavior() {
|
||||
let publisher_root = test_env_dir("ownership-sync-publisher");
|
||||
|
|
|
|||
2
tests/fixtures/ksei_balancepos_20260130_excerpt.txt
vendored
Normal file
2
tests/fixtures/ksei_balancepos_20260130_excerpt.txt
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
Date|Code|Type|Sec. Num|Price|Local IS|Local CP|Local PF|Local IB|Local ID|Local MF|Local SC|Local FD|Local OT|Total|Foreign IS|Foreign CP|Foreign PF|Foreign IB|Foreign ID|Foreign MF|Foreign SC|Foreign FD|Foreign OT|Total
|
||||
30-JAN-2026|AADI|EQUITY|7786891760|7600|58704495|5046580437|13135269|41|1868661103|91475881|27984191|1966845|85275|7108593537|1150700|55820749|58897143|207783974|2127405|239139003|39765137|2493100|71121012|678298223
|
||||
2
tests/fixtures/ksei_balancepos_20260227_excerpt.txt
vendored
Normal file
2
tests/fixtures/ksei_balancepos_20260227_excerpt.txt
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
Date|Code|Type|Sec. Num|Price|Local IS|Local CP|Local PF|Local IB|Local ID|Local MF|Local SC|Local FD|Local OT|Total|Foreign IS|Foreign CP|Foreign PF|Foreign IB|Foreign ID|Foreign MF|Foreign SC|Foreign FD|Foreign OT|Total
|
||||
27-FEB-2026|AADI|EQUITY|7786891760|9250|100730965|5035745466|15622269|41|1817092689|117982099|22932698|2060140|105275|7112271642|1150700|66705786|43311343|204936195|1261705|262226503|42723690|2493100|49811096|674620118
|
||||
Loading…
Add table
Add a link
Reference in a new issue