From 6671e229762df31007acb8ec458bcbf9f223242b Mon Sep 17 00:00:00 2001 From: Ciphercat <78522797+0xrsydn@users.noreply.github.com> Date: Sat, 7 Mar 2026 17:31:04 +0000 Subject: [PATCH] feat(ownership): complete ownership intelligence module Sprints 0-8: Types, schema, KSEI parser, entity resolution, DB CRUD, Bing API client, import pipeline, query commands, graph traversal, changes diff, entity resolution CLI, FTS5 search. - KSEI PDF parser (mutool stext + quick-xml, 99.2% accuracy) - 7 query commands: ticker, entity, search, cross-holders, concentration, flow, releases - Ownership graph with recursive CTE (ASCII tree + Graphviz DOT) - Release diff/changes between KSEI snapshots - Entity resolution CLI: unresolved, map, merge - FTS5 trigram search on entity names - Feature-gated under 'ownership' (default-on) - 82 tests passing --- .gitignore | 1 + AGENTS.md | 92 +- Cargo.lock | 221 +++- Cargo.toml | 7 + docs/ARCHITECTURE.md | 94 ++ docs/CONVENTIONS.md | 127 +++ flake.nix | 1 + src/api/msn/bing.rs | 406 +++++++ src/api/msn/mod.rs | 2 + src/cli/mod.rs | 5 + src/cli/ownership.rs | 888 +++++++++++++++ src/error.rs | 8 + src/main.rs | 9 + src/ownership/db.rs | 1447 +++++++++++++++++++++++++ src/ownership/entities.rs | 404 +++++++ src/ownership/graph.rs | 332 ++++++ src/ownership/mod.rs | 6 + src/ownership/parser.rs | 355 ++++++ src/ownership/search.rs | 111 ++ src/ownership/types.rs | 451 ++++++++ tests/fixtures/bing_buyers_bbca.json | 47 + tests/fixtures/bing_holders_bbca.json | 56 + tests/fixtures/ksei_stext_sample.xml | 328 ++++++ 23 files changed, 5351 insertions(+), 47 deletions(-) create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/CONVENTIONS.md create mode 100644 src/api/msn/bing.rs create mode 100644 src/cli/ownership.rs create mode 100644 src/ownership/db.rs create mode 100644 src/ownership/entities.rs create mode 100644 src/ownership/graph.rs create mode 100644 src/ownership/mod.rs create mode 100644 src/ownership/parser.rs create mode 100644 src/ownership/search.rs create mode 100644 src/ownership/types.rs create mode 100644 tests/fixtures/bing_buyers_bbca.json create mode 100644 tests/fixtures/bing_holders_bbca.json create mode 100644 tests/fixtures/ksei_stext_sample.xml diff --git a/.gitignore b/.gitignore index d9bc5d2..8076db4 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ research/ # Added by cargo /target +out.txt diff --git a/AGENTS.md b/AGENTS.md index a7a71f6..af599f4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,78 +1,78 @@ # AGENTS.md ## Project -`idx-cli` — CLI tool for Indonesian stock market (IDX) analysis. Built in Rust for humans and AI agents. Single binary, zero runtime deps. +`idx-cli` — CLI tool for Indonesian stock market (IDX) analysis. Built in Rust for humans and AI agents. Single binary, schema-driven, functional architecture. ## Stack - **Language:** Rust (stable, via rust-overlay) - **CLI:** clap 4 (derive) - **HTTP:** ureq 3 (sync, no async runtime) - **Output:** comfy-table, owo-colors +- **DB:** rusqlite (bundled SQLite, FTS5) — ownership module - **Config:** TOML (`~/.config/idx/config.toml`) - **Cache:** JSON file-based (`~/.cache/idx/`) -- **Testing:** cargo nextest, assert_cmd, predicates +- **Testing:** cargo test, assert_cmd, predicates - **Hooks:** prek (pre-commit: fmt+clippy, pre-push: test) +- **VCS:** jj (Jujutsu, colocated with git) ## Structure ``` src/ -├── main.rs # Entry point, clap setup, command dispatch -├── cli/ # Command definitions (clap structs + handlers) -│ ├── stocks.rs # stocks quote, history commands +├── main.rs # Entry point, command dispatch +├── cli/ # Command handlers (clap derive structs) +│ ├── stocks.rs # stocks quote/history/technical/fundamental/... │ ├── config.rs # config get/set/init/path │ └── cache.rs # cache info/clear -├── api/ # Data provider abstraction + implementations -│ ├── mod.rs # MarketDataProvider trait -│ ├── yahoo.rs # Yahoo Finance provider (query2 endpoint) -│ └── types.rs # Quote, OHLC, Period, Interval types -├── output/ # Rendering layer (table, json) -│ ├── table.rs # comfy-table + owo-colors -│ └── json.rs # serde_json pretty print +├── api/ # Data providers (trait-based abstraction) +│ ├── mod.rs # MarketDataProvider trait + factory functions +│ ├── types.rs # All domain types (Quote, Ohlc, Fundamentals, ...) +│ ├── yahoo/ # Yahoo Finance provider (history/OHLCV) +│ └── msn/ # MSN Finance provider (quotes, fundamentals, ++) +├── analysis/ # Technical & fundamental analysis (pure functions) +├── ownership/ # Ownership intelligence module (SQLite-backed) +│ ├── types.rs # Ownership domain types +│ └── db.rs # Schema, migrations, queries +├── output/ # Rendering (table, json) ├── cache.rs # File-based TTL cache -├── config.rs # Config loading + merge (flags > env > file > defaults) +├── config.rs # Config loading (flags > env > file > defaults) └── error.rs # IdxError enum (thiserror) -tests/ -├── cli.rs # Integration tests (assert_cmd, mock provider) -docs-internal/ # (gitignored) Specs, research, business strategy ``` +## Providers +- **MSN** = default provider (quotes, fundamentals, profile, earnings, financials, sentiment, insights, news, screener) +- **Yahoo** = automatic fallback for history/OHLCV (MSN doesn't support IDX history) +- Configurable: `IDX_PROVIDER=msn|yahoo`, `IDX_HISTORY_PROVIDER=auto|yahoo|msn` + ## Development ```bash -# Enter dev shell (requires Nix + direnv) -direnv allow # or: nix develop - -# Build -cargo build - -# Run -cargo run -- stocks quote BBCA -cargo run -- -o json stocks quote BBCA,BBRI -cargo run -- stocks history BBCA --period 3mo - -# Test -cargo nextest run # or: cargo test - -# Lint -cargo fmt --check -cargo clippy -- -D warnings +nix develop # enter dev shell +cargo build # build +cargo run -- stocks quote BBCA # run +cargo run -- -o json stocks history BBCA # JSON output +cargo test # test +cargo fmt --check && cargo clippy -- -D warnings # lint ``` -## Docs -- `docs-internal/SPEC.md` — system design, command tree, milestones (gitignored) -- `docs-internal/TODO.md` — task breakdown with checklist (gitignored) -- `docs-internal/OWNERSHIP_FEATURE_DESIGN.md` — ownership intelligence feature design (gitignored) - ## Verification ```bash cargo build # must compile cargo clippy -- -D warnings # zero warnings -cargo nextest run # all tests pass +cargo test # all tests pass ``` -Hooks enforce this: prek runs fmt+clippy on commit, tests on push. -## Rules -1. **Provider abstraction** — all data access goes through `MarketDataProvider` trait, never call Yahoo directly from commands -2. **Sync only** — no tokio/async, this is a CLI tool using ureq -3. **Test with fixtures** — never hit live APIs in tests, use mock provider + fixture JSON -4. **Output contract** — table mode to stdout for humans, `--json` for machines, errors to stderr -5. **Symbol resolution** — always normalize symbols (`BBCA` → `BBCA.JK`) before API calls +## Principles +1. **Schema-driven** — define types first, build logic around them. Types are the spec. +2. **Functional approach** — pure parse/transform functions (`parse_*`, `normalize_*`), no hidden state. +3. **Data types heavy** — rich enums, newtypes, composite structs. Precision via integer representations (basis points for %, i64 for shares). +4. **Provider abstraction** — all data access through traits, never call Yahoo/MSN directly from commands. +5. **Sync only** — no tokio/async. CLI tool, ureq is sufficient. +6. **Test with fixtures** — never hit live APIs in tests. Mock provider + fixture JSON. +7. **Output contract** — table to stdout (humans), `--output json` (machines), errors to stderr. +8. **Feature-gated modules** — `ownership` feature for SQLite dep, keeps base binary lean. + +## Docs +Detailed specs live in `docs-internal/` (gitignored — internal strategy): +- `docs-internal/SPEC.md` — system design, command tree, milestones +- `docs-internal/TODO.md` — sprint breakdown +- `docs-internal/ownership/SPEC.md` — ownership module architecture +- `docs-internal/ownership/TODO.md` — ownership sprint plan diff --git a/Cargo.lock b/Cargo.lock index b5fb300..af7b2f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,18 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -109,6 +121,15 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bstr" version = "1.12.1" @@ -263,6 +284,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -295,6 +325,37 @@ dependencies = [ "winapi", ] +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "deranged" version = "0.5.8" @@ -310,6 +371,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "directories" version = "5.0.1" @@ -367,6 +438,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.3.0" @@ -407,6 +490,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -418,12 +511,30 @@ dependencies = [ "wasi", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "heck" version = "0.5.0" @@ -585,8 +696,11 @@ dependencies = [ "fastrand", "owo-colors", "predicates", + "quick-xml", + "rusqlite", "serde", "serde_json", + "sha2", "thiserror 2.0.18", "toml", "ureq", @@ -599,7 +713,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.16.1", ] [[package]] @@ -639,6 +753,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -762,6 +887,12 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "potential_utf" version = "0.1.4" @@ -816,6 +947,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.45" @@ -888,6 +1028,26 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags", + "chrono", + "csv", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "serde_json", + "smallvec", + "time", + "url", + "uuid", +] + [[package]] name = "rustix" version = "1.1.4" @@ -942,6 +1102,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "scopeguard" version = "1.2.0" @@ -1000,6 +1166,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "1.3.0" @@ -1186,6 +1363,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1273,6 +1456,22 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -1615,6 +1814,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.6" diff --git a/Cargo.toml b/Cargo.toml index e431e87..ff18fde 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,10 @@ homepage = "https://github.com/0xrsydn/idx-cli" keywords = ["idx", "stocks", "indonesia", "cli", "finance"] categories = ["command-line-utilities", "finance"] +[features] +default = ["ownership"] +ownership = ["dep:rusqlite", "dep:quick-xml", "dep:sha2"] + [dependencies] clap = { version = "4", features = ["derive"] } clap_complete = "4" @@ -22,6 +26,9 @@ directories = "5" chrono = { version = "0.4", features = ["serde"] } thiserror = "2" fastrand = "2" +rusqlite = { version = "0.32", features = ["bundled-full"], optional = true } +quick-xml = { version = "0.37", optional = true } +sha2 = { version = "0.10", optional = true } [dev-dependencies] assert_cmd = "2" diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..3f11106 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,94 @@ +# Architecture + +## Domain Map + +``` +┌─────────────────────────────────────────────────────────┐ +│ CLI Layer │ +│ main.rs → cli/stocks.rs, cli/ownership.rs, cli/... │ +│ Clap derive structs, command dispatch, arg validation │ +└──────────────────────┬──────────────────────────────────┘ + │ + ┌──────────────┼──────────────┐ + │ │ │ +┌───────▼───────┐ ┌────▼────┐ ┌──────▼──────┐ +│ API Layer │ │ Analysis│ │ Ownership │ +│ (providers) │ │ Module │ │ Module │ +│ │ │ │ │ │ +│ MarketData │ │ techni- │ │ SQLite DB │ +│ Provider trait│ │ cal.rs │ │ KSEI parser │ +│ │ │ signals │ │ Bing client │ +│ Yahoo (OHLCV) │ │ fund.rs │ │ entity res. │ +│ MSN (rich) │ │ │ │ FTS5 search │ +└───────┬───────┘ └────┬────┘ └──────┬──────┘ + │ │ │ +┌───────▼──────────────▼──────────────▼──────┐ +│ Output Layer │ +│ table.rs (comfy-table) │ json.rs (serde) │ +└─────────────────────────────────────────────┘ +``` + +## Provider Architecture + +### Dual Provider Model +- **MSN Finance** — default provider. Rich data: quotes, fundamentals, profile, earnings, financials, sentiment, insights, news, screener. No history for IDX stocks. +- **Yahoo Finance** — history fallback. Reliable OHLCV data via `/v8/finance/chart/`. + +### Hybrid History Strategy +When `history_provider = auto` (default): +1. Check if current provider supports `HistoryProvider` trait +2. MSN doesn't → transparently fallback to Yahoo +3. Log info message: `"history provider fallback active (msn -> yahoo)"` + +### Capability Gating +``` +MarketDataProvider = QuoteProvider + FundamentalsProvider +HistoryProvider = separate trait, not all providers implement + +Factory functions: + default_provider(kind) → Box + history_provider(kind, mode, verbose) → Result<(ProviderKind, Box)> +``` + +## Ownership Module (SQLite-backed) + +Unlike the `stocks` module (live-fetch), ownership is **import-then-query**: + +1. `idx ownership import` — ETL pipeline: fetch PDF/API → parse → normalize → load SQLite +2. All query commands read from local `~/.local/share/idx/ownership.db` +3. Fully offline after import + +### Data Sources +- **KSEI** — official ≥1% shareholder registry (monthly PDF from IDX) +- **Bing Finance** — global institutional ownership (REST API, quarterly) + +### Parser Pipeline +``` +KSEI PDF → mutool stext (XML with coordinates) → quick-xml parse → KseiRawRow + → normalize (ID locale numbers, dates, entity names) → KseiHolding + → SQLite INSERT (within transaction) +``` + +## Data Flow Patterns + +### Live Query (stocks module) +``` +CLI args → resolve symbol → provider.quote/history/fundamentals → render table/json +``` + +### Import-Query (ownership module) +``` +Import: PDF/API → parse → normalize → resolve entities → SQLite INSERT +Query: CLI args → SQLite SELECT → render table/json (no network) +``` + +## Configuration Precedence +``` +CLI flags > environment variables > config file > defaults +``` + +## Error Strategy +- `IdxError` enum (thiserror) with structured error codes +- Table mode: human-readable error on stderr +- JSON mode: `{"error": true, "code": "...", "message": "..."}` +- Exit code 0 on success, non-zero on failure diff --git a/docs/CONVENTIONS.md b/docs/CONVENTIONS.md new file mode 100644 index 0000000..a8786dc --- /dev/null +++ b/docs/CONVENTIONS.md @@ -0,0 +1,127 @@ +# Conventions + +## Design Philosophy + +### Schema-Driven Development +Define data types FIRST, then build logic around them. Types are the spec. + +```rust +// ✅ Good: rich type with documented fields, integer precision +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KseiHolding { + /// Ownership percentage in basis points: 41.10% → 4110. + pub percentage_bps: i64, + /// Total shares held (absolute count). + pub total_shares: i64, +} + +// ❌ Bad: stringly-typed, float precision +pub struct Holding { + pub percentage: f64, // what unit? what precision? + pub shares: String, // why string? +} +``` + +### Functional Approach +Parse/transform functions are **pure**: take input, return `Result`, no side effects. + +```rust +// ✅ Good: pure function, testable in isolation +pub fn parse_id_number(s: &str) -> Result { ... } +pub fn normalize_name(raw: &str) -> String { ... } +pub fn parse_quote_from_str(symbol: &str, raw: &str) -> Result { ... } + +// ❌ Bad: function with side effects, hard to test +pub fn fetch_and_save_quote(symbol: &str) -> Result<(), IdxError> { ... } +``` + +### Types Over Primitives +Use newtypes, enums, and rich structs. Avoid `String` where a domain type exists. + +```rust +// ✅ Good +pub struct InvestorTypeCode(pub String); +pub enum Locality { Local, Foreign } +pub enum FlowSignal { Holder, Buyer, Seller, NewPosition, Exited } + +// ❌ Bad +pub type InvestorType = String; +pub type Locality = String; +``` + +## Naming + +### Files +- `types.rs` — domain data types for a module +- `mod.rs` — module declarations and re-exports +- `client.rs` — HTTP client code +- `map.rs` / `parse.rs` — response mapping / parsing functions +- `raw_types.rs` — raw API response shapes (before normalization) + +### Functions +- `parse_*` — deserialize raw data into domain types +- `normalize_*` — clean/transform data (names, numbers, dates) +- `resolve_*` — lookup/match entities +- `query_*` — read from database +- `fetch_*` — HTTP requests to external APIs +- `render_*` — output formatting (tables, JSON) +- `handle` — CLI command dispatch entry point + +### Types +- `*Raw` / `*RawRow` — pre-normalization data (strings from API/PDF) +- `*Holding` — ownership fact row +- `*Metrics` — computed analytics +- `*Row` — display-ready composite type +- `*Args` — clap command arguments + +## Patterns + +### Provider Trait Pattern +```rust +pub trait QuoteProvider { + fn quote(&self, symbol: &str) -> Result; +} + +// Factory function, not direct construction +pub fn default_provider(kind: ProviderKind) -> Box { ... } +``` + +### Parse Pipeline Pattern +```rust +// Raw API response → domain type, always via parse function +let raw: &str = &response_body; +let quote = yahoo::parse_quote_from_str("BBCA.JK", raw)?; +``` + +### DB Function Pattern +```rust +// Take &Connection, caller manages lifetime. Use transactions for bulk. +pub fn insert_ksei_holdings(conn: &Connection, holdings: &[KseiHolding]) -> Result { ... } +pub fn query_ticker_holdings(conn: &Connection, code: &str) -> Result { ... } +``` + +### Error Propagation +```rust +// Map external errors to IdxError variants +let conn = Connection::open(path) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; +``` + +## Integer Precision +- **Percentages:** basis points (i64). `41.10%` → `4110` +- **Shares:** absolute count (i64). No floats. +- **Prices:** whole IDR (i64). Rounded from float at parse boundary. +- **Money (USD):** whole dollars (i64) for Bing data. + +## Testing +- **Unit tests:** pure functions (parsers, normalizers, signals) +- **Integration tests:** in-memory SQLite (`Connection::open_in_memory()`), mock providers +- **Fixtures:** `tests/fixtures/*.json` — real API responses, sanitized +- **No live API calls in CI** — `IDX_USE_MOCK_PROVIDER=1` +- **Test naming:** `test__` (e.g., `test_parse_id_number_with_dots`) + +## Git / VCS +- **jj (Jujutsu)** as local workflow, colocated with git +- Push via `nix develop --command git push` (for prek hooks) +- Branch naming: `feat/`, `fix/` +- Commit messages: conventional commits (`feat:`, `fix:`, `refactor:`, `docs:`) diff --git a/flake.nix b/flake.nix index a16ae5a..2e6df0d 100644 --- a/flake.nix +++ b/flake.nix @@ -29,6 +29,7 @@ cargo-nextest prek curl-impersonate # required for Yahoo Finance auth (curl_chrome* binaries for TLS fingerprinting) + mupdf # mutool for KSEI PDF parsing (ownership module) ]; env = { diff --git a/src/api/msn/bing.rs b/src/api/msn/bing.rs new file mode 100644 index 0000000..c13ad5f --- /dev/null +++ b/src/api/msn/bing.rs @@ -0,0 +1,406 @@ +//! Bing Finance institutional ownership HTTP client. +//! +//! Provides access to 5 ownership signal endpoints from the Bing hedge fund data +//! provider API. Each endpoint returns a list of institutional holders grouped by +//! signal type (holders, buyers, sellers, new positions, exits). +//! +//! # Usage +//! ```rust,ignore +//! use crate::api::msn::bing::{BingEndpoint, fetch_all_ownership}; +//! +//! // Fetch all 5 signals for BBCA (instrument ID bn91jc) +//! let results = fetch_all_ownership("bn91jc", false)?; +//! for (signal, holders) in results { +//! println!("{:?}: {} rows", signal, holders.len()); +//! } +//! ``` + +use std::time::Duration; + +use crate::error::IdxError; +use crate::ownership::types::{BingHolderRaw, FlowSignal}; + +/// Base URL for the Bing hedge fund data provider API. +const BING_OWNERSHIP_BASE: &str = + "https://services.bingapis.com/contentservices-finance.hedgefunddataprovider/api/v1"; + +/// Browser-like User-Agent matching existing MSN client pattern. +const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"; + +// ── Endpoint enum ──────────────────────────────────────────────────────────── + +/// Bing ownership API endpoint variants. +/// +/// Each variant maps to one Bing hedge-fund endpoint path and the +/// [`FlowSignal`] it represents in the domain model. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BingEndpoint { + /// Existing top institutional holders. + TopShareHolders, + /// Net buyers over the most recent reporting period. + TopBuyers, + /// Net sellers over the most recent reporting period. + TopSellers, + /// Institutions that opened a new position this period. + TopNewShareHolders, + /// Institutions that fully exited their position this period. + TopExitedShareHolders, +} + +impl BingEndpoint { + /// URL path segment used in the Bing API request. + /// + /// The full URL is constructed as: + /// `{BING_OWNERSHIP_BASE}/{path}/{instrument_id}` + pub fn path(&self) -> &'static str { + match self { + Self::TopShareHolders => "GetSecurityTopShareHolders", + Self::TopBuyers => "GetSecurityTopBuyers", + Self::TopSellers => "GetSecurityTopSellers", + Self::TopNewShareHolders => "GetSecurityTopNewShareHolders", + Self::TopExitedShareHolders => "GetSecurityTopExitedShareHolders", + } + } + + /// The [`FlowSignal`] this endpoint represents. + pub fn signal(&self) -> FlowSignal { + match self { + Self::TopShareHolders => FlowSignal::Holder, + Self::TopBuyers => FlowSignal::Buyer, + Self::TopSellers => FlowSignal::Seller, + Self::TopNewShareHolders => FlowSignal::NewPosition, + Self::TopExitedShareHolders => FlowSignal::Exited, + } + } + + /// All endpoint variants in a canonical order. + pub fn all() -> &'static [BingEndpoint] { + &[ + Self::TopShareHolders, + Self::TopBuyers, + Self::TopSellers, + Self::TopNewShareHolders, + Self::TopExitedShareHolders, + ] + } +} + +// ── HTTP helpers ───────────────────────────────────────────────────────────── + +/// Build a `ureq::Agent` with timeouts matching existing MSN client conventions. +fn build_agent() -> ureq::Agent { + ureq::Agent::config_builder() + .timeout_connect(Some(Duration::from_secs(5))) + .timeout_recv_body(Some(Duration::from_secs(10))) + .build() + .into() +} + +/// Wrapper type used to handle both bare-array and wrapped-object responses +/// from the Bing API. Some endpoints may return `[...]` directly, others may +/// wrap in `{ "value": [...] }` or similar. We first try bare Vec, then +/// fall back to the wrapper. +#[derive(Debug, serde::Deserialize)] +#[serde(untagged)] +enum BingResponse { + /// Direct JSON array of holder objects. + Array(Vec), + /// Object wrapper with a nested array under a common key. + Wrapped(BingWrappedResponse), +} + +#[derive(Debug, serde::Deserialize)] +struct BingWrappedResponse { + /// `value` key used by some Bing OData-style responses. + #[serde(alias = "value", alias = "Value", alias = "data", alias = "Data")] + items: Option>, +} + +impl BingResponse { + fn into_holders(self) -> Vec { + match self { + BingResponse::Array(v) => v, + BingResponse::Wrapped(w) => w.items.unwrap_or_default(), + } + } +} + +// ── Public API ──────────────────────────────────────────────────────────────── + +/// Fetch ownership data from a single Bing endpoint. +/// +/// `instrument_id` is the MSN instrument identifier (for example `bn91jc` for BBCA). +/// Use [`crate::api::msn::symbols::resolve_msn_id`] to obtain this from a ticker code. +/// +/// Empty responses (valid HTTP 200 with empty array) are returned as `Ok(vec![])`. +/// This is expected for some IDX stocks that Bing does not cover. +/// +/// # Errors +/// Returns [`IdxError::Http`] on network or non-retriable HTTP errors. +/// Returns [`IdxError::RateLimited`] after exhausting retries on 429. +/// Returns [`IdxError::ParseError`] if the response body cannot be deserialized. +pub fn fetch_holders( + instrument_id: &str, + endpoint: &BingEndpoint, + verbose: bool, +) -> Result, IdxError> { + let url = format!("{BING_OWNERSHIP_BASE}/{}/{instrument_id}", endpoint.path()); + + if verbose { + eprintln!("[bing] GET {url}"); + } + + let agent = build_agent(); + let endpoint_name = endpoint.path(); + let mut wait = Duration::from_millis(500); + + for attempt in 0..3 { + let response = agent + .get(&url) + .header("User-Agent", USER_AGENT) + .header("Accept", "application/json") + .header("Accept-Language", "en-US,en;q=0.9") + .header("Origin", "https://www.bing.com") + .header("Referer", "https://www.bing.com/") + .call(); + + match response { + Ok(ok) => { + let body = ok + .into_body() + .read_to_string() + .map_err(|e| IdxError::Http(format!("bing {endpoint_name}: read body: {e}")))?; + + if body.trim().is_empty() || body.trim() == "null" { + return Ok(vec![]); + } + + let parsed: BingResponse = serde_json::from_str(&body) + .map_err(|e| IdxError::ParseError(format!("bing {endpoint_name}: {e}")))?; + + return Ok(parsed.into_holders()); + } + Err(ureq::Error::StatusCode(404)) => { + // Instrument not found on Bing — return empty, not an error. + return Ok(vec![]); + } + Err(ureq::Error::StatusCode(429)) => { + if attempt < 2 { + std::thread::sleep(wait); + wait *= 2; + continue; + } + return Err(IdxError::RateLimited); + } + Err(ureq::Error::StatusCode(code)) if code >= 500 => { + if attempt < 2 { + std::thread::sleep(wait); + wait *= 2; + continue; + } + return Err(IdxError::Http(format!( + "bing {endpoint_name}: status {code}" + ))); + } + Err(err) => { + return Err(IdxError::Http(format!("bing {endpoint_name}: {err}"))); + } + } + } + + Err(IdxError::RateLimited) +} + +/// Fetch all 5 ownership endpoints for a given MSN instrument ID. +/// +/// Returns results grouped by [`FlowSignal`] in canonical endpoint order: +/// `Holder`, `Buyer`, `Seller`, `NewPosition`, `Exited`. +/// +/// Endpoints that return empty data are still included as `(signal, vec![])`. +/// +/// # Errors +/// Propagates any non-empty error from [`fetch_holders`]. +pub fn fetch_all_ownership( + instrument_id: &str, + verbose: bool, +) -> Result)>, IdxError> { + let mut results = Vec::with_capacity(5); + for endpoint in BingEndpoint::all() { + let holders = fetch_holders(instrument_id, endpoint, verbose)?; + results.push((endpoint.signal(), holders)); + } + Ok(results) +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + // ── BingEndpoint::path() ────────────────────────────────────────────── + + #[test] + fn test_endpoint_path_top_share_holders() { + assert_eq!( + BingEndpoint::TopShareHolders.path(), + "GetSecurityTopShareHolders" + ); + } + + #[test] + fn test_endpoint_path_top_buyers() { + assert_eq!(BingEndpoint::TopBuyers.path(), "GetSecurityTopBuyers"); + } + + #[test] + fn test_endpoint_path_top_sellers() { + assert_eq!(BingEndpoint::TopSellers.path(), "GetSecurityTopSellers"); + } + + #[test] + fn test_endpoint_path_top_new_share_holders() { + assert_eq!( + BingEndpoint::TopNewShareHolders.path(), + "GetSecurityTopNewShareHolders" + ); + } + + #[test] + fn test_endpoint_path_top_exited_share_holders() { + assert_eq!( + BingEndpoint::TopExitedShareHolders.path(), + "GetSecurityTopExitedShareHolders" + ); + } + + // ── BingEndpoint::signal() ──────────────────────────────────────────── + + #[test] + fn test_endpoint_signal_mapping() { + assert_eq!(BingEndpoint::TopShareHolders.signal(), FlowSignal::Holder); + assert_eq!(BingEndpoint::TopBuyers.signal(), FlowSignal::Buyer); + assert_eq!(BingEndpoint::TopSellers.signal(), FlowSignal::Seller); + assert_eq!( + BingEndpoint::TopNewShareHolders.signal(), + FlowSignal::NewPosition + ); + assert_eq!( + BingEndpoint::TopExitedShareHolders.signal(), + FlowSignal::Exited + ); + } + + // ── BingEndpoint::all() ─────────────────────────────────────────────── + + #[test] + fn test_endpoint_all_has_five_variants() { + assert_eq!(BingEndpoint::all().len(), 5); + } + + #[test] + fn test_endpoint_all_covers_all_signals() { + use std::collections::HashSet; + let signals: HashSet = BingEndpoint::all() + .iter() + .map(|e| format!("{:?}", e.signal())) + .collect(); + assert!(signals.contains("Holder")); + assert!(signals.contains("Buyer")); + assert!(signals.contains("Seller")); + assert!(signals.contains("NewPosition")); + assert!(signals.contains("Exited")); + } + + // ── Fixture deserialization ─────────────────────────────────────────── + + #[test] + fn test_deserialize_holders_fixture() { + let json = include_str!("../../../tests/fixtures/bing_holders_bbca.json"); + let holders: Vec = serde_json::from_str(json) + .expect("bing_holders_bbca.json should deserialize into Vec"); + assert!( + !holders.is_empty(), + "fixture should have at least one holder" + ); + // First holder should have a name + assert!( + holders[0].investor_name.is_some(), + "first holder should have investor_name" + ); + } + + #[test] + fn test_deserialize_buyers_fixture() { + let json = include_str!("../../../tests/fixtures/bing_buyers_bbca.json"); + let holders: Vec = serde_json::from_str(json) + .expect("bing_buyers_bbca.json should deserialize into Vec"); + assert!( + !holders.is_empty(), + "fixture should have at least one buyer" + ); + } + + // ── Empty response handling ─────────────────────────────────────────── + + #[test] + fn test_deserialize_empty_array_is_ok() { + let json = "[]"; + let holders: Vec = serde_json::from_str(json).expect("empty array is valid"); + assert!(holders.is_empty()); + } + + #[test] + fn test_bing_response_bare_array() { + let json = r#"[ + { + "investorName": "Vanguard Group", + "investorType": "Institutional", + "sharesHeld": 1234567.0, + "reportDate": "2024-12-31" + } + ]"#; + let resp: BingResponse = serde_json::from_str(json).unwrap(); + let holders = resp.into_holders(); + assert_eq!(holders.len(), 1); + assert_eq!(holders[0].investor_name.as_deref(), Some("Vanguard Group")); + } + + #[test] + fn test_bing_response_wrapped_value_key() { + let json = r#"{ + "value": [ + { + "InvestorName": "BlackRock", + "InvestorType": "Institutional", + "SharesHeld": 9876543.0 + } + ] + }"#; + let resp: BingResponse = serde_json::from_str(json).unwrap(); + let holders = resp.into_holders(); + assert_eq!(holders.len(), 1); + assert_eq!(holders[0].investor_name.as_deref(), Some("BlackRock")); + } + + #[test] + fn test_bing_response_wrapped_empty_value() { + let json = r#"{ "value": [] }"#; + let resp: BingResponse = serde_json::from_str(json).unwrap(); + let holders = resp.into_holders(); + assert!(holders.is_empty()); + } + + #[test] + fn test_holder_optional_fields_are_none() { + let json = r#"[{"investorName": "Some Fund"}]"#; + let holders: Vec = serde_json::from_str(json).unwrap(); + assert_eq!(holders.len(), 1); + assert!(holders[0].shares_held.is_none()); + assert!(holders[0].shares_changed.is_none()); + assert!(holders[0].pct_outstanding.is_none()); + assert!(holders[0].value.is_none()); + assert!(holders[0].report_date.is_none()); + } +} diff --git a/src/api/msn/mod.rs b/src/api/msn/mod.rs index 3231ff8..39f0fda 100644 --- a/src/api/msn/mod.rs +++ b/src/api/msn/mod.rs @@ -1,3 +1,5 @@ +#[allow(dead_code, clippy::enum_variant_names)] +pub mod bing; mod client; mod map; mod parse; diff --git a/src/cli/mod.rs b/src/cli/mod.rs index ab4200d..a0e5384 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -1,5 +1,7 @@ pub mod cache; pub mod config; +#[cfg(feature = "ownership")] +pub mod ownership; pub mod stocks; use clap::{Parser, Subcommand, ValueEnum}; @@ -45,6 +47,9 @@ pub enum Commands { Config(config::ConfigCmd), #[command(about = "Manage local cache")] Cache(cache::CacheCmd), + #[cfg(feature = "ownership")] + #[command(about = "Ownership intelligence (KSEI + Bing)")] + Ownership(ownership::OwnershipCmd), #[command(about = "Generate shell completions")] Completions { shell: Shell }, #[command(about = "Show idx-cli version")] diff --git a/src/cli/ownership.rs b/src/cli/ownership.rs new file mode 100644 index 0000000..2cbafb9 --- /dev/null +++ b/src/cli/ownership.rs @@ -0,0 +1,888 @@ +use std::collections::HashSet; +use std::fs; +use std::path::{Path, PathBuf}; + +use chrono::Utc; +use clap::{Args, Subcommand}; +use comfy_table::{Cell, ContentArrangement, Table, presets::UTF8_FULL}; +use directories::ProjectDirs; +use owo_colors::OwoColorize; +use serde::Serialize; +use sha2::{Digest, Sha256}; + +use crate::config::IdxConfig; +use crate::error::IdxError; +use crate::output::OutputFormat; +use crate::output::json; +use crate::output::table::format_idr; +use crate::ownership::types::{ + ChangeType, FlowSignal, HolderRow, KseiHolding, OwnershipRelease, OwnershipSource, +}; +use crate::ownership::{db, entities, graph, parser, search}; + +#[derive(Debug, Args)] +pub struct OwnershipCmd { + #[command(subcommand)] + pub command: OwnershipCommand, +} + +#[derive(Debug, Subcommand)] +pub enum OwnershipCommand { + /// Import ownership data from KSEI PDF or Bing API. + Import(ImportArgs), + /// Show all holders for a ticker (KSEI + Bing combined). + Ticker(TickerArgs), + /// Show all holdings for an entity across tickers. + Entity(EntityArgs), + /// Search entities by name. + Search(SearchArgs), + /// Rank entities by cross-ownership breadth. + CrossHolders(CrossHolderArgs), + /// Rank tickers by ownership concentration. + Concentration(ConcentrationArgs), + /// Show Bing institutional flow for a ticker. + Flow(FlowArgs), + /// Diff two KSEI releases. + Changes(ChangesArgs), + /// Ownership network graph. + Graph(GraphArgs), + /// Manual entity resolution workflow. + Resolve(ResolveArgs), + /// List imported KSEI releases. + Releases, +} + +#[derive(Debug, Args)] +pub struct ImportArgs { + /// URL to KSEI ownership PDF. + #[arg(long)] + pub url: Option, + /// Path to local KSEI PDF file. + #[arg(long)] + pub file: Option, + /// Fetch Bing institutional data for these symbols. + #[arg(long, value_delimiter = ',')] + pub fetch_bing: Option>, + /// Re-import even if already imported. + #[arg(long)] + pub force: bool, +} + +#[derive(Debug, Args)] +pub struct TickerArgs { + pub symbol: String, + #[arg(long, default_value = "all")] + pub source: String, +} + +#[derive(Debug, Args)] +pub struct EntityArgs { + pub name: String, +} + +#[derive(Debug, Args)] +pub struct SearchArgs { + pub query: String, + #[arg(long, default_value_t = 20)] + pub limit: usize, +} + +#[derive(Debug, Args)] +pub struct CrossHolderArgs { + #[arg(long, default_value_t = 20)] + pub top: usize, + #[arg(long, default_value_t = 2)] + pub min_tickers: usize, +} + +#[derive(Debug, Args)] +pub struct ConcentrationArgs { + #[arg(long, default_value = "hhi")] + pub by: String, + #[arg(long, default_value_t = 20)] + pub top: usize, +} + +#[derive(Debug, Args)] +pub struct FlowArgs { + pub symbol: String, +} + +#[derive(Debug, Args)] +pub struct ChangesArgs { + #[arg(long)] + pub from: String, + #[arg(long)] + pub to: String, +} + +#[derive(Debug, Args)] +pub struct GraphArgs { + /// Ticker code or entity name to start from. + pub root: String, + /// Traversal depth (default 2). + #[arg(long, default_value_t = 2)] + pub depth: usize, + /// Output format: table or dot (Graphviz). + #[arg(long, default_value = "table")] + pub format: String, +} + +#[derive(Debug, Args)] +pub struct ResolveArgs { + #[command(subcommand)] + pub command: ResolveCommand, +} + +#[derive(Debug, Subcommand)] +pub enum ResolveCommand { + /// List unresolved or low-confidence entity aliases. + Unresolved { + #[arg(long, default_value_t = 50)] + limit: usize, + }, + /// Manually map a raw investor name to a canonical entity name. + Map { + /// The raw name as it appears in the data. + alias: String, + /// The canonical entity name to map to. + entity: String, + }, + /// Merge two entities (keep first, merge second into it). + Merge { + /// Entity ID to keep. + keep: i64, + /// Entity ID to merge and delete. + merge: i64, + }, +} + +pub fn handle(cmd: &OwnershipCommand, config: &IdxConfig) -> Result<(), IdxError> { + match cmd { + OwnershipCommand::Import(args) => handle_import(args, config), + OwnershipCommand::Ticker(args) => handle_ticker(args, config), + OwnershipCommand::Entity(args) => handle_entity(args, config), + OwnershipCommand::Search(args) => handle_search(args, config), + OwnershipCommand::CrossHolders(args) => handle_cross_holders(args, config), + OwnershipCommand::Concentration(args) => handle_concentration(args, config), + OwnershipCommand::Flow(args) => handle_flow(args, config), + OwnershipCommand::Changes(args) => handle_changes(args, config), + OwnershipCommand::Graph(args) => handle_graph(args, config), + OwnershipCommand::Resolve(args) => handle_resolve(args, config), + OwnershipCommand::Releases => handle_releases(config), + } +} + +fn handle_ticker(args: &TickerArgs, config: &IdxConfig) -> Result<(), IdxError> { + let conn = db::open_db(config)?; + let symbol = args.symbol.trim().to_uppercase(); + let source = args.source.trim().to_lowercase(); + if !["all", "ksei", "bing"].contains(&source.as_str()) { + return Err(IdxError::ParseError( + "invalid --source, expected: all|ksei|bing".to_string(), + )); + } + + let mut data = db::query_ticker_holdings(&conn, &symbol)?; + if data.holders.is_empty() { + return Err(IdxError::ParseError(format!( + "no ownership data found for symbol {symbol}" + ))); + } + + if source != "all" { + data.holders.retain(|h| match source.as_str() { + "ksei" => matches!(h.source, OwnershipSource::Ksei), + "bing" => matches!(h.source, OwnershipSource::Bing), + _ => true, + }); + for (i, row) in data.holders.iter_mut().enumerate() { + row.rank = i + 1; + } + let percentages: Vec = data.holders.iter().map(|h| h.percentage_bps).collect(); + data.concentration = db::compute_concentration(&percentages); + } + + if data.holders.is_empty() { + return Err(IdxError::ParseError(format!( + "no {source} ownership data found for symbol {symbol}" + ))); + } + + if matches!(config.output, OutputFormat::Json) { + return json::print_json(&data); + } + + println!( + "{} {} KSEI={} Bing={}", + "Ownership:".bold(), + data.ticker.code.bold(), + data.ksei_as_of + .map(|d| d.format("%Y-%m-%d").to_string()) + .unwrap_or_else(|| "-".to_string()), + data.bing_as_of.unwrap_or_else(|| "-".to_string()) + ); + + let mut table = Table::new(); + table + .load_preset(UTF8_FULL) + .set_content_arrangement(ContentArrangement::Dynamic) + .set_header(vec![ + "RANK", "SOURCE", "INVESTOR", "TYPE", "L/F", "SHARES", "%", "SIGNAL", + ]); + + for h in &data.holders { + table.add_row(vec![ + Cell::new(h.rank), + Cell::new(match h.source { + OwnershipSource::Ksei => "KSEI", + OwnershipSource::Bing => "BING", + }), + Cell::new(&h.name), + Cell::new(h.investor_type.clone().unwrap_or_else(|| "-".to_string())), + Cell::new(match h.locality { + Some(crate::ownership::types::Locality::Local) => "L", + Some(crate::ownership::types::Locality::Foreign) => "F", + None => "-", + }), + Cell::new(format_idr(h.shares)), + Cell::new(format_bps(h.percentage_bps)), + Cell::new(format_signal(h.signal)), + ]); + } + println!("{table}"); + + println!( + "{} top1={} top3={} hhi={} free_float={} holders={}", + "Concentration:".bold(), + format_bps(data.concentration.top1_bps), + format_bps(data.concentration.top3_bps), + data.concentration.hhi, + format_bps(data.concentration.free_float_bps), + data.concentration.holder_count + ); + + Ok(()) +} + +fn handle_entity(args: &EntityArgs, config: &IdxConfig) -> Result<(), IdxError> { + let conn = db::open_db(config)?; + let q = format!("%{}%", args.name.trim()); + + let entity = conn + .query_row( + "SELECT id, canonical_name FROM entities WHERE canonical_name LIKE ?1 COLLATE NOCASE ORDER BY LENGTH(canonical_name) ASC LIMIT 1", + rusqlite::params![q], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)), + ) + .map_err(|_| IdxError::ParseError(format!("entity not found: {}", args.name)))?; + + let data = db::query_entity_holdings(&conn, entity.0)?; + + if matches!(config.output, OutputFormat::Json) { + return json::print_json(&data); + } + + println!("{} {}", "Entity:".bold(), entity.1.bold()); + let mut table = Table::new(); + table + .load_preset(UTF8_FULL) + .set_content_arrangement(ContentArrangement::Dynamic) + .set_header(vec!["TICKER", "SOURCE", "SHARES", "%", "REPORT_DATE"]); + + for row in &data.holdings { + table.add_row(vec![ + Cell::new(&row.ticker.code), + Cell::new(match row.source { + OwnershipSource::Ksei => "KSEI", + OwnershipSource::Bing => "BING", + }), + Cell::new(format_idr(row.shares)), + Cell::new(format_bps(row.percentage_bps)), + Cell::new(row.report_date.format("%Y-%m-%d").to_string()), + ]); + } + println!("{table}"); + println!("{} {}", "Total tickers:".bold(), data.ticker_count); + + Ok(()) +} + +fn handle_search(args: &SearchArgs, config: &IdxConfig) -> Result<(), IdxError> { + let conn = db::open_db(config)?; + let entities = search::fts_search(&conn, &args.query, args.limit)?; + + if matches!(config.output, OutputFormat::Json) { + return json::print_json(&entities); + } + + if entities.is_empty() { + println!("No entities found for query: {}", args.query); + return Ok(()); + } + + let mut table = Table::new(); + table + .load_preset(UTF8_FULL) + .set_content_arrangement(ContentArrangement::Dynamic) + .set_header(vec!["ENTITY", "TYPE", "COUNTRY"]); + + for entity in &entities { + table.add_row(vec![ + Cell::new(&entity.canonical_name), + Cell::new( + entity + .entity_type + .clone() + .unwrap_or_else(|| "-".to_string()), + ), + Cell::new(entity.country.clone().unwrap_or_else(|| "-".to_string())), + ]); + } + println!("{table}"); + Ok(()) +} + +fn handle_cross_holders(args: &CrossHolderArgs, config: &IdxConfig) -> Result<(), IdxError> { + let conn = db::open_db(config)?; + let rows = db::query_cross_holders(&conn, args.min_tickers, args.top)?; + + if matches!(config.output, OutputFormat::Json) { + return json::print_json(&rows); + } + + let mut table = Table::new(); + table + .load_preset(UTF8_FULL) + .set_content_arrangement(ContentArrangement::Dynamic) + .set_header(vec!["RANK", "ENTITY", "TICKERS", "TOTAL_BPS"]); + + for (i, row) in rows.iter().enumerate() { + table.add_row(vec![ + Cell::new(i + 1), + Cell::new(&row.entity.canonical_name), + Cell::new(row.ticker_count), + Cell::new(format_bps(row.total_bps)), + ]); + } + + println!("{table}"); + Ok(()) +} + +fn handle_concentration(args: &ConcentrationArgs, config: &IdxConfig) -> Result<(), IdxError> { + let conn = db::open_db(config)?; + let by = args.by.trim().to_lowercase(); + if !["top1", "top3", "hhi"].contains(&by.as_str()) { + return Err(IdxError::ParseError( + "invalid --by, expected: top1|top3|hhi".to_string(), + )); + } + + let rows = db::query_concentration(&conn, &by, args.top)?; + + if matches!(config.output, OutputFormat::Json) { + return json::print_json(&rows); + } + + let mut table = Table::new(); + table + .load_preset(UTF8_FULL) + .set_content_arrangement(ContentArrangement::Dynamic) + .set_header(vec![ + "RANK", + "TICKER", + "TOP1%", + "TOP3%", + "HHI", + "FREE_FLOAT%", + "HOLDERS", + ]); + + for (idx, (ticker, m)) in rows.iter().enumerate() { + table.add_row(vec![ + Cell::new(idx + 1), + Cell::new(ticker), + Cell::new(format_bps(m.top1_bps)), + Cell::new(format_bps(m.top3_bps)), + Cell::new(m.hhi), + Cell::new(format_bps(m.free_float_bps)), + Cell::new(m.holder_count), + ]); + } + + println!("{table}"); + Ok(()) +} + +fn handle_flow(args: &FlowArgs, config: &IdxConfig) -> Result<(), IdxError> { + let conn = db::open_db(config)?; + let symbol = args.symbol.trim().to_uppercase(); + let ticker_id = db::get_ticker_id(&conn, &symbol)? + .ok_or_else(|| IdxError::SymbolNotFound(symbol.clone()))?; + + let flow = db::query_bing_flow(&conn, ticker_id)?; + let Some(flow) = flow else { + println!( + "No institutional flow data. Run: idx ownership import --fetch-bing {}", + symbol + ); + return Ok(()); + }; + + if matches!(config.output, OutputFormat::Json) { + return json::print_json(&flow); + } + + println!("{} {} ({})", "Flow:".bold(), symbol.bold(), flow.period); + print_flow_section("TOP BUYERS", &flow.top_buyers); + print_flow_section("TOP SELLERS", &flow.top_sellers); + print_flow_section("NEW POSITIONS", &flow.new_positions); + print_flow_section("EXITED", &flow.exited); + Ok(()) +} + +fn print_flow_section(title: &str, rows: &[HolderRow]) { + println!("\n{}", title.bold()); + if rows.is_empty() { + println!("-"); + return; + } + + let mut table = Table::new(); + table + .load_preset(UTF8_FULL) + .set_content_arrangement(ContentArrangement::Dynamic) + .set_header(vec!["RANK", "INVESTOR", "SHARES", "%", "SIGNAL"]); + + for (i, row) in rows.iter().enumerate() { + table.add_row(vec![ + Cell::new(i + 1), + Cell::new(&row.name), + Cell::new(format_idr(row.shares)), + Cell::new(format_bps(row.percentage_bps)), + Cell::new(format_signal(row.signal)), + ]); + } + println!("{table}"); +} + +fn handle_graph(args: &GraphArgs, config: &IdxConfig) -> Result<(), IdxError> { + let conn = db::open_db(config)?; + let (nodes, edges) = graph::query_ownership_graph(&conn, &args.root, args.depth)?; + + if matches!(config.output, OutputFormat::Json) { + #[derive(Serialize)] + struct GraphOutput<'a> { + nodes: &'a [crate::ownership::types::GraphNode], + edges: &'a [crate::ownership::types::GraphEdge], + } + return json::print_json(&GraphOutput { + nodes: &nodes, + edges: &edges, + }); + } + + match args.format.trim().to_lowercase().as_str() { + "table" => { + print!("{}", graph::format_graph_text(&nodes, &edges)); + Ok(()) + } + "dot" => { + print!("{}", graph::format_graph_dot(&nodes, &edges)); + Ok(()) + } + other => Err(IdxError::ParseError(format!( + "invalid --format '{other}', expected table|dot" + ))), + } +} + +fn handle_changes(args: &ChangesArgs, config: &IdxConfig) -> Result<(), IdxError> { + let conn = db::open_db(config)?; + let changes = db::query_changes(&conn, &args.from, &args.to)?; + + if matches!(config.output, OutputFormat::Json) { + return json::print_json(&changes); + } + + if changes.is_empty() { + println!( + "No changes found between {} and {}.", + args.from.trim(), + args.to.trim() + ); + return Ok(()); + } + + let mut table = Table::new(); + table + .load_preset(UTF8_FULL) + .set_content_arrangement(ContentArrangement::Dynamic) + .set_header(vec!["TICKER", "ENTITY", "CHANGE", "OLD%", "NEW%", "DELTA%"]); + + for row in changes { + table.add_row(vec![ + Cell::new(row.ticker_code), + Cell::new(row.entity_name), + Cell::new(format_change_type(row.change_type)), + Cell::new( + row.old_bps + .map(format_bps) + .unwrap_or_else(|| "-".to_string()), + ), + Cell::new( + row.new_bps + .map(format_bps) + .unwrap_or_else(|| "-".to_string()), + ), + Cell::new(format_signed_bps(row.delta_bps)), + ]); + } + + println!("{table}"); + Ok(()) +} + +fn handle_resolve(args: &ResolveArgs, config: &IdxConfig) -> Result<(), IdxError> { + let conn = db::open_db(config)?; + + match &args.command { + ResolveCommand::Unresolved { limit } => { + let rows = db::list_unresolved(&conn, *limit)?; + + if matches!(config.output, OutputFormat::Json) { + return json::print_json(&rows); + } + + if rows.is_empty() { + println!("No unresolved or low-confidence aliases found."); + return Ok(()); + } + + let mut table = Table::new(); + table + .load_preset(UTF8_FULL) + .set_content_arrangement(ContentArrangement::Dynamic) + .set_header(vec![ + "RAW_NAME", + "SOURCE", + "TICKER", + "CURRENT_ENTITY", + "CONFIDENCE", + ]); + + for row in rows { + table.add_row(vec![ + Cell::new(row.raw_name), + Cell::new(row.source), + Cell::new(row.ticker_code), + Cell::new(row.current_entity.unwrap_or_else(|| "-".to_string())), + Cell::new( + row.confidence + .map(|v| format!("{v:.2}")) + .unwrap_or_else(|| "-".to_string()), + ), + ]); + } + println!("{table}"); + Ok(()) + } + ResolveCommand::Map { alias, entity } => { + db::manual_map(&conn, alias, entity)?; + println!("Mapped alias '{}' -> '{}'.", alias, entity); + Ok(()) + } + ResolveCommand::Merge { keep, merge } => { + let alias_updates: i64 = conn + .query_row( + "SELECT COUNT(*) FROM entity_aliases WHERE entity_id = ?1", + rusqlite::params![merge], + |row| row.get(0), + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + let ksei_updates: i64 = conn + .query_row( + "SELECT COUNT(*) FROM ksei_holdings WHERE entity_id = ?1", + rusqlite::params![merge], + |row| row.get(0), + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + let bing_updates: i64 = conn + .query_row( + "SELECT COUNT(*) FROM bing_holdings WHERE entity_id = ?1", + rusqlite::params![merge], + |row| row.get(0), + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + db::merge_entities(&conn, *keep, *merge)?; + + println!( + "Merged entity {} into {} (aliases: {}, ksei_holdings: {}, bing_holdings: {}).", + merge, keep, alias_updates, ksei_updates, bing_updates + ); + Ok(()) + } + } +} + +fn handle_import(args: &ImportArgs, config: &IdxConfig) -> Result<(), IdxError> { + if args.file.is_none() && args.url.is_none() && args.fetch_bing.is_none() { + return Err(IdxError::ParseError( + "provide one of: --file, --url, or --fetch-bing".to_string(), + )); + } + + if args.file.is_some() && args.url.is_some() { + return Err(IdxError::ParseError( + "--file and --url are mutually exclusive".to_string(), + )); + } + + if let Some(symbols) = &args.fetch_bing { + let clean: Vec = symbols + .iter() + .flat_map(|v| v.split(',')) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(|s| s.to_uppercase()) + .collect(); + + if !clean.is_empty() { + eprintln!( + "info: --fetch-bing requested for {} symbol(s), implementation deferred for Sprint 6", + clean.len() + ); + for symbol in &clean { + eprintln!(" - {symbol}"); + } + } + } + + let Some(pdf_path) = resolve_pdf_input(args)? else { + return Ok(()); + }; + + let conn = db::open_db(config)?; + + let sha256 = sha256_file(&pdf_path)?; + if !args.force && db::release_exists(&conn, &sha256)? { + println!("Release already imported (sha256: {sha256}). Use --force to re-import."); + return Ok(()); + } + + let raw_rows = parser::parse_ksei_pdf(&pdf_path)?; + if raw_rows.is_empty() { + return Err(IdxError::ParseError( + "no KSEI rows parsed from PDF".to_string(), + )); + } + + let mut holdings = Vec::with_capacity(raw_rows.len()); + let mut ticker_ids = HashSet::new(); + + for raw in &raw_rows { + let draft = entities::normalize_ksei_row(raw)?; + let ticker_id = db::upsert_ticker(&conn, &draft.ticker_code, draft.issuer_name.as_deref())?; + let entity_id = + entities::resolve_entity(&conn, &draft.raw_investor_name, OwnershipSource::Ksei)?; + + ticker_ids.insert(ticker_id); + holdings.push(KseiHolding { + id: 0, + ticker_id, + entity_id: Some(entity_id), + raw_investor_name: draft.raw_investor_name, + investor_type: draft.investor_type, + locality: draft.locality, + nationality: draft.nationality, + domicile: draft.domicile, + holdings_scripless: draft.holdings_scripless, + holdings_scrip: draft.holdings_scrip, + total_shares: draft.total_shares, + percentage_bps: draft.percentage_bps, + report_date: draft.report_date, + release_sha256: sha256.clone(), + }); + } + + let inserted_rows = db::insert_ksei_holdings(&conn, &holdings)?; + let as_of_date = holdings + .iter() + .map(|h| h.report_date) + .max() + .ok_or_else(|| { + IdxError::ParseError("missing report_date in parsed holdings".to_string()) + })?; + + let release = OwnershipRelease { + id: 0, + source_url: args.url.clone(), + sha256, + as_of_date, + row_count: inserted_rows, + imported_at: Utc::now().timestamp(), + }; + let _ = db::insert_release(&conn, &release)?; + + println!( + "Imported {} rows for {} tickers (as of {}).", + inserted_rows, + ticker_ids.len(), + as_of_date.format("%Y-%m-%d") + ); + + Ok(()) +} + +fn handle_releases(config: &IdxConfig) -> Result<(), IdxError> { + let conn = db::open_db(config)?; + let releases = db::query_releases(&conn)?; + + if matches!(config.output, OutputFormat::Json) { + return json::print_json(&releases); + } + + if releases.is_empty() { + println!("No ownership releases imported yet."); + return Ok(()); + } + + let mut table = Table::new(); + table + .load_preset(UTF8_FULL) + .set_content_arrangement(ContentArrangement::Dynamic) + .set_header(vec!["AS_OF", "ROWS", "SHA256", "SOURCE", "IMPORTED_AT"]); + + for r in releases { + table.add_row(vec![ + Cell::new(r.as_of_date.format("%Y-%m-%d").to_string()), + Cell::new(r.row_count), + Cell::new(r.sha256), + Cell::new(r.source_url.unwrap_or_else(|| "-".to_string())), + Cell::new(r.imported_at), + ]); + } + + println!("{table}"); + Ok(()) +} + +fn resolve_pdf_input(args: &ImportArgs) -> Result, IdxError> { + if let Some(path) = &args.file { + if !path.exists() { + return Err(IdxError::Io(format!( + "input PDF not found: {}", + path.display() + ))); + } + return Ok(Some(path.clone())); + } + + if let Some(url) = &args.url { + let target = cache_pdf_path(url)?; + download_pdf(url, &target)?; + return Ok(Some(target)); + } + + Ok(None) +} + +fn cache_pdf_path(url: &str) -> Result { + let dirs = ProjectDirs::from("", "", "idx") + .ok_or_else(|| IdxError::Io("unable to resolve cache directory".to_string()))?; + let raw_dir = dirs.cache_dir().join("ownership").join("raw"); + fs::create_dir_all(&raw_dir).map_err(|e| IdxError::Io(e.to_string()))?; + + let mut file_name = url + .rsplit('/') + .next() + .unwrap_or("ksei-ownership.pdf") + .split('?') + .next() + .unwrap_or("ksei-ownership.pdf") + .to_string(); + + if file_name.is_empty() || file_name == "/" { + file_name = format!("ksei-{}.pdf", Utc::now().timestamp()); + } + if !file_name.to_ascii_lowercase().ends_with(".pdf") { + file_name.push_str(".pdf"); + } + + Ok(raw_dir.join(file_name)) +} + +fn download_pdf(url: &str, target: &Path) -> Result<(), IdxError> { + let response = ureq::get(url) + .header( + "User-Agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", + ) + .header("Accept", "application/pdf,application/octet-stream,*/*;q=0.8") + .header("Accept-Language", "en-US,en;q=0.9") + .header("Referer", "https://www.idx.co.id/") + .call() + .map_err(|e| IdxError::Http(format!("failed to download PDF: {e}")))?; + + let mut body = response.into_body(); + let bytes = body + .read_to_vec() + .map_err(|e| IdxError::Http(format!("failed reading PDF body: {e}")))?; + + fs::write(target, &bytes).map_err(|e| { + IdxError::Io(format!( + "failed writing cached PDF {}: {e}", + target.display() + )) + })?; + + Ok(()) +} + +fn sha256_file(path: &Path) -> Result { + let bytes = fs::read(path).map_err(|e| { + IdxError::Io(format!( + "failed to read file for sha256 {}: {e}", + path.display() + )) + })?; + let mut hasher = Sha256::new(); + hasher.update(bytes); + let digest = hasher.finalize(); + Ok(digest.iter().map(|b| format!("{b:02x}")).collect()) +} + +fn format_bps(bps: i64) -> String { + format!("{:.2}%", bps as f64 / 100.0) +} + +fn format_signal(signal: Option) -> String { + match signal { + Some(FlowSignal::Buyer) => "BUYER".green().to_string(), + Some(FlowSignal::Seller) => "SELLER".red().to_string(), + Some(FlowSignal::NewPosition) => "NEW".blue().to_string(), + Some(FlowSignal::Exited) => "EXITED".yellow().to_string(), + Some(FlowSignal::Holder) => "HOLDER".to_string(), + None => "-".to_string(), + } +} + +fn format_change_type(change_type: ChangeType) -> String { + match change_type { + ChangeType::New => "NEW".green().to_string(), + ChangeType::Exited => "EXITED".red().to_string(), + ChangeType::Increased => "INCREASED".green().to_string(), + ChangeType::Decreased => "DECREASED".red().to_string(), + } +} + +fn format_signed_bps(bps: i64) -> String { + let pct = bps as f64 / 100.0; + if bps > 0 { + format!("+{pct:.2}%") + } else { + format!("{pct:.2}%") + } +} diff --git a/src/error.rs b/src/error.rs index c659aa8..5f882ed 100644 --- a/src/error.rs +++ b/src/error.rs @@ -26,6 +26,10 @@ pub enum IdxError { Http(String), #[error("auth error: {0}")] AuthError(String), + #[error("database error: {0}")] + DatabaseError(String), + #[error("PDF parse error: {0}")] + PdfParseError(String), } #[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] @@ -41,6 +45,8 @@ pub enum ErrorCode { Io, Http, AuthError, + DatabaseError, + PdfParseError, } impl IdxError { @@ -57,6 +63,8 @@ impl IdxError { Self::Io(_) => ErrorCode::Io, Self::Http(_) => ErrorCode::Http, Self::AuthError(_) => ErrorCode::AuthError, + Self::DatabaseError(_) => ErrorCode::DatabaseError, + Self::PdfParseError(_) => ErrorCode::PdfParseError, } } diff --git a/src/main.rs b/src/main.rs index fed1d65..23286fa 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,6 +5,8 @@ mod cli; mod config; mod error; mod output; +#[cfg(feature = "ownership")] +pub mod ownership; use clap::CommandFactory; use clap::Parser; @@ -70,6 +72,13 @@ fn run() -> Result<(), IdxError> { return Err(err); } } + #[cfg(feature = "ownership")] + Commands::Ownership(cmd) => { + if let Err(err) = cli::ownership::handle(&cmd.command, &config) { + emit_error(&err, &config.output); + return Err(err); + } + } } Ok(()) diff --git a/src/ownership/db.rs b/src/ownership/db.rs new file mode 100644 index 0000000..ca899aa --- /dev/null +++ b/src/ownership/db.rs @@ -0,0 +1,1447 @@ +use std::fs; +use std::path::PathBuf; + +use chrono::NaiveDate; +use directories::ProjectDirs; +use rusqlite::{Connection, params}; + +use crate::config::{IdxConfig, get_config_value}; +use crate::error::IdxError; +use crate::ownership::search; +use crate::ownership::types::{ + BingHolding, ChangeRow, ChangeType, ConcentrationMetrics, CrossHolderRow, Entity, + EntityHoldings, EntityTickerRow, FlowSignal, HolderRow, InstitutionalFlow, KseiHolding, + Locality, OwnershipRelease, OwnershipSource, Ticker, TickerOwnership, UnresolvedRow, +}; + +/// Ownership schema version 1 DDL. +pub const SCHEMA_V1: &str = r#" +PRAGMA foreign_keys = ON; + +CREATE TABLE IF NOT EXISTS ownership_releases ( + id INTEGER PRIMARY KEY, + source_url TEXT, + sha256 TEXT NOT NULL UNIQUE, + as_of_date TEXT NOT NULL, + row_count INTEGER NOT NULL, + imported_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS entities ( + id INTEGER PRIMARY KEY, + canonical_name TEXT NOT NULL, + entity_type TEXT, + country TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE VIRTUAL TABLE IF NOT EXISTS entity_fts USING fts5( + canonical_name, + aliases, + content='entities', + content_rowid='id', + tokenize='trigram' +); + +CREATE TABLE IF NOT EXISTS entity_aliases ( + id INTEGER PRIMARY KEY, + entity_id INTEGER NOT NULL REFERENCES entities(id), + raw_name TEXT NOT NULL, + source TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 1.0, + method TEXT NOT NULL, + UNIQUE(raw_name, source) +); +CREATE INDEX IF NOT EXISTS idx_aliases_entity ON entity_aliases(entity_id); +CREATE INDEX IF NOT EXISTS idx_aliases_name ON entity_aliases(raw_name); + +CREATE TABLE IF NOT EXISTS tickers ( + id INTEGER PRIMARY KEY, + code TEXT NOT NULL UNIQUE, + name TEXT, + sector TEXT +); + +CREATE TABLE IF NOT EXISTS ksei_holdings ( + id INTEGER PRIMARY KEY, + ticker_id INTEGER NOT NULL REFERENCES tickers(id), + entity_id INTEGER REFERENCES entities(id), + raw_investor_name TEXT NOT NULL, + investor_type TEXT, + locality TEXT, + nationality TEXT, + domicile TEXT, + holdings_scripless INTEGER NOT NULL, + holdings_scrip INTEGER NOT NULL, + total_shares INTEGER NOT NULL, + percentage_bps INTEGER NOT NULL, + report_date TEXT NOT NULL, + release_sha256 TEXT NOT NULL, + UNIQUE(release_sha256, ticker_id, raw_investor_name) +); +CREATE INDEX IF NOT EXISTS idx_ksei_ticker ON ksei_holdings(ticker_id); +CREATE INDEX IF NOT EXISTS idx_ksei_entity ON ksei_holdings(entity_id); +CREATE INDEX IF NOT EXISTS idx_ksei_date ON ksei_holdings(report_date); +CREATE INDEX IF NOT EXISTS idx_ksei_pct ON ksei_holdings(percentage_bps DESC); + +CREATE TABLE IF NOT EXISTS bing_holdings ( + id INTEGER PRIMARY KEY, + ticker_id INTEGER NOT NULL REFERENCES tickers(id), + entity_id INTEGER REFERENCES entities(id), + raw_investor_name TEXT NOT NULL, + investor_type TEXT, + shares_held INTEGER, + shares_changed INTEGER, + pct_ownership_bps INTEGER, + value_usd INTEGER, + report_date TEXT NOT NULL, + signal TEXT NOT NULL, + fetched_at INTEGER NOT NULL, + UNIQUE(ticker_id, raw_investor_name, report_date, signal) +); +CREATE INDEX IF NOT EXISTS idx_bing_ticker ON bing_holdings(ticker_id); +CREATE INDEX IF NOT EXISTS idx_bing_entity ON bing_holdings(entity_id); +CREATE INDEX IF NOT EXISTS idx_bing_date ON bing_holdings(report_date); +"#; + +/// Ensure ownership schema is migrated and ready for use. +pub fn ensure_schema(conn: &Connection) -> Result<(), IdxError> { + let user_version: i64 = conn + .query_row("PRAGMA user_version", [], |row| row.get(0)) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + if conn.path().is_some() { + let _ = conn.pragma_update(None, "journal_mode", "WAL"); + } + conn.pragma_update(None, "foreign_keys", "ON") + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + if user_version < 1 { + conn.execute_batch(SCHEMA_V1) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + conn.pragma_update(None, "user_version", 1) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + } + + Ok(()) +} + +/// Open ownership database connection and run idempotent schema migration. +pub fn open_db(_config: &IdxConfig) -> Result { + let db_path = resolve_db_path()?; + + if let Some(parent) = db_path.parent() { + fs::create_dir_all(parent).map_err(|e| IdxError::DatabaseError(e.to_string()))?; + } + + let conn = Connection::open(&db_path).map_err(|e| IdxError::DatabaseError(e.to_string()))?; + ensure_schema(&conn)?; + + Ok(conn) +} + +/// Insert or get a ticker by code. Returns ticker_id. +pub fn upsert_ticker(conn: &Connection, code: &str, name: Option<&str>) -> Result { + conn.execute( + "INSERT INTO tickers (code, name) VALUES (?1, ?2) + ON CONFLICT(code) DO UPDATE SET + name = COALESCE(excluded.name, tickers.name)", + params![code, name], + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + conn.query_row( + "SELECT id FROM tickers WHERE code = ?1", + params![code], + |row| row.get(0), + ) + .map_err(|e| IdxError::DatabaseError(e.to_string())) +} + +/// Bulk insert KSEI holdings within a transaction. +/// Uses INSERT OR IGNORE for dedup (unique on release_sha256 + ticker_id + raw_investor_name). +pub fn insert_ksei_holdings( + conn: &Connection, + holdings: &[KseiHolding], +) -> Result { + if holdings.is_empty() { + return Ok(0); + } + + conn.execute("BEGIN IMMEDIATE", []) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let mut inserted = 0usize; + for h in holdings { + let locality = h.locality.map(locality_to_db); + let investor_type = h.investor_type.as_ref().map(|v| v.0.as_str()); + let changed = conn + .execute( + "INSERT OR IGNORE INTO ksei_holdings ( + ticker_id, entity_id, raw_investor_name, investor_type, locality, + nationality, domicile, holdings_scripless, holdings_scrip, total_shares, + percentage_bps, report_date, release_sha256 + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", + params![ + h.ticker_id, + h.entity_id, + h.raw_investor_name, + investor_type, + locality, + h.nationality, + h.domicile, + h.holdings_scripless, + h.holdings_scrip, + h.total_shares, + h.percentage_bps, + h.report_date.format("%Y-%m-%d").to_string(), + h.release_sha256, + ], + ) + .map_err(|e| IdxError::DatabaseError(e.to_string())); + + match changed { + Ok(n) => inserted += n, + Err(err) => { + let _ = conn.execute("ROLLBACK", []); + return Err(err); + } + } + } + + conn.execute("COMMIT", []) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + Ok(inserted) +} + +/// Bulk insert Bing holdings within a transaction. +pub fn insert_bing_holdings( + conn: &Connection, + holdings: &[BingHolding], +) -> Result { + if holdings.is_empty() { + return Ok(0); + } + + conn.execute("BEGIN IMMEDIATE", []) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let mut inserted = 0usize; + for h in holdings { + let changed = conn + .execute( + "INSERT OR IGNORE INTO bing_holdings ( + ticker_id, entity_id, raw_investor_name, investor_type, shares_held, + shares_changed, pct_ownership_bps, value_usd, report_date, signal, fetched_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", + params![ + h.ticker_id, + h.entity_id, + h.raw_investor_name, + h.investor_type, + h.shares_held, + h.shares_changed, + h.pct_ownership_bps, + h.value_usd, + h.report_date.format("%Y-%m-%d").to_string(), + flow_signal_to_db(h.signal), + h.fetched_at, + ], + ) + .map_err(|e| IdxError::DatabaseError(e.to_string())); + + match changed { + Ok(n) => inserted += n, + Err(err) => { + let _ = conn.execute("ROLLBACK", []); + return Err(err); + } + } + } + + conn.execute("COMMIT", []) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + Ok(inserted) +} + +/// Record a KSEI release import. Returns release_id. +pub fn insert_release(conn: &Connection, release: &OwnershipRelease) -> Result { + conn.execute( + "INSERT INTO ownership_releases (source_url, sha256, as_of_date, row_count, imported_at) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + release.source_url, + release.sha256, + release.as_of_date.format("%Y-%m-%d").to_string(), + i64::try_from(release.row_count) + .map_err(|e| IdxError::DatabaseError(format!("invalid row_count: {e}")))?, + release.imported_at, + ], + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + Ok(conn.last_insert_rowid()) +} + +/// Check if a release with this SHA-256 already exists. +pub fn release_exists(conn: &Connection, sha256: &str) -> Result { + let exists: i64 = conn + .query_row( + "SELECT EXISTS(SELECT 1 FROM ownership_releases WHERE sha256 = ?1)", + params![sha256], + |row| row.get(0), + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + Ok(exists == 1) +} + +/// Get combined KSEI + Bing holdings for a ticker. +/// Returns TickerOwnership with merged holders sorted by percentage desc, +/// concentration metrics, and Bing flow data. +pub fn query_ticker_holdings(conn: &Connection, code: &str) -> Result { + let ticker = + query_ticker(conn, code)?.ok_or_else(|| IdxError::SymbolNotFound(code.to_string()))?; + + let ksei_as_of = conn + .query_row( + "SELECT MAX(report_date) FROM ksei_holdings WHERE ticker_id = ?1", + params![ticker.id], + |row| row.get::<_, Option>(0), + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))? + .map(|s| parse_iso_date(&s)) + .transpose()?; + + let bing_as_of = conn + .query_row( + "SELECT MAX(report_date) FROM bing_holdings WHERE ticker_id = ?1", + params![ticker.id], + |row| row.get::<_, Option>(0), + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let mut holders: Vec = Vec::new(); + + { + let mut stmt = conn + .prepare( + "SELECT k.entity_id, COALESCE(e.canonical_name, k.raw_investor_name), + k.investor_type, k.locality, k.total_shares, k.percentage_bps + FROM ksei_holdings k + LEFT JOIN entities e ON e.id = k.entity_id + WHERE k.ticker_id = ?1 + ORDER BY k.percentage_bps DESC, k.total_shares DESC", + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let rows = stmt + .query_map(params![ticker.id], |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, i64>(5)?, + )) + }) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + for row in rows { + let (entity_id, name, investor_type, locality_raw, shares, percentage_bps) = + row.map_err(|e| IdxError::DatabaseError(e.to_string()))?; + holders.push(HolderRow { + rank: 0, + source: OwnershipSource::Ksei, + name, + entity_id, + investor_type, + locality: locality_raw.as_deref().and_then(locality_from_db), + shares, + percentage_bps, + signal: None, + }); + } + } + + { + let mut stmt = conn + .prepare( + "SELECT b.entity_id, COALESCE(e.canonical_name, b.raw_investor_name), + b.investor_type, COALESCE(b.shares_held, 0), COALESCE(b.pct_ownership_bps, 0), b.signal + FROM bing_holdings b + LEFT JOIN entities e ON e.id = b.entity_id + WHERE b.ticker_id = ?1 + ORDER BY COALESCE(b.pct_ownership_bps, 0) DESC, COALESCE(b.shares_held, 0) DESC", + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let rows = stmt + .query_map(params![ticker.id], |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, String>(5)?, + )) + }) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + for row in rows { + let (entity_id, name, investor_type, shares, percentage_bps, signal_raw) = + row.map_err(|e| IdxError::DatabaseError(e.to_string()))?; + holders.push(HolderRow { + rank: 0, + source: OwnershipSource::Bing, + name, + entity_id, + investor_type, + locality: None, + shares, + percentage_bps, + signal: flow_signal_from_db(&signal_raw), + }); + } + } + + holders.sort_by(|a, b| { + b.percentage_bps + .cmp(&a.percentage_bps) + .then_with(|| b.shares.cmp(&a.shares)) + }); + for (idx, holder) in holders.iter_mut().enumerate() { + holder.rank = idx + 1; + } + + let percentages = holders.iter().map(|h| h.percentage_bps).collect::>(); + let concentration = compute_concentration(&percentages); + let flow = query_bing_flow(conn, ticker.id)?; + + Ok(TickerOwnership { + ticker, + ksei_as_of, + bing_as_of, + holders, + concentration, + flow, + }) +} + +/// Get all holdings for an entity across tickers. +pub fn query_entity_holdings( + conn: &Connection, + entity_id: i64, +) -> Result { + let entity = query_entity(conn, entity_id)? + .ok_or_else(|| IdxError::SymbolNotFound(format!("entity:{entity_id}")))?; + + let mut holdings = Vec::new(); + + { + let mut stmt = conn + .prepare( + "SELECT t.id, t.code, t.name, t.sector, k.total_shares, k.percentage_bps, k.report_date + FROM ksei_holdings k + JOIN tickers t ON t.id = k.ticker_id + WHERE k.entity_id = ?1", + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let rows = stmt + .query_map(params![entity_id], |row| { + Ok(EntityTickerRow { + ticker: Ticker { + id: row.get(0)?, + code: row.get(1)?, + name: row.get(2)?, + sector: row.get(3)?, + }, + source: OwnershipSource::Ksei, + shares: row.get(4)?, + percentage_bps: row.get(5)?, + report_date: parse_iso_date(&row.get::<_, String>(6)?) + .map_err(|e| rusqlite::Error::ToSqlConversionFailure(Box::new(e)))?, + }) + }) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + for row in rows { + holdings.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?); + } + } + + { + let mut stmt = conn + .prepare( + "SELECT t.id, t.code, t.name, t.sector, + COALESCE(b.shares_held, 0), COALESCE(b.pct_ownership_bps, 0), b.report_date + FROM bing_holdings b + JOIN tickers t ON t.id = b.ticker_id + WHERE b.entity_id = ?1", + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let rows = stmt + .query_map(params![entity_id], |row| { + Ok(EntityTickerRow { + ticker: Ticker { + id: row.get(0)?, + code: row.get(1)?, + name: row.get(2)?, + sector: row.get(3)?, + }, + source: OwnershipSource::Bing, + shares: row.get(4)?, + percentage_bps: row.get(5)?, + report_date: parse_iso_date(&row.get::<_, String>(6)?) + .map_err(|e| rusqlite::Error::ToSqlConversionFailure(Box::new(e)))?, + }) + }) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + for row in rows { + holdings.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?); + } + } + + holdings.sort_by(|a, b| { + b.percentage_bps + .cmp(&a.percentage_bps) + .then_with(|| a.ticker.code.cmp(&b.ticker.code)) + }); + + let ticker_count = holdings + .iter() + .map(|h| h.ticker.id) + .collect::>() + .len(); + + Ok(EntityHoldings { + entity, + ticker_count, + holdings, + }) +} + +/// Rank entities by number of tickers they hold (cross-ownership breadth). +pub fn query_cross_holders( + conn: &Connection, + min_tickers: usize, + limit: usize, +) -> Result, IdxError> { + let mut stmt = conn + .prepare( + "SELECT e.id, e.canonical_name, e.entity_type, e.country, + COUNT(DISTINCT u.ticker_id) AS ticker_count, + SUM(u.percentage_bps) AS total_bps + FROM entities e + JOIN ( + SELECT entity_id, ticker_id, percentage_bps + FROM ksei_holdings + WHERE entity_id IS NOT NULL + UNION ALL + SELECT entity_id, ticker_id, COALESCE(pct_ownership_bps, 0) AS percentage_bps + FROM bing_holdings + WHERE entity_id IS NOT NULL + ) u ON u.entity_id = e.id + GROUP BY e.id + HAVING COUNT(DISTINCT u.ticker_id) >= ?1 + ORDER BY ticker_count DESC, total_bps DESC, e.canonical_name ASC + LIMIT ?2", + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let rows = stmt + .query_map( + params![ + i64::try_from(min_tickers) + .map_err(|e| IdxError::DatabaseError(format!("invalid min_tickers: {e}")))?, + i64::try_from(limit) + .map_err(|e| IdxError::DatabaseError(format!("invalid limit: {e}")))?, + ], + |row| { + Ok(CrossHolderRow { + entity: Entity { + id: row.get(0)?, + canonical_name: row.get(1)?, + entity_type: row.get(2)?, + country: row.get(3)?, + }, + ticker_count: row.get::<_, i64>(4)? as usize, + total_bps: row.get(5)?, + }) + }, + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let mut out = Vec::new(); + for row in rows { + out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?); + } + Ok(out) +} + +/// Rank tickers by ownership concentration. +/// sort_by: "top1", "top3", "hhi" +pub fn query_concentration( + conn: &Connection, + sort_by: &str, + limit: usize, +) -> Result, IdxError> { + let valid = ["top1", "top3", "hhi"]; + if !valid.contains(&sort_by) { + return Err(IdxError::ParseError(format!( + "invalid sort_by '{sort_by}', expected one of: top1, top3, hhi" + ))); + } + + let mut stmt = conn + .prepare( + "SELECT t.code, k.percentage_bps + FROM tickers t + JOIN ksei_holdings k ON k.ticker_id = t.id + ORDER BY t.code ASC, k.percentage_bps DESC", + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let rows = stmt + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + }) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let mut grouped: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for row in rows { + let (code, bps) = row.map_err(|e| IdxError::DatabaseError(e.to_string()))?; + grouped.entry(code).or_default().push(bps); + } + + let mut ranked: Vec<(String, ConcentrationMetrics)> = grouped + .into_iter() + .map(|(code, bps)| (code, compute_concentration(&bps))) + .collect(); + + ranked.sort_by(|a, b| { + let am = &a.1; + let bm = &b.1; + match sort_by { + "top1" => bm.top1_bps.cmp(&am.top1_bps), + "top3" => bm.top3_bps.cmp(&am.top3_bps), + "hhi" => bm.hhi.cmp(&am.hhi), + _ => std::cmp::Ordering::Equal, + } + .then_with(|| a.0.cmp(&b.0)) + }); + + ranked.truncate(limit); + Ok(ranked) +} + +/// List all imported releases. +pub fn query_releases(conn: &Connection) -> Result, IdxError> { + let mut stmt = conn + .prepare( + "SELECT id, source_url, sha256, as_of_date, row_count, imported_at + FROM ownership_releases + ORDER BY as_of_date DESC, imported_at DESC", + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let rows = stmt + .query_map([], |row| { + Ok(OwnershipRelease { + id: row.get(0)?, + source_url: row.get(1)?, + sha256: row.get(2)?, + as_of_date: parse_iso_date(&row.get::<_, String>(3)?) + .map_err(|e| rusqlite::Error::ToSqlConversionFailure(Box::new(e)))?, + row_count: row.get::<_, i64>(4)? as usize, + imported_at: row.get(5)?, + }) + }) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let mut out = Vec::new(); + for row in rows { + out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?); + } + Ok(out) +} + +/// Get Bing institutional flow for a ticker. +pub fn query_bing_flow( + conn: &Connection, + ticker_id: i64, +) -> Result, IdxError> { + let latest = conn + .query_row( + "SELECT MAX(report_date) FROM bing_holdings WHERE ticker_id = ?1", + params![ticker_id], + |row| row.get::<_, Option>(0), + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let Some(period) = latest else { + return Ok(None); + }; + + let mut stmt = conn + .prepare( + "SELECT b.entity_id, COALESCE(e.canonical_name, b.raw_investor_name), + b.investor_type, COALESCE(b.shares_held, 0), COALESCE(b.pct_ownership_bps, 0), b.signal + FROM bing_holdings b + LEFT JOIN entities e ON e.id = b.entity_id + WHERE b.ticker_id = ?1 AND b.report_date = ?2 + ORDER BY COALESCE(b.pct_ownership_bps, 0) DESC, COALESCE(b.shares_held, 0) DESC", + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let rows = stmt + .query_map(params![ticker_id, &period], |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, String>(5)?, + )) + }) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let mut top_buyers = Vec::new(); + let mut top_sellers = Vec::new(); + let mut new_positions = Vec::new(); + let mut exited = Vec::new(); + + let mut rank_counter = 1usize; + for row in rows { + let (entity_id, name, investor_type, shares, percentage_bps, signal_raw) = + row.map_err(|e| IdxError::DatabaseError(e.to_string()))?; + let Some(signal) = flow_signal_from_db(&signal_raw) else { + continue; + }; + + let holder = HolderRow { + rank: rank_counter, + source: OwnershipSource::Bing, + name, + entity_id, + investor_type, + locality: None, + shares, + percentage_bps, + signal: Some(signal), + }; + rank_counter += 1; + + match signal { + FlowSignal::Buyer => top_buyers.push(holder), + FlowSignal::Seller => top_sellers.push(holder), + FlowSignal::NewPosition => new_positions.push(holder), + FlowSignal::Exited => exited.push(holder), + FlowSignal::Holder => {} + } + } + + Ok(Some(InstitutionalFlow { + period, + top_buyers, + top_sellers, + new_positions, + exited, + })) +} + +/// Compare two KSEI snapshots by date and return ownership changes. +/// Finds: new holders, exited holders, percentage increases/decreases. +pub fn query_changes( + conn: &Connection, + from_date: &str, + to_date: &str, +) -> Result, IdxError> { + let mut stmt = conn + .prepare( + "WITH + from_snapshot AS ( + SELECT + t.code AS ticker_code, + COALESCE(e.canonical_name, k.raw_investor_name) AS entity_name, + CASE + WHEN k.entity_id IS NOT NULL THEN 'id:' || k.entity_id + ELSE 'raw:' || UPPER(TRIM(k.raw_investor_name)) + END AS holder_key, + k.percentage_bps AS old_bps + FROM ksei_holdings k + JOIN tickers t ON t.id = k.ticker_id + LEFT JOIN entities e ON e.id = k.entity_id + WHERE k.report_date = ?1 + ), + to_snapshot AS ( + SELECT + t.code AS ticker_code, + COALESCE(e.canonical_name, k.raw_investor_name) AS entity_name, + CASE + WHEN k.entity_id IS NOT NULL THEN 'id:' || k.entity_id + ELSE 'raw:' || UPPER(TRIM(k.raw_investor_name)) + END AS holder_key, + k.percentage_bps AS new_bps + FROM ksei_holdings k + JOIN tickers t ON t.id = k.ticker_id + LEFT JOIN entities e ON e.id = k.entity_id + WHERE k.report_date = ?2 + ), + keys AS ( + SELECT ticker_code, holder_key FROM from_snapshot + UNION + SELECT ticker_code, holder_key FROM to_snapshot + ) + SELECT + k.ticker_code, + COALESCE(ts.entity_name, fs.entity_name) AS entity_name, + fs.old_bps, + ts.new_bps + FROM keys k + LEFT JOIN from_snapshot fs + ON fs.ticker_code = k.ticker_code AND fs.holder_key = k.holder_key + LEFT JOIN to_snapshot ts + ON ts.ticker_code = k.ticker_code AND ts.holder_key = k.holder_key + WHERE + (fs.old_bps IS NULL AND ts.new_bps IS NOT NULL) + OR (fs.old_bps IS NOT NULL AND ts.new_bps IS NULL) + OR (fs.old_bps IS NOT NULL AND ts.new_bps IS NOT NULL AND fs.old_bps <> ts.new_bps) + ORDER BY k.ticker_code ASC, ABS(COALESCE(ts.new_bps, 0) - COALESCE(fs.old_bps, 0)) DESC, entity_name ASC", + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let rows = stmt + .query_map(params![from_date, to_date], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + )) + }) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let mut out = Vec::new(); + for row in rows { + let (ticker_code, entity_name, old_bps, new_bps) = + row.map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let change_type = match (old_bps, new_bps) { + (None, Some(_)) => ChangeType::New, + (Some(_), None) => ChangeType::Exited, + (Some(old), Some(new)) if new > old => ChangeType::Increased, + (Some(_), Some(_)) => ChangeType::Decreased, + (None, None) => continue, + }; + + let delta_bps = new_bps.unwrap_or(0) - old_bps.unwrap_or(0); + out.push(ChangeRow { + ticker_code, + entity_name, + change_type, + old_bps, + new_bps, + delta_bps, + }); + } + + Ok(out) +} + +/// List aliases that are unresolved in holdings or have low-confidence mappings. +pub fn list_unresolved(conn: &Connection, limit: usize) -> Result, IdxError> { + let limit_i64 = + i64::try_from(limit).map_err(|e| IdxError::DatabaseError(format!("invalid limit: {e}")))?; + + let mut stmt = conn + .prepare( + "SELECT DISTINCT + h.raw_name, + h.source, + t.code, + e.canonical_name, + a.confidence + FROM ( + SELECT raw_investor_name AS raw_name, 'ksei' AS source, ticker_id, entity_id + FROM ksei_holdings + UNION ALL + SELECT raw_investor_name AS raw_name, 'bing' AS source, ticker_id, entity_id + FROM bing_holdings + ) h + JOIN tickers t ON t.id = h.ticker_id + LEFT JOIN entity_aliases a ON a.raw_name = h.raw_name AND a.source = h.source + LEFT JOIN entities e ON e.id = COALESCE(a.entity_id, h.entity_id) + WHERE h.entity_id IS NULL OR a.entity_id IS NULL OR COALESCE(a.confidence, 0.0) < 0.8 + ORDER BY COALESCE(a.confidence, 0.0) ASC, h.raw_name ASC + LIMIT ?1", + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let rows = stmt + .query_map(params![limit_i64], |row| { + Ok(UnresolvedRow { + raw_name: row.get(0)?, + source: row.get(1)?, + ticker_code: row.get(2)?, + current_entity: row.get(3)?, + confidence: row.get(4)?, + }) + }) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let mut out = Vec::new(); + for row in rows { + out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?); + } + Ok(out) +} + +/// Manually map a raw investor name to a canonical entity. +/// Creates the entity if it does not exist, then creates/updates aliases and unresolved holdings. +pub fn manual_map(conn: &Connection, raw_name: &str, canonical_name: &str) -> Result<(), IdxError> { + let raw_name = raw_name.trim(); + let canonical_name = canonical_name.trim(); + + if raw_name.is_empty() || canonical_name.is_empty() { + return Err(IdxError::ParseError( + "alias and canonical entity must be non-empty".to_string(), + )); + } + + conn.execute("BEGIN IMMEDIATE", []) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let tx_result: Result<(), IdxError> = (|| { + let entity_id: i64 = match conn.query_row( + "SELECT id FROM entities WHERE canonical_name = ?1", + params![canonical_name], + |row| row.get(0), + ) { + Ok(id) => id, + Err(rusqlite::Error::QueryReturnedNoRows) => { + let now = chrono::Utc::now().timestamp(); + conn.execute( + "INSERT INTO entities (canonical_name, entity_type, country, created_at, updated_at) + VALUES (?1, NULL, NULL, ?2, ?2)", + params![canonical_name, now], + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + conn.last_insert_rowid() + } + Err(e) => return Err(IdxError::DatabaseError(e.to_string())), + }; + + for source in ["ksei", "bing"] { + conn.execute( + "INSERT INTO entity_aliases (entity_id, raw_name, source, confidence, method) + VALUES (?1, ?2, ?3, 1.0, 'manual') + ON CONFLICT(raw_name, source) DO UPDATE SET + entity_id = excluded.entity_id, + confidence = 1.0, + method = 'manual'", + params![entity_id, raw_name, source], + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + } + + conn.execute( + "UPDATE ksei_holdings SET entity_id = ?1 WHERE raw_investor_name = ?2", + params![entity_id, raw_name], + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + conn.execute( + "UPDATE bing_holdings SET entity_id = ?1 WHERE raw_investor_name = ?2", + params![entity_id, raw_name], + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let _ = search::rebuild_fts(conn); + Ok(()) + })(); + + match tx_result { + Ok(()) => { + conn.execute("COMMIT", []) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + Ok(()) + } + Err(err) => { + let _ = conn.execute("ROLLBACK", []); + Err(err) + } + } +} + +/// Merge two entities by re-pointing aliases and holdings from merge_id to keep_id, then deleting merge_id. +pub fn merge_entities(conn: &Connection, keep_id: i64, merge_id: i64) -> Result<(), IdxError> { + if keep_id == merge_id { + return Err(IdxError::ParseError( + "keep_id and merge_id must be different".to_string(), + )); + } + + conn.execute("BEGIN IMMEDIATE", []) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let tx_result: Result<(), IdxError> = (|| { + let keep_exists: i64 = conn + .query_row( + "SELECT EXISTS(SELECT 1 FROM entities WHERE id = ?1)", + params![keep_id], + |row| row.get(0), + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + let merge_exists: i64 = conn + .query_row( + "SELECT EXISTS(SELECT 1 FROM entities WHERE id = ?1)", + params![merge_id], + |row| row.get(0), + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + if keep_exists != 1 || merge_exists != 1 { + return Err(IdxError::ParseError(format!( + "entity not found: keep_id={keep_id}, merge_id={merge_id}" + ))); + } + + conn.execute( + "UPDATE entity_aliases SET entity_id = ?1 WHERE entity_id = ?2", + params![keep_id, merge_id], + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + conn.execute( + "UPDATE ksei_holdings SET entity_id = ?1 WHERE entity_id = ?2", + params![keep_id, merge_id], + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + conn.execute( + "UPDATE bing_holdings SET entity_id = ?1 WHERE entity_id = ?2", + params![keep_id, merge_id], + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + conn.execute("DELETE FROM entities WHERE id = ?1", params![merge_id]) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let _ = search::rebuild_fts(conn); + Ok(()) + })(); + + match tx_result { + Ok(()) => { + conn.execute("COMMIT", []) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + Ok(()) + } + Err(err) => { + let _ = conn.execute("ROLLBACK", []); + Err(err) + } + } +} + +/// Get ticker_id by code. +pub fn get_ticker_id(conn: &Connection, code: &str) -> Result, IdxError> { + conn.query_row( + "SELECT id FROM tickers WHERE code = ?1", + params![code], + |row| row.get(0), + ) + .map(Some) + .or_else(|e| match e { + rusqlite::Error::QueryReturnedNoRows => Ok(None), + _ => Err(IdxError::DatabaseError(e.to_string())), + }) +} + +/// Compute concentration metrics from a list of percentage_bps values. +pub fn compute_concentration(percentages_bps: &[i64]) -> ConcentrationMetrics { + let mut values = percentages_bps.to_vec(); + values.sort_by(|a, b| b.cmp(a)); + + let top1_bps = values.first().copied().unwrap_or(0); + let top3_bps: i64 = values.iter().take(3).sum(); + let total_bps: i64 = values.iter().sum(); + let hhi: i64 = values.iter().map(|p| (p * p) / 10000).sum(); + + ConcentrationMetrics { + top1_bps, + top3_bps, + hhi, + free_float_bps: (10000 - total_bps).max(0), + holder_count: values.iter().filter(|p| **p >= 100).count(), + } +} + +fn resolve_db_path() -> Result { + if let Some(custom_path) = get_config_value("ownership.db_path")? { + let trimmed = custom_path.trim(); + if !trimmed.is_empty() { + return Ok(PathBuf::from(trimmed)); + } + } + + ProjectDirs::from("", "", "idx") + .map(|dirs| dirs.data_local_dir().join("ownership.db")) + .ok_or_else(|| IdxError::DatabaseError("unable to resolve ownership db path".to_string())) +} + +fn query_ticker(conn: &Connection, code: &str) -> Result, IdxError> { + conn.query_row( + "SELECT id, code, name, sector FROM tickers WHERE code = ?1", + params![code], + |row| { + Ok(Ticker { + id: row.get(0)?, + code: row.get(1)?, + name: row.get(2)?, + sector: row.get(3)?, + }) + }, + ) + .map(Some) + .or_else(|e| match e { + rusqlite::Error::QueryReturnedNoRows => Ok(None), + _ => Err(IdxError::DatabaseError(e.to_string())), + }) +} + +fn query_entity(conn: &Connection, entity_id: i64) -> Result, IdxError> { + conn.query_row( + "SELECT id, canonical_name, entity_type, country FROM entities WHERE id = ?1", + params![entity_id], + |row| { + Ok(Entity { + id: row.get(0)?, + canonical_name: row.get(1)?, + entity_type: row.get(2)?, + country: row.get(3)?, + }) + }, + ) + .map(Some) + .or_else(|e| match e { + rusqlite::Error::QueryReturnedNoRows => Ok(None), + _ => Err(IdxError::DatabaseError(e.to_string())), + }) +} + +fn parse_iso_date(s: &str) -> Result { + NaiveDate::parse_from_str(s, "%Y-%m-%d") + .map_err(|e| IdxError::ParseError(format!("invalid ISO date '{s}': {e}"))) +} + +fn locality_to_db(locality: Locality) -> &'static str { + match locality { + Locality::Local => "L", + Locality::Foreign => "F", + } +} + +fn locality_from_db(value: &str) -> Option { + match value { + "L" => Some(Locality::Local), + "F" | "A" => Some(Locality::Foreign), + _ => None, + } +} + +fn flow_signal_to_db(signal: FlowSignal) -> &'static str { + match signal { + FlowSignal::Holder => "holder", + FlowSignal::Buyer => "buyer", + FlowSignal::Seller => "seller", + FlowSignal::NewPosition => "new_position", + FlowSignal::Exited => "exited", + } +} + +fn flow_signal_from_db(value: &str) -> Option { + match value { + "holder" => Some(FlowSignal::Holder), + "buyer" => Some(FlowSignal::Buyer), + "seller" => Some(FlowSignal::Seller), + "new_position" => Some(FlowSignal::NewPosition), + "exited" => Some(FlowSignal::Exited), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use chrono::NaiveDate; + use rusqlite::{Connection, params}; + + use crate::ownership::db::{ + compute_concentration, ensure_schema, get_ticker_id, insert_bing_holdings, + insert_ksei_holdings, insert_release, query_concentration, query_cross_holders, + query_ticker_holdings, release_exists, upsert_ticker, + }; + use crate::ownership::types::{ + BingHolding, FlowSignal, KseiHolding, Locality, OwnershipRelease, + }; + + fn setup() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + ensure_schema(&conn).unwrap(); + conn + } + + #[test] + fn test_compute_concentration_pure_function() { + let metrics = compute_concentration(&[4000, 2500, 1000, 500]); + assert_eq!(metrics.top1_bps, 4000); + assert_eq!(metrics.top3_bps, 7500); + assert_eq!(metrics.hhi, 2350); + assert_eq!(metrics.free_float_bps, 2000); + assert_eq!(metrics.holder_count, 4); + } + + #[test] + fn test_release_exists_before_and_after_insert() { + let conn = setup(); + assert!(!release_exists(&conn, "abc").unwrap()); + + let release = OwnershipRelease { + id: 0, + source_url: Some("https://example.com/r.pdf".to_string()), + sha256: "abc".to_string(), + as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(), + row_count: 10, + imported_at: 1_700_000_000, + }; + let _ = insert_release(&conn, &release).unwrap(); + + assert!(release_exists(&conn, "abc").unwrap()); + } + + #[test] + fn test_insert_fixture_ksei_and_query_ticker() { + let conn = setup(); + let bbca_id = upsert_ticker(&conn, "BBCA", Some("BCA")).unwrap(); + + conn.execute( + "INSERT INTO entities (canonical_name, entity_type, country, created_at, updated_at) + VALUES ('ALPHA FUND', NULL, NULL, 0, 0)", + [], + ) + .unwrap(); + let alpha_id = conn.last_insert_rowid(); + + let holdings = vec![ + KseiHolding { + id: 0, + ticker_id: bbca_id, + entity_id: Some(alpha_id), + raw_investor_name: "PT ALPHA FUND".to_string(), + investor_type: None, + locality: Some(Locality::Local), + nationality: None, + domicile: None, + holdings_scripless: 1_000, + holdings_scrip: 0, + total_shares: 1_000, + percentage_bps: 4000, + report_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(), + release_sha256: "r1".to_string(), + }, + KseiHolding { + id: 0, + ticker_id: bbca_id, + entity_id: None, + raw_investor_name: "BETA".to_string(), + investor_type: None, + locality: Some(Locality::Foreign), + nationality: None, + domicile: None, + holdings_scripless: 600, + holdings_scrip: 0, + total_shares: 600, + percentage_bps: 2400, + report_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(), + release_sha256: "r1".to_string(), + }, + ]; + + assert_eq!(insert_ksei_holdings(&conn, &holdings).unwrap(), 2); + + let data = query_ticker_holdings(&conn, "BBCA").unwrap(); + assert_eq!(data.ticker.code, "BBCA"); + assert_eq!(data.holders.len(), 2); + assert_eq!(data.holders[0].percentage_bps, 4000); + assert_eq!(data.concentration.top1_bps, 4000); + assert_eq!(data.concentration.top3_bps, 6400); + } + + #[test] + fn test_duplicate_insert_ignored() { + let conn = setup(); + let bbri_id = upsert_ticker(&conn, "BBRI", None).unwrap(); + + let h = KseiHolding { + id: 0, + ticker_id: bbri_id, + entity_id: None, + raw_investor_name: "DUP".to_string(), + investor_type: None, + locality: None, + nationality: None, + domicile: None, + holdings_scripless: 10, + holdings_scrip: 0, + total_shares: 10, + percentage_bps: 100, + report_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(), + release_sha256: "same-release".to_string(), + }; + + assert_eq!( + insert_ksei_holdings(&conn, std::slice::from_ref(&h)).unwrap(), + 1 + ); + assert_eq!(insert_ksei_holdings(&conn, &[h]).unwrap(), 0); + + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM ksei_holdings", [], |row| row.get(0)) + .unwrap(); + assert_eq!(count, 1); + } + + #[test] + fn test_cross_holders_same_entity_three_tickers() { + let conn = setup(); + conn.execute( + "INSERT INTO entities (canonical_name, entity_type, country, created_at, updated_at) + VALUES ('OMEGA', NULL, NULL, 0, 0)", + [], + ) + .unwrap(); + let eid = conn.last_insert_rowid(); + + for (code, bps) in [("BBCA", 1000), ("BBRI", 1100), ("BMRI", 1200)] { + let tid = upsert_ticker(&conn, code, None).unwrap(); + let h = KseiHolding { + id: 0, + ticker_id: tid, + entity_id: Some(eid), + raw_investor_name: "OMEGA".to_string(), + investor_type: None, + locality: None, + nationality: None, + domicile: None, + holdings_scripless: 10, + holdings_scrip: 0, + total_shares: 10, + percentage_bps: bps, + report_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(), + release_sha256: format!("r-{code}"), + }; + insert_ksei_holdings(&conn, &[h]).unwrap(); + } + + let rows = query_cross_holders(&conn, 3, 10).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].entity.id, eid); + assert_eq!(rows[0].ticker_count, 3); + } + + #[test] + fn test_concentration_sorting() { + let conn = setup(); + let aa = upsert_ticker(&conn, "AA", None).unwrap(); + let bb = upsert_ticker(&conn, "BB", None).unwrap(); + + let rows = vec![ + (aa, 6000, "A1"), + (aa, 1000, "A2"), + (bb, 5000, "B1"), + (bb, 3000, "B2"), + ]; + + for (tid, bps, name) in rows { + insert_ksei_holdings( + &conn, + &[KseiHolding { + id: 0, + ticker_id: tid, + entity_id: None, + raw_investor_name: name.to_string(), + investor_type: None, + locality: None, + nationality: None, + domicile: None, + holdings_scripless: 1, + holdings_scrip: 0, + total_shares: 1, + percentage_bps: bps, + report_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(), + release_sha256: format!("{tid}-{name}"), + }], + ) + .unwrap(); + } + + let by_top1 = query_concentration(&conn, "top1", 10).unwrap(); + assert_eq!(by_top1[0].0, "AA"); + + let by_top3 = query_concentration(&conn, "top3", 10).unwrap(); + assert_eq!(by_top3[0].0, "BB"); + + let by_hhi = query_concentration(&conn, "hhi", 10).unwrap(); + assert_eq!(by_hhi[0].0, "AA"); + } + + #[test] + fn test_insert_bing_and_query_empty_db_paths() { + let conn = setup(); + + assert!(get_ticker_id(&conn, "NONE").unwrap().is_none()); + assert!(query_cross_holders(&conn, 1, 10).unwrap().is_empty()); + assert!(query_concentration(&conn, "top1", 10).unwrap().is_empty()); + + let tid = upsert_ticker(&conn, "TLKM", None).unwrap(); + let inserted = insert_bing_holdings( + &conn, + &[BingHolding { + id: 0, + ticker_id: tid, + entity_id: None, + raw_investor_name: "FLOW".to_string(), + investor_type: None, + shares_held: Some(100), + shares_changed: Some(50), + pct_ownership_bps: Some(123), + value_usd: Some(10), + report_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(), + signal: FlowSignal::Buyer, + fetched_at: 12345, + }], + ) + .unwrap(); + assert_eq!(inserted, 1); + + let buyers: i64 = conn + .query_row( + "SELECT COUNT(*) FROM bing_holdings WHERE ticker_id = ?1 AND signal = 'buyer'", + params![tid], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(buyers, 1); + } +} diff --git a/src/ownership/entities.rs b/src/ownership/entities.rs new file mode 100644 index 0000000..a6847d8 --- /dev/null +++ b/src/ownership/entities.rs @@ -0,0 +1,404 @@ +use chrono::NaiveDate; +use rusqlite::{Connection, params}; + +use crate::error::IdxError; +use crate::ownership::types::{ + InvestorTypeCode, KseiHoldingDraft, KseiRawRow, Locality, OwnershipSource, +}; + +/// Parse Indonesian locale number string to i64. +/// "1.533.682.440" → 1533682440, "0" → 0 +pub fn parse_id_number(s: &str) -> Result { + let trimmed = s.trim(); + if trimmed.is_empty() { + return Err(IdxError::ParseError( + "empty Indonesian number string".to_string(), + )); + } + + let normalized = trimmed.replace(['.', ' '], ""); + if normalized.is_empty() || !normalized.chars().all(|c| c.is_ascii_digit()) { + return Err(IdxError::ParseError(format!( + "invalid Indonesian number format: {s}" + ))); + } + + normalized + .parse::() + .map_err(|e| IdxError::ParseError(format!("failed to parse Indonesian number '{s}': {e}"))) +} + +/// Parse Indonesian locale percentage to basis points (i64). +/// "54,94" → 5494, "0,00" → 0, "100,00" → 10000 +pub fn parse_id_percentage(s: &str) -> Result { + let trimmed = s.trim(); + if trimmed.is_empty() { + return Err(IdxError::ParseError( + "empty Indonesian percentage string".to_string(), + )); + } + + let mut parts = trimmed.split(','); + let whole = parts.next().unwrap_or_default(); + let frac = parts.next().unwrap_or("00"); + + if parts.next().is_some() { + return Err(IdxError::ParseError(format!( + "invalid Indonesian percentage format: {s}" + ))); + } + + if whole.is_empty() || !whole.chars().all(|c| c.is_ascii_digit()) { + return Err(IdxError::ParseError(format!( + "invalid Indonesian percentage whole part: {s}" + ))); + } + + if !frac.chars().all(|c| c.is_ascii_digit()) || frac.len() > 2 { + return Err(IdxError::ParseError(format!( + "invalid Indonesian percentage fractional part: {s}" + ))); + } + + let whole_i = whole.parse::().map_err(|e| { + IdxError::ParseError(format!("failed to parse Indonesian percentage '{s}': {e}")) + })?; + + let frac_padded = if frac.len() == 1 { + format!("{frac}0") + } else { + frac.to_string() + }; + let frac_i = frac_padded.parse::().map_err(|e| { + IdxError::ParseError(format!("failed to parse Indonesian percentage '{s}': {e}")) + })?; + + Ok((whole_i * 100) + frac_i) +} + +/// Parse KSEI date format to NaiveDate. +/// "27-Feb-2026" → NaiveDate(2026, 2, 27) +pub fn parse_ksei_date(s: &str) -> Result { + let trimmed = s.trim(); + if trimmed.is_empty() { + return Err(IdxError::ParseError("empty KSEI date string".to_string())); + } + + NaiveDate::parse_from_str(trimmed, "%d-%b-%Y") + .map_err(|e| IdxError::ParseError(format!("invalid KSEI date '{s}': {e}"))) +} + +/// Normalize an investor name for entity matching. +/// Strips: "PT.", "PT ", "TBK", "Tbk", "(PERSERO)", "LIMITED", "PTE", "LTD" +/// Collapses whitespace, trims, uppercases. +pub fn normalize_name(raw: &str) -> String { + let upper = collapse_whitespace(raw).to_uppercase(); + if upper.is_empty() { + return upper; + } + + let mut tokens: Vec<&str> = upper.split_whitespace().collect(); + + loop { + let mut changed = false; + + if let Some(first) = tokens.first().copied() + && matches!(first, "PT" | "PT.") + { + tokens.remove(0); + changed = true; + } + + if let Some(last) = tokens.last().copied() + && matches!( + last, + "TBK" | "Tbk" | "(PERSERO)" | "LIMITED" | "PTE" | "LTD" + ) + { + let _ = tokens.pop(); + changed = true; + } + + if !changed { + break; + } + } + + collapse_whitespace(&tokens.join(" ")) +} + +/// Convert a KseiRawRow into a normalized KseiHoldingDraft. +/// Applies all parsing functions above. +pub fn normalize_ksei_row(raw: &KseiRawRow) -> Result { + let investor_type = normalize_investor_type(&raw.investor_type); + let locality = normalize_locality(&raw.local_foreign); + + Ok(KseiHoldingDraft { + ticker_code: raw.share_code.trim().to_string(), + issuer_name: optional_string(&raw.issuer_name), + raw_investor_name: raw.investor_name.trim().to_string(), + investor_type, + locality, + nationality: optional_string(&raw.nationality), + domicile: optional_string(&raw.domicile), + holdings_scripless: parse_id_number(&raw.holdings_scripless)?, + holdings_scrip: parse_id_number(&raw.holdings_scrip)?, + total_shares: parse_id_number(&raw.total_holding_shares)?, + percentage_bps: parse_id_percentage(&raw.percentage)?, + report_date: parse_ksei_date(&raw.date)?, + }) +} + +/// Find or create a canonical entity for a raw investor name. +/// Strategy: exact match on normalized name → rule-based normalization → create new. +/// Returns entity_id. +pub fn resolve_entity( + conn: &Connection, + raw_name: &str, + source: OwnershipSource, +) -> Result { + let source_db = source_to_db(source); + let raw_trimmed = raw_name.trim(); + + if raw_trimmed.is_empty() { + return Err(IdxError::ParseError("empty raw entity name".to_string())); + } + + let exact_normalized = collapse_whitespace(raw_trimmed).to_uppercase(); + let rule_normalized = normalize_name(raw_trimmed); + + if let Some(entity_id) = find_entity_by_alias(conn, raw_trimmed, source_db)? { + return Ok(entity_id); + } + + if let Some(entity_id) = find_entity_by_canonical(conn, &exact_normalized)? { + insert_alias(conn, entity_id, raw_trimmed, source_db, "exact")?; + return Ok(entity_id); + } + + if rule_normalized != exact_normalized + && let Some(entity_id) = find_entity_by_canonical(conn, &rule_normalized)? + { + insert_alias(conn, entity_id, raw_trimmed, source_db, "rule")?; + return Ok(entity_id); + } + + let now: i64 = chrono::Utc::now().timestamp(); + conn.execute( + "INSERT INTO entities (canonical_name, entity_type, country, created_at, updated_at) + VALUES (?1, NULL, NULL, ?2, ?2)", + params![ + if rule_normalized.is_empty() { + &exact_normalized + } else { + &rule_normalized + }, + now + ], + ) + .map_err(|e| IdxError::DatabaseError(format!("insert entity failed: {e}")))?; + + let entity_id = conn.last_insert_rowid(); + insert_alias(conn, entity_id, raw_trimmed, source_db, "exact")?; + + Ok(entity_id) +} + +fn normalize_investor_type(raw: &str) -> Option { + let value = raw.trim(); + if value.is_empty() { + None + } else { + Some(InvestorTypeCode(value.to_uppercase())) + } +} + +fn normalize_locality(raw: &str) -> Option { + match raw.trim().to_uppercase().as_str() { + "L" => Some(Locality::Local), + "F" | "A" => Some(Locality::Foreign), + _ => None, + } +} + +fn optional_string(raw: &str) -> Option { + let value = raw.trim(); + if value.is_empty() { + None + } else { + Some(value.to_string()) + } +} + +fn collapse_whitespace(input: &str) -> String { + input.split_whitespace().collect::>().join(" ") +} + +fn source_to_db(source: OwnershipSource) -> &'static str { + match source { + OwnershipSource::Ksei => "ksei", + OwnershipSource::Bing => "bing", + } +} + +fn find_entity_by_alias( + conn: &Connection, + raw_name: &str, + source: &str, +) -> Result, IdxError> { + conn.query_row( + "SELECT entity_id FROM entity_aliases WHERE raw_name = ?1 AND source = ?2", + params![raw_name, source], + |row| row.get(0), + ) + .map(Some) + .or_else(|e| match e { + rusqlite::Error::QueryReturnedNoRows => Ok(None), + _ => Err(IdxError::DatabaseError(e.to_string())), + }) +} + +fn find_entity_by_canonical( + conn: &Connection, + canonical_name: &str, +) -> Result, IdxError> { + conn.query_row( + "SELECT id FROM entities WHERE canonical_name = ?1", + params![canonical_name], + |row| row.get(0), + ) + .map(Some) + .or_else(|e| match e { + rusqlite::Error::QueryReturnedNoRows => Ok(None), + _ => Err(IdxError::DatabaseError(e.to_string())), + }) +} + +fn insert_alias( + conn: &Connection, + entity_id: i64, + raw_name: &str, + source: &str, + method: &str, +) -> Result<(), IdxError> { + conn.execute( + "INSERT OR IGNORE INTO entity_aliases (entity_id, raw_name, source, confidence, method) + VALUES (?1, ?2, ?3, 1.0, ?4)", + params![entity_id, raw_name, source, method], + ) + .map_err(|e| IdxError::DatabaseError(format!("insert alias failed: {e}")))?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use chrono::NaiveDate; + use rusqlite::Connection; + + use crate::ownership::entities::{ + normalize_name, parse_id_number, parse_id_percentage, parse_ksei_date, resolve_entity, + }; + use crate::ownership::types::OwnershipSource; + + #[test] + fn test_parse_id_number() { + assert_eq!(parse_id_number("1.533.682.440").unwrap(), 1_533_682_440); + assert_eq!(parse_id_number("0").unwrap(), 0); + assert_eq!(parse_id_number("3.200.142.830").unwrap(), 3_200_142_830); + assert!(parse_id_number("").is_err()); + } + + #[test] + fn test_parse_id_percentage() { + assert_eq!(parse_id_percentage("54,94").unwrap(), 5494); + assert_eq!(parse_id_percentage("0,00").unwrap(), 0); + assert_eq!(parse_id_percentage("100,00").unwrap(), 10000); + assert_eq!(parse_id_percentage("41,10").unwrap(), 4110); + } + + #[test] + fn test_parse_ksei_date_various_months() { + let samples = [ + ("01-Jan-2026", NaiveDate::from_ymd_opt(2026, 1, 1).unwrap()), + ("01-Feb-2026", NaiveDate::from_ymd_opt(2026, 2, 1).unwrap()), + ("01-Mar-2026", NaiveDate::from_ymd_opt(2026, 3, 1).unwrap()), + ("01-Apr-2026", NaiveDate::from_ymd_opt(2026, 4, 1).unwrap()), + ("01-May-2026", NaiveDate::from_ymd_opt(2026, 5, 1).unwrap()), + ("01-Jun-2026", NaiveDate::from_ymd_opt(2026, 6, 1).unwrap()), + ("01-Jul-2026", NaiveDate::from_ymd_opt(2026, 7, 1).unwrap()), + ("01-Aug-2026", NaiveDate::from_ymd_opt(2026, 8, 1).unwrap()), + ("01-Sep-2026", NaiveDate::from_ymd_opt(2026, 9, 1).unwrap()), + ("01-Oct-2026", NaiveDate::from_ymd_opt(2026, 10, 1).unwrap()), + ("01-Nov-2026", NaiveDate::from_ymd_opt(2026, 11, 1).unwrap()), + ("01-Dec-2026", NaiveDate::from_ymd_opt(2026, 12, 1).unwrap()), + ]; + + for (input, expected) in samples { + assert_eq!(parse_ksei_date(input).unwrap(), expected); + } + + assert_eq!( + parse_ksei_date("27-Feb-2026").unwrap(), + NaiveDate::from_ymd_opt(2026, 2, 27).unwrap() + ); + } + + #[test] + fn test_normalize_name() { + assert_eq!( + normalize_name("PT. ASTRA INTERNATIONAL TBK"), + "ASTRA INTERNATIONAL" + ); + assert_eq!( + normalize_name("PT BANK CENTRAL ASIA Tbk"), + "BANK CENTRAL ASIA" + ); + assert_eq!( + normalize_name("UOB KAY HIAN PRIVATE LIMITED"), + "UOB KAY HIAN PRIVATE" + ); + assert_eq!( + normalize_name("DJS Ketenagakerjaan (JHT)"), + "DJS KETENAGAKERJAAN (JHT)" + ); + assert_eq!( + normalize_name("BPJS KETENAGAKERJAAN"), + "BPJS KETENAGAKERJAAN" + ); + } + + #[test] + fn test_resolve_entity_create_then_reuse() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + r#" + CREATE TABLE entities ( + id INTEGER PRIMARY KEY, + canonical_name TEXT NOT NULL, + entity_type TEXT, + country TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE entity_aliases ( + id INTEGER PRIMARY KEY, + entity_id INTEGER NOT NULL, + raw_name TEXT NOT NULL, + source TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 1.0, + method TEXT NOT NULL, + UNIQUE(raw_name, source) + ); + "#, + ) + .unwrap(); + + let id1 = + resolve_entity(&conn, "PT. ASTRA INTERNATIONAL TBK", OwnershipSource::Ksei).unwrap(); + let id2 = + resolve_entity(&conn, "PT. ASTRA INTERNATIONAL TBK", OwnershipSource::Ksei).unwrap(); + + assert_eq!(id1, id2); + } +} diff --git a/src/ownership/graph.rs b/src/ownership/graph.rs new file mode 100644 index 0000000..411dbb6 --- /dev/null +++ b/src/ownership/graph.rs @@ -0,0 +1,332 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +use rusqlite::{Connection, params}; + +use crate::error::IdxError; +use crate::ownership::types::{GraphEdge, GraphNode, GraphNodeType, OwnershipSource}; + +/// Query ownership graph starting from a ticker or entity, traversing N hops. +/// Root can be ticker code (`BBCA`) or entity name — auto-detect. +pub fn query_ownership_graph( + conn: &Connection, + root: &str, + depth: usize, +) -> Result<(Vec, Vec), IdxError> { + let root = root.trim(); + if root.is_empty() { + return Err(IdxError::ParseError( + "graph root cannot be empty".to_string(), + )); + } + + let root_node_id = detect_root_node(conn, root)?; + + let mut visited: BTreeSet = BTreeSet::new(); + { + let mut stmt = conn + .prepare( + "WITH RECURSIVE + all_edges AS ( + SELECT + 'entity:' || k.entity_id AS from_id, + 'ticker:' || t.code AS to_id + FROM ksei_holdings k + JOIN tickers t ON t.id = k.ticker_id + WHERE k.entity_id IS NOT NULL + + UNION + + SELECT + 'entity:' || b.entity_id AS from_id, + 'ticker:' || t.code AS to_id + FROM bing_holdings b + JOIN tickers t ON t.id = b.ticker_id + WHERE b.entity_id IS NOT NULL + ), + neighbors AS ( + SELECT from_id AS a, to_id AS b FROM all_edges + UNION + SELECT to_id AS a, from_id AS b FROM all_edges + ), + walk(node_id, depth, path) AS ( + SELECT ?1 AS node_id, 0 AS depth, ?1 AS path + UNION ALL + SELECT n.b, w.depth + 1, w.path || '>' || n.b + FROM walk w + JOIN neighbors n ON n.a = w.node_id + WHERE w.depth < ?2 + AND instr(w.path, n.b) = 0 + ) + SELECT DISTINCT node_id + FROM walk", + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let rows = stmt + .query_map(params![root_node_id, depth as i64], |row| { + row.get::<_, String>(0) + }) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + for row in rows { + visited.insert(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?); + } + } + + let mut edges = query_all_edges(conn)?; + edges.retain(|edge| visited.contains(&edge.from) && visited.contains(&edge.to)); + + let nodes = build_nodes(conn, &visited)?; + + Ok((nodes, edges)) +} + +/// Format graph as ASCII tree for terminal display. +pub fn format_graph_text(nodes: &[GraphNode], edges: &[GraphEdge]) -> String { + if nodes.is_empty() { + return "(empty graph)".to_string(); + } + + let labels: HashMap<&str, (&str, GraphNodeType)> = nodes + .iter() + .map(|n| (n.id.as_str(), (n.label.as_str(), n.node_type))) + .collect(); + + let mut ticker_to_entities: BTreeMap<&str, Vec<&GraphEdge>> = BTreeMap::new(); + for edge in edges { + if let Some((_, GraphNodeType::Ticker)) = labels.get(edge.to.as_str()) { + ticker_to_entities + .entry(edge.to.as_str()) + .or_default() + .push(edge); + } + } + + let mut out = String::new(); + out.push_str(&format!("nodes: {} edges: {}\n", nodes.len(), edges.len())); + + for (ticker_id, mut rels) in ticker_to_entities { + rels.sort_by(|a, b| b.percentage_bps.cmp(&a.percentage_bps)); + let ticker_label = labels.get(ticker_id).map(|v| v.0).unwrap_or(ticker_id); + out.push_str(&format!("\n{ticker_label} [TICKER]\n")); + + for (idx, edge) in rels.iter().enumerate() { + let holder_label = labels.get(edge.from.as_str()).map(|v| v.0).unwrap_or("?"); + let branch = if idx + 1 == rels.len() { + "└─" + } else { + "├─" + }; + out.push_str(&format!( + "{branch} {holder_label} ({:.2}%, {})\n", + edge.percentage_bps as f64 / 100.0, + source_label(edge.source) + )); + } + } + + if edges.is_empty() { + out.push_str("\n(no ownership edges found)\n"); + } + + out +} + +/// Format graph as Graphviz DOT for export. +pub fn format_graph_dot(nodes: &[GraphNode], edges: &[GraphEdge]) -> String { + let mut out = String::new(); + out.push_str("digraph ownership {\n"); + out.push_str(" rankdir=LR;\n"); + out.push_str(" graph [fontname=\"Helvetica\"];\n"); + out.push_str(" node [fontname=\"Helvetica\"];\n"); + out.push_str(" edge [fontname=\"Helvetica\"];\n\n"); + + for node in nodes { + let shape = match node.node_type { + GraphNodeType::Entity => "ellipse", + GraphNodeType::Ticker => "box", + }; + out.push_str(&format!( + " \"{}\" [label=\"{}\", shape={}];\n", + escape_dot(&node.id), + escape_dot(&node.label), + shape + )); + } + + out.push('\n'); + for edge in edges { + out.push_str(&format!( + " \"{}\" -> \"{}\" [label=\"{:.2}% ({})\"];\n", + escape_dot(&edge.from), + escape_dot(&edge.to), + edge.percentage_bps as f64 / 100.0, + source_label(edge.source) + )); + } + + out.push_str("}\n"); + out +} + +fn detect_root_node(conn: &Connection, root: &str) -> Result { + let maybe_ticker = conn + .query_row( + "SELECT code FROM tickers WHERE UPPER(code) = UPPER(?1) LIMIT 1", + params![root], + |row| row.get::<_, String>(0), + ) + .ok(); + + if let Some(code) = maybe_ticker { + return Ok(format!("ticker:{code}")); + } + + let q = format!("%{}%", root); + let maybe_entity_id = conn + .query_row( + "SELECT id + FROM entities + WHERE canonical_name LIKE ?1 COLLATE NOCASE + ORDER BY LENGTH(canonical_name) ASC, canonical_name ASC + LIMIT 1", + params![q], + |row| row.get::<_, i64>(0), + ) + .ok(); + + if let Some(entity_id) = maybe_entity_id { + return Ok(format!("entity:{entity_id}")); + } + + Err(IdxError::ParseError(format!( + "graph root not found as ticker or entity: {root}" + ))) +} + +fn query_all_edges(conn: &Connection) -> Result, IdxError> { + let mut out = Vec::new(); + + { + let mut stmt = conn + .prepare( + "SELECT 'entity:' || k.entity_id, + 'ticker:' || t.code, + k.percentage_bps + FROM ksei_holdings k + JOIN tickers t ON t.id = k.ticker_id + WHERE k.entity_id IS NOT NULL", + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let rows = stmt + .query_map([], |row| { + Ok(GraphEdge { + from: row.get(0)?, + to: row.get(1)?, + percentage_bps: row.get(2)?, + source: OwnershipSource::Ksei, + }) + }) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + for row in rows { + out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?); + } + } + + { + let mut stmt = conn + .prepare( + "SELECT 'entity:' || b.entity_id, + 'ticker:' || t.code, + COALESCE(b.pct_ownership_bps, 0) + FROM bing_holdings b + JOIN tickers t ON t.id = b.ticker_id + WHERE b.entity_id IS NOT NULL", + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let rows = stmt + .query_map([], |row| { + Ok(GraphEdge { + from: row.get(0)?, + to: row.get(1)?, + percentage_bps: row.get(2)?, + source: OwnershipSource::Bing, + }) + }) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + for row in rows { + out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?); + } + } + + dedup_edges(out) +} + +fn dedup_edges(edges: Vec) -> Result, IdxError> { + let mut seen: HashSet<(String, String, i64, &'static str)> = HashSet::new(); + let mut out = Vec::new(); + + for edge in edges { + let key = ( + edge.from.clone(), + edge.to.clone(), + edge.percentage_bps, + source_label(edge.source), + ); + if seen.insert(key) { + out.push(edge); + } + } + + Ok(out) +} + +fn build_nodes(conn: &Connection, node_ids: &BTreeSet) -> Result, IdxError> { + let mut nodes = Vec::new(); + + for node_id in node_ids { + if let Some(entity_id) = node_id.strip_prefix("entity:") { + let entity_id_num = entity_id.parse::().map_err(|e| { + IdxError::ParseError(format!("invalid entity node id '{node_id}': {e}")) + })?; + let label = conn + .query_row( + "SELECT canonical_name FROM entities WHERE id = ?1", + params![entity_id_num], + |row| row.get::<_, String>(0), + ) + .unwrap_or_else(|_| format!("entity:{entity_id_num}")); + nodes.push(GraphNode { + id: node_id.clone(), + label, + node_type: GraphNodeType::Entity, + }); + continue; + } + + if let Some(code) = node_id.strip_prefix("ticker:") { + nodes.push(GraphNode { + id: node_id.clone(), + label: code.to_string(), + node_type: GraphNodeType::Ticker, + }); + } + } + + Ok(nodes) +} + +fn source_label(source: OwnershipSource) -> &'static str { + match source { + OwnershipSource::Ksei => "ksei", + OwnershipSource::Bing => "bing", + } +} + +fn escape_dot(value: &str) -> String { + value.replace('"', "\\\"") +} diff --git a/src/ownership/mod.rs b/src/ownership/mod.rs new file mode 100644 index 0000000..c2c67c7 --- /dev/null +++ b/src/ownership/mod.rs @@ -0,0 +1,6 @@ +pub mod db; +pub mod entities; +pub mod graph; +pub mod parser; +pub mod search; +pub mod types; diff --git a/src/ownership/parser.rs b/src/ownership/parser.rs new file mode 100644 index 0000000..a414f2b --- /dev/null +++ b/src/ownership/parser.rs @@ -0,0 +1,355 @@ +use std::collections::HashMap; +use std::path::Path; +use std::process::Command; + +use quick_xml::Reader; +use quick_xml::events::Event; + +use crate::error::IdxError; +use crate::ownership::types::KseiRawRow; + +/// Character grid for a single PDF page: y-coord → column-index → sorted (x, char) pairs. +type PageGrid = HashMap>>; + +const Y_TOLERANCE: f32 = 0.8; + +/// Inclusive-left, exclusive-right X ranges for each KSEI data column. +const COLUMN_BOUNDS: [(f32, f32); 12] = [ + (15.0, 52.0), // date + (52.0, 70.0), // share_code + (70.0, 167.0), // issuer_name + (167.0, 432.0), // investor_name + (432.0, 463.0), // investor_type + (463.0, 497.0), // local_foreign + (497.0, 558.0), // nationality + (558.0, 615.0), // domicile + (615.0, 653.0), // holdings_scripless + (653.0, 692.0), // holdings_scrip + (692.0, 745.0), // total_holding_shares + (745.0, 800.0), // percentage +]; + +/// Parse a KSEI ownership PDF into raw rows. +/// Shells out to `mutool` for XML extraction, then parses with quick-xml. +pub fn parse_ksei_pdf(path: &Path) -> Result, IdxError> { + check_mutool()?; + + let output = Command::new("mutool") + .arg("convert") + .arg("-F") + .arg("stext") + .arg("-o") + .arg("-") + .arg(path) + .output() + .map_err(|e| IdxError::PdfParseError(format!("failed to run mutool: {e}")))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(IdxError::PdfParseError(format!( + "mutool convert failed: {}", + stderr.trim() + ))); + } + + let xml = String::from_utf8(output.stdout) + .map_err(|e| IdxError::PdfParseError(format!("invalid utf-8 stext output: {e}")))?; + + parse_stext_xml(&xml) +} + +/// Parse mutool stext XML output into raw rows. +/// Pure function — takes XML string, returns parsed rows. +pub fn parse_stext_xml(xml: &str) -> Result, IdxError> { + let mut reader = Reader::from_str(xml); + reader.config_mut().trim_text(false); + + let mut rows: Vec = Vec::new(); + let mut current_page: Option = None; + let mut buf = Vec::new(); + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(e)) if e.name().as_ref() == b"page" => { + current_page = Some(HashMap::new()); + } + Ok(Event::Empty(e)) if e.name().as_ref() == b"char" => { + if let Some(page) = current_page.as_mut() { + let mut x: Option = None; + let mut y: Option = None; + let mut c: Option = None; + + for attr_result in e.attributes().with_checks(false) { + let attr = attr_result.map_err(|err| { + IdxError::PdfParseError(format!("invalid XML attribute: {err}")) + })?; + + match attr.key.as_ref() { + b"x" => { + let s = attr.decode_and_unescape_value(reader.decoder()).map_err( + |err| { + IdxError::PdfParseError(format!( + "invalid XML x attribute: {err}" + )) + }, + )?; + x = s.parse::().ok(); + } + b"y" => { + let s = attr.decode_and_unescape_value(reader.decoder()).map_err( + |err| { + IdxError::PdfParseError(format!( + "invalid XML y attribute: {err}" + )) + }, + )?; + y = s.parse::().ok(); + } + b"c" => { + let s = attr.decode_and_unescape_value(reader.decoder()).map_err( + |err| { + IdxError::PdfParseError(format!( + "invalid XML char attribute: {err}" + )) + }, + )?; + c = s.chars().next(); + } + _ => {} + } + } + + if let (Some(x_val), Some(y_val), Some(ch)) = (x, y, c) + && let Some(col_idx) = x_to_column(x_val) + { + let yb = y_bucket(y_val); + let xi = (x_val * 100.0).round() as i32; + page.entry(yb) + .or_default() + .entry(col_idx) + .or_default() + .push((xi, ch)); + } + } + } + Ok(Event::End(e)) if e.name().as_ref() == b"page" => { + if let Some(page) = current_page.take() { + rows.extend(extract_rows_from_page(page)); + } + } + Ok(Event::Eof) => break, + Err(err) => { + return Err(IdxError::PdfParseError(format!( + "failed to parse stext XML: {err}" + ))); + } + _ => {} + } + buf.clear(); + } + + Ok(rows) +} + +/// Check if mutool is available in PATH. +pub fn check_mutool() -> Result<(), IdxError> { + // mutool with no args prints usage to stderr and exits non-zero, + // so we just check that the binary is found and executable. + Command::new("mutool") + .arg("--help") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map_err(|e| IdxError::PdfParseError(format!("mutool not found in PATH: {e}")))?; + Ok(()) +} + +fn extract_rows_from_page(page: PageGrid) -> Vec { + let mut page_rows = Vec::new(); + + let mut y_keys: Vec = page.keys().copied().collect(); + y_keys.sort_unstable(); + + for y in y_keys { + let mut row = KseiRawRow { + date: String::new(), + share_code: String::new(), + issuer_name: String::new(), + investor_name: String::new(), + investor_type: String::new(), + local_foreign: String::new(), + nationality: String::new(), + domicile: String::new(), + holdings_scripless: String::new(), + holdings_scrip: String::new(), + total_holding_shares: String::new(), + percentage: String::new(), + }; + + if let Some(col_map) = page.get(&y) { + for (col_idx, chars) in col_map { + let mut sorted = chars.clone(); + sorted.sort_by_key(|(x, _)| *x); + let text = normalize_spaces(&sorted.iter().map(|(_, c)| c).collect::()); + assign_column(&mut row, *col_idx, text); + } + } + + if is_data_row(&row) { + page_rows.push(row); + } + } + + page_rows +} + +fn assign_column(row: &mut KseiRawRow, col_idx: usize, value: String) { + match col_idx { + 0 => row.date = value, + 1 => row.share_code = value, + 2 => row.issuer_name = value, + 3 => row.investor_name = value, + 4 => row.investor_type = value, + 5 => row.local_foreign = value, + 6 => row.nationality = value, + 7 => row.domicile = value, + 8 => row.holdings_scripless = value, + 9 => row.holdings_scrip = value, + 10 => row.total_holding_shares = value, + 11 => row.percentage = value, + _ => {} + } +} + +fn x_to_column(x: f32) -> Option { + COLUMN_BOUNDS + .iter() + .position(|(left, right)| x >= *left && x < *right) +} + +fn y_bucket(y: f32) -> i32 { + (y / Y_TOLERANCE).round() as i32 +} + +fn normalize_spaces(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + let mut prev_space = false; + + for ch in input.chars() { + if ch == ' ' { + if !prev_space { + out.push(' '); + } + prev_space = true; + } else { + out.push(ch); + prev_space = false; + } + } + + out.trim().to_string() +} + +fn is_data_row(row: &KseiRawRow) -> bool { + is_ksei_date(&row.date) + && is_percentage_like(&row.percentage) + && !row.share_code.trim().is_empty() + && !row.investor_name.trim().is_empty() +} + +fn is_ksei_date(s: &str) -> bool { + if s.len() != 11 { + return false; + } + + let mut parts = s.split('-'); + let day = parts.next(); + let mon = parts.next(); + let year = parts.next(); + + if parts.next().is_some() { + return false; + } + + match (day, mon, year) { + (Some(d), Some(m), Some(y)) => { + d.len() == 2 + && d.chars().all(|c| c.is_ascii_digit()) + && m.len() == 3 + && m.chars().all(|c| c.is_ascii_alphabetic()) + && y.len() == 4 + && y.chars().all(|c| c.is_ascii_digit()) + } + _ => false, + } +} + +fn is_percentage_like(s: &str) -> bool { + let cleaned = s.trim(); + if cleaned.is_empty() { + return false; + } + + let mut parts = cleaned.split(','); + let left = parts.next(); + let right = parts.next(); + + if parts.next().is_some() { + return false; + } + + match (left, right) { + (Some(l), Some(r)) => { + !l.is_empty() + && l.chars().all(|c| c.is_ascii_digit()) + && !r.is_empty() + && r.chars().all(|c| c.is_ascii_digit()) + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::path::Path; + + use super::{check_mutool, parse_ksei_pdf, parse_stext_xml}; + + #[test] + fn test_parse_stext_xml_fixture_extracts_rows() { + let fixture_path = Path::new("tests/fixtures/ksei_stext_sample.xml"); + let xml = fs::read_to_string(fixture_path).expect("failed to read stext fixture"); + + let rows = parse_stext_xml(&xml).expect("failed to parse fixture XML"); + assert_eq!(rows.len(), 3); + + let first = &rows[0]; + assert_eq!(first.date, "27-Feb-2026"); + assert_eq!(first.share_code, "BBCA"); + assert_eq!(first.investor_name, "PT DWIMURIA INVESTAMA ANDALAN"); + assert_eq!(first.percentage, "54,94"); + } + + #[test] + fn test_parse_ksei_pdf_real_file_row_count() { + if check_mutool().is_err() { + eprintln!("skipping mutool-dependent test: mutool not available"); + return; + } + + let pdf_path = + Path::new("/var/lib/openclaw/projects/idx-cli/research/ownership_202603.pdf"); + if !pdf_path.exists() { + eprintln!("skipping mutool-dependent test: sample PDF not found"); + return; + } + + let rows = parse_ksei_pdf(pdf_path).expect("failed to parse real KSEI PDF"); + assert!( + rows.len() >= 7_200, + "expected at least 7200 rows, got {}", + rows.len() + ); + } +} diff --git a/src/ownership/search.rs b/src/ownership/search.rs new file mode 100644 index 0000000..6e63532 --- /dev/null +++ b/src/ownership/search.rs @@ -0,0 +1,111 @@ +use rusqlite::{Connection, params}; + +use crate::error::IdxError; +use crate::ownership::types::Entity; + +pub fn fts_search(conn: &Connection, query: &str, limit: usize) -> Result, IdxError> { + let q = query.trim(); + if q.is_empty() { + return Ok(Vec::new()); + } + + let limit_i64 = + i64::try_from(limit).map_err(|e| IdxError::DatabaseError(format!("invalid limit: {e}")))?; + + // Ensure entity_fts has content. For external-content FTS table, populate manually. + let fts_count: i64 = conn + .query_row("SELECT COUNT(*) FROM entity_fts", [], |row| row.get(0)) + .unwrap_or(0); + + if fts_count == 0 { + let _ = rebuild_fts(conn); + } + + if let Ok(rows) = fts_query(conn, q, limit_i64) + && !rows.is_empty() + { + return Ok(rows); + } + + like_query(conn, q, limit_i64) +} + +fn fts_query(conn: &Connection, query: &str, limit: i64) -> Result, IdxError> { + let mut stmt = conn + .prepare( + "SELECT e.id, e.canonical_name, e.entity_type, e.country + FROM entity_fts f + JOIN entities e ON e.id = f.rowid + WHERE entity_fts MATCH ?1 + ORDER BY rank + LIMIT ?2", + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let rows = stmt + .query_map(params![query, limit], |row| { + Ok(Entity { + id: row.get(0)?, + canonical_name: row.get(1)?, + entity_type: row.get(2)?, + country: row.get(3)?, + }) + }) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let mut out = Vec::new(); + for row in rows { + out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?); + } + Ok(out) +} + +fn like_query(conn: &Connection, query: &str, limit: i64) -> Result, IdxError> { + let mut stmt = conn + .prepare( + "SELECT id, canonical_name, entity_type, country + FROM entities + WHERE canonical_name LIKE ?1 COLLATE NOCASE + ORDER BY canonical_name ASC + LIMIT ?2", + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let like = format!("%{query}%"); + let rows = stmt + .query_map(params![like, limit], |row| { + Ok(Entity { + id: row.get(0)?, + canonical_name: row.get(1)?, + entity_type: row.get(2)?, + country: row.get(3)?, + }) + }) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + let mut out = Vec::new(); + for row in rows { + out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?); + } + Ok(out) +} + +pub fn rebuild_fts(conn: &Connection) -> Result<(), IdxError> { + conn.execute("DELETE FROM entity_fts", []) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + conn.execute( + "INSERT INTO entity_fts(rowid, canonical_name, aliases) + SELECT e.id, e.canonical_name, COALESCE(a.aliases, '') + FROM entities e + LEFT JOIN ( + SELECT entity_id, GROUP_CONCAT(raw_name, ' ') AS aliases + FROM entity_aliases + GROUP BY entity_id + ) a ON a.entity_id = e.id", + [], + ) + .map_err(|e| IdxError::DatabaseError(e.to_string()))?; + + Ok(()) +} diff --git a/src/ownership/types.rs b/src/ownership/types.rs new file mode 100644 index 0000000..acc9b7f --- /dev/null +++ b/src/ownership/types.rs @@ -0,0 +1,451 @@ +use chrono::NaiveDate; +use serde::{Deserialize, Serialize}; + +/// Which data source a holding originates from. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum OwnershipSource { + /// KSEI shareholder registry source. + Ksei, + /// Bing institutional ownership source. + Bing, +} + +/// Investor locality classification from KSEI. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum Locality { + /// Local Indonesian investor. + Local, + /// Foreign investor. + Foreign, +} + +/// Bing institutional flow signal. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FlowSignal { + /// Existing top holder. + Holder, + /// Net buyer for the period. + Buyer, + /// Net seller for the period. + Seller, + /// New position opened this period. + NewPosition, + /// Fully exited this period. + Exited, +} + +/// Method used to resolve an entity alias to a canonical entity. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ResolutionMethod { + /// Exact string match after normalization. + Exact, + /// Rule-based normalization match. + Rule, + /// Fuzzy or similarity-based match. + Fuzzy, + /// Human-curated manual mapping. + Manual, +} + +/// Graph node category used in ownership network output. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum GraphNodeType { + /// Canonical investor entity node. + Entity, + /// Listed issuer ticker node. + Ticker, +} + +/// KSEI investor type code (for example: `CP`, `ID`, `IB`, `MF`, `SC`, `IS`, `OT`). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InvestorTypeCode(pub String); + +/// Canonical resolved entity (investor or shareholder). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Entity { + /// Internal entity identifier. + pub id: i64, + /// Canonical normalized display name. + pub canonical_name: String, + /// Optional coarse entity type (`fund`, `bank`, `conglomerate`, `govt`, `individual`). + pub entity_type: Option, + /// Optional ISO-like country tag. + pub country: Option, +} + +/// A raw name variant mapped to a canonical entity. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EntityAlias { + /// Internal alias identifier. + pub id: i64, + /// Referenced canonical entity id. + pub entity_id: i64, + /// Raw alias text from source. + pub raw_name: String, + /// Source that produced this alias. + pub source: OwnershipSource, + /// Resolution confidence score in `0.0..=1.0`. + pub confidence: f64, + /// Matching method used for resolution. + pub method: ResolutionMethod, +} + +/// Ticker (issuer) reference metadata. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Ticker { + /// Internal ticker identifier. + pub id: i64, + /// Exchange ticker code, for example `BBCA`. + pub code: String, + /// Optional long issuer name. + pub name: Option, + /// Optional sector classification. + pub sector: Option, +} + +/// Single KSEI ownership row for one holder in one ticker and release. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KseiHolding { + /// Internal holding row identifier. + pub id: i64, + /// Referenced ticker id. + pub ticker_id: i64, + /// Referenced entity id, unresolved when `None`. + pub entity_id: Option, + /// Raw investor name exactly as imported. + pub raw_investor_name: String, + /// Optional KSEI investor type code. + pub investor_type: Option, + /// Optional local/foreign classification. + pub locality: Option, + /// Optional nationality text. + pub nationality: Option, + /// Optional domicile text. + pub domicile: Option, + /// Scripless holdings in shares. + pub holdings_scripless: i64, + /// Script holdings in shares. + pub holdings_scrip: i64, + /// Total shares held. + pub total_shares: i64, + /// Ownership percentage in basis points (`41.10% -> 4110`). + pub percentage_bps: i64, + /// Snapshot as-of date. + pub report_date: NaiveDate, + /// SHA-256 hash of source release for deduplication. + pub release_sha256: String, +} + +/// Draft holding before entity resolution (without persisted row id and entity id). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KseiHoldingDraft { + /// Exchange ticker code, for example `BBCA`. + pub ticker_code: String, + /// Optional issuer name as provided by source row. + pub issuer_name: Option, + /// Raw investor name exactly as imported. + pub raw_investor_name: String, + /// Optional KSEI investor type code. + pub investor_type: Option, + /// Optional local/foreign classification. + pub locality: Option, + /// Optional nationality text. + pub nationality: Option, + /// Optional domicile text. + pub domicile: Option, + /// Scripless holdings in shares. + pub holdings_scripless: i64, + /// Script holdings in shares. + pub holdings_scrip: i64, + /// Total shares held. + pub total_shares: i64, + /// Ownership percentage in basis points (`41.10% -> 4110`). + pub percentage_bps: i64, + /// Snapshot as-of date. + pub report_date: NaiveDate, +} + +/// Single Bing institutional ownership row. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BingHolding { + /// Internal holding row identifier. + pub id: i64, + /// Referenced ticker id. + pub ticker_id: i64, + /// Referenced entity id, unresolved when `None`. + pub entity_id: Option, + /// Raw investor name exactly as imported. + pub raw_investor_name: String, + /// Optional Bing investor type text. + pub investor_type: Option, + /// Shares currently held. + pub shares_held: Option, + /// Share delta over period (`+buy`, `-sell`). + pub shares_changed: Option, + /// Ownership percentage in basis points. + pub pct_ownership_bps: Option, + /// Position value in USD. + pub value_usd: Option, + /// Source report date. + pub report_date: NaiveDate, + /// Institutional flow signal for the row. + pub signal: FlowSignal, + /// Import timestamp (unix epoch seconds). + pub fetched_at: i64, +} + +/// Raw row extracted from KSEI PDF before normalization. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KseiRawRow { + /// Date string as appears in PDF (for example `27-Feb-2026`). + pub date: String, + /// Share code string (for example `BBCA`). + pub share_code: String, + /// Issuer name as appears in source. + pub issuer_name: String, + /// Investor name as appears in source. + pub investor_name: String, + /// Investor type code string. + pub investor_type: String, + /// Local or foreign marker (`L` or `F`). + pub local_foreign: String, + /// Nationality text. + pub nationality: String, + /// Domicile text. + pub domicile: String, + /// Scripless holdings string in Indonesian locale format. + pub holdings_scripless: String, + /// Script holdings string in Indonesian locale format. + pub holdings_scrip: String, + /// Total holdings string in Indonesian locale format. + pub total_holding_shares: String, + /// Percentage string in Indonesian locale decimal format. + pub percentage: String, +} + +/// Raw holder object returned by Bing ownership endpoints. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BingHolderRaw { + /// Investor name field from Bing payload. + #[serde(alias = "investorName", alias = "InvestorName")] + pub investor_name: Option, + /// Investor type field from Bing payload. + #[serde(alias = "investorType", alias = "InvestorType")] + pub investor_type: Option, + /// Shares held field from Bing payload. + #[serde(alias = "sharesHeld", alias = "SharesHeld")] + pub shares_held: Option, + /// Shares changed field from Bing payload. + #[serde(alias = "sharesChanged", alias = "SharesChanged")] + pub shares_changed: Option, + /// Percentage of shares outstanding field from Bing payload. + #[serde( + alias = "percentageOfSharesOutstanding", + alias = "PercentageOfSharesOutstanding" + )] + pub pct_outstanding: Option, + /// Position value field from Bing payload. + #[serde(alias = "value", alias = "Value")] + pub value: Option, + /// Report date field from Bing payload. + #[serde(alias = "reportDate", alias = "ReportDate")] + pub report_date: Option, +} + +/// Combined ownership view for a ticker (KSEI and Bing merged). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TickerOwnership { + /// Ticker metadata. + pub ticker: Ticker, + /// Latest KSEI as-of date. + pub ksei_as_of: Option, + /// Latest Bing reporting period label. + pub bing_as_of: Option, + /// Combined holder rows for display. + pub holders: Vec, + /// Concentration metrics calculated for the ticker. + pub concentration: ConcentrationMetrics, + /// Optional institutional flow breakdown. + pub flow: Option, +} + +/// Single row in a combined holders table. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HolderRow { + /// Ranking position in result set. + pub rank: usize, + /// Source dataset for the row. + pub source: OwnershipSource, + /// Canonical holder name or unresolved raw name. + pub name: String, + /// Optional referenced canonical entity id. + pub entity_id: Option, + /// Optional investor type label. + pub investor_type: Option, + /// Optional local/foreign marker. + pub locality: Option, + /// Held shares quantity. + pub shares: i64, + /// Ownership percentage in basis points. + pub percentage_bps: i64, + /// Optional flow signal (Bing rows only). + pub signal: Option, +} + +/// Concentration metrics for ownership distribution of a ticker. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ConcentrationMetrics { + /// Largest single holder percentage in basis points. + pub top1_bps: i64, + /// Sum of top 3 holder percentages in basis points. + pub top3_bps: i64, + /// Herfindahl-Hirschman index value. + pub hhi: i64, + /// Estimated free float in basis points (`10000 - known holders`). + pub free_float_bps: i64, + /// Number of KSEI holders above or equal to 1%. + pub holder_count: usize, +} + +/// Institutional flow summary from Bing for one ticker and period. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InstitutionalFlow { + /// Reporting period label. + pub period: String, + /// Top buyer rows. + pub top_buyers: Vec, + /// Top seller rows. + pub top_sellers: Vec, + /// New position rows. + pub new_positions: Vec, + /// Exited position rows. + pub exited: Vec, +} + +/// Cross-holding summary for a single entity. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EntityHoldings { + /// Canonical entity metadata. + pub entity: Entity, + /// Number of distinct tickers held. + pub ticker_count: usize, + /// Per-ticker ownership rows. + pub holdings: Vec, +} + +/// One ticker row within an entity holdings summary. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EntityTickerRow { + /// Ticker metadata. + pub ticker: Ticker, + /// Source dataset for the holding. + pub source: OwnershipSource, + /// Shares held quantity. + pub shares: i64, + /// Ownership percentage in basis points. + pub percentage_bps: i64, + /// Report as-of date. + pub report_date: NaiveDate, +} + +/// Cross-holder leaderboard row across multiple tickers. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CrossHolderRow { + /// Canonical entity metadata. + pub entity: Entity, + /// Number of distinct tickers held. + pub ticker_count: usize, + /// Summed ownership basis points across tickers. + pub total_bps: i64, +} + +/// Graph node used for ownership network visualization. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GraphNode { + /// Stable graph node id (`entity:42` or `ticker:BBCA`). + pub id: String, + /// Display label. + pub label: String, + /// Node category. + pub node_type: GraphNodeType, +} + +/// Graph edge used for ownership network visualization. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GraphEdge { + /// Source node id. + pub from: String, + /// Target node id. + pub to: String, + /// Edge weight in basis points. + pub percentage_bps: i64, + /// Source dataset of the edge. + pub source: OwnershipSource, +} + +/// Imported ownership release metadata. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OwnershipRelease { + /// Internal release id. + pub id: i64, + /// Optional source URL where release was obtained. + pub source_url: Option, + /// SHA-256 hash for release-level deduplication. + pub sha256: String, + /// Release as-of date. + pub as_of_date: NaiveDate, + /// Parsed row count imported. + pub row_count: usize, + /// Import timestamp (unix epoch seconds). + pub imported_at: i64, +} + +/// Type of ownership change between two snapshots. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ChangeType { + /// Holder appears in `to` snapshot but not in `from`. + New, + /// Holder appears in `from` snapshot but not in `to`. + Exited, + /// Holder exists in both snapshots and percentage increased. + Increased, + /// Holder exists in both snapshots and percentage decreased. + Decreased, +} + +/// One ownership change row for a ticker-holder pair. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ChangeRow { + /// Exchange ticker code. + pub ticker_code: String, + /// Canonical or raw holder name. + pub entity_name: String, + /// Change classification. + pub change_type: ChangeType, + /// Old percentage in basis points (from snapshot). + pub old_bps: Option, + /// New percentage in basis points (to snapshot). + pub new_bps: Option, + /// Delta in basis points (`new - old`, missing treated as 0). + pub delta_bps: i64, +} + +/// Row describing unresolved or low-confidence alias mappings. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UnresolvedRow { + /// Raw investor name captured from source data. + pub raw_name: String, + /// Source marker (`ksei` or `bing`). + pub source: String, + /// Ticker code where the unresolved alias appears. + pub ticker_code: String, + /// Current canonical entity name when available. + pub current_entity: Option, + /// Resolution confidence score. + pub confidence: Option, +} diff --git a/tests/fixtures/bing_buyers_bbca.json b/tests/fixtures/bing_buyers_bbca.json new file mode 100644 index 0000000..348ef39 --- /dev/null +++ b/tests/fixtures/bing_buyers_bbca.json @@ -0,0 +1,47 @@ +[ + { + "investorName": "Norges Bank Investment Management", + "investorType": "Government / Sovereign Wealth", + "sharesHeld": 278330000.0, + "sharesChanged": 3200000.0, + "percentageOfSharesOutstanding": 1.13, + "value": 275196270000.0, + "reportDate": "2024-12-31" + }, + { + "investorName": "Dimensional Fund Advisors LP", + "investorType": "Institutional", + "sharesHeld": 156920000.0, + "sharesChanged": 14750000.0, + "percentageOfSharesOutstanding": 0.64, + "value": 155110680000.0, + "reportDate": "2024-12-31" + }, + { + "investorName": "Baillie Gifford & Co", + "investorType": "Institutional", + "sharesHeld": 132450000.0, + "sharesChanged": 11200000.0, + "percentageOfSharesOutstanding": 0.54, + "value": 130965450000.0, + "reportDate": "2024-12-31" + }, + { + "investorName": "BlackRock Inc", + "investorType": "Institutional", + "sharesHeld": 498112750.0, + "sharesChanged": 5600000.0, + "percentageOfSharesOutstanding": 2.02, + "value": 492523521250.0, + "reportDate": "2024-12-31" + }, + { + "investorName": "Fidelity Management & Research", + "investorType": "Institutional", + "sharesHeld": 89340000.0, + "sharesChanged": 4100000.0, + "percentageOfSharesOutstanding": 0.36, + "value": 88346060000.0, + "reportDate": "2024-12-31" + } +] diff --git a/tests/fixtures/bing_holders_bbca.json b/tests/fixtures/bing_holders_bbca.json new file mode 100644 index 0000000..3f58efa --- /dev/null +++ b/tests/fixtures/bing_holders_bbca.json @@ -0,0 +1,56 @@ +[ + { + "investorName": "PT Dwimuria Investama Andalan", + "investorType": "Corporate", + "sharesHeld": 13533682440.0, + "sharesChanged": 0.0, + "percentageOfSharesOutstanding": 54.94, + "value": 13380524934600.0, + "reportDate": "2024-12-31" + }, + { + "investorName": "Vanguard Group Inc", + "investorType": "Institutional", + "sharesHeld": 621438200.0, + "sharesChanged": -12000000.0, + "percentageOfSharesOutstanding": 2.52, + "value": 614321778000.0, + "reportDate": "2024-12-31" + }, + { + "investorName": "BlackRock Inc", + "investorType": "Institutional", + "sharesHeld": 498112750.0, + "sharesChanged": 5600000.0, + "percentageOfSharesOutstanding": 2.02, + "value": 492523521250.0, + "reportDate": "2024-12-31" + }, + { + "investorName": "Government Pension Investment Fund (Japan)", + "investorType": "Government / Sovereign Wealth", + "sharesHeld": 312045000.0, + "sharesChanged": 0.0, + "percentageOfSharesOutstanding": 1.27, + "value": 308564505000.0, + "reportDate": "2024-12-31" + }, + { + "investorName": "Norges Bank Investment Management", + "investorType": "Government / Sovereign Wealth", + "sharesHeld": 278330000.0, + "sharesChanged": 3200000.0, + "percentageOfSharesOutstanding": 1.13, + "value": 275196270000.0, + "reportDate": "2024-12-31" + }, + { + "investorName": "Capital Research and Management", + "investorType": "Institutional", + "sharesHeld": 248760000.0, + "sharesChanged": -8000000.0, + "percentageOfSharesOutstanding": 1.01, + "value": 245971800000.0, + "reportDate": "2024-12-31" + } +] diff --git a/tests/fixtures/ksei_stext_sample.xml b/tests/fixtures/ksei_stext_sample.xml new file mode 100644 index 0000000..1c5cba9 --- /dev/null +++ b/tests/fixtures/ksei_stext_sample.xml @@ -0,0 +1,328 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +