From 1d8b615857462e9ec285a2d6e3a50f42e5307039 Mon Sep 17 00:00:00 2001 From: hermes Date: Wed, 14 Oct 2026 09:52:17 +0000 Subject: [PATCH] refactor(core): move ownership queries, import, and sync into idx-core Phase 3c of docs/DESIGN.md. `ownership` handlers are now argument parsing, one idx-core call, and rendering. - investor.rs: Investor (also Idx::investor()) with releases, ticker, entity, search, cross_holders, concentration, flow, changes, graph, unresolved, map_alias, merge_entities, discover, sync, and import. Import orchestration (input resolution, download, SHA-256 dedup, PDF/XLSX/archive parsing, entity resolution, release write) moves from the CLI unchanged. - Typed inputs: ImportSource, HolderSource, ConcentrationRank, and parse_report_family, with the same error messages. - CoreConfig gains an ownership_db override; db::open_db_at opens an explicit path. idx-cli no longer depends on rusqlite, sha2, or ureq at runtime. - Behavior note: an invalid `ticker --source` or `concentration --by` is now rejected before the database file is opened (and created); the error and exit code are unchanged. - Fix: building idx-cli without the ownership feature no longer warns about the ownership-only `blue` style. All 143 goldens unchanged; 170 core + 7 CLI unit tests and 62 CLI tests pass; clippy clean for default, release, testing, and no-default feature sets. Live A/B against main: 16/16 stocks identical; ownership discover got IDX 403 on both (exit 4 vs 1 is the phase 1 change). Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 + Cargo.lock | 1 - crates/idx-cli/Cargo.toml | 6 +- crates/idx-cli/src/cli/ownership.rs | 587 +++++--------------------- crates/idx-cli/src/output/style.rs | 2 + crates/idx-core/src/client.rs | 5 + crates/idx-core/src/config.rs | 4 + crates/idx-core/src/investor.rs | 633 ++++++++++++++++++++++++++++ crates/idx-core/src/lib.rs | 2 + crates/idx-core/src/ownership/db.rs | 9 +- docs/ARCHITECTURE.md | 7 +- docs/DESIGN.md | 6 +- 12 files changed, 774 insertions(+), 490 deletions(-) create mode 100644 crates/idx-core/src/investor.rs diff --git a/AGENTS.md b/AGENTS.md index aedb82f..b8a03c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,6 +6,8 @@ ## Navigation ```text crates/idx-core/ library: all data access and analysis +├── src/client.rs Idx entry point, CachePolicy, Fetched +├── src/{market,financial,investor}.rs domain APIs over providers and the DB ├── src/api/ providers: msn/ (default), yahoo/ (history/OHLCV) ├── src/analysis/ technical, fundamental, signals ├── src/ownership/ xlsx + pdf parsers, archive (ZIP/TXT), remote discovery, diff --git a/Cargo.lock b/Cargo.lock index f8f727a..35f189f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -726,7 +726,6 @@ dependencies = [ "serde_json", "sha2", "toml", - "ureq", "zip", ] diff --git a/crates/idx-cli/Cargo.toml b/crates/idx-cli/Cargo.toml index debf2a1..ddb3186 100644 --- a/crates/idx-cli/Cargo.toml +++ b/crates/idx-cli/Cargo.toml @@ -18,8 +18,7 @@ path = "src/main.rs" [features] default = ["ownership"] -# rusqlite/sha2/ureq back ownership import code that phase 3 moves into idx-core. -ownership = ["idx-core/ownership", "dep:rusqlite", "dep:sha2", "dep:ureq"] +ownership = ["idx-core/ownership"] [dependencies] idx-core = { workspace = true, features = ["clap"] } @@ -31,9 +30,6 @@ comfy-table = "7" owo-colors = "4" toml.workspace = true chrono.workspace = true -rusqlite = { workspace = true, optional = true } -sha2 = { workspace = true, optional = true } -ureq = { workspace = true, optional = true } [dev-dependencies] # Tests drive the binary with the mock provider; release builds exclude it. diff --git a/crates/idx-cli/src/cli/ownership.rs b/crates/idx-cli/src/cli/ownership.rs index f5d33b5..7836c5b 100644 --- a/crates/idx-cli/src/cli/ownership.rs +++ b/crates/idx-cli/src/cli/ownership.rs @@ -1,22 +1,20 @@ -use std::collections::HashSet; -use std::fs; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; -use crate::output::style::Paint; -use chrono::Utc; use clap::{Args, Subcommand}; use comfy_table::{Cell, ContentArrangement, Table, presets::UTF8_FULL}; use serde::Serialize; -use sha2::{Digest, Sha256}; use crate::config::IdxConfig; use crate::output::json; +use crate::output::style::Paint; use crate::output::table::format_idr; use idx_core::error::IdxError; -use idx_core::ownership::types::{ - ChangeType, FlowSignal, HolderRow, KseiHolding, OwnershipRelease, OwnershipSource, +use idx_core::investor::{ + ConcentrationRank, HolderSource, ImportSource, Investor, parse_report_family, }; -use idx_core::ownership::{archive, db, entities, graph, parser, remote, search, snapshot, xlsx}; +use idx_core::ownership::graph; +use idx_core::ownership::snapshot; +use idx_core::ownership::types::{ChangeType, FlowSignal, HolderRow, OwnershipSource}; #[derive(Debug, Args)] #[command( @@ -196,27 +194,27 @@ pub enum ResolveCommand { } pub fn handle(cmd: &OwnershipCommand, config: &IdxConfig) -> Result<(), IdxError> { + let investor = Investor::new(&config.core); + let inv = &investor; match cmd { - OwnershipCommand::Discover(args) => handle_discover(args, config), - OwnershipCommand::Import(args) => handle_import(args, config), - OwnershipCommand::Sync(args) => handle_sync(args, config), - OwnershipCommand::Ticker(args) => handle_ticker(args, config), - OwnershipCommand::Entity(args) => handle_entity(args, config), - OwnershipCommand::Search(args) => handle_search(args, config), - OwnershipCommand::CrossHolders(args) => handle_cross_holders(args, config), - OwnershipCommand::Concentration(args) => handle_concentration(args, config), - OwnershipCommand::Flow(args) => handle_flow(args, config), - OwnershipCommand::Changes(args) => handle_changes(args, config), - OwnershipCommand::Graph(args) => handle_graph(args, config), - OwnershipCommand::Resolve(args) => handle_resolve(args, config), - OwnershipCommand::Releases => handle_releases(config), + OwnershipCommand::Discover(args) => handle_discover(inv, args, config), + OwnershipCommand::Import(args) => handle_import(inv, args, config), + OwnershipCommand::Sync(args) => handle_sync(inv, args, config), + OwnershipCommand::Ticker(args) => handle_ticker(inv, args, config), + OwnershipCommand::Entity(args) => handle_entity(inv, args, config), + OwnershipCommand::Search(args) => handle_search(inv, args, config), + OwnershipCommand::CrossHolders(args) => handle_cross_holders(inv, args, config), + OwnershipCommand::Concentration(args) => handle_concentration(inv, args, config), + OwnershipCommand::Flow(args) => handle_flow(inv, args, config), + OwnershipCommand::Changes(args) => handle_changes(inv, args, config), + OwnershipCommand::Graph(args) => handle_graph(inv, args, config), + OwnershipCommand::Resolve(args) => handle_resolve(inv, args, config), + OwnershipCommand::Releases => handle_releases(inv, config), } } -fn handle_sync(args: &SyncArgs, config: &IdxConfig) -> Result<(), IdxError> { - let manifest_source = snapshot::resolve_manifest_source(args.manifest.as_deref())?; - let db_path = db::db_path()?; - let result = snapshot::sync_snapshot(&manifest_source, &db_path, args.force)?; +fn handle_sync(investor: &Investor, args: &SyncArgs, config: &IdxConfig) -> Result<(), IdxError> { + let result = investor.sync(args.manifest.as_deref(), args.force)?; if config.output.is_json() { return json::print_json(&result); @@ -258,15 +256,19 @@ fn handle_sync(args: &SyncArgs, config: &IdxConfig) -> Result<(), IdxError> { Ok(()) } -fn handle_discover(args: &DiscoverArgs, config: &IdxConfig) -> Result<(), IdxError> { +fn handle_discover( + investor: &Investor, + args: &DiscoverArgs, + config: &IdxConfig, +) -> Result<(), IdxError> { if args.limit == 0 { return Err(IdxError::ParseError( "--limit must be greater than 0".to_string(), )); } - let family = parse_discovery_family(&args.family)?; - let reports = remote::discover_idx_ownership_reports(family, args.limit)?; + let family = parse_report_family(&args.family)?; + let reports = investor.discover(family, args.limit)?; if config.output.is_json() { return json::print_json(&reports); @@ -300,41 +302,13 @@ fn handle_discover(args: &DiscoverArgs, config: &IdxConfig) -> Result<(), IdxErr Ok(()) } -fn handle_ticker(args: &TickerArgs, config: &IdxConfig) -> Result<(), IdxError> { - let conn = db::open_db()?; - let symbol = args.symbol.trim().to_uppercase(); - let source = args.source.trim().to_lowercase(); - if !["all", "ksei", "bing"].contains(&source.as_str()) { - return Err(IdxError::ParseError( - "invalid --source, expected: all|ksei|bing".to_string(), - )); - } - - let mut data = db::query_ticker_holdings(&conn, &symbol)?; - if data.holders.is_empty() { - return Err(IdxError::ParseError(format!( - "no ownership data found for symbol {symbol}" - ))); - } - - if source != "all" { - data.holders.retain(|h| match source.as_str() { - "ksei" => matches!(h.source, OwnershipSource::Ksei), - "bing" => matches!(h.source, OwnershipSource::Bing), - _ => true, - }); - for (i, row) in data.holders.iter_mut().enumerate() { - row.rank = i + 1; - } - let percentages: Vec = data.holders.iter().map(|h| h.percentage_bps).collect(); - data.concentration = db::compute_concentration(&percentages); - } - - if data.holders.is_empty() { - return Err(IdxError::ParseError(format!( - "no {source} ownership data found for symbol {symbol}" - ))); - } +fn handle_ticker( + investor: &Investor, + args: &TickerArgs, + config: &IdxConfig, +) -> Result<(), IdxError> { + let source: HolderSource = args.source.parse()?; + let data = investor.ticker(&args.symbol, source)?; if config.output.is_json() { return json::print_json(&data); @@ -392,25 +366,22 @@ fn handle_ticker(args: &TickerArgs, config: &IdxConfig) -> Result<(), IdxError> Ok(()) } -fn handle_entity(args: &EntityArgs, config: &IdxConfig) -> Result<(), IdxError> { - let conn = db::open_db()?; - let q = format!("%{}%", args.name.trim()); - - let entity = conn - .query_row( - "SELECT id, canonical_name FROM entities WHERE canonical_name LIKE ?1 COLLATE NOCASE ORDER BY LENGTH(canonical_name) ASC LIMIT 1", - rusqlite::params![q], - |row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)), - ) - .map_err(|_| IdxError::ParseError(format!("entity not found: {}", args.name)))?; - - let data = db::query_entity_holdings(&conn, entity.0)?; +fn handle_entity( + investor: &Investor, + args: &EntityArgs, + config: &IdxConfig, +) -> Result<(), IdxError> { + let data = investor.entity(&args.name)?; if config.output.is_json() { return json::print_json(&data); } - println!("{} {}", "Entity:".bold(), entity.1.bold()); + println!( + "{} {}", + "Entity:".bold(), + data.entity.canonical_name.as_str().bold() + ); let mut table = Table::new(); table .load_preset(UTF8_FULL) @@ -435,9 +406,12 @@ fn handle_entity(args: &EntityArgs, config: &IdxConfig) -> Result<(), IdxError> Ok(()) } -fn handle_search(args: &SearchArgs, config: &IdxConfig) -> Result<(), IdxError> { - let conn = db::open_db()?; - let entities = search::fts_search(&conn, &args.query, args.limit)?; +fn handle_search( + investor: &Investor, + args: &SearchArgs, + config: &IdxConfig, +) -> Result<(), IdxError> { + let entities = investor.search(&args.query, args.limit)?; if config.output.is_json() { return json::print_json(&entities); @@ -470,9 +444,12 @@ fn handle_search(args: &SearchArgs, config: &IdxConfig) -> Result<(), IdxError> Ok(()) } -fn handle_cross_holders(args: &CrossHolderArgs, config: &IdxConfig) -> Result<(), IdxError> { - let conn = db::open_db()?; - let rows = db::query_cross_holders(&conn, args.min_tickers, args.top)?; +fn handle_cross_holders( + investor: &Investor, + args: &CrossHolderArgs, + config: &IdxConfig, +) -> Result<(), IdxError> { + let rows = investor.cross_holders(args.min_tickers, args.top)?; if config.output.is_json() { return json::print_json(&rows); @@ -497,16 +474,13 @@ fn handle_cross_holders(args: &CrossHolderArgs, config: &IdxConfig) -> Result<() Ok(()) } -fn handle_concentration(args: &ConcentrationArgs, config: &IdxConfig) -> Result<(), IdxError> { - let conn = db::open_db()?; - let by = args.by.trim().to_lowercase(); - if !["top1", "top3", "hhi"].contains(&by.as_str()) { - return Err(IdxError::ParseError( - "invalid --by, expected: top1|top3|hhi".to_string(), - )); - } - - let rows = db::query_concentration(&conn, &by, args.top)?; +fn handle_concentration( + investor: &Investor, + args: &ConcentrationArgs, + config: &IdxConfig, +) -> Result<(), IdxError> { + let by: ConcentrationRank = args.by.parse()?; + let rows = investor.concentration(by, args.top)?; if config.output.is_json() { return json::print_json(&rows); @@ -542,13 +516,9 @@ fn handle_concentration(args: &ConcentrationArgs, config: &IdxConfig) -> Result< Ok(()) } -fn handle_flow(args: &FlowArgs, config: &IdxConfig) -> Result<(), IdxError> { - let conn = db::open_db()?; +fn handle_flow(investor: &Investor, args: &FlowArgs, config: &IdxConfig) -> Result<(), IdxError> { let symbol = args.symbol.trim().to_uppercase(); - let ticker_id = db::get_ticker_id(&conn, &symbol)? - .ok_or_else(|| IdxError::SymbolNotFound(symbol.clone()))?; - - let flow = db::query_bing_flow(&conn, ticker_id)?; + let flow = investor.flow(&symbol)?; let Some(flow) = flow else { if config.output.is_json() { return json::print_json( @@ -596,9 +566,8 @@ fn print_flow_section(title: &str, rows: &[HolderRow]) { println!("{table}"); } -fn handle_graph(args: &GraphArgs, config: &IdxConfig) -> Result<(), IdxError> { - let conn = db::open_db()?; - let (nodes, edges) = graph::query_ownership_graph(&conn, &args.root, args.depth)?; +fn handle_graph(investor: &Investor, args: &GraphArgs, config: &IdxConfig) -> Result<(), IdxError> { + let (nodes, edges) = investor.graph(&args.root, args.depth)?; if config.output.is_json() { #[derive(Serialize)] @@ -627,9 +596,12 @@ fn handle_graph(args: &GraphArgs, config: &IdxConfig) -> Result<(), IdxError> { } } -fn handle_changes(args: &ChangesArgs, config: &IdxConfig) -> Result<(), IdxError> { - let conn = db::open_db()?; - let changes = db::query_changes(&conn, &args.from, &args.to)?; +fn handle_changes( + investor: &Investor, + args: &ChangesArgs, + config: &IdxConfig, +) -> Result<(), IdxError> { + let changes = investor.changes(&args.from, &args.to)?; if config.output.is_json() { return json::print_json(&changes); @@ -673,12 +645,14 @@ fn handle_changes(args: &ChangesArgs, config: &IdxConfig) -> Result<(), IdxError Ok(()) } -fn handle_resolve(args: &ResolveArgs, config: &IdxConfig) -> Result<(), IdxError> { - let conn = db::open_db()?; - +fn handle_resolve( + investor: &Investor, + args: &ResolveArgs, + config: &IdxConfig, +) -> Result<(), IdxError> { match &args.command { ResolveCommand::Unresolved { limit } => { - let rows = db::list_unresolved(&conn, *limit)?; + let rows = investor.unresolved(*limit)?; if config.output.is_json() { return json::print_json(&rows); @@ -718,7 +692,7 @@ fn handle_resolve(args: &ResolveArgs, config: &IdxConfig) -> Result<(), IdxError Ok(()) } ResolveCommand::Map { alias, entity } => { - db::manual_map(&conn, alias, entity)?; + investor.map_alias(alias, entity)?; if config.output.is_json() { return json::print_json(&serde_json::json!({ "action": "mapped", @@ -730,29 +704,10 @@ fn handle_resolve(args: &ResolveArgs, config: &IdxConfig) -> Result<(), IdxError Ok(()) } ResolveCommand::Merge { keep, merge } => { - let alias_updates: i64 = conn - .query_row( - "SELECT COUNT(*) FROM entity_aliases WHERE entity_id = ?1", - rusqlite::params![merge], - |row| row.get(0), - ) - .map_err(|e| IdxError::DatabaseError(e.to_string()))?; - let ksei_updates: i64 = conn - .query_row( - "SELECT COUNT(*) FROM ksei_holdings WHERE entity_id = ?1", - rusqlite::params![merge], - |row| row.get(0), - ) - .map_err(|e| IdxError::DatabaseError(e.to_string()))?; - let bing_updates: i64 = conn - .query_row( - "SELECT COUNT(*) FROM bing_holdings WHERE entity_id = ?1", - rusqlite::params![merge], - |row| row.get(0), - ) - .map_err(|e| IdxError::DatabaseError(e.to_string()))?; - - db::merge_entities(&conn, *keep, *merge)?; + let merged = investor.merge_entities(*keep, *merge)?; + let alias_updates = merged.alias_updates; + let ksei_updates = merged.ksei_holdings_updates; + let bing_updates = merged.bing_holdings_updates; if config.output.is_json() { return json::print_json(&serde_json::json!({ @@ -773,7 +728,11 @@ fn handle_resolve(args: &ResolveArgs, config: &IdxConfig) -> Result<(), IdxError } } -fn handle_import(args: &ImportArgs, config: &IdxConfig) -> Result<(), IdxError> { +fn handle_import( + investor: &Investor, + args: &ImportArgs, + config: &IdxConfig, +) -> Result<(), IdxError> { if args.file.is_none() && args.url.is_none() && args.fetch_bing.is_none() { return Err(IdxError::ParseError( "provide one of: --file, --url, or --fetch-bing".to_string(), @@ -797,124 +756,34 @@ fn handle_import(args: &ImportArgs, config: &IdxConfig) -> Result<(), IdxError> )); } - let Some(import_input) = resolve_import_input(args)? else { - return Ok(()); - }; - - let conn = db::open_db()?; - - let sha256 = sha256_file(&import_input.import_path)?; - if !args.force && db::release_exists(&conn, &sha256)? { - let result = OwnershipImportResult { - action: "skipped_existing", - inserted_rows: 0, - ticker_count: 0, - as_of_date: None, - sha256, - source_url: import_input.source_url, - }; - - if config.output.is_json() { - return json::print_json(&result); - } - - println!( - "Release already imported (sha256: {}). Use --force to re-import.", - result.sha256 - ); - return Ok(()); - } - - let drafts = match import_input.format { - ImportInputFormat::Pdf => { - let raw_rows = parser::parse_ksei_pdf(&import_input.import_path)?; - if raw_rows.is_empty() { - return Err(IdxError::ParseError( - "no KSEI rows parsed from PDF".to_string(), - )); - } - - let mut drafts = Vec::with_capacity(raw_rows.len()); - for raw in &raw_rows { - drafts.push(entities::normalize_ksei_row(raw)?); - } - drafts - } - ImportInputFormat::Archive => archive::parse_balancepos_file(&import_input.import_path)?, - ImportInputFormat::Xlsx => xlsx::parse_above1_xlsx_file(&import_input.import_path)?, - }; - - let mut holdings = Vec::with_capacity(drafts.len()); - let mut ticker_ids = HashSet::new(); - - for draft in drafts { - let ticker_id = db::upsert_ticker(&conn, &draft.ticker_code, draft.issuer_name.as_deref())?; - let entity_id = - entities::resolve_entity(&conn, &draft.raw_investor_name, OwnershipSource::Ksei)?; - - ticker_ids.insert(ticker_id); - holdings.push(KseiHolding { - id: 0, - ticker_id, - entity_id: Some(entity_id), - raw_investor_name: draft.raw_investor_name, - investor_type: draft.investor_type, - locality: draft.locality, - nationality: draft.nationality, - domicile: draft.domicile, - holdings_scripless: draft.holdings_scripless, - holdings_scrip: draft.holdings_scrip, - total_shares: draft.total_shares, - percentage_bps: draft.percentage_bps, - report_date: draft.report_date, - release_sha256: sha256.clone(), - }); - } - - let as_of_date = holdings - .iter() - .map(|h| h.report_date) - .max() - .ok_or_else(|| { - IdxError::ParseError("missing report_date in parsed holdings".to_string()) - })?; - - let release = OwnershipRelease { - id: 0, - source_url: import_input.source_url, - sha256, - as_of_date, - row_count: holdings.len(), - imported_at: Utc::now().timestamp(), - }; - let inserted_rows = db::write_ksei_release(&conn, &release, &holdings, args.force)?; - - let result = OwnershipImportResult { - action: if args.force { "reimported" } else { "imported" }, - inserted_rows, - ticker_count: ticker_ids.len(), - as_of_date: Some(as_of_date.format("%Y-%m-%d").to_string()), - sha256: release.sha256.clone(), - source_url: release.source_url.clone(), + let source = match (&args.file, &args.url) { + (Some(path), _) => ImportSource::File(path.clone()), + (None, Some(url)) => ImportSource::Url(url.clone()), + // An empty --fetch-bing list has nothing to import. + (None, None) => return Ok(()), }; + let result = investor.import(&source, args.force)?; if config.output.is_json() { return json::print_json(&result); } - println!( - "Imported {} rows for {} tickers (as of {}).", - result.inserted_rows, - result.ticker_count, - as_of_date.format("%Y-%m-%d") - ); + match result.as_of_date.as_deref() { + None => println!( + "Release already imported (sha256: {}). Use --force to re-import.", + result.sha256 + ), + Some(as_of_date) => println!( + "Imported {} rows for {} tickers (as of {}).", + result.inserted_rows, result.ticker_count, as_of_date + ), + } Ok(()) } -fn handle_releases(config: &IdxConfig) -> Result<(), IdxError> { - let conn = db::open_db()?; - let releases = db::query_releases(&conn)?; +fn handle_releases(investor: &Investor, config: &IdxConfig) -> Result<(), IdxError> { + let releases = investor.releases()?; if config.output.is_json() { return json::print_json(&releases); @@ -945,238 +814,6 @@ fn handle_releases(config: &IdxConfig) -> Result<(), IdxError> { Ok(()) } -fn parse_discovery_family(raw: &str) -> Result, IdxError> { - match raw.trim().to_ascii_lowercase().as_str() { - "all" => Ok(None), - "above1" | "above-1" | "above_1" => { - Ok(Some(remote::OwnershipReportFamily::AboveOnePercent)) - } - "above5" | "above-5" | "above_5" => { - Ok(Some(remote::OwnershipReportFamily::AboveFivePercent)) - } - "investor-type" | "investor_type" | "investortype" => { - Ok(Some(remote::OwnershipReportFamily::InvestorTypeBreakdown)) - } - _ => Err(IdxError::ParseError( - "invalid --family, expected: all|above1|above5|investor-type".to_string(), - )), - } -} - -struct ResolvedImportInput { - import_path: PathBuf, - source_url: Option, - format: ImportInputFormat, -} - -#[derive(Debug, Serialize)] -struct OwnershipImportResult { - action: &'static str, - inserted_rows: usize, - ticker_count: usize, - as_of_date: Option, - sha256: String, - source_url: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ImportInputFormat { - Pdf, - Archive, - Xlsx, -} - -fn resolve_import_input(args: &ImportArgs) -> Result, IdxError> { - if let Some(path) = &args.file { - if !path.exists() { - return Err(IdxError::Io(format!( - "input ownership file not found: {}", - path.display() - ))); - } - - return Ok(Some(ResolvedImportInput { - import_path: path.clone(), - source_url: None, - format: detect_local_import_format(path)?, - })); - } - - if let Some(url) = &args.url { - let trimmed = url.trim(); - let format = validate_import_url(trimmed)?; - let target = cache_download_path(trimmed, format)?; - match format { - ImportInputFormat::Xlsx => download_xlsx(trimmed, &target)?, - _ => download_pdf(trimmed, &target)?, - } - return Ok(Some(ResolvedImportInput { - import_path: target, - source_url: Some(trimmed.to_string()), - format, - })); - } - - Ok(None) -} - -fn detect_local_import_format(path: &Path) -> Result { - match path - .extension() - .and_then(|value| value.to_str()) - .map(|value| value.trim().to_ascii_lowercase()) - .as_deref() - { - Some("pdf") => Ok(ImportInputFormat::Pdf), - Some("xlsx") if xlsx::supports_xlsx_file(path) => Ok(ImportInputFormat::Xlsx), - Some("zip") | Some("txt") if archive::supports_local_archive_file(path) => { - Ok(ImportInputFormat::Archive) - } - _ => Err(IdxError::InvalidInput(format!( - "unsupported local ownership file {}; expected a .xlsx, .pdf, .zip, or .txt input", - path.display() - ))), - } -} - -fn cache_download_path(url: &str, format: ImportInputFormat) -> Result { - let extension = match format { - ImportInputFormat::Xlsx => ".xlsx", - _ => ".pdf", - }; - let raw_dir = idx_core::cache::cache_dir()?.join("ownership").join("raw"); - fs::create_dir_all(&raw_dir).map_err(|e| IdxError::Io(e.to_string()))?; - - let mut file_name = url - .rsplit('/') - .next() - .unwrap_or("ksei-ownership.pdf") - .split('?') - .next() - .unwrap_or("ksei-ownership.pdf") - .to_string(); - - if file_name.is_empty() || file_name == "/" { - file_name = format!("ksei-{}{extension}", Utc::now().timestamp()); - } - if !file_name.to_ascii_lowercase().ends_with(extension) { - file_name.push_str(extension); - } - - Ok(raw_dir.join(file_name)) -} - -fn validate_import_url(url: &str) -> Result { - let trimmed = url.trim(); - if trimmed.is_empty() { - return Err(IdxError::InvalidInput( - "ownership import --url accepts direct XLSX or PDF URLs only".to_string(), - )); - } - - let normalized = trimmed.to_ascii_lowercase(); - let normalized = normalized - .split(['?', '#']) - .next() - .unwrap_or(normalized.as_str()); - if normalized.ends_with(".pdf") { - return Ok(ImportInputFormat::Pdf); - } - if normalized.ends_with(".xlsx") { - return Ok(ImportInputFormat::Xlsx); - } - - Err(IdxError::InvalidInput( - "ownership import --url accepts direct XLSX or PDF URLs only; run `idx ownership discover` first to find the current supported file".to_string(), - )) -} - -fn download_pdf(url: &str, target: &Path) -> Result<(), IdxError> { - if is_idx_url(url) { - return remote::download_idx_pdf(url, target); - } - - let response = ureq::get(url) - .header( - "User-Agent", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", - ) - .header("Accept", "application/pdf,application/octet-stream,*/*;q=0.8") - .header("Accept-Language", "en-US,en;q=0.9") - .header("Referer", "https://www.idx.co.id/") - .call() - .map_err(|e| IdxError::Http(format!("failed to download PDF: {e}")))?; - - let bytes = response - .into_body() - .with_config() - .limit(REMOTE_REPORT_LIMIT_BYTES) - .read_to_vec() - .map_err(|e| IdxError::Http(format!("failed reading PDF body: {e}")))?; - remote::validate_pdf_payload(&bytes)?; - - fs::write(target, &bytes).map_err(|e| { - IdxError::Io(format!( - "failed writing cached PDF {}: {e}", - target.display() - )) - })?; - - Ok(()) -} - -/// Above ureq's 10 MiB default: the monthly holder-register PDF can grow past it. -const REMOTE_REPORT_LIMIT_BYTES: u64 = 256 * 1024 * 1024; - -fn download_xlsx(url: &str, target: &Path) -> Result<(), IdxError> { - if is_idx_url(url) { - return remote::download_idx_xlsx(url, target); - } - - let response = ureq::get(url) - .header( - "Accept", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/octet-stream,*/*;q=0.8", - ) - .call() - .map_err(|e| IdxError::Http(format!("failed to download XLSX: {e}")))?; - let bytes = response - .into_body() - .with_config() - .limit(REMOTE_REPORT_LIMIT_BYTES) - .read_to_vec() - .map_err(|e| IdxError::Http(format!("failed reading XLSX body: {e}")))?; - remote::validate_xlsx_payload(&bytes)?; - - fs::write(target, &bytes).map_err(|e| { - IdxError::Io(format!( - "failed writing cached XLSX {}: {e}", - target.display() - )) - }) -} - -fn is_idx_url(url: &str) -> bool { - let normalized = url.trim().to_ascii_lowercase(); - normalized.starts_with("https://www.idx.co.id/") - || normalized.starts_with("http://www.idx.co.id/") - || normalized.starts_with("https://idx.co.id/") - || normalized.starts_with("http://idx.co.id/") -} - -fn sha256_file(path: &Path) -> Result { - let bytes = fs::read(path).map_err(|e| { - IdxError::Io(format!( - "failed to read file for sha256 {}: {e}", - path.display() - )) - })?; - let mut hasher = Sha256::new(); - hasher.update(bytes); - let digest = hasher.finalize(); - Ok(digest.iter().map(|b| format!("{b:02x}")).collect()) -} - fn format_bps(bps: i64) -> String { format!("{:.2}%", bps as f64 / 100.0) } diff --git a/crates/idx-cli/src/output/style.rs b/crates/idx-cli/src/output/style.rs index ee72e50..cf6359c 100644 --- a/crates/idx-cli/src/output/style.rs +++ b/crates/idx-cli/src/output/style.rs @@ -33,6 +33,8 @@ pub trait Paint: Display + Sized { fn yellow(self) -> String { paint(self, |v| OwoColorize::yellow(v).to_string()) } + // Only ownership output uses blue. + #[cfg_attr(not(feature = "ownership"), allow(dead_code))] fn blue(self) -> String { paint(self, |v| OwoColorize::blue(v).to_string()) } diff --git a/crates/idx-core/src/client.rs b/crates/idx-core/src/client.rs index b73f49c..c92d58d 100644 --- a/crates/idx-core/src/client.rs +++ b/crates/idx-core/src/client.rs @@ -94,6 +94,11 @@ impl Idx { crate::analysis::Analysis { idx: self } } + #[cfg(feature = "ownership")] + pub fn investor(&self) -> crate::investor::Investor { + crate::investor::Investor::new(&self.config) + } + pub(crate) fn resolve(&self, symbol: &str) -> Result { crate::api::resolve_symbol(symbol, &self.config.exchange) } diff --git a/crates/idx-core/src/config.rs b/crates/idx-core/src/config.rs index 4f28071..a76c104 100644 --- a/crates/idx-core/src/config.rs +++ b/crates/idx-core/src/config.rs @@ -85,6 +85,9 @@ pub struct CoreConfig { pub fundamental_ttl: u64, /// Cache root; `None` resolves `$XDG_CACHE_HOME/idx` or the platform default. pub cache_dir: Option, + /// Ownership database; `None` uses `ownership.db_path` from `config.toml`, + /// else `$XDG_DATA_HOME/idx/ownership.db`. + pub ownership_db: Option, /// Provider diagnostics on stderr. pub verbose: bool, } @@ -98,6 +101,7 @@ impl Default for CoreConfig { quote_ttl: 300, fundamental_ttl: 3600, cache_dir: None, + ownership_db: None, verbose: false, } } diff --git a/crates/idx-core/src/investor.rs b/crates/idx-core/src/investor.rs new file mode 100644 index 0000000..eaef76b --- /dev/null +++ b/crates/idx-core/src/investor.rs @@ -0,0 +1,633 @@ +//! Investor (ownership) data: queries over the local SQLite database, plus +//! the import, sync, and discovery that populate it. + +use std::collections::HashSet; +use std::fmt; +use std::fs; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use chrono::Utc; +use rusqlite::Connection; +use serde::Serialize; +use sha2::{Digest, Sha256}; + +use crate::config::CoreConfig; +use crate::error::IdxError; +use crate::ownership::remote::{self, DiscoveredOwnershipPdf, OwnershipReportFamily}; +use crate::ownership::snapshot::{self, OwnershipSyncResult}; +use crate::ownership::types::{ + ChangeRow, ConcentrationMetrics, CrossHolderRow, Entity, EntityHoldings, GraphEdge, GraphNode, + InstitutionalFlow, KseiHolding, OwnershipRelease, OwnershipSource, TickerOwnership, + UnresolvedRow, +}; +use crate::ownership::{archive, db, entities, graph, parser, search, xlsx}; + +/// Ownership database access. Opening is lazy: each call opens (and, if +/// needed, creates and migrates) the database at the configured path. +pub struct Investor { + db_path: Option, + cache_dir: Option, +} + +impl Investor { + pub fn new(config: &CoreConfig) -> Self { + Self { + db_path: config.ownership_db.clone(), + cache_dir: config.cache_dir.clone(), + } + } + + /// The database path: `CoreConfig::ownership_db`, else `ownership.db_path` + /// from `config.toml`, else `$XDG_DATA_HOME/idx/ownership.db`. + pub fn db_path(&self) -> Result { + match &self.db_path { + Some(path) => Ok(path.clone()), + None => db::db_path(), + } + } + + fn connect(&self) -> Result { + db::open_db_at(&self.db_path()?) + } + + pub fn releases(&self) -> Result, IdxError> { + db::query_releases(&self.connect()?) + } + + /// Holders of `symbol` in the latest release, optionally limited to one + /// source (ranks and concentration are recomputed for that source). + pub fn ticker(&self, symbol: &str, source: HolderSource) -> Result { + let conn = self.connect()?; + let symbol = symbol.trim().to_uppercase(); + let mut data = db::query_ticker_holdings(&conn, &symbol)?; + if data.holders.is_empty() { + return Err(IdxError::ParseError(format!( + "no ownership data found for symbol {symbol}" + ))); + } + + if source != HolderSource::All { + data.holders.retain(|h| match source { + HolderSource::Ksei => matches!(h.source, OwnershipSource::Ksei), + HolderSource::Bing => matches!(h.source, OwnershipSource::Bing), + HolderSource::All => true, + }); + for (i, row) in data.holders.iter_mut().enumerate() { + row.rank = i + 1; + } + let percentages: Vec = data.holders.iter().map(|h| h.percentage_bps).collect(); + data.concentration = db::compute_concentration(&percentages); + } + + if data.holders.is_empty() { + return Err(IdxError::ParseError(format!( + "no {source} ownership data found for symbol {symbol}" + ))); + } + Ok(data) + } + + /// Holdings of the entity whose name best matches `name` (shortest + /// canonical name containing it). + pub fn entity(&self, name: &str) -> Result { + let conn = self.connect()?; + let pattern = format!("%{}%", name.trim()); + let entity_id = conn + .query_row( + "SELECT id FROM entities WHERE canonical_name LIKE ?1 COLLATE NOCASE ORDER BY LENGTH(canonical_name) ASC LIMIT 1", + rusqlite::params![pattern], + |row| row.get::<_, i64>(0), + ) + .map_err(|_| IdxError::ParseError(format!("entity not found: {name}")))?; + db::query_entity_holdings(&conn, entity_id) + } + + pub fn search(&self, query: &str, limit: usize) -> Result, IdxError> { + search::fts_search(&self.connect()?, query, limit) + } + + pub fn cross_holders( + &self, + min_tickers: usize, + top: usize, + ) -> Result, IdxError> { + db::query_cross_holders(&self.connect()?, min_tickers, top) + } + + pub fn concentration( + &self, + by: ConcentrationRank, + top: usize, + ) -> Result, IdxError> { + db::query_concentration(&self.connect()?, by.as_str(), top) + } + + /// Institutional (Bing) flow; `None` when no flow data exists. + pub fn flow(&self, symbol: &str) -> Result, IdxError> { + let conn = self.connect()?; + let symbol = symbol.trim().to_uppercase(); + let ticker_id = db::get_ticker_id(&conn, &symbol)? + .ok_or_else(|| IdxError::SymbolNotFound(symbol.clone()))?; + db::query_bing_flow(&conn, ticker_id) + } + + pub fn changes(&self, from: &str, to: &str) -> Result, IdxError> { + db::query_changes(&self.connect()?, from, to) + } + + pub fn graph( + &self, + root: &str, + depth: usize, + ) -> Result<(Vec, Vec), IdxError> { + graph::query_ownership_graph(&self.connect()?, root, depth) + } + + pub fn unresolved(&self, limit: usize) -> Result, IdxError> { + db::list_unresolved(&self.connect()?, limit) + } + + pub fn map_alias(&self, alias: &str, entity: &str) -> Result<(), IdxError> { + db::manual_map(&self.connect()?, alias, entity) + } + + /// Merges entity `merge` into `keep`, reporting how many rows moved. + pub fn merge_entities(&self, keep: i64, merge: i64) -> Result { + let conn = self.connect()?; + let count = |table: &str| -> Result { + conn.query_row( + &format!("SELECT COUNT(*) FROM {table} WHERE entity_id = ?1"), + rusqlite::params![merge], + |row| row.get(0), + ) + .map_err(|e| IdxError::DatabaseError(e.to_string())) + }; + let alias_updates = count("entity_aliases")?; + let ksei_holdings_updates = count("ksei_holdings")?; + let bing_holdings_updates = count("bing_holdings")?; + db::merge_entities(&conn, keep, merge)?; + Ok(MergeResult { + keep, + merge, + alias_updates, + ksei_holdings_updates, + bing_holdings_updates, + }) + } + + /// Lists IDX ownership reports (newest first), live from IDX. + pub fn discover( + &self, + family: Option, + limit: usize, + ) -> Result, IdxError> { + remote::discover_idx_ownership_reports(family, limit) + } + + /// Installs or refreshes the maintained snapshot. `manifest` overrides + /// the configured manifest location. + pub fn sync( + &self, + manifest: Option<&str>, + force: bool, + ) -> Result { + let manifest_source = snapshot::resolve_manifest_source(manifest)?; + let db_path = self.db_path()?; + snapshot::sync_snapshot(&manifest_source, &db_path, force) + } + + /// Imports one report. A release already imported (same SHA-256) is + /// skipped unless `force`, which replaces it. + pub fn import(&self, source: &ImportSource, force: bool) -> Result { + let input = self.resolve_import_input(source)?; + let conn = self.connect()?; + + let sha256 = sha256_file(&input.path)?; + if !force && db::release_exists(&conn, &sha256)? { + return Ok(ImportResult { + action: "skipped_existing", + inserted_rows: 0, + ticker_count: 0, + as_of_date: None, + sha256, + source_url: input.source_url, + }); + } + + let drafts = match input.format { + ImportFormat::Pdf => { + let raw_rows = parser::parse_ksei_pdf(&input.path)?; + if raw_rows.is_empty() { + return Err(IdxError::ParseError( + "no KSEI rows parsed from PDF".to_string(), + )); + } + let mut drafts = Vec::with_capacity(raw_rows.len()); + for raw in &raw_rows { + drafts.push(entities::normalize_ksei_row(raw)?); + } + drafts + } + ImportFormat::Archive => archive::parse_balancepos_file(&input.path)?, + ImportFormat::Xlsx => xlsx::parse_above1_xlsx_file(&input.path)?, + }; + + let mut holdings = Vec::with_capacity(drafts.len()); + let mut ticker_ids = HashSet::new(); + for draft in drafts { + let ticker_id = + db::upsert_ticker(&conn, &draft.ticker_code, draft.issuer_name.as_deref())?; + let entity_id = + entities::resolve_entity(&conn, &draft.raw_investor_name, OwnershipSource::Ksei)?; + ticker_ids.insert(ticker_id); + holdings.push(KseiHolding { + id: 0, + ticker_id, + entity_id: Some(entity_id), + raw_investor_name: draft.raw_investor_name, + investor_type: draft.investor_type, + locality: draft.locality, + nationality: draft.nationality, + domicile: draft.domicile, + holdings_scripless: draft.holdings_scripless, + holdings_scrip: draft.holdings_scrip, + total_shares: draft.total_shares, + percentage_bps: draft.percentage_bps, + report_date: draft.report_date, + release_sha256: sha256.clone(), + }); + } + + let as_of_date = holdings + .iter() + .map(|h| h.report_date) + .max() + .ok_or_else(|| { + IdxError::ParseError("missing report_date in parsed holdings".to_string()) + })?; + + let release = OwnershipRelease { + id: 0, + source_url: input.source_url, + sha256, + as_of_date, + row_count: holdings.len(), + imported_at: Utc::now().timestamp(), + }; + let inserted_rows = db::write_ksei_release(&conn, &release, &holdings, force)?; + + Ok(ImportResult { + action: if force { "reimported" } else { "imported" }, + inserted_rows, + ticker_count: ticker_ids.len(), + as_of_date: Some(as_of_date.format("%Y-%m-%d").to_string()), + sha256: release.sha256, + source_url: release.source_url, + }) + } + + fn resolve_import_input(&self, source: &ImportSource) -> Result { + match source { + ImportSource::File(path) => { + if !path.exists() { + return Err(IdxError::Io(format!( + "input ownership file not found: {}", + path.display() + ))); + } + Ok(ImportInput { + path: path.clone(), + source_url: None, + format: detect_local_import_format(path)?, + }) + } + ImportSource::Url(url) => { + let trimmed = url.trim(); + let format = validate_import_url(trimmed)?; + let target = self.download_path(trimmed, format)?; + match format { + ImportFormat::Xlsx => download_xlsx(trimmed, &target)?, + _ => download_pdf(trimmed, &target)?, + } + Ok(ImportInput { + path: target, + source_url: Some(trimmed.to_string()), + format, + }) + } + } + } + + fn download_path(&self, url: &str, format: ImportFormat) -> Result { + let extension = match format { + ImportFormat::Xlsx => ".xlsx", + _ => ".pdf", + }; + let cache_dir = match &self.cache_dir { + Some(dir) => dir.clone(), + None => crate::cache::cache_dir()?, + }; + let raw_dir = cache_dir.join("ownership").join("raw"); + fs::create_dir_all(&raw_dir).map_err(|e| IdxError::Io(e.to_string()))?; + + let mut file_name = url + .rsplit('/') + .next() + .unwrap_or("ksei-ownership.pdf") + .split('?') + .next() + .unwrap_or("ksei-ownership.pdf") + .to_string(); + + if file_name.is_empty() || file_name == "/" { + file_name = format!("ksei-{}{extension}", Utc::now().timestamp()); + } + if !file_name.to_ascii_lowercase().ends_with(extension) { + file_name.push_str(extension); + } + + Ok(raw_dir.join(file_name)) + } +} + +/// Where an import reads its report from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ImportSource { + /// Local `.xlsx`, `.pdf`, `.zip`, or `.txt`. + File(PathBuf), + /// Direct XLSX or PDF URL; IDX URLs use browser impersonation. + Url(String), +} + +#[derive(Debug, Clone, Serialize)] +pub struct ImportResult { + /// `imported`, `reimported`, or `skipped_existing`. + pub action: &'static str, + pub inserted_rows: usize, + pub ticker_count: usize, + pub as_of_date: Option, + pub sha256: String, + pub source_url: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub struct MergeResult { + pub keep: i64, + pub merge: i64, + pub alias_updates: i64, + pub ksei_holdings_updates: i64, + pub bing_holdings_updates: i64, +} + +/// Holder sources for `Investor::ticker`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HolderSource { + All, + Ksei, + Bing, +} + +impl fmt::Display for HolderSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::All => "all", + Self::Ksei => "ksei", + Self::Bing => "bing", + }) + } +} + +impl FromStr for HolderSource { + type Err = IdxError; + + fn from_str(value: &str) -> Result { + match value.trim().to_lowercase().as_str() { + "all" => Ok(Self::All), + "ksei" => Ok(Self::Ksei), + "bing" => Ok(Self::Bing), + _ => Err(IdxError::ParseError( + "invalid --source, expected: all|ksei|bing".to_string(), + )), + } + } +} + +/// Ranking metric for `Investor::concentration`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConcentrationRank { + Top1, + Top3, + Hhi, +} + +impl ConcentrationRank { + fn as_str(self) -> &'static str { + match self { + Self::Top1 => "top1", + Self::Top3 => "top3", + Self::Hhi => "hhi", + } + } +} + +impl FromStr for ConcentrationRank { + type Err = IdxError; + + fn from_str(value: &str) -> Result { + match value.trim().to_lowercase().as_str() { + "top1" => Ok(Self::Top1), + "top3" => Ok(Self::Top3), + "hhi" => Ok(Self::Hhi), + _ => Err(IdxError::ParseError( + "invalid --by, expected: top1|top3|hhi".to_string(), + )), + } + } +} + +/// `all` (None) or one report family, with the aliases the CLI accepts. +pub fn parse_report_family(raw: &str) -> Result, IdxError> { + match raw.trim().to_ascii_lowercase().as_str() { + "all" => Ok(None), + "above1" | "above-1" | "above_1" => Ok(Some(OwnershipReportFamily::AboveOnePercent)), + "above5" | "above-5" | "above_5" => Ok(Some(OwnershipReportFamily::AboveFivePercent)), + "investor-type" | "investor_type" | "investortype" => { + Ok(Some(OwnershipReportFamily::InvestorTypeBreakdown)) + } + _ => Err(IdxError::ParseError( + "invalid --family, expected: all|above1|above5|investor-type".to_string(), + )), + } +} + +struct ImportInput { + path: PathBuf, + source_url: Option, + format: ImportFormat, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ImportFormat { + Pdf, + Archive, + Xlsx, +} + +fn detect_local_import_format(path: &Path) -> Result { + match path + .extension() + .and_then(|value| value.to_str()) + .map(|value| value.trim().to_ascii_lowercase()) + .as_deref() + { + Some("pdf") => Ok(ImportFormat::Pdf), + Some("xlsx") if xlsx::supports_xlsx_file(path) => Ok(ImportFormat::Xlsx), + Some("zip") | Some("txt") if archive::supports_local_archive_file(path) => { + Ok(ImportFormat::Archive) + } + _ => Err(IdxError::InvalidInput(format!( + "unsupported local ownership file {}; expected a .xlsx, .pdf, .zip, or .txt input", + path.display() + ))), + } +} + +fn validate_import_url(url: &str) -> Result { + let trimmed = url.trim(); + if trimmed.is_empty() { + return Err(IdxError::InvalidInput( + "ownership import --url accepts direct XLSX or PDF URLs only".to_string(), + )); + } + + let normalized = trimmed.to_ascii_lowercase(); + let normalized = normalized + .split(['?', '#']) + .next() + .unwrap_or(normalized.as_str()); + if normalized.ends_with(".pdf") { + return Ok(ImportFormat::Pdf); + } + if normalized.ends_with(".xlsx") { + return Ok(ImportFormat::Xlsx); + } + + Err(IdxError::InvalidInput( + "ownership import --url accepts direct XLSX or PDF URLs only; run `idx ownership discover` first to find the current supported file".to_string(), + )) +} + +/// Above ureq's 10 MiB default: the monthly holder-register PDF can grow past it. +const REMOTE_REPORT_LIMIT_BYTES: u64 = 256 * 1024 * 1024; + +fn download_pdf(url: &str, target: &Path) -> Result<(), IdxError> { + if is_idx_url(url) { + return remote::download_idx_pdf(url, target); + } + + let response = ureq::get(url) + .header( + "User-Agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", + ) + .header("Accept", "application/pdf,application/octet-stream,*/*;q=0.8") + .header("Accept-Language", "en-US,en;q=0.9") + .header("Referer", "https://www.idx.co.id/") + .call() + .map_err(|e| IdxError::Http(format!("failed to download PDF: {e}")))?; + + let bytes = response + .into_body() + .with_config() + .limit(REMOTE_REPORT_LIMIT_BYTES) + .read_to_vec() + .map_err(|e| IdxError::Http(format!("failed reading PDF body: {e}")))?; + remote::validate_pdf_payload(&bytes)?; + + fs::write(target, &bytes).map_err(|e| { + IdxError::Io(format!( + "failed writing cached PDF {}: {e}", + target.display() + )) + }) +} + +fn download_xlsx(url: &str, target: &Path) -> Result<(), IdxError> { + if is_idx_url(url) { + return remote::download_idx_xlsx(url, target); + } + + let response = ureq::get(url) + .header( + "Accept", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/octet-stream,*/*;q=0.8", + ) + .call() + .map_err(|e| IdxError::Http(format!("failed to download XLSX: {e}")))?; + let bytes = response + .into_body() + .with_config() + .limit(REMOTE_REPORT_LIMIT_BYTES) + .read_to_vec() + .map_err(|e| IdxError::Http(format!("failed reading XLSX body: {e}")))?; + remote::validate_xlsx_payload(&bytes)?; + + fs::write(target, &bytes).map_err(|e| { + IdxError::Io(format!( + "failed writing cached XLSX {}: {e}", + target.display() + )) + }) +} + +fn is_idx_url(url: &str) -> bool { + let normalized = url.trim().to_ascii_lowercase(); + normalized.starts_with("https://www.idx.co.id/") + || normalized.starts_with("http://www.idx.co.id/") + || normalized.starts_with("https://idx.co.id/") + || normalized.starts_with("http://idx.co.id/") +} + +fn sha256_file(path: &Path) -> Result { + let bytes = fs::read(path).map_err(|e| { + IdxError::Io(format!( + "failed to read file for sha256 {}: {e}", + path.display() + )) + })?; + let mut hasher = Sha256::new(); + hasher.update(bytes); + let digest = hasher.finalize(); + Ok(digest.iter().map(|b| format!("{b:02x}")).collect()) +} + +#[cfg(test)] +mod tests { + use super::{ConcentrationRank, HolderSource, parse_report_family}; + use crate::ownership::remote::OwnershipReportFamily; + + #[test] + fn parses_holder_sources_and_ranks() { + assert_eq!( + " KSEI ".parse::().unwrap(), + HolderSource::Ksei + ); + assert!("x".parse::().is_err()); + assert_eq!( + "HHI".parse::().unwrap(), + ConcentrationRank::Hhi + ); + assert!("top2".parse::().is_err()); + } + + #[test] + fn parses_report_family_aliases() { + assert_eq!(parse_report_family("all").unwrap(), None); + assert_eq!( + parse_report_family("above_1").unwrap(), + Some(OwnershipReportFamily::AboveOnePercent) + ); + assert!(parse_report_family("above10").is_err()); + } +} diff --git a/crates/idx-core/src/lib.rs b/crates/idx-core/src/lib.rs index 703911e..bcc2e52 100644 --- a/crates/idx-core/src/lib.rs +++ b/crates/idx-core/src/lib.rs @@ -10,6 +10,8 @@ pub mod config; pub mod curl_impersonate; pub mod error; pub mod financial; +#[cfg(feature = "ownership")] +pub mod investor; pub mod market; #[cfg(feature = "ownership")] pub mod ownership; diff --git a/crates/idx-core/src/ownership/db.rs b/crates/idx-core/src/ownership/db.rs index d98dbce..f298383 100644 --- a/crates/idx-core/src/ownership/db.rs +++ b/crates/idx-core/src/ownership/db.rs @@ -1,5 +1,5 @@ use std::fs; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use chrono::NaiveDate; use directories::ProjectDirs; @@ -129,13 +129,16 @@ pub fn ensure_schema(conn: &Connection) -> Result<(), IdxError> { /// Open ownership database connection and run idempotent schema migration. pub fn open_db() -> Result { - let db_path = db_path()?; + open_db_at(&db_path()?) +} +/// Open the ownership database at `db_path`, creating it and migrating the schema. +pub fn open_db_at(db_path: &Path) -> Result { if let Some(parent) = db_path.parent() { fs::create_dir_all(parent).map_err(|e| IdxError::DatabaseError(e.to_string()))?; } - let conn = Connection::open(&db_path).map_err(|e| IdxError::DatabaseError(e.to_string()))?; + let conn = Connection::open(db_path).map_err(|e| IdxError::DatabaseError(e.to_string()))?; ensure_schema(&conn)?; Ok(conn) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 38e2dd1..d5c10ae 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -5,7 +5,8 @@ The repo is a Cargo workspace with one shared version: - `crates/idx-core` (library): the `Idx` entry point with `market()`, - `financial()`, and `analysis()` (cache policy and `Fetched` in `client.rs`), + `financial()`, `analysis()`, and `investor()` (cache policy and `Fetched` in + `client.rs`; ownership queries, import, sync, and discovery in `investor.rs`), providers (`api/`), `analysis/`, `cache`, `config` (`CoreConfig`), `ownership/` (SQLite, parsers, discovery, snapshot sync), `error`, and `runtime` (quiet/info/warn). It owns the shared test fixtures in @@ -14,8 +15,8 @@ The repo is a Cargo workspace with one shared version: color resolution (`config.rs`, `IdxConfig`), and rendering (`output/`). Its `tests/` hold the CLI integration and golden output suites. -Ownership import/sync orchestration still lives in -`crates/idx-cli/src/cli/ownership.rs` until phase 3c of `docs/DESIGN.md`. +`crates/idx-cli/src/cli/` holds no fetching, caching, or database logic: each +command parses arguments, makes one idx-core call, and renders the result. ## Domain Map diff --git a/docs/DESIGN.md b/docs/DESIGN.md index b728fec..c726772 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -1,6 +1,6 @@ # System Design: idx-core and its frontends -**Status:** Accepted 2026-10-14. Implementation on `refactor/idx-core`; phases 0-2 done. +**Status:** Accepted 2026-10-14. Implementation on `refactor/idx-core`; phases 0-3 done. ## Goal @@ -119,8 +119,8 @@ Key decisions: | ownership command orchestration (`cli/ownership.rs`) | `investor` | | `IdxConfig.output`, `no_color` | stay in `idx-cli` | -After the move, `cli/stocks.rs` and `cli/ownership.rs` should be argument -parsing plus a call into core plus rendering. +Done in phase 3: `cli/stocks.rs` and `cli/ownership.rs` are argument parsing, +one idx-core call, and rendering. ## Error Model