mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-11-07 09:45:36 +00:00
From the Codex re-gate of the 0.3.3 fixes: - Imports recorded row_count as the rows parsed, but INSERT OR IGNORE drops a file's repeated ticker/investor rows, so the count could exceed what is stored, and the new stored-row validation would then reject a snapshot the publisher built from such a file. Record the rows stored. - A local manifest over the 1 MiB limit was read truncated rather than rejected; read one byte more and fail when it does not fit. - The backup-recovery test also passed on the old code; it now fails an install after the leftover backup was found and checks the old data is back. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V3syn2fN2LNo5pvTSqc6eh
2315 lines
78 KiB
Rust
2315 lines
78 KiB
Rust
use std::fs;
|
|
use std::path::{Path, PathBuf};
|
|
|
|
use chrono::NaiveDate;
|
|
use directories::ProjectDirs;
|
|
use rusqlite::{Connection, params};
|
|
|
|
use crate::config::get_config_value;
|
|
use crate::error::IdxError;
|
|
use crate::ownership::types::{
|
|
BingHolding, ChangeRow, ChangeType, ConcentrationMetrics, CrossHolderRow, Entity,
|
|
EntityHoldings, EntityTickerRow, FlowSignal, HolderRow, InstitutionalFlow, KseiHolding,
|
|
Locality, OwnershipRelease, OwnershipSource, Ticker, TickerOwnership, UnresolvedRow,
|
|
};
|
|
|
|
/// Ownership schema version 1 DDL.
|
|
pub const SCHEMA_V1: &str = r#"
|
|
PRAGMA foreign_keys = ON;
|
|
|
|
CREATE TABLE IF NOT EXISTS ownership_releases (
|
|
id INTEGER PRIMARY KEY,
|
|
source_url TEXT,
|
|
sha256 TEXT NOT NULL UNIQUE,
|
|
as_of_date TEXT NOT NULL,
|
|
row_count INTEGER NOT NULL,
|
|
imported_at INTEGER NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS entities (
|
|
id INTEGER PRIMARY KEY,
|
|
canonical_name TEXT NOT NULL,
|
|
entity_type TEXT,
|
|
country TEXT,
|
|
created_at INTEGER NOT NULL,
|
|
updated_at INTEGER NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS entity_aliases (
|
|
id INTEGER PRIMARY KEY,
|
|
entity_id INTEGER NOT NULL REFERENCES entities(id),
|
|
raw_name TEXT NOT NULL,
|
|
source TEXT NOT NULL,
|
|
confidence REAL NOT NULL DEFAULT 1.0,
|
|
method TEXT NOT NULL,
|
|
UNIQUE(raw_name, source)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_aliases_entity ON entity_aliases(entity_id);
|
|
CREATE INDEX IF NOT EXISTS idx_aliases_name ON entity_aliases(raw_name);
|
|
|
|
CREATE TABLE IF NOT EXISTS tickers (
|
|
id INTEGER PRIMARY KEY,
|
|
code TEXT NOT NULL UNIQUE,
|
|
name TEXT,
|
|
sector TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS ksei_holdings (
|
|
id INTEGER PRIMARY KEY,
|
|
ticker_id INTEGER NOT NULL REFERENCES tickers(id),
|
|
entity_id INTEGER REFERENCES entities(id),
|
|
raw_investor_name TEXT NOT NULL,
|
|
investor_type TEXT,
|
|
locality TEXT,
|
|
nationality TEXT,
|
|
domicile TEXT,
|
|
holdings_scripless INTEGER NOT NULL,
|
|
holdings_scrip INTEGER NOT NULL,
|
|
total_shares INTEGER NOT NULL,
|
|
percentage_bps INTEGER NOT NULL,
|
|
report_date TEXT NOT NULL,
|
|
release_sha256 TEXT NOT NULL,
|
|
UNIQUE(release_sha256, ticker_id, raw_investor_name)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_ksei_ticker ON ksei_holdings(ticker_id);
|
|
CREATE INDEX IF NOT EXISTS idx_ksei_entity ON ksei_holdings(entity_id);
|
|
CREATE INDEX IF NOT EXISTS idx_ksei_date ON ksei_holdings(report_date);
|
|
CREATE INDEX IF NOT EXISTS idx_ksei_pct ON ksei_holdings(percentage_bps DESC);
|
|
|
|
CREATE TABLE IF NOT EXISTS bing_holdings (
|
|
id INTEGER PRIMARY KEY,
|
|
ticker_id INTEGER NOT NULL REFERENCES tickers(id),
|
|
entity_id INTEGER REFERENCES entities(id),
|
|
raw_investor_name TEXT NOT NULL,
|
|
investor_type TEXT,
|
|
shares_held INTEGER,
|
|
shares_changed INTEGER,
|
|
pct_ownership_bps INTEGER,
|
|
value_usd INTEGER,
|
|
report_date TEXT NOT NULL,
|
|
signal TEXT NOT NULL,
|
|
fetched_at INTEGER NOT NULL,
|
|
UNIQUE(ticker_id, raw_investor_name, report_date, signal)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_bing_ticker ON bing_holdings(ticker_id);
|
|
CREATE INDEX IF NOT EXISTS idx_bing_entity ON bing_holdings(entity_id);
|
|
CREATE INDEX IF NOT EXISTS idx_bing_date ON bing_holdings(report_date);
|
|
"#;
|
|
|
|
/// Ensure ownership schema is migrated and ready for use.
|
|
pub fn ensure_schema(conn: &Connection) -> Result<(), IdxError> {
|
|
let user_version: i64 = conn
|
|
.query_row("PRAGMA user_version", [], |row| row.get(0))
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
if conn.path().is_some() {
|
|
let _ = conn.pragma_update(None, "journal_mode", "WAL");
|
|
}
|
|
conn.pragma_update(None, "foreign_keys", "ON")
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
if user_version < 1 {
|
|
conn.execute_batch(SCHEMA_V1)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
conn.pragma_update(None, "user_version", 1)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Open ownership database connection and run idempotent schema migration.
|
|
pub fn open_db() -> Result<Connection, IdxError> {
|
|
open_db_at(&db_path()?)
|
|
}
|
|
|
|
/// Open the ownership database at `db_path`, creating it and migrating the schema.
|
|
/// Where the database file really lives: symlinks are followed by reading
|
|
/// the link, so this works even while the target does not exist (a fresh
|
|
/// install, or mid-replacement during sync). Creates the parent directory.
|
|
pub(crate) fn resolve_db_path(path: &Path) -> Result<PathBuf, IdxError> {
|
|
let mut current = path.to_path_buf();
|
|
for _ in 0..16 {
|
|
let parent = match current.parent() {
|
|
Some(parent) if !parent.as_os_str().is_empty() => parent.to_path_buf(),
|
|
_ => PathBuf::from("."),
|
|
};
|
|
fs::create_dir_all(&parent).map_err(|e| IdxError::Io(e.to_string()))?;
|
|
let parent = fs::canonicalize(&parent).map_err(|e| IdxError::Io(e.to_string()))?;
|
|
let name = current.file_name().ok_or_else(|| {
|
|
IdxError::ConfigError(format!(
|
|
"ownership database path has no file name: {}",
|
|
path.display()
|
|
))
|
|
})?;
|
|
let candidate = parent.join(name);
|
|
match fs::symlink_metadata(&candidate) {
|
|
Ok(meta) if meta.file_type().is_symlink() => {
|
|
let target = fs::read_link(&candidate).map_err(|e| IdxError::Io(e.to_string()))?;
|
|
current = if target.is_absolute() {
|
|
target
|
|
} else {
|
|
parent.join(target)
|
|
};
|
|
}
|
|
_ => return Ok(candidate),
|
|
}
|
|
}
|
|
Err(IdxError::Io(format!(
|
|
"too many symlink levels resolving {}",
|
|
path.display()
|
|
)))
|
|
}
|
|
|
|
pub fn open_db_at(db_path: &Path) -> Result<Connection, IdxError> {
|
|
if let Some(parent) = db_path.parent() {
|
|
fs::create_dir_all(parent).map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
}
|
|
|
|
let conn = Connection::open(db_path).map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
ensure_schema(&conn)?;
|
|
|
|
Ok(conn)
|
|
}
|
|
|
|
/// Insert or get a ticker by code. Returns ticker_id.
|
|
pub fn upsert_ticker(conn: &Connection, code: &str, name: Option<&str>) -> Result<i64, IdxError> {
|
|
conn.execute(
|
|
"INSERT INTO tickers (code, name) VALUES (?1, ?2)
|
|
ON CONFLICT(code) DO UPDATE SET
|
|
name = COALESCE(excluded.name, tickers.name)",
|
|
params![code, name],
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
conn.query_row(
|
|
"SELECT id FROM tickers WHERE code = ?1",
|
|
params![code],
|
|
|row| row.get(0),
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))
|
|
}
|
|
|
|
fn insert_ksei_holdings_rows(
|
|
conn: &Connection,
|
|
holdings: &[KseiHolding],
|
|
) -> Result<usize, IdxError> {
|
|
if holdings.is_empty() {
|
|
return Ok(0);
|
|
}
|
|
|
|
let mut inserted = 0usize;
|
|
for h in holdings {
|
|
let locality = h.locality.map(locality_to_db);
|
|
let investor_type = h.investor_type.as_ref().map(|v| v.0.as_str());
|
|
let changed = conn
|
|
.execute(
|
|
"INSERT OR IGNORE INTO ksei_holdings (
|
|
ticker_id, entity_id, raw_investor_name, investor_type, locality,
|
|
nationality, domicile, holdings_scripless, holdings_scrip, total_shares,
|
|
percentage_bps, report_date, release_sha256
|
|
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)",
|
|
params![
|
|
h.ticker_id,
|
|
h.entity_id,
|
|
h.raw_investor_name,
|
|
investor_type,
|
|
locality,
|
|
h.nationality,
|
|
h.domicile,
|
|
h.holdings_scripless,
|
|
h.holdings_scrip,
|
|
h.total_shares,
|
|
h.percentage_bps,
|
|
h.report_date.format("%Y-%m-%d").to_string(),
|
|
h.release_sha256,
|
|
],
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()));
|
|
|
|
inserted += changed?;
|
|
}
|
|
|
|
Ok(inserted)
|
|
}
|
|
|
|
/// Bulk insert KSEI holdings within a transaction.
|
|
/// Uses INSERT OR IGNORE for dedup (unique on release_sha256 + ticker_id + raw_investor_name).
|
|
pub fn insert_ksei_holdings(
|
|
conn: &Connection,
|
|
holdings: &[KseiHolding],
|
|
) -> Result<usize, IdxError> {
|
|
if holdings.is_empty() {
|
|
return Ok(0);
|
|
}
|
|
|
|
conn.execute("BEGIN IMMEDIATE", [])
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let inserted = match insert_ksei_holdings_rows(conn, holdings) {
|
|
Ok(inserted) => inserted,
|
|
Err(err) => {
|
|
let _ = conn.execute("ROLLBACK", []);
|
|
return Err(err);
|
|
}
|
|
};
|
|
|
|
conn.execute("COMMIT", [])
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
Ok(inserted)
|
|
}
|
|
|
|
/// Bulk insert Bing holdings within a transaction.
|
|
pub fn insert_bing_holdings(
|
|
conn: &Connection,
|
|
holdings: &[BingHolding],
|
|
) -> Result<usize, IdxError> {
|
|
if holdings.is_empty() {
|
|
return Ok(0);
|
|
}
|
|
|
|
conn.execute("BEGIN IMMEDIATE", [])
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let mut inserted = 0usize;
|
|
for h in holdings {
|
|
let changed = conn
|
|
.execute(
|
|
"INSERT OR IGNORE INTO bing_holdings (
|
|
ticker_id, entity_id, raw_investor_name, investor_type, shares_held,
|
|
shares_changed, pct_ownership_bps, value_usd, report_date, signal, fetched_at
|
|
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)",
|
|
params![
|
|
h.ticker_id,
|
|
h.entity_id,
|
|
h.raw_investor_name,
|
|
h.investor_type,
|
|
h.shares_held,
|
|
h.shares_changed,
|
|
h.pct_ownership_bps,
|
|
h.value_usd,
|
|
h.report_date.format("%Y-%m-%d").to_string(),
|
|
flow_signal_to_db(h.signal),
|
|
h.fetched_at,
|
|
],
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()));
|
|
|
|
match changed {
|
|
Ok(n) => inserted += n,
|
|
Err(err) => {
|
|
let _ = conn.execute("ROLLBACK", []);
|
|
return Err(err);
|
|
}
|
|
}
|
|
}
|
|
|
|
conn.execute("COMMIT", [])
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
Ok(inserted)
|
|
}
|
|
|
|
/// Record a KSEI release import. Returns release_id.
|
|
pub fn insert_release(conn: &Connection, release: &OwnershipRelease) -> Result<i64, IdxError> {
|
|
conn.execute(
|
|
"INSERT INTO ownership_releases (source_url, sha256, as_of_date, row_count, imported_at)
|
|
VALUES (?1, ?2, ?3, ?4, ?5)",
|
|
params![
|
|
release.source_url,
|
|
release.sha256,
|
|
release.as_of_date.format("%Y-%m-%d").to_string(),
|
|
i64::try_from(release.row_count)
|
|
.map_err(|e| IdxError::DatabaseError(format!("invalid row_count: {e}")))?,
|
|
release.imported_at,
|
|
],
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
Ok(conn.last_insert_rowid())
|
|
}
|
|
|
|
/// Insert or replace a complete KSEI release and its rows atomically.
|
|
pub fn write_ksei_release(
|
|
conn: &Connection,
|
|
release: &OwnershipRelease,
|
|
holdings: &[KseiHolding],
|
|
replace_existing: bool,
|
|
) -> Result<usize, IdxError> {
|
|
conn.execute("BEGIN IMMEDIATE", [])
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let result = (|| {
|
|
if replace_existing {
|
|
conn.execute(
|
|
"DELETE FROM ksei_holdings WHERE release_sha256 = ?1",
|
|
params![&release.sha256],
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
conn.execute(
|
|
"DELETE FROM ownership_releases WHERE sha256 = ?1",
|
|
params![&release.sha256],
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
}
|
|
|
|
let inserted = insert_ksei_holdings_rows(conn, holdings)?;
|
|
// Record the rows stored, not parsed: INSERT OR IGNORE drops repeated
|
|
// ticker/investor rows, and snapshot validation counts what is stored.
|
|
let stored = OwnershipRelease {
|
|
row_count: inserted,
|
|
..release.clone()
|
|
};
|
|
let _ = insert_release(conn, &stored)?;
|
|
Ok(inserted)
|
|
})();
|
|
|
|
match result {
|
|
Ok(inserted) => {
|
|
conn.execute("COMMIT", [])
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
Ok(inserted)
|
|
}
|
|
Err(err) => {
|
|
let _ = conn.execute("ROLLBACK", []);
|
|
Err(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Check if a release with this SHA-256 already exists.
|
|
pub fn release_exists(conn: &Connection, sha256: &str) -> Result<bool, IdxError> {
|
|
let exists: i64 = conn
|
|
.query_row(
|
|
"SELECT EXISTS(SELECT 1 FROM ownership_releases WHERE sha256 = ?1)",
|
|
params![sha256],
|
|
|row| row.get(0),
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
Ok(exists == 1)
|
|
}
|
|
|
|
/// Get combined KSEI + Bing holdings for a ticker.
|
|
/// Returns TickerOwnership with merged holders sorted by percentage desc,
|
|
/// concentration metrics, and Bing flow data.
|
|
pub fn query_ticker_holdings(conn: &Connection, code: &str) -> Result<TickerOwnership, IdxError> {
|
|
let ticker =
|
|
query_ticker(conn, code)?.ok_or_else(|| IdxError::SymbolNotFound(code.to_string()))?;
|
|
|
|
let latest_ksei = latest_ksei_release(conn)?;
|
|
let ksei_as_of = latest_ksei.as_ref().map(|release| release.as_of_date);
|
|
|
|
let bing_as_of = conn
|
|
.query_row(
|
|
"SELECT MAX(report_date) FROM bing_holdings WHERE ticker_id = ?1",
|
|
params![ticker.id],
|
|
|row| row.get::<_, Option<String>>(0),
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let mut holders: Vec<HolderRow> = Vec::new();
|
|
|
|
if let Some(latest_ksei) = latest_ksei.as_ref() {
|
|
let mut stmt = conn
|
|
.prepare(
|
|
"SELECT k.entity_id, COALESCE(e.canonical_name, k.raw_investor_name),
|
|
k.investor_type, k.locality, k.total_shares, k.percentage_bps
|
|
FROM ksei_holdings k
|
|
LEFT JOIN entities e ON e.id = k.entity_id
|
|
WHERE k.ticker_id = ?1 AND k.release_sha256 = ?2
|
|
ORDER BY k.percentage_bps DESC, k.total_shares DESC",
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let rows = stmt
|
|
.query_map(params![ticker.id, &latest_ksei.sha256], |row| {
|
|
Ok((
|
|
row.get::<_, Option<i64>>(0)?,
|
|
row.get::<_, String>(1)?,
|
|
row.get::<_, Option<String>>(2)?,
|
|
row.get::<_, Option<String>>(3)?,
|
|
row.get::<_, i64>(4)?,
|
|
row.get::<_, i64>(5)?,
|
|
))
|
|
})
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
for row in rows {
|
|
let (entity_id, name, investor_type, locality_raw, shares, percentage_bps) =
|
|
row.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
holders.push(HolderRow {
|
|
rank: 0,
|
|
source: OwnershipSource::Ksei,
|
|
name,
|
|
entity_id,
|
|
investor_type,
|
|
locality: locality_raw.as_deref().and_then(locality_from_db),
|
|
shares,
|
|
percentage_bps,
|
|
signal: None,
|
|
});
|
|
}
|
|
}
|
|
|
|
if let Some(bing_as_of) = bing_as_of.as_ref() {
|
|
let mut stmt = conn
|
|
.prepare(
|
|
"SELECT b.entity_id, COALESCE(e.canonical_name, b.raw_investor_name),
|
|
b.investor_type, COALESCE(b.shares_held, 0), COALESCE(b.pct_ownership_bps, 0), b.signal
|
|
FROM bing_holdings b
|
|
LEFT JOIN entities e ON e.id = b.entity_id
|
|
WHERE b.ticker_id = ?1 AND b.report_date = ?2 AND b.signal = 'holder'
|
|
ORDER BY COALESCE(b.pct_ownership_bps, 0) DESC, COALESCE(b.shares_held, 0) DESC",
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let rows = stmt
|
|
.query_map(params![ticker.id, bing_as_of], |row| {
|
|
Ok((
|
|
row.get::<_, Option<i64>>(0)?,
|
|
row.get::<_, String>(1)?,
|
|
row.get::<_, Option<String>>(2)?,
|
|
row.get::<_, i64>(3)?,
|
|
row.get::<_, i64>(4)?,
|
|
row.get::<_, String>(5)?,
|
|
))
|
|
})
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
for row in rows {
|
|
let (entity_id, name, investor_type, shares, percentage_bps, signal_raw) =
|
|
row.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
holders.push(HolderRow {
|
|
rank: 0,
|
|
source: OwnershipSource::Bing,
|
|
name,
|
|
entity_id,
|
|
investor_type,
|
|
locality: None,
|
|
shares,
|
|
percentage_bps,
|
|
signal: flow_signal_from_db(&signal_raw),
|
|
});
|
|
}
|
|
}
|
|
|
|
holders.sort_by(|a, b| {
|
|
b.percentage_bps
|
|
.cmp(&a.percentage_bps)
|
|
.then_with(|| b.shares.cmp(&a.shares))
|
|
});
|
|
for (idx, holder) in holders.iter_mut().enumerate() {
|
|
holder.rank = idx + 1;
|
|
}
|
|
|
|
let percentages = holders.iter().map(|h| h.percentage_bps).collect::<Vec<_>>();
|
|
let concentration = compute_concentration(&percentages);
|
|
let flow = query_bing_flow(conn, ticker.id)?;
|
|
|
|
Ok(TickerOwnership {
|
|
ticker,
|
|
ksei_as_of,
|
|
bing_as_of,
|
|
holders,
|
|
concentration,
|
|
flow,
|
|
})
|
|
}
|
|
|
|
/// Get all holdings for an entity across tickers.
|
|
pub fn query_entity_holdings(
|
|
conn: &Connection,
|
|
entity_id: i64,
|
|
) -> Result<EntityHoldings, IdxError> {
|
|
let entity = query_entity(conn, entity_id)?
|
|
.ok_or_else(|| IdxError::SymbolNotFound(format!("entity:{entity_id}")))?;
|
|
|
|
let mut holdings = Vec::new();
|
|
|
|
if let Some(latest_ksei) = latest_ksei_release(conn)? {
|
|
let mut stmt = conn
|
|
.prepare(
|
|
"SELECT t.id, t.code, t.name, t.sector, k.total_shares, k.percentage_bps, k.report_date
|
|
FROM ksei_holdings k
|
|
JOIN tickers t ON t.id = k.ticker_id
|
|
WHERE k.entity_id = ?1 AND k.release_sha256 = ?2",
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let rows = stmt
|
|
.query_map(params![entity_id, latest_ksei.sha256], |row| {
|
|
Ok(EntityTickerRow {
|
|
ticker: Ticker {
|
|
id: row.get(0)?,
|
|
code: row.get(1)?,
|
|
name: row.get(2)?,
|
|
sector: row.get(3)?,
|
|
},
|
|
source: OwnershipSource::Ksei,
|
|
shares: row.get(4)?,
|
|
percentage_bps: row.get(5)?,
|
|
report_date: parse_iso_date(&row.get::<_, String>(6)?)
|
|
.map_err(|e| rusqlite::Error::ToSqlConversionFailure(Box::new(e)))?,
|
|
})
|
|
})
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
for row in rows {
|
|
holdings.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
|
}
|
|
}
|
|
|
|
{
|
|
let mut stmt = conn
|
|
.prepare(
|
|
"WITH latest_bing AS (
|
|
SELECT ticker_id, MAX(report_date) AS report_date
|
|
FROM bing_holdings
|
|
GROUP BY ticker_id
|
|
)
|
|
SELECT t.id, t.code, t.name, t.sector,
|
|
COALESCE(b.shares_held, 0), COALESCE(b.pct_ownership_bps, 0), b.report_date
|
|
FROM bing_holdings b
|
|
JOIN latest_bing lb
|
|
ON lb.ticker_id = b.ticker_id
|
|
AND lb.report_date = b.report_date
|
|
JOIN tickers t ON t.id = b.ticker_id
|
|
WHERE b.entity_id = ?1 AND b.signal = 'holder'",
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let rows = stmt
|
|
.query_map(params![entity_id], |row| {
|
|
Ok(EntityTickerRow {
|
|
ticker: Ticker {
|
|
id: row.get(0)?,
|
|
code: row.get(1)?,
|
|
name: row.get(2)?,
|
|
sector: row.get(3)?,
|
|
},
|
|
source: OwnershipSource::Bing,
|
|
shares: row.get(4)?,
|
|
percentage_bps: row.get(5)?,
|
|
report_date: parse_iso_date(&row.get::<_, String>(6)?)
|
|
.map_err(|e| rusqlite::Error::ToSqlConversionFailure(Box::new(e)))?,
|
|
})
|
|
})
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
for row in rows {
|
|
holdings.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
|
}
|
|
}
|
|
|
|
holdings.sort_by(|a, b| {
|
|
b.percentage_bps
|
|
.cmp(&a.percentage_bps)
|
|
.then_with(|| a.ticker.code.cmp(&b.ticker.code))
|
|
});
|
|
|
|
let ticker_count = holdings
|
|
.iter()
|
|
.map(|h| h.ticker.id)
|
|
.collect::<std::collections::HashSet<_>>()
|
|
.len();
|
|
|
|
Ok(EntityHoldings {
|
|
entity,
|
|
ticker_count,
|
|
holdings,
|
|
})
|
|
}
|
|
|
|
/// Rank entities by number of tickers they hold (cross-ownership breadth).
|
|
pub fn query_cross_holders(
|
|
conn: &Connection,
|
|
min_tickers: usize,
|
|
limit: usize,
|
|
) -> Result<Vec<CrossHolderRow>, IdxError> {
|
|
let mut aggregates: std::collections::BTreeMap<
|
|
i64,
|
|
(Entity, std::collections::BTreeSet<i64>, i64),
|
|
> = std::collections::BTreeMap::new();
|
|
|
|
if let Some(latest_ksei) = latest_ksei_release(conn)? {
|
|
let mut stmt = conn
|
|
.prepare(
|
|
"SELECT e.id, e.canonical_name, e.entity_type, e.country, k.ticker_id, k.percentage_bps
|
|
FROM ksei_holdings k
|
|
JOIN entities e ON e.id = k.entity_id
|
|
WHERE k.entity_id IS NOT NULL AND k.release_sha256 = ?1",
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let rows = stmt
|
|
.query_map(params![latest_ksei.sha256], |row| {
|
|
Ok((
|
|
Entity {
|
|
id: row.get(0)?,
|
|
canonical_name: row.get(1)?,
|
|
entity_type: row.get(2)?,
|
|
country: row.get(3)?,
|
|
},
|
|
row.get::<_, i64>(4)?,
|
|
row.get::<_, i64>(5)?,
|
|
))
|
|
})
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
for row in rows {
|
|
let (entity, ticker_id, percentage_bps) =
|
|
row.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
let entry = aggregates
|
|
.entry(entity.id)
|
|
.or_insert_with(|| (entity, std::collections::BTreeSet::new(), 0));
|
|
entry.1.insert(ticker_id);
|
|
entry.2 += percentage_bps;
|
|
}
|
|
}
|
|
|
|
{
|
|
let mut stmt = conn
|
|
.prepare(
|
|
"WITH latest_bing AS (
|
|
SELECT ticker_id, MAX(report_date) AS report_date
|
|
FROM bing_holdings
|
|
GROUP BY ticker_id
|
|
)
|
|
SELECT e.id, e.canonical_name, e.entity_type, e.country, b.ticker_id,
|
|
COALESCE(b.pct_ownership_bps, 0)
|
|
FROM bing_holdings b
|
|
JOIN latest_bing lb
|
|
ON lb.ticker_id = b.ticker_id
|
|
AND lb.report_date = b.report_date
|
|
JOIN entities e ON e.id = b.entity_id
|
|
WHERE b.entity_id IS NOT NULL AND b.signal = 'holder'",
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let rows = stmt
|
|
.query_map([], |row| {
|
|
Ok((
|
|
Entity {
|
|
id: row.get(0)?,
|
|
canonical_name: row.get(1)?,
|
|
entity_type: row.get(2)?,
|
|
country: row.get(3)?,
|
|
},
|
|
row.get::<_, i64>(4)?,
|
|
row.get::<_, i64>(5)?,
|
|
))
|
|
})
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
for row in rows {
|
|
let (entity, ticker_id, percentage_bps) =
|
|
row.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
let entry = aggregates
|
|
.entry(entity.id)
|
|
.or_insert_with(|| (entity, std::collections::BTreeSet::new(), 0));
|
|
entry.1.insert(ticker_id);
|
|
entry.2 += percentage_bps;
|
|
}
|
|
}
|
|
|
|
let mut out = aggregates
|
|
.into_values()
|
|
.filter_map(|(entity, tickers, total_bps)| {
|
|
(tickers.len() >= min_tickers).then_some(CrossHolderRow {
|
|
entity,
|
|
ticker_count: tickers.len(),
|
|
total_bps,
|
|
})
|
|
})
|
|
.collect::<Vec<_>>();
|
|
|
|
out.sort_by(|a, b| {
|
|
b.ticker_count
|
|
.cmp(&a.ticker_count)
|
|
.then_with(|| b.total_bps.cmp(&a.total_bps))
|
|
.then_with(|| a.entity.canonical_name.cmp(&b.entity.canonical_name))
|
|
});
|
|
out.truncate(limit);
|
|
Ok(out)
|
|
}
|
|
|
|
/// Rank tickers by ownership concentration.
|
|
/// sort_by: "top1", "top3", "hhi"
|
|
pub fn query_concentration(
|
|
conn: &Connection,
|
|
sort_by: &str,
|
|
limit: usize,
|
|
) -> Result<Vec<(String, ConcentrationMetrics)>, IdxError> {
|
|
let valid = ["top1", "top3", "hhi"];
|
|
if !valid.contains(&sort_by) {
|
|
return Err(IdxError::ParseError(format!(
|
|
"invalid sort_by '{sort_by}', expected one of: top1, top3, hhi"
|
|
)));
|
|
}
|
|
|
|
let Some(latest_ksei) = latest_ksei_release(conn)? else {
|
|
return Ok(Vec::new());
|
|
};
|
|
|
|
let mut stmt = conn
|
|
.prepare(
|
|
"SELECT t.code, k.percentage_bps
|
|
FROM tickers t
|
|
JOIN ksei_holdings k ON k.ticker_id = t.id
|
|
WHERE k.release_sha256 = ?1
|
|
ORDER BY t.code ASC, k.percentage_bps DESC",
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let rows = stmt
|
|
.query_map(params![latest_ksei.sha256], |row| {
|
|
Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
|
|
})
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let mut grouped: std::collections::BTreeMap<String, Vec<i64>> =
|
|
std::collections::BTreeMap::new();
|
|
for row in rows {
|
|
let (code, bps) = row.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
grouped.entry(code).or_default().push(bps);
|
|
}
|
|
|
|
let mut ranked: Vec<(String, ConcentrationMetrics)> = grouped
|
|
.into_iter()
|
|
.map(|(code, bps)| (code, compute_concentration(&bps)))
|
|
.collect();
|
|
|
|
ranked.sort_by(|a, b| {
|
|
let am = &a.1;
|
|
let bm = &b.1;
|
|
match sort_by {
|
|
"top1" => bm.top1_bps.cmp(&am.top1_bps),
|
|
"top3" => bm.top3_bps.cmp(&am.top3_bps),
|
|
"hhi" => bm.hhi.cmp(&am.hhi),
|
|
_ => std::cmp::Ordering::Equal,
|
|
}
|
|
.then_with(|| a.0.cmp(&b.0))
|
|
});
|
|
|
|
ranked.truncate(limit);
|
|
Ok(ranked)
|
|
}
|
|
|
|
/// List all imported releases.
|
|
pub fn query_releases(conn: &Connection) -> Result<Vec<OwnershipRelease>, IdxError> {
|
|
let mut stmt = conn
|
|
.prepare(
|
|
"SELECT id, source_url, sha256, as_of_date, row_count, imported_at
|
|
FROM ownership_releases
|
|
ORDER BY as_of_date DESC, imported_at DESC, id DESC",
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let rows = stmt
|
|
.query_map([], |row| {
|
|
Ok(OwnershipRelease {
|
|
id: row.get(0)?,
|
|
source_url: row.get(1)?,
|
|
sha256: row.get(2)?,
|
|
as_of_date: parse_iso_date(&row.get::<_, String>(3)?)
|
|
.map_err(|e| rusqlite::Error::ToSqlConversionFailure(Box::new(e)))?,
|
|
row_count: row.get::<_, i64>(4)? as usize,
|
|
imported_at: row.get(5)?,
|
|
})
|
|
})
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let mut out = Vec::new();
|
|
for row in rows {
|
|
out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
|
}
|
|
Ok(out)
|
|
}
|
|
|
|
/// Get Bing institutional flow for a ticker.
|
|
pub fn query_bing_flow(
|
|
conn: &Connection,
|
|
ticker_id: i64,
|
|
) -> Result<Option<InstitutionalFlow>, IdxError> {
|
|
let latest = conn
|
|
.query_row(
|
|
"SELECT MAX(report_date) FROM bing_holdings WHERE ticker_id = ?1",
|
|
params![ticker_id],
|
|
|row| row.get::<_, Option<String>>(0),
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let Some(period) = latest else {
|
|
return Ok(None);
|
|
};
|
|
|
|
let mut stmt = conn
|
|
.prepare(
|
|
"SELECT b.entity_id, COALESCE(e.canonical_name, b.raw_investor_name),
|
|
b.investor_type, COALESCE(b.shares_held, 0), COALESCE(b.pct_ownership_bps, 0), b.signal
|
|
FROM bing_holdings b
|
|
LEFT JOIN entities e ON e.id = b.entity_id
|
|
WHERE b.ticker_id = ?1 AND b.report_date = ?2
|
|
ORDER BY COALESCE(b.pct_ownership_bps, 0) DESC, COALESCE(b.shares_held, 0) DESC",
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let rows = stmt
|
|
.query_map(params![ticker_id, &period], |row| {
|
|
Ok((
|
|
row.get::<_, Option<i64>>(0)?,
|
|
row.get::<_, String>(1)?,
|
|
row.get::<_, Option<String>>(2)?,
|
|
row.get::<_, i64>(3)?,
|
|
row.get::<_, i64>(4)?,
|
|
row.get::<_, String>(5)?,
|
|
))
|
|
})
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let mut top_buyers = Vec::new();
|
|
let mut top_sellers = Vec::new();
|
|
let mut new_positions = Vec::new();
|
|
let mut exited = Vec::new();
|
|
|
|
let mut rank_counter = 1usize;
|
|
for row in rows {
|
|
let (entity_id, name, investor_type, shares, percentage_bps, signal_raw) =
|
|
row.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
let Some(signal) = flow_signal_from_db(&signal_raw) else {
|
|
continue;
|
|
};
|
|
|
|
let holder = HolderRow {
|
|
rank: rank_counter,
|
|
source: OwnershipSource::Bing,
|
|
name,
|
|
entity_id,
|
|
investor_type,
|
|
locality: None,
|
|
shares,
|
|
percentage_bps,
|
|
signal: Some(signal),
|
|
};
|
|
rank_counter += 1;
|
|
|
|
match signal {
|
|
FlowSignal::Buyer => top_buyers.push(holder),
|
|
FlowSignal::Seller => top_sellers.push(holder),
|
|
FlowSignal::NewPosition => new_positions.push(holder),
|
|
FlowSignal::Exited => exited.push(holder),
|
|
FlowSignal::Holder => {}
|
|
}
|
|
}
|
|
|
|
Ok(Some(InstitutionalFlow {
|
|
period,
|
|
top_buyers,
|
|
top_sellers,
|
|
new_positions,
|
|
exited,
|
|
}))
|
|
}
|
|
|
|
/// The release used for a report date: the latest imported one, as
|
|
/// `latest_ksei_release` picks, so a corrected file supersedes the original.
|
|
/// The latest import holding rows reported on `date`. A file can carry rows
|
|
/// from several report dates, so a release is matched by its rows rather
|
|
/// than by its `as_of_date` (the latest date in it).
|
|
fn release_for_date(conn: &Connection, date: &str) -> Result<Option<String>, IdxError> {
|
|
conn.query_row(
|
|
"SELECT r.sha256 FROM ownership_releases r
|
|
WHERE EXISTS (
|
|
SELECT 1 FROM ksei_holdings k
|
|
WHERE k.release_sha256 = r.sha256 AND k.report_date = ?1
|
|
)
|
|
ORDER BY r.imported_at DESC, r.id DESC
|
|
LIMIT 1",
|
|
params![date],
|
|
|row| row.get::<_, String>(0),
|
|
)
|
|
.map(Some)
|
|
.or_else(|e| match e {
|
|
rusqlite::Error::QueryReturnedNoRows => Ok(None),
|
|
other => Err(IdxError::DatabaseError(other.to_string())),
|
|
})
|
|
}
|
|
|
|
/// Compare two KSEI releases by date and return ownership changes.
|
|
/// Finds: new holders, exited holders, percentage increases/decreases.
|
|
/// Each date reads its rows from one release, and a holder's accounts in a ticker are
|
|
/// summed, so an investor with two accounts compares by its total stake.
|
|
pub fn query_changes(
|
|
conn: &Connection,
|
|
from_date: &str,
|
|
to_date: &str,
|
|
) -> Result<Vec<ChangeRow>, IdxError> {
|
|
let release = |date: &str| {
|
|
release_for_date(conn, date)?.ok_or_else(|| {
|
|
IdxError::InvalidInput(format!(
|
|
"no KSEI release imported for {date}; list the imported dates with `idx ownership releases` or the ownership_status tool"
|
|
))
|
|
})
|
|
};
|
|
let (from_release, to_release) = (release(from_date)?, release(to_date)?);
|
|
let mut stmt = conn
|
|
.prepare(
|
|
"WITH
|
|
from_snapshot AS (
|
|
SELECT
|
|
t.code AS ticker_code,
|
|
MIN(COALESCE(e.canonical_name, k.raw_investor_name)) AS entity_name,
|
|
CASE
|
|
WHEN k.entity_id IS NOT NULL THEN 'id:' || k.entity_id
|
|
ELSE 'raw:' || UPPER(TRIM(k.raw_investor_name))
|
|
END AS holder_key,
|
|
SUM(k.percentage_bps) AS old_bps
|
|
FROM ksei_holdings k
|
|
JOIN tickers t ON t.id = k.ticker_id
|
|
LEFT JOIN entities e ON e.id = k.entity_id
|
|
WHERE k.release_sha256 = ?1 AND k.report_date = ?3
|
|
GROUP BY t.code, holder_key
|
|
),
|
|
to_snapshot AS (
|
|
SELECT
|
|
t.code AS ticker_code,
|
|
MIN(COALESCE(e.canonical_name, k.raw_investor_name)) AS entity_name,
|
|
CASE
|
|
WHEN k.entity_id IS NOT NULL THEN 'id:' || k.entity_id
|
|
ELSE 'raw:' || UPPER(TRIM(k.raw_investor_name))
|
|
END AS holder_key,
|
|
SUM(k.percentage_bps) AS new_bps
|
|
FROM ksei_holdings k
|
|
JOIN tickers t ON t.id = k.ticker_id
|
|
LEFT JOIN entities e ON e.id = k.entity_id
|
|
WHERE k.release_sha256 = ?2 AND k.report_date = ?4
|
|
GROUP BY t.code, holder_key
|
|
),
|
|
keys AS (
|
|
SELECT ticker_code, holder_key FROM from_snapshot
|
|
UNION
|
|
SELECT ticker_code, holder_key FROM to_snapshot
|
|
)
|
|
SELECT
|
|
k.ticker_code,
|
|
COALESCE(ts.entity_name, fs.entity_name) AS entity_name,
|
|
fs.old_bps,
|
|
ts.new_bps
|
|
FROM keys k
|
|
LEFT JOIN from_snapshot fs
|
|
ON fs.ticker_code = k.ticker_code AND fs.holder_key = k.holder_key
|
|
LEFT JOIN to_snapshot ts
|
|
ON ts.ticker_code = k.ticker_code AND ts.holder_key = k.holder_key
|
|
WHERE
|
|
(fs.old_bps IS NULL AND ts.new_bps IS NOT NULL)
|
|
OR (fs.old_bps IS NOT NULL AND ts.new_bps IS NULL)
|
|
OR (fs.old_bps IS NOT NULL AND ts.new_bps IS NOT NULL AND fs.old_bps <> ts.new_bps)
|
|
ORDER BY k.ticker_code ASC, ABS(COALESCE(ts.new_bps, 0) - COALESCE(fs.old_bps, 0)) DESC, entity_name ASC",
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let rows = stmt
|
|
.query_map(
|
|
params![from_release, to_release, from_date, to_date],
|
|
|row| {
|
|
Ok((
|
|
row.get::<_, String>(0)?,
|
|
row.get::<_, String>(1)?,
|
|
row.get::<_, Option<i64>>(2)?,
|
|
row.get::<_, Option<i64>>(3)?,
|
|
))
|
|
},
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let mut out = Vec::new();
|
|
for row in rows {
|
|
let (ticker_code, entity_name, old_bps, new_bps) =
|
|
row.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let change_type = match (old_bps, new_bps) {
|
|
(None, Some(_)) => ChangeType::New,
|
|
(Some(_), None) => ChangeType::Exited,
|
|
(Some(old), Some(new)) if new > old => ChangeType::Increased,
|
|
(Some(_), Some(_)) => ChangeType::Decreased,
|
|
(None, None) => continue,
|
|
};
|
|
|
|
let delta_bps = new_bps.unwrap_or(0) - old_bps.unwrap_or(0);
|
|
out.push(ChangeRow {
|
|
ticker_code,
|
|
entity_name,
|
|
change_type,
|
|
old_bps,
|
|
new_bps,
|
|
delta_bps,
|
|
});
|
|
}
|
|
|
|
Ok(out)
|
|
}
|
|
|
|
/// List aliases that are unresolved in holdings or have low-confidence mappings.
|
|
pub fn list_unresolved(conn: &Connection, limit: usize) -> Result<Vec<UnresolvedRow>, IdxError> {
|
|
let limit_i64 =
|
|
i64::try_from(limit).map_err(|e| IdxError::DatabaseError(format!("invalid limit: {e}")))?;
|
|
|
|
let mut stmt = conn
|
|
.prepare(
|
|
"SELECT DISTINCT
|
|
h.raw_name,
|
|
h.source,
|
|
t.code,
|
|
e.canonical_name,
|
|
a.confidence
|
|
FROM (
|
|
SELECT raw_investor_name AS raw_name, 'ksei' AS source, ticker_id, entity_id
|
|
FROM ksei_holdings
|
|
UNION ALL
|
|
SELECT raw_investor_name AS raw_name, 'bing' AS source, ticker_id, entity_id
|
|
FROM bing_holdings
|
|
) h
|
|
JOIN tickers t ON t.id = h.ticker_id
|
|
LEFT JOIN entity_aliases a ON a.raw_name = h.raw_name AND a.source = h.source
|
|
LEFT JOIN entities e ON e.id = COALESCE(a.entity_id, h.entity_id)
|
|
WHERE h.entity_id IS NULL OR a.entity_id IS NULL OR COALESCE(a.confidence, 0.0) < 0.8
|
|
ORDER BY COALESCE(a.confidence, 0.0) ASC, h.raw_name ASC
|
|
LIMIT ?1",
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let rows = stmt
|
|
.query_map(params![limit_i64], |row| {
|
|
Ok(UnresolvedRow {
|
|
raw_name: row.get(0)?,
|
|
source: row.get(1)?,
|
|
ticker_code: row.get(2)?,
|
|
current_entity: row.get(3)?,
|
|
confidence: row.get(4)?,
|
|
})
|
|
})
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let mut out = Vec::new();
|
|
for row in rows {
|
|
out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
|
}
|
|
Ok(out)
|
|
}
|
|
|
|
/// Manually map a raw investor name to a canonical entity.
|
|
/// Creates the entity if it does not exist, then creates/updates aliases and unresolved holdings.
|
|
pub fn manual_map(conn: &Connection, raw_name: &str, canonical_name: &str) -> Result<(), IdxError> {
|
|
let raw_name = raw_name.trim();
|
|
let canonical_name = canonical_name.trim();
|
|
|
|
if raw_name.is_empty() || canonical_name.is_empty() {
|
|
return Err(IdxError::ParseError(
|
|
"alias and canonical entity must be non-empty".to_string(),
|
|
));
|
|
}
|
|
|
|
conn.execute("BEGIN IMMEDIATE", [])
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let tx_result: Result<(), IdxError> = (|| {
|
|
let entity_id: i64 = match conn.query_row(
|
|
"SELECT id FROM entities WHERE canonical_name = ?1",
|
|
params![canonical_name],
|
|
|row| row.get(0),
|
|
) {
|
|
Ok(id) => id,
|
|
Err(rusqlite::Error::QueryReturnedNoRows) => {
|
|
let now = chrono::Utc::now().timestamp();
|
|
conn.execute(
|
|
"INSERT INTO entities (canonical_name, entity_type, country, created_at, updated_at)
|
|
VALUES (?1, NULL, NULL, ?2, ?2)",
|
|
params![canonical_name, now],
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
conn.last_insert_rowid()
|
|
}
|
|
Err(e) => return Err(IdxError::DatabaseError(e.to_string())),
|
|
};
|
|
|
|
for source in ["ksei", "bing"] {
|
|
conn.execute(
|
|
"INSERT INTO entity_aliases (entity_id, raw_name, source, confidence, method)
|
|
VALUES (?1, ?2, ?3, 1.0, 'manual')
|
|
ON CONFLICT(raw_name, source) DO UPDATE SET
|
|
entity_id = excluded.entity_id,
|
|
confidence = 1.0,
|
|
method = 'manual'",
|
|
params![entity_id, raw_name, source],
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
}
|
|
|
|
conn.execute(
|
|
"UPDATE ksei_holdings SET entity_id = ?1 WHERE raw_investor_name = ?2",
|
|
params![entity_id, raw_name],
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
conn.execute(
|
|
"UPDATE bing_holdings SET entity_id = ?1 WHERE raw_investor_name = ?2",
|
|
params![entity_id, raw_name],
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
Ok(())
|
|
})();
|
|
|
|
match tx_result {
|
|
Ok(()) => {
|
|
conn.execute("COMMIT", [])
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
Ok(())
|
|
}
|
|
Err(err) => {
|
|
let _ = conn.execute("ROLLBACK", []);
|
|
Err(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Merge two entities by re-pointing aliases and holdings from merge_id to keep_id, then deleting merge_id.
|
|
pub fn merge_entities(conn: &Connection, keep_id: i64, merge_id: i64) -> Result<(), IdxError> {
|
|
if keep_id == merge_id {
|
|
return Err(IdxError::ParseError(
|
|
"keep_id and merge_id must be different".to_string(),
|
|
));
|
|
}
|
|
|
|
conn.execute("BEGIN IMMEDIATE", [])
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
let tx_result: Result<(), IdxError> = (|| {
|
|
let keep_exists: i64 = conn
|
|
.query_row(
|
|
"SELECT EXISTS(SELECT 1 FROM entities WHERE id = ?1)",
|
|
params![keep_id],
|
|
|row| row.get(0),
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
let merge_exists: i64 = conn
|
|
.query_row(
|
|
"SELECT EXISTS(SELECT 1 FROM entities WHERE id = ?1)",
|
|
params![merge_id],
|
|
|row| row.get(0),
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
if keep_exists != 1 || merge_exists != 1 {
|
|
return Err(IdxError::ParseError(format!(
|
|
"entity not found: keep_id={keep_id}, merge_id={merge_id}"
|
|
)));
|
|
}
|
|
|
|
conn.execute(
|
|
"UPDATE entity_aliases SET entity_id = ?1 WHERE entity_id = ?2",
|
|
params![keep_id, merge_id],
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
conn.execute(
|
|
"UPDATE ksei_holdings SET entity_id = ?1 WHERE entity_id = ?2",
|
|
params![keep_id, merge_id],
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
conn.execute(
|
|
"UPDATE bing_holdings SET entity_id = ?1 WHERE entity_id = ?2",
|
|
params![keep_id, merge_id],
|
|
)
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
conn.execute("DELETE FROM entities WHERE id = ?1", params![merge_id])
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
|
|
Ok(())
|
|
})();
|
|
|
|
match tx_result {
|
|
Ok(()) => {
|
|
conn.execute("COMMIT", [])
|
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
Ok(())
|
|
}
|
|
Err(err) => {
|
|
let _ = conn.execute("ROLLBACK", []);
|
|
Err(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Get ticker_id by code.
|
|
pub fn get_ticker_id(conn: &Connection, code: &str) -> Result<Option<i64>, IdxError> {
|
|
conn.query_row(
|
|
"SELECT id FROM tickers WHERE code = ?1",
|
|
params![code],
|
|
|row| row.get(0),
|
|
)
|
|
.map(Some)
|
|
.or_else(|e| match e {
|
|
rusqlite::Error::QueryReturnedNoRows => Ok(None),
|
|
_ => Err(IdxError::DatabaseError(e.to_string())),
|
|
})
|
|
}
|
|
|
|
/// Compute concentration metrics from a list of percentage_bps values.
|
|
pub fn compute_concentration(percentages_bps: &[i64]) -> ConcentrationMetrics {
|
|
let mut values = percentages_bps.to_vec();
|
|
values.sort_by(|a, b| b.cmp(a));
|
|
|
|
let top1_bps = values.first().copied().unwrap_or(0);
|
|
let top3_bps: i64 = values.iter().take(3).sum();
|
|
let total_bps: i64 = values.iter().sum();
|
|
let hhi: i64 = values
|
|
.iter()
|
|
.map(|p| p.saturating_mul(*p) / 10000)
|
|
.fold(0, i64::saturating_add);
|
|
|
|
ConcentrationMetrics {
|
|
top1_bps,
|
|
top3_bps,
|
|
hhi,
|
|
free_float_bps: (10000 - total_bps).max(0),
|
|
holder_count: values.iter().filter(|p| **p >= 100).count(),
|
|
}
|
|
}
|
|
|
|
pub fn db_path() -> Result<PathBuf, IdxError> {
|
|
if let Some(custom_path) = get_config_value("ownership.db_path")? {
|
|
let trimmed = custom_path.trim();
|
|
if !trimmed.is_empty() {
|
|
return Ok(PathBuf::from(trimmed));
|
|
}
|
|
}
|
|
|
|
if let Ok(dir) = std::env::var("XDG_DATA_HOME")
|
|
&& !dir.is_empty()
|
|
{
|
|
let path = PathBuf::from(dir);
|
|
if path.is_absolute() {
|
|
return Ok(path.join("idx").join("ownership.db"));
|
|
}
|
|
}
|
|
|
|
ProjectDirs::from("", "", "idx")
|
|
.map(|dirs| dirs.data_local_dir().join("ownership.db"))
|
|
.ok_or_else(|| IdxError::DatabaseError("unable to resolve ownership db path".to_string()))
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct LatestKseiRelease {
|
|
sha256: String,
|
|
as_of_date: NaiveDate,
|
|
}
|
|
|
|
fn latest_ksei_release(conn: &Connection) -> Result<Option<LatestKseiRelease>, IdxError> {
|
|
conn.query_row(
|
|
"SELECT sha256, as_of_date
|
|
FROM ownership_releases
|
|
ORDER BY as_of_date DESC, imported_at DESC, id DESC
|
|
LIMIT 1",
|
|
[],
|
|
|row| {
|
|
Ok(LatestKseiRelease {
|
|
sha256: row.get(0)?,
|
|
as_of_date: parse_iso_date(&row.get::<_, String>(1)?)
|
|
.map_err(|e| rusqlite::Error::ToSqlConversionFailure(Box::new(e)))?,
|
|
})
|
|
},
|
|
)
|
|
.map(Some)
|
|
.or_else(|e| match e {
|
|
rusqlite::Error::QueryReturnedNoRows => Ok(None),
|
|
_ => Err(IdxError::DatabaseError(e.to_string())),
|
|
})
|
|
}
|
|
|
|
fn query_ticker(conn: &Connection, code: &str) -> Result<Option<Ticker>, IdxError> {
|
|
conn.query_row(
|
|
"SELECT id, code, name, sector FROM tickers WHERE code = ?1",
|
|
params![code],
|
|
|row| {
|
|
Ok(Ticker {
|
|
id: row.get(0)?,
|
|
code: row.get(1)?,
|
|
name: row.get(2)?,
|
|
sector: row.get(3)?,
|
|
})
|
|
},
|
|
)
|
|
.map(Some)
|
|
.or_else(|e| match e {
|
|
rusqlite::Error::QueryReturnedNoRows => Ok(None),
|
|
_ => Err(IdxError::DatabaseError(e.to_string())),
|
|
})
|
|
}
|
|
|
|
fn query_entity(conn: &Connection, entity_id: i64) -> Result<Option<Entity>, IdxError> {
|
|
conn.query_row(
|
|
"SELECT id, canonical_name, entity_type, country FROM entities WHERE id = ?1",
|
|
params![entity_id],
|
|
|row| {
|
|
Ok(Entity {
|
|
id: row.get(0)?,
|
|
canonical_name: row.get(1)?,
|
|
entity_type: row.get(2)?,
|
|
country: row.get(3)?,
|
|
})
|
|
},
|
|
)
|
|
.map(Some)
|
|
.or_else(|e| match e {
|
|
rusqlite::Error::QueryReturnedNoRows => Ok(None),
|
|
_ => Err(IdxError::DatabaseError(e.to_string())),
|
|
})
|
|
}
|
|
|
|
fn parse_iso_date(s: &str) -> Result<NaiveDate, IdxError> {
|
|
NaiveDate::parse_from_str(s, "%Y-%m-%d")
|
|
.map_err(|e| IdxError::ParseError(format!("invalid ISO date '{s}': {e}")))
|
|
}
|
|
|
|
fn locality_to_db(locality: Locality) -> &'static str {
|
|
match locality {
|
|
Locality::Local => "L",
|
|
Locality::Foreign => "F",
|
|
}
|
|
}
|
|
|
|
fn locality_from_db(value: &str) -> Option<Locality> {
|
|
match value {
|
|
"L" => Some(Locality::Local),
|
|
"F" | "A" => Some(Locality::Foreign),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
fn flow_signal_to_db(signal: FlowSignal) -> &'static str {
|
|
match signal {
|
|
FlowSignal::Holder => "holder",
|
|
FlowSignal::Buyer => "buyer",
|
|
FlowSignal::Seller => "seller",
|
|
FlowSignal::NewPosition => "new_position",
|
|
FlowSignal::Exited => "exited",
|
|
}
|
|
}
|
|
|
|
fn flow_signal_from_db(value: &str) -> Option<FlowSignal> {
|
|
match value {
|
|
"holder" => Some(FlowSignal::Holder),
|
|
"buyer" => Some(FlowSignal::Buyer),
|
|
"seller" => Some(FlowSignal::Seller),
|
|
"new_position" => Some(FlowSignal::NewPosition),
|
|
"exited" => Some(FlowSignal::Exited),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use chrono::NaiveDate;
|
|
use rusqlite::{Connection, params};
|
|
|
|
use crate::error::IdxError;
|
|
use crate::ownership::db::{
|
|
compute_concentration, ensure_schema, get_ticker_id, insert_bing_holdings,
|
|
insert_ksei_holdings, insert_release, query_changes, query_concentration,
|
|
query_cross_holders, query_ticker_holdings, release_exists, upsert_ticker,
|
|
write_ksei_release,
|
|
};
|
|
use crate::ownership::types::{
|
|
BingHolding, ChangeType, FlowSignal, KseiHolding, Locality, OwnershipRelease,
|
|
};
|
|
|
|
fn setup() -> Connection {
|
|
let conn = Connection::open_in_memory().unwrap();
|
|
ensure_schema(&conn).unwrap();
|
|
conn
|
|
}
|
|
|
|
/// Writes one release of `(entity_id, raw_name, bps)` rows for ticker_id.
|
|
fn write_release(
|
|
conn: &Connection,
|
|
sha256: &str,
|
|
date: &str,
|
|
imported_at: i64,
|
|
ticker_id: i64,
|
|
rows: &[(Option<i64>, &str, i64)],
|
|
) {
|
|
let as_of_date = NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap();
|
|
let holdings: Vec<KseiHolding> = rows
|
|
.iter()
|
|
.map(|&(entity_id, raw, bps)| KseiHolding {
|
|
id: 0,
|
|
ticker_id,
|
|
entity_id,
|
|
raw_investor_name: raw.to_string(),
|
|
investor_type: None,
|
|
locality: None,
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 1,
|
|
holdings_scrip: 0,
|
|
total_shares: 1,
|
|
percentage_bps: bps,
|
|
report_date: as_of_date,
|
|
release_sha256: sha256.to_string(),
|
|
})
|
|
.collect();
|
|
let release = OwnershipRelease {
|
|
id: 0,
|
|
source_url: None,
|
|
sha256: sha256.to_string(),
|
|
as_of_date,
|
|
row_count: holdings.len(),
|
|
imported_at,
|
|
};
|
|
write_ksei_release(conn, &release, &holdings, false).unwrap();
|
|
}
|
|
|
|
fn entity(conn: &Connection, name: &str) -> i64 {
|
|
conn.execute(
|
|
"INSERT INTO entities (canonical_name, entity_type, country, created_at, updated_at)
|
|
VALUES (?1, NULL, NULL, 0, 0)",
|
|
params![name],
|
|
)
|
|
.unwrap();
|
|
conn.last_insert_rowid()
|
|
}
|
|
|
|
#[test]
|
|
fn changes_sum_an_investors_accounts_in_a_ticker() {
|
|
// KREN, 2026-09: a second account took KRESNA from 9.44% to 13.45%.
|
|
let conn = setup();
|
|
let kren = upsert_ticker(&conn, "KREN", None).unwrap();
|
|
let kresna = entity(&conn, "KRESNA PRIMA INVEST");
|
|
write_release(
|
|
&conn,
|
|
"aug",
|
|
"2026-08-31",
|
|
1,
|
|
kren,
|
|
&[(Some(kresna), "PT KRESNA PRIMA INVEST", 944)],
|
|
);
|
|
write_release(
|
|
&conn,
|
|
"sep",
|
|
"2026-09-30",
|
|
2,
|
|
kren,
|
|
&[
|
|
(Some(kresna), "PT KRESNA PRIMA INVEST", 944),
|
|
(Some(kresna), "PT. KRESNA PRIMA INVEST", 401),
|
|
],
|
|
);
|
|
|
|
let rows = query_changes(&conn, "2026-08-31", "2026-09-30").unwrap();
|
|
|
|
assert_eq!(rows.len(), 1, "{rows:?}");
|
|
assert_eq!(rows[0].change_type, ChangeType::Increased);
|
|
assert_eq!(
|
|
(rows[0].old_bps, rows[0].new_bps, rows[0].delta_bps),
|
|
(Some(944), Some(1345), 401)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn changes_read_the_latest_import_for_a_date() {
|
|
// A corrected January file supersedes the original, so Jan -> Feb at
|
|
// the corrected 12% shows no change, not a spurious increase.
|
|
let conn = setup();
|
|
let bbca = upsert_ticker(&conn, "BBCA", None).unwrap();
|
|
let alpha = entity(&conn, "ALPHA");
|
|
write_release(
|
|
&conn,
|
|
"jan",
|
|
"2026-01-30",
|
|
1,
|
|
bbca,
|
|
&[(Some(alpha), "ALPHA", 1000)],
|
|
);
|
|
write_release(
|
|
&conn,
|
|
"jan-fix",
|
|
"2026-01-30",
|
|
2,
|
|
bbca,
|
|
&[(Some(alpha), "ALPHA", 1200)],
|
|
);
|
|
write_release(
|
|
&conn,
|
|
"feb",
|
|
"2026-02-27",
|
|
3,
|
|
bbca,
|
|
&[(Some(alpha), "ALPHA", 1200)],
|
|
);
|
|
|
|
assert!(
|
|
query_changes(&conn, "2026-01-30", "2026-02-27")
|
|
.unwrap()
|
|
.is_empty()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn changes_keep_each_date_to_its_own_rows() {
|
|
// A file with a January ALPHA row and February BETA row is the
|
|
// February release; ALPHA was never held in February, so Feb -> Mar
|
|
// must not report it as exited.
|
|
let conn = setup();
|
|
let bbca = upsert_ticker(&conn, "BBCA", None).unwrap();
|
|
write_release(
|
|
&conn,
|
|
"mixed",
|
|
"2026-02-27",
|
|
1,
|
|
bbca,
|
|
&[(None, "ALPHA", 1000), (None, "BETA", 200)],
|
|
);
|
|
conn.execute(
|
|
"UPDATE ksei_holdings SET report_date = '2026-01-30' WHERE raw_investor_name = 'ALPHA'",
|
|
[],
|
|
)
|
|
.unwrap();
|
|
write_release(&conn, "mar", "2026-03-31", 2, bbca, &[(None, "BETA", 200)]);
|
|
|
|
let rows = query_changes(&conn, "2026-02-27", "2026-03-31").unwrap();
|
|
assert!(rows.is_empty(), "{rows:?}");
|
|
|
|
let rows = query_changes(&conn, "2026-01-30", "2026-03-31").unwrap();
|
|
let alpha = rows.iter().find(|r| r.entity_name == "ALPHA").unwrap();
|
|
assert_eq!(alpha.change_type, ChangeType::Exited);
|
|
}
|
|
|
|
#[test]
|
|
fn same_second_imports_resolve_to_the_later_file_everywhere() {
|
|
// Original and corrected file for one date imported in the same
|
|
// second: the ticker view and `changes` must read the same one.
|
|
let conn = setup();
|
|
let bbca = upsert_ticker(&conn, "BBCA", None).unwrap();
|
|
let alpha = entity(&conn, "ALPHA");
|
|
write_release(
|
|
&conn,
|
|
"jan",
|
|
"2026-01-30",
|
|
7,
|
|
bbca,
|
|
&[(Some(alpha), "ALPHA", 1000)],
|
|
);
|
|
write_release(
|
|
&conn,
|
|
"jan-fix",
|
|
"2026-01-30",
|
|
7,
|
|
bbca,
|
|
&[(Some(alpha), "ALPHA", 1200)],
|
|
);
|
|
|
|
let ticker = query_ticker_holdings(&conn, "BBCA").unwrap();
|
|
|
|
assert_eq!(
|
|
ticker.holders[0].percentage_bps, 1200,
|
|
"{:?}",
|
|
ticker.holders
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_release_records_the_rows_it_stored() {
|
|
// A file listing the same ticker/investor twice stores one row; the
|
|
// release metadata must say 1, or snapshot validation rejects it.
|
|
let conn = setup();
|
|
let bbca = upsert_ticker(&conn, "BBCA", None).unwrap();
|
|
write_release(
|
|
&conn,
|
|
"dup",
|
|
"2026-02-27",
|
|
1,
|
|
bbca,
|
|
&[(None, "ALPHA", 1000), (None, "ALPHA", 1000)],
|
|
);
|
|
|
|
let row_count: i64 = conn
|
|
.query_row(
|
|
"SELECT row_count FROM ownership_releases WHERE sha256 = 'dup'",
|
|
[],
|
|
|row| row.get(0),
|
|
)
|
|
.unwrap();
|
|
let stored: i64 = conn
|
|
.query_row(
|
|
"SELECT COUNT(*) FROM ksei_holdings WHERE release_sha256 = 'dup'",
|
|
[],
|
|
|row| row.get(0),
|
|
)
|
|
.unwrap();
|
|
assert_eq!((row_count, stored), (1, 1));
|
|
}
|
|
|
|
#[test]
|
|
fn changes_reject_a_date_without_a_release() {
|
|
let conn = setup();
|
|
let bbca = upsert_ticker(&conn, "BBCA", None).unwrap();
|
|
write_release(
|
|
&conn,
|
|
"feb",
|
|
"2026-02-27",
|
|
1,
|
|
bbca,
|
|
&[(None, "ALPHA", 1200)],
|
|
);
|
|
|
|
let err = query_changes(&conn, "2026-01-30", "2026-02-27").unwrap_err();
|
|
|
|
assert!(
|
|
matches!(err, IdxError::InvalidInput(ref m) if m.contains("2026-01-30")),
|
|
"{err:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_compute_concentration_pure_function() {
|
|
let metrics = compute_concentration(&[4000, 2500, 1000, 500]);
|
|
assert_eq!(metrics.top1_bps, 4000);
|
|
assert_eq!(metrics.top3_bps, 7500);
|
|
assert_eq!(metrics.hhi, 2350);
|
|
assert_eq!(metrics.free_float_bps, 2000);
|
|
assert_eq!(metrics.holder_count, 4);
|
|
}
|
|
|
|
#[test]
|
|
fn test_release_exists_before_and_after_insert() {
|
|
let conn = setup();
|
|
assert!(!release_exists(&conn, "abc").unwrap());
|
|
|
|
let release = OwnershipRelease {
|
|
id: 0,
|
|
source_url: Some("https://example.com/r.pdf".to_string()),
|
|
sha256: "abc".to_string(),
|
|
as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
|
row_count: 10,
|
|
imported_at: 1_700_000_000,
|
|
};
|
|
let _ = insert_release(&conn, &release).unwrap();
|
|
|
|
assert!(release_exists(&conn, "abc").unwrap());
|
|
}
|
|
|
|
#[test]
|
|
fn test_insert_fixture_ksei_and_query_ticker() {
|
|
let conn = setup();
|
|
let bbca_id = upsert_ticker(&conn, "BBCA", Some("BCA")).unwrap();
|
|
|
|
conn.execute(
|
|
"INSERT INTO entities (canonical_name, entity_type, country, created_at, updated_at)
|
|
VALUES ('ALPHA FUND', NULL, NULL, 0, 0)",
|
|
[],
|
|
)
|
|
.unwrap();
|
|
let alpha_id = conn.last_insert_rowid();
|
|
|
|
let holdings = vec![
|
|
KseiHolding {
|
|
id: 0,
|
|
ticker_id: bbca_id,
|
|
entity_id: Some(alpha_id),
|
|
raw_investor_name: "PT ALPHA FUND".to_string(),
|
|
investor_type: None,
|
|
locality: Some(Locality::Local),
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 1_000,
|
|
holdings_scrip: 0,
|
|
total_shares: 1_000,
|
|
percentage_bps: 4000,
|
|
report_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
|
release_sha256: "r1".to_string(),
|
|
},
|
|
KseiHolding {
|
|
id: 0,
|
|
ticker_id: bbca_id,
|
|
entity_id: None,
|
|
raw_investor_name: "BETA".to_string(),
|
|
investor_type: None,
|
|
locality: Some(Locality::Foreign),
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 600,
|
|
holdings_scrip: 0,
|
|
total_shares: 600,
|
|
percentage_bps: 2400,
|
|
report_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
|
release_sha256: "r1".to_string(),
|
|
},
|
|
];
|
|
|
|
let release = OwnershipRelease {
|
|
id: 0,
|
|
source_url: None,
|
|
sha256: "r1".to_string(),
|
|
as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
|
row_count: holdings.len(),
|
|
imported_at: 1,
|
|
};
|
|
assert_eq!(
|
|
write_ksei_release(&conn, &release, &holdings, false).unwrap(),
|
|
2
|
|
);
|
|
|
|
let data = query_ticker_holdings(&conn, "BBCA").unwrap();
|
|
assert_eq!(data.ticker.code, "BBCA");
|
|
assert_eq!(data.holders.len(), 2);
|
|
assert_eq!(data.holders[0].percentage_bps, 4000);
|
|
assert_eq!(data.concentration.top1_bps, 4000);
|
|
assert_eq!(data.concentration.top3_bps, 6400);
|
|
}
|
|
|
|
#[test]
|
|
fn test_duplicate_insert_ignored() {
|
|
let conn = setup();
|
|
let bbri_id = upsert_ticker(&conn, "BBRI", None).unwrap();
|
|
|
|
let h = KseiHolding {
|
|
id: 0,
|
|
ticker_id: bbri_id,
|
|
entity_id: None,
|
|
raw_investor_name: "DUP".to_string(),
|
|
investor_type: None,
|
|
locality: None,
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 10,
|
|
holdings_scrip: 0,
|
|
total_shares: 10,
|
|
percentage_bps: 100,
|
|
report_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
|
release_sha256: "same-release".to_string(),
|
|
};
|
|
|
|
assert_eq!(
|
|
insert_ksei_holdings(&conn, std::slice::from_ref(&h)).unwrap(),
|
|
1
|
|
);
|
|
assert_eq!(insert_ksei_holdings(&conn, &[h]).unwrap(), 0);
|
|
|
|
let count: i64 = conn
|
|
.query_row("SELECT COUNT(*) FROM ksei_holdings", [], |row| row.get(0))
|
|
.unwrap();
|
|
assert_eq!(count, 1);
|
|
}
|
|
|
|
#[test]
|
|
fn test_write_ksei_release_replaces_existing_sha_when_requested() {
|
|
let conn = setup();
|
|
let bbri_id = upsert_ticker(&conn, "BBRI", None).unwrap();
|
|
|
|
let initial = KseiHolding {
|
|
id: 0,
|
|
ticker_id: bbri_id,
|
|
entity_id: None,
|
|
raw_investor_name: "FORCE".to_string(),
|
|
investor_type: None,
|
|
locality: None,
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 10,
|
|
holdings_scrip: 0,
|
|
total_shares: 10,
|
|
percentage_bps: 100,
|
|
report_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
|
release_sha256: "force-sha".to_string(),
|
|
};
|
|
let initial_release = OwnershipRelease {
|
|
id: 0,
|
|
source_url: None,
|
|
sha256: "force-sha".to_string(),
|
|
as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
|
row_count: 1,
|
|
imported_at: 1,
|
|
};
|
|
|
|
assert_eq!(
|
|
write_ksei_release(
|
|
&conn,
|
|
&initial_release,
|
|
std::slice::from_ref(&initial),
|
|
false
|
|
)
|
|
.unwrap(),
|
|
1
|
|
);
|
|
|
|
let replacement = KseiHolding {
|
|
percentage_bps: 250,
|
|
total_shares: 25,
|
|
holdings_scripless: 25,
|
|
..initial
|
|
};
|
|
let replacement_release = OwnershipRelease {
|
|
imported_at: 2,
|
|
..initial_release
|
|
};
|
|
|
|
assert_eq!(
|
|
write_ksei_release(
|
|
&conn,
|
|
&replacement_release,
|
|
std::slice::from_ref(&replacement),
|
|
true,
|
|
)
|
|
.unwrap(),
|
|
1
|
|
);
|
|
|
|
let release_count: i64 = conn
|
|
.query_row(
|
|
"SELECT COUNT(*) FROM ownership_releases WHERE sha256 = 'force-sha'",
|
|
[],
|
|
|row| row.get(0),
|
|
)
|
|
.unwrap();
|
|
assert_eq!(release_count, 1);
|
|
|
|
let latest_bps: i64 = conn
|
|
.query_row(
|
|
"SELECT percentage_bps FROM ksei_holdings WHERE release_sha256 = 'force-sha'",
|
|
[],
|
|
|row| row.get(0),
|
|
)
|
|
.unwrap();
|
|
assert_eq!(latest_bps, 250);
|
|
}
|
|
|
|
#[test]
|
|
fn test_query_ticker_holdings_uses_latest_release_only() {
|
|
let conn = setup();
|
|
let tlkm_id = upsert_ticker(&conn, "TLKM", None).unwrap();
|
|
|
|
let older_release = OwnershipRelease {
|
|
id: 0,
|
|
source_url: None,
|
|
sha256: "old-release".to_string(),
|
|
as_of_date: NaiveDate::from_ymd_opt(2026, 1, 30).unwrap(),
|
|
row_count: 2,
|
|
imported_at: 1,
|
|
};
|
|
let older_rows = vec![
|
|
KseiHolding {
|
|
id: 0,
|
|
ticker_id: tlkm_id,
|
|
entity_id: None,
|
|
raw_investor_name: "OLDER A".to_string(),
|
|
investor_type: None,
|
|
locality: None,
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 40,
|
|
holdings_scrip: 0,
|
|
total_shares: 40,
|
|
percentage_bps: 4000,
|
|
report_date: older_release.as_of_date,
|
|
release_sha256: older_release.sha256.clone(),
|
|
},
|
|
KseiHolding {
|
|
id: 0,
|
|
ticker_id: tlkm_id,
|
|
entity_id: None,
|
|
raw_investor_name: "OLDER B".to_string(),
|
|
investor_type: None,
|
|
locality: None,
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 30,
|
|
holdings_scrip: 0,
|
|
total_shares: 30,
|
|
percentage_bps: 3000,
|
|
report_date: older_release.as_of_date,
|
|
release_sha256: older_release.sha256.clone(),
|
|
},
|
|
];
|
|
write_ksei_release(&conn, &older_release, &older_rows, false).unwrap();
|
|
|
|
let latest_release = OwnershipRelease {
|
|
id: 0,
|
|
source_url: None,
|
|
sha256: "latest-release".to_string(),
|
|
as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
|
row_count: 1,
|
|
imported_at: 2,
|
|
};
|
|
let latest_rows = vec![KseiHolding {
|
|
id: 0,
|
|
ticker_id: tlkm_id,
|
|
entity_id: None,
|
|
raw_investor_name: "LATEST".to_string(),
|
|
investor_type: None,
|
|
locality: None,
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 25,
|
|
holdings_scrip: 0,
|
|
total_shares: 25,
|
|
percentage_bps: 2500,
|
|
report_date: latest_release.as_of_date,
|
|
release_sha256: latest_release.sha256.clone(),
|
|
}];
|
|
write_ksei_release(&conn, &latest_release, &latest_rows, false).unwrap();
|
|
|
|
let data = query_ticker_holdings(&conn, "TLKM").unwrap();
|
|
assert_eq!(data.ksei_as_of, Some(latest_release.as_of_date));
|
|
assert_eq!(data.holders.len(), 1);
|
|
assert_eq!(data.holders[0].name, "LATEST");
|
|
assert_eq!(data.concentration.top1_bps, 2500);
|
|
}
|
|
|
|
#[test]
|
|
fn test_cross_holders_same_entity_three_tickers() {
|
|
let conn = setup();
|
|
conn.execute(
|
|
"INSERT INTO entities (canonical_name, entity_type, country, created_at, updated_at)
|
|
VALUES ('OMEGA', NULL, NULL, 0, 0)",
|
|
[],
|
|
)
|
|
.unwrap();
|
|
let eid = conn.last_insert_rowid();
|
|
|
|
let mut holdings = Vec::new();
|
|
for (code, bps) in [("BBCA", 1000), ("BBRI", 1100), ("BMRI", 1200)] {
|
|
let tid = upsert_ticker(&conn, code, None).unwrap();
|
|
holdings.push(KseiHolding {
|
|
id: 0,
|
|
ticker_id: tid,
|
|
entity_id: Some(eid),
|
|
raw_investor_name: "OMEGA".to_string(),
|
|
investor_type: None,
|
|
locality: None,
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 10,
|
|
holdings_scrip: 0,
|
|
total_shares: 10,
|
|
percentage_bps: bps,
|
|
report_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
|
release_sha256: "current-release".to_string(),
|
|
});
|
|
}
|
|
let release = OwnershipRelease {
|
|
id: 0,
|
|
source_url: None,
|
|
sha256: "current-release".to_string(),
|
|
as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
|
row_count: holdings.len(),
|
|
imported_at: 1,
|
|
};
|
|
write_ksei_release(&conn, &release, &holdings, false).unwrap();
|
|
|
|
let rows = query_cross_holders(&conn, 3, 10).unwrap();
|
|
assert_eq!(rows.len(), 1);
|
|
assert_eq!(rows[0].entity.id, eid);
|
|
assert_eq!(rows[0].ticker_count, 3);
|
|
}
|
|
|
|
#[test]
|
|
fn test_cross_holders_use_latest_release_only() {
|
|
let conn = setup();
|
|
conn.execute(
|
|
"INSERT INTO entities (canonical_name, entity_type, country, created_at, updated_at)
|
|
VALUES ('OMEGA', NULL, NULL, 0, 0)",
|
|
[],
|
|
)
|
|
.unwrap();
|
|
let eid = conn.last_insert_rowid();
|
|
|
|
let aa = upsert_ticker(&conn, "AA", None).unwrap();
|
|
let bb = upsert_ticker(&conn, "BB", None).unwrap();
|
|
|
|
let old_release = OwnershipRelease {
|
|
id: 0,
|
|
source_url: None,
|
|
sha256: "old-cross".to_string(),
|
|
as_of_date: NaiveDate::from_ymd_opt(2026, 1, 30).unwrap(),
|
|
row_count: 2,
|
|
imported_at: 1,
|
|
};
|
|
let old_rows = vec![
|
|
KseiHolding {
|
|
id: 0,
|
|
ticker_id: aa,
|
|
entity_id: Some(eid),
|
|
raw_investor_name: "OMEGA".to_string(),
|
|
investor_type: None,
|
|
locality: None,
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 1,
|
|
holdings_scrip: 0,
|
|
total_shares: 1,
|
|
percentage_bps: 1000,
|
|
report_date: old_release.as_of_date,
|
|
release_sha256: old_release.sha256.clone(),
|
|
},
|
|
KseiHolding {
|
|
id: 0,
|
|
ticker_id: bb,
|
|
entity_id: Some(eid),
|
|
raw_investor_name: "OMEGA".to_string(),
|
|
investor_type: None,
|
|
locality: None,
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 1,
|
|
holdings_scrip: 0,
|
|
total_shares: 1,
|
|
percentage_bps: 1200,
|
|
report_date: old_release.as_of_date,
|
|
release_sha256: old_release.sha256.clone(),
|
|
},
|
|
];
|
|
write_ksei_release(&conn, &old_release, &old_rows, false).unwrap();
|
|
|
|
let latest_release = OwnershipRelease {
|
|
id: 0,
|
|
source_url: None,
|
|
sha256: "latest-cross".to_string(),
|
|
as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
|
row_count: 1,
|
|
imported_at: 2,
|
|
};
|
|
let latest_rows = vec![KseiHolding {
|
|
id: 0,
|
|
ticker_id: aa,
|
|
entity_id: Some(eid),
|
|
raw_investor_name: "OMEGA".to_string(),
|
|
investor_type: None,
|
|
locality: None,
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 1,
|
|
holdings_scrip: 0,
|
|
total_shares: 1,
|
|
percentage_bps: 900,
|
|
report_date: latest_release.as_of_date,
|
|
release_sha256: latest_release.sha256.clone(),
|
|
}];
|
|
write_ksei_release(&conn, &latest_release, &latest_rows, false).unwrap();
|
|
|
|
assert!(query_cross_holders(&conn, 2, 10).unwrap().is_empty());
|
|
let rows = query_cross_holders(&conn, 1, 10).unwrap();
|
|
assert_eq!(rows.len(), 1);
|
|
assert_eq!(rows[0].ticker_count, 1);
|
|
assert_eq!(rows[0].total_bps, 900);
|
|
}
|
|
|
|
#[test]
|
|
fn test_concentration_sorting() {
|
|
let conn = setup();
|
|
let aa = upsert_ticker(&conn, "AA", None).unwrap();
|
|
let bb = upsert_ticker(&conn, "BB", None).unwrap();
|
|
|
|
let rows = vec![
|
|
(aa, 6000, "A1"),
|
|
(aa, 1000, "A2"),
|
|
(bb, 5000, "B1"),
|
|
(bb, 3000, "B2"),
|
|
];
|
|
|
|
let holdings = rows
|
|
.into_iter()
|
|
.map(|(tid, bps, name)| KseiHolding {
|
|
id: 0,
|
|
ticker_id: tid,
|
|
entity_id: None,
|
|
raw_investor_name: name.to_string(),
|
|
investor_type: None,
|
|
locality: None,
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 1,
|
|
holdings_scrip: 0,
|
|
total_shares: 1,
|
|
percentage_bps: bps,
|
|
report_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
|
release_sha256: "current-release".to_string(),
|
|
})
|
|
.collect::<Vec<_>>();
|
|
let release = OwnershipRelease {
|
|
id: 0,
|
|
source_url: None,
|
|
sha256: "current-release".to_string(),
|
|
as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
|
row_count: holdings.len(),
|
|
imported_at: 1,
|
|
};
|
|
write_ksei_release(&conn, &release, &holdings, false).unwrap();
|
|
|
|
let by_top1 = query_concentration(&conn, "top1", 10).unwrap();
|
|
assert_eq!(by_top1[0].0, "AA");
|
|
|
|
let by_top3 = query_concentration(&conn, "top3", 10).unwrap();
|
|
assert_eq!(by_top3[0].0, "BB");
|
|
|
|
let by_hhi = query_concentration(&conn, "hhi", 10).unwrap();
|
|
assert_eq!(by_hhi[0].0, "AA");
|
|
}
|
|
|
|
#[test]
|
|
fn test_concentration_uses_latest_release_only() {
|
|
let conn = setup();
|
|
let aa = upsert_ticker(&conn, "AA", None).unwrap();
|
|
|
|
let old_release = OwnershipRelease {
|
|
id: 0,
|
|
source_url: None,
|
|
sha256: "old-concentration".to_string(),
|
|
as_of_date: NaiveDate::from_ymd_opt(2026, 1, 30).unwrap(),
|
|
row_count: 1,
|
|
imported_at: 1,
|
|
};
|
|
let old_rows = vec![KseiHolding {
|
|
id: 0,
|
|
ticker_id: aa,
|
|
entity_id: None,
|
|
raw_investor_name: "OLD".to_string(),
|
|
investor_type: None,
|
|
locality: None,
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 1,
|
|
holdings_scrip: 0,
|
|
total_shares: 1,
|
|
percentage_bps: 6000,
|
|
report_date: old_release.as_of_date,
|
|
release_sha256: old_release.sha256.clone(),
|
|
}];
|
|
write_ksei_release(&conn, &old_release, &old_rows, false).unwrap();
|
|
|
|
let latest_release = OwnershipRelease {
|
|
id: 0,
|
|
source_url: None,
|
|
sha256: "latest-concentration".to_string(),
|
|
as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
|
row_count: 2,
|
|
imported_at: 2,
|
|
};
|
|
let latest_rows = vec![
|
|
KseiHolding {
|
|
id: 0,
|
|
ticker_id: aa,
|
|
entity_id: None,
|
|
raw_investor_name: "NEW A".to_string(),
|
|
investor_type: None,
|
|
locality: None,
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 1,
|
|
holdings_scrip: 0,
|
|
total_shares: 1,
|
|
percentage_bps: 2000,
|
|
report_date: latest_release.as_of_date,
|
|
release_sha256: latest_release.sha256.clone(),
|
|
},
|
|
KseiHolding {
|
|
id: 0,
|
|
ticker_id: aa,
|
|
entity_id: None,
|
|
raw_investor_name: "NEW B".to_string(),
|
|
investor_type: None,
|
|
locality: None,
|
|
nationality: None,
|
|
domicile: None,
|
|
holdings_scripless: 1,
|
|
holdings_scrip: 0,
|
|
total_shares: 1,
|
|
percentage_bps: 1500,
|
|
report_date: latest_release.as_of_date,
|
|
release_sha256: latest_release.sha256.clone(),
|
|
},
|
|
];
|
|
write_ksei_release(&conn, &latest_release, &latest_rows, false).unwrap();
|
|
|
|
let rows = query_concentration(&conn, "top1", 10).unwrap();
|
|
assert_eq!(rows.len(), 1);
|
|
assert_eq!(rows[0].1.top1_bps, 2000);
|
|
assert_eq!(rows[0].1.top3_bps, 3500);
|
|
}
|
|
|
|
#[test]
|
|
fn test_insert_bing_and_query_empty_db_paths() {
|
|
let conn = setup();
|
|
|
|
assert!(get_ticker_id(&conn, "NONE").unwrap().is_none());
|
|
assert!(query_cross_holders(&conn, 1, 10).unwrap().is_empty());
|
|
assert!(query_concentration(&conn, "top1", 10).unwrap().is_empty());
|
|
|
|
let tid = upsert_ticker(&conn, "TLKM", None).unwrap();
|
|
let inserted = insert_bing_holdings(
|
|
&conn,
|
|
&[BingHolding {
|
|
id: 0,
|
|
ticker_id: tid,
|
|
entity_id: None,
|
|
raw_investor_name: "FLOW".to_string(),
|
|
investor_type: None,
|
|
shares_held: Some(100),
|
|
shares_changed: Some(50),
|
|
pct_ownership_bps: Some(123),
|
|
value_usd: Some(10),
|
|
report_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
|
signal: FlowSignal::Buyer,
|
|
fetched_at: 12345,
|
|
}],
|
|
)
|
|
.unwrap();
|
|
assert_eq!(inserted, 1);
|
|
|
|
let buyers: i64 = conn
|
|
.query_row(
|
|
"SELECT COUNT(*) FROM bing_holdings WHERE ticker_id = ?1 AND signal = 'buyer'",
|
|
params![tid],
|
|
|row| row.get(0),
|
|
)
|
|
.unwrap();
|
|
assert_eq!(buyers, 1);
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[test]
|
|
fn resolve_db_path_follows_symlinks_even_when_the_target_is_missing() {
|
|
let root = std::env::temp_dir().join(format!(
|
|
"idx-resolve-{}-{}",
|
|
std::process::id(),
|
|
fastrand::u64(..)
|
|
));
|
|
let real_dir = root.join("real");
|
|
std::fs::create_dir_all(&real_dir).unwrap();
|
|
let real_dir = std::fs::canonicalize(&real_dir).unwrap();
|
|
let link = root.join("link.db");
|
|
// Dangling: the target does not exist yet (fresh install, mid-swap).
|
|
std::os::unix::fs::symlink(real_dir.join("ownership.db"), &link).unwrap();
|
|
assert_eq!(
|
|
super::resolve_db_path(&link).unwrap(),
|
|
real_dir.join("ownership.db")
|
|
);
|
|
// Relative link, resolved against the link's directory.
|
|
let relative = root.join("relative.db");
|
|
std::os::unix::fs::symlink("real/ownership.db", &relative).unwrap();
|
|
assert_eq!(
|
|
super::resolve_db_path(&relative).unwrap(),
|
|
real_dir.join("ownership.db")
|
|
);
|
|
// A plain path is returned as-is (canonical parent).
|
|
assert_eq!(
|
|
super::resolve_db_path(&real_dir.join("ownership.db")).unwrap(),
|
|
real_dir.join("ownership.db")
|
|
);
|
|
let _ = std::fs::remove_dir_all(&root);
|
|
}
|
|
}
|