0xrsydn/idx-cli
1
mirror of https://github.com/0xrsydn/idx-cli.git synced 2026-11-07 09:45:36 +00:00
Code Issues Projects Releases 1 Packages Wiki Activity

fix(ownership): correct changes, alias search, graph, and name lookups

Browse source 
From a full-repo Codex audit, each reproduced on the synced ownership DB:
- `changes` paired an investor's accounts across releases instead of
  comparing totals: KREN's KRESNA PRIMA INVEST went 9.44% -> 13.45% (a
  second 4.01% account) but was reported as decreased 944 -> 401 bps.
  Sum a holder's accounts per ticker, and read one release per date (the
  latest imported, as other queries do), so a corrected file supersedes
  the original instead of mixing with it.
- `changes` / get_ownership_changes accepted any dates: `from: ""`
  reported all 7,154 holders as new, unknown dates returned no changes.
  Dates must be YYYY-MM-DD imported releases with `from` before `to`.
- get_investor_holdings / `entity` with an empty name matched every
  investor and returned an arbitrary one ("AMY"); `%` and `_` acted as
  LIKE wildcards. Reject empty names; match wildcards literally.
- Alias search never worked: the FTS table declared an `aliases` column
  its content table lacks (`no such column: T.aliases`), so searches
  fell back to canonical names and a raw KSEI name such as
  "PT ASTRA INTERNATIONAL TBK" found nothing. Search names and aliases
  directly, ranked exact name, exact alias, prefix, then shorter names.
- `graph` enumerated every path (exponential in depth) and its cycle
  check matched substrings, dropping entity:1 when entity:10 was on the
  path. Traverse breadth-first with an exact visited set.
- Name normalization removed leading PT tokens from the front of a Vec,
  quadratic in hostile names; narrow a slice instead.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V3syn2fN2LNo5pvTSqc6eh
This commit is contained in:
hermes 2026-10-17 04:09:51 +00:00
commit 1d3fe5ca88
6 changed files with 437 additions and 196 deletions
Download patch file Download diff file Expand all files Collapse all files

211
crates/idx-core/src/ownership/db.rs
View file

@ -7,7 +7,6 @@ use rusqlite::{Connection, params};
use crate::config::get_config_value;
use crate::error::IdxError;
use crate::ownership::search;
use crate::ownership::types::{
BingHolding, ChangeRow, ChangeType, ConcentrationMetrics, CrossHolderRow, Entity,
EntityHoldings, EntityTickerRow, FlowSignal, HolderRow, InstitutionalFlow, KseiHolding,
@ -36,14 +35,6 @@ CREATE TABLE IF NOT EXISTS entities (
updated_at INTEGER NOT NULL
);
CREATE VIRTUAL TABLE IF NOT EXISTS entity_fts USING fts5(
canonical_name,
aliases,
content='entities',
content_rowid='id',
tokenize='trigram'
);
CREATE TABLE IF NOT EXISTS entity_aliases (
id INTEGER PRIMARY KEY,
entity_id INTEGER NOT NULL REFERENCES entities(id),
@ -916,43 +907,73 @@ pub fn query_bing_flow(
}))
}
/// Compare two KSEI snapshots by date and return ownership changes.
/// The release used for a report date: the latest imported one, as
/// `latest_ksei_release` picks, so a corrected file supersedes the original.
fn release_for_date(conn: &Connection, date: &str) -> Result<Option<String>, IdxError> {
conn.query_row(
"SELECT sha256 FROM ownership_releases
WHERE as_of_date = ?1
ORDER BY imported_at DESC, id DESC
LIMIT 1",
params![date],
|row| row.get::<_, String>(0),
)
.map(Some)
.or_else(|e| match e {
rusqlite::Error::QueryReturnedNoRows => Ok(None),
other => Err(IdxError::DatabaseError(other.to_string())),
})
}
/// Compare two KSEI releases by date and return ownership changes.
/// Finds: new holders, exited holders, percentage increases/decreases.
/// Each date reads one release, and a holder's accounts in a ticker are
/// summed, so an investor with two accounts compares by its total stake.
pub fn query_changes(
conn: &Connection,
from_date: &str,
to_date: &str,
) -> Result<Vec<ChangeRow>, IdxError> {
let release = |date: &str| {
release_for_date(conn, date)?.ok_or_else(|| {
IdxError::InvalidInput(format!(
"no KSEI release imported for {date}; list the imported dates with `idx ownership releases` or the ownership_status tool"
))
})
};
let (from_release, to_release) = (release(from_date)?, release(to_date)?);
let mut stmt = conn
.prepare(
"WITH
from_snapshot AS (
SELECT
t.code AS ticker_code,
COALESCE(e.canonical_name, k.raw_investor_name) AS entity_name,
MIN(COALESCE(e.canonical_name, k.raw_investor_name)) AS entity_name,
CASE
WHEN k.entity_id IS NOT NULL THEN 'id:' || k.entity_id
ELSE 'raw:' || UPPER(TRIM(k.raw_investor_name))
END AS holder_key,
k.percentage_bps AS old_bps
SUM(k.percentage_bps) AS old_bps
FROM ksei_holdings k
JOIN tickers t ON t.id = k.ticker_id
LEFT JOIN entities e ON e.id = k.entity_id
WHERE k.report_date = ?1
WHERE k.release_sha256 = ?1
GROUP BY t.code, holder_key
),
to_snapshot AS (
SELECT
t.code AS ticker_code,
COALESCE(e.canonical_name, k.raw_investor_name) AS entity_name,
MIN(COALESCE(e.canonical_name, k.raw_investor_name)) AS entity_name,
CASE
WHEN k.entity_id IS NOT NULL THEN 'id:' || k.entity_id
ELSE 'raw:' || UPPER(TRIM(k.raw_investor_name))
END AS holder_key,
k.percentage_bps AS new_bps
SUM(k.percentage_bps) AS new_bps
FROM ksei_holdings k
JOIN tickers t ON t.id = k.ticker_id
LEFT JOIN entities e ON e.id = k.entity_id
WHERE k.report_date = ?2
WHERE k.release_sha256 = ?2
GROUP BY t.code, holder_key
),
keys AS (
SELECT ticker_code, holder_key FROM from_snapshot
@ -978,7 +999,7 @@ pub fn query_changes(
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
let rows = stmt
.query_map(params![from_date, to_date], |row| {
.query_map(params![from_release, to_release], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
@ -1123,7 +1144,6 @@ pub fn manual_map(conn: &Connection, raw_name: &str, canonical_name: &str) -> Re
)
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
let _ = search::rebuild_fts(conn);
Ok(())
})();
@ -1194,7 +1214,6 @@ pub fn merge_entities(conn: &Connection, keep_id: i64, merge_id: i64) -> Result<
conn.execute("DELETE FROM entities WHERE id = ?1", params![merge_id])
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
let _ = search::rebuild_fts(conn);
Ok(())
})();
@ -1383,13 +1402,15 @@ mod tests {
use chrono::NaiveDate;
use rusqlite::{Connection, params};
use crate::error::IdxError;
use crate::ownership::db::{
compute_concentration, ensure_schema, get_ticker_id, insert_bing_holdings,
insert_ksei_holdings, insert_release, query_concentration, query_cross_holders,
query_ticker_holdings, release_exists, upsert_ticker, write_ksei_release,
insert_ksei_holdings, insert_release, query_changes, query_concentration,
query_cross_holders, query_ticker_holdings, release_exists, upsert_ticker,
write_ksei_release,
};
use crate::ownership::types::{
BingHolding, FlowSignal, KseiHolding, Locality, OwnershipRelease,
BingHolding, ChangeType, FlowSignal, KseiHolding, Locality, OwnershipRelease,
};
fn setup() -> Connection {
@ -1398,6 +1419,152 @@ mod tests {
conn
}
/// Writes one release of `(entity_id, raw_name, bps)` rows for ticker_id.
fn write_release(
conn: &Connection,
sha256: &str,
date: &str,
imported_at: i64,
ticker_id: i64,
rows: &[(Option<i64>, &str, i64)],
) {
let as_of_date = NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap();
let holdings: Vec<KseiHolding> = rows
.iter()
.map(|&(entity_id, raw, bps)| KseiHolding {
id: 0,
ticker_id,
entity_id,
raw_investor_name: raw.to_string(),
investor_type: None,
locality: None,
nationality: None,
domicile: None,
holdings_scripless: 1,
holdings_scrip: 0,
total_shares: 1,
percentage_bps: bps,
report_date: as_of_date,
release_sha256: sha256.to_string(),
})
.collect();
let release = OwnershipRelease {
id: 0,
source_url: None,
sha256: sha256.to_string(),
as_of_date,
row_count: holdings.len(),
imported_at,
};
write_ksei_release(conn, &release, &holdings, false).unwrap();
}
fn entity(conn: &Connection, name: &str) -> i64 {
conn.execute(
"INSERT INTO entities (canonical_name, entity_type, country, created_at, updated_at)
VALUES (?1, NULL, NULL, 0, 0)",
params![name],
)
.unwrap();
conn.last_insert_rowid()
}
#[test]
fn changes_sum_an_investors_accounts_in_a_ticker() {
// KREN, 2026-09: a second account took KRESNA from 9.44% to 13.45%.
let conn = setup();
let kren = upsert_ticker(&conn, "KREN", None).unwrap();
let kresna = entity(&conn, "KRESNA PRIMA INVEST");
write_release(
&conn,
"aug",
"2026-08-31",
1,
kren,
&[(Some(kresna), "PT KRESNA PRIMA INVEST", 944)],
);
write_release(
&conn,
"sep",
"2026-09-30",
2,
kren,
&[
(Some(kresna), "PT KRESNA PRIMA INVEST", 944),
(Some(kresna), "PT. KRESNA PRIMA INVEST", 401),
],
);
let rows = query_changes(&conn, "2026-08-31", "2026-09-30").unwrap();
assert_eq!(rows.len(), 1, "{rows:?}");
assert_eq!(rows[0].change_type, ChangeType::Increased);
assert_eq!(
(rows[0].old_bps, rows[0].new_bps, rows[0].delta_bps),
(Some(944), Some(1345), 401)
);
}
#[test]
fn changes_read_the_latest_import_for_a_date() {
// A corrected January file supersedes the original, so Jan -> Feb at
// the corrected 12% shows no change, not a spurious increase.
let conn = setup();
let bbca = upsert_ticker(&conn, "BBCA", None).unwrap();
let alpha = entity(&conn, "ALPHA");
write_release(
&conn,
"jan",
"2026-01-30",
1,
bbca,
&[(Some(alpha), "ALPHA", 1000)],
);
write_release(
&conn,
"jan-fix",
"2026-01-30",
2,
bbca,
&[(Some(alpha), "ALPHA", 1200)],
);
write_release(
&conn,
"feb",
"2026-02-27",
3,
bbca,
&[(Some(alpha), "ALPHA", 1200)],
);
assert!(
query_changes(&conn, "2026-01-30", "2026-02-27")
.unwrap()
.is_empty()
);
}
#[test]
fn changes_reject_a_date_without_a_release() {
let conn = setup();
let bbca = upsert_ticker(&conn, "BBCA", None).unwrap();
write_release(
&conn,
"feb",
"2026-02-27",
1,
bbca,
&[(None, "ALPHA", 1200)],
);
let err = query_changes(&conn, "2026-01-30", "2026-02-27").unwrap_err();
assert!(
matches!(err, IdxError::InvalidInput(ref m) if m.contains("2026-01-30")),
"{err:?}"
);
}
#[test]
fn test_compute_concentration_pure_function() {
let metrics = compute_concentration(&[4000, 2500, 1000, 500]);