mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-11-07 09:45:36 +00:00
fix(ownership): correct changes, alias search, graph, and name lookups
From a full-repo Codex audit, each reproduced on the synced ownership DB:
- `changes` paired an investor's accounts across releases instead of
comparing totals: KREN's KRESNA PRIMA INVEST went 9.44% -> 13.45% (a
second 4.01% account) but was reported as decreased 944 -> 401 bps.
Sum a holder's accounts per ticker, and read one release per date (the
latest imported, as other queries do), so a corrected file supersedes
the original instead of mixing with it.
- `changes` / get_ownership_changes accepted any dates: `from: ""`
reported all 7,154 holders as new, unknown dates returned no changes.
Dates must be YYYY-MM-DD imported releases with `from` before `to`.
- get_investor_holdings / `entity` with an empty name matched every
investor and returned an arbitrary one ("AMY"); `%` and `_` acted as
LIKE wildcards. Reject empty names; match wildcards literally.
- Alias search never worked: the FTS table declared an `aliases` column
its content table lacks (`no such column: T.aliases`), so searches
fell back to canonical names and a raw KSEI name such as
"PT ASTRA INTERNATIONAL TBK" found nothing. Search names and aliases
directly, ranked exact name, exact alias, prefix, then shorter names.
- `graph` enumerated every path (exponential in depth) and its cycle
check matched substrings, dropping entity:1 when entity:10 was on the
path. Traverse breadth-first with an exact visited set.
- Name normalization removed leading PT tokens from the front of a Vec,
quadratic in hostile names; narrow a slice instead.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V3syn2fN2LNo5pvTSqc6eh
This commit is contained in:
parent
f5da3b745e
commit
1d3fe5ca88
6 changed files with 437 additions and 196 deletions
|
|
@ -134,11 +134,23 @@ impl Investor {
|
||||||
/// Holdings of the entity whose name best matches `name` (shortest
|
/// Holdings of the entity whose name best matches `name` (shortest
|
||||||
/// canonical name containing it).
|
/// canonical name containing it).
|
||||||
pub fn entity(&self, name: &str) -> Result<EntityHoldings, IdxError> {
|
pub fn entity(&self, name: &str) -> Result<EntityHoldings, IdxError> {
|
||||||
|
let name = name.trim();
|
||||||
|
// An empty pattern would match every investor and return an arbitrary one.
|
||||||
|
if name.is_empty() {
|
||||||
|
return Err(IdxError::InvalidInput(
|
||||||
|
"investor name must not be empty".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
let conn = self.connect()?;
|
let conn = self.connect()?;
|
||||||
let pattern = format!("%{}%", name.trim());
|
// `%` and `_` in the name are literal characters, not LIKE wildcards.
|
||||||
|
let escaped = name
|
||||||
|
.replace('\\', "\\\\")
|
||||||
|
.replace('%', "\\%")
|
||||||
|
.replace('_', "\\_");
|
||||||
|
let pattern = format!("%{escaped}%");
|
||||||
let entity_id = conn
|
let entity_id = conn
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT id FROM entities WHERE canonical_name LIKE ?1 COLLATE NOCASE ORDER BY LENGTH(canonical_name) ASC LIMIT 1",
|
"SELECT id FROM entities WHERE canonical_name LIKE ?1 ESCAPE '\\' COLLATE NOCASE ORDER BY LENGTH(canonical_name) ASC LIMIT 1",
|
||||||
rusqlite::params![pattern],
|
rusqlite::params![pattern],
|
||||||
|row| row.get::<_, i64>(0),
|
|row| row.get::<_, i64>(0),
|
||||||
)
|
)
|
||||||
|
|
@ -147,7 +159,7 @@ impl Investor {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn search(&self, query: &str, limit: usize) -> Result<Vec<Entity>, IdxError> {
|
pub fn search(&self, query: &str, limit: usize) -> Result<Vec<Entity>, IdxError> {
|
||||||
search::fts_search(&*self.connect()?, query, limit)
|
search::search_entities(&*self.connect()?, query, limit)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn cross_holders(
|
pub fn cross_holders(
|
||||||
|
|
@ -175,14 +187,29 @@ impl Investor {
|
||||||
db::query_bing_flow(&conn, ticker_id)
|
db::query_bing_flow(&conn, ticker_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Holder changes between two release dates, optionally for one ticker.
|
/// Holder changes between two release dates (`from` earlier than `to`,
|
||||||
|
/// both imported releases), optionally for one ticker.
|
||||||
pub fn changes(
|
pub fn changes(
|
||||||
&self,
|
&self,
|
||||||
from: &str,
|
from: &str,
|
||||||
to: &str,
|
to: &str,
|
||||||
ticker: Option<&str>,
|
ticker: Option<&str>,
|
||||||
) -> Result<Vec<ChangeRow>, IdxError> {
|
) -> Result<Vec<ChangeRow>, IdxError> {
|
||||||
let mut rows = db::query_changes(&*self.connect()?, from, to)?;
|
let date = |field: &str, value: &str| {
|
||||||
|
NaiveDate::parse_from_str(value.trim(), "%Y-%m-%d").map_err(|_| {
|
||||||
|
IdxError::InvalidInput(format!(
|
||||||
|
"{field} must be a release date as YYYY-MM-DD (got {value:?})"
|
||||||
|
))
|
||||||
|
})
|
||||||
|
};
|
||||||
|
let (from_date, to_date) = (date("from", from)?, date("to", to)?);
|
||||||
|
if from_date >= to_date {
|
||||||
|
return Err(IdxError::InvalidInput(format!(
|
||||||
|
"from ({from_date}) must be earlier than to ({to_date})"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let (from, to) = (from_date.to_string(), to_date.to_string());
|
||||||
|
let mut rows = db::query_changes(&*self.connect()?, &from, &to)?;
|
||||||
if let Some(ticker) = ticker {
|
if let Some(ticker) = ticker {
|
||||||
let ticker = ticker.trim().to_uppercase();
|
let ticker = ticker.trim().to_uppercase();
|
||||||
rows.retain(|row| row.ticker_code == ticker);
|
rows.retain(|row| row.ticker_code == ticker);
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,6 @@ use rusqlite::{Connection, params};
|
||||||
|
|
||||||
use crate::config::get_config_value;
|
use crate::config::get_config_value;
|
||||||
use crate::error::IdxError;
|
use crate::error::IdxError;
|
||||||
use crate::ownership::search;
|
|
||||||
use crate::ownership::types::{
|
use crate::ownership::types::{
|
||||||
BingHolding, ChangeRow, ChangeType, ConcentrationMetrics, CrossHolderRow, Entity,
|
BingHolding, ChangeRow, ChangeType, ConcentrationMetrics, CrossHolderRow, Entity,
|
||||||
EntityHoldings, EntityTickerRow, FlowSignal, HolderRow, InstitutionalFlow, KseiHolding,
|
EntityHoldings, EntityTickerRow, FlowSignal, HolderRow, InstitutionalFlow, KseiHolding,
|
||||||
|
|
@ -36,14 +35,6 @@ CREATE TABLE IF NOT EXISTS entities (
|
||||||
updated_at INTEGER NOT NULL
|
updated_at INTEGER NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE VIRTUAL TABLE IF NOT EXISTS entity_fts USING fts5(
|
|
||||||
canonical_name,
|
|
||||||
aliases,
|
|
||||||
content='entities',
|
|
||||||
content_rowid='id',
|
|
||||||
tokenize='trigram'
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS entity_aliases (
|
CREATE TABLE IF NOT EXISTS entity_aliases (
|
||||||
id INTEGER PRIMARY KEY,
|
id INTEGER PRIMARY KEY,
|
||||||
entity_id INTEGER NOT NULL REFERENCES entities(id),
|
entity_id INTEGER NOT NULL REFERENCES entities(id),
|
||||||
|
|
@ -916,43 +907,73 @@ pub fn query_bing_flow(
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Compare two KSEI snapshots by date and return ownership changes.
|
/// The release used for a report date: the latest imported one, as
|
||||||
|
/// `latest_ksei_release` picks, so a corrected file supersedes the original.
|
||||||
|
fn release_for_date(conn: &Connection, date: &str) -> Result<Option<String>, IdxError> {
|
||||||
|
conn.query_row(
|
||||||
|
"SELECT sha256 FROM ownership_releases
|
||||||
|
WHERE as_of_date = ?1
|
||||||
|
ORDER BY imported_at DESC, id DESC
|
||||||
|
LIMIT 1",
|
||||||
|
params![date],
|
||||||
|
|row| row.get::<_, String>(0),
|
||||||
|
)
|
||||||
|
.map(Some)
|
||||||
|
.or_else(|e| match e {
|
||||||
|
rusqlite::Error::QueryReturnedNoRows => Ok(None),
|
||||||
|
other => Err(IdxError::DatabaseError(other.to_string())),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compare two KSEI releases by date and return ownership changes.
|
||||||
/// Finds: new holders, exited holders, percentage increases/decreases.
|
/// Finds: new holders, exited holders, percentage increases/decreases.
|
||||||
|
/// Each date reads one release, and a holder's accounts in a ticker are
|
||||||
|
/// summed, so an investor with two accounts compares by its total stake.
|
||||||
pub fn query_changes(
|
pub fn query_changes(
|
||||||
conn: &Connection,
|
conn: &Connection,
|
||||||
from_date: &str,
|
from_date: &str,
|
||||||
to_date: &str,
|
to_date: &str,
|
||||||
) -> Result<Vec<ChangeRow>, IdxError> {
|
) -> Result<Vec<ChangeRow>, IdxError> {
|
||||||
|
let release = |date: &str| {
|
||||||
|
release_for_date(conn, date)?.ok_or_else(|| {
|
||||||
|
IdxError::InvalidInput(format!(
|
||||||
|
"no KSEI release imported for {date}; list the imported dates with `idx ownership releases` or the ownership_status tool"
|
||||||
|
))
|
||||||
|
})
|
||||||
|
};
|
||||||
|
let (from_release, to_release) = (release(from_date)?, release(to_date)?);
|
||||||
let mut stmt = conn
|
let mut stmt = conn
|
||||||
.prepare(
|
.prepare(
|
||||||
"WITH
|
"WITH
|
||||||
from_snapshot AS (
|
from_snapshot AS (
|
||||||
SELECT
|
SELECT
|
||||||
t.code AS ticker_code,
|
t.code AS ticker_code,
|
||||||
COALESCE(e.canonical_name, k.raw_investor_name) AS entity_name,
|
MIN(COALESCE(e.canonical_name, k.raw_investor_name)) AS entity_name,
|
||||||
CASE
|
CASE
|
||||||
WHEN k.entity_id IS NOT NULL THEN 'id:' || k.entity_id
|
WHEN k.entity_id IS NOT NULL THEN 'id:' || k.entity_id
|
||||||
ELSE 'raw:' || UPPER(TRIM(k.raw_investor_name))
|
ELSE 'raw:' || UPPER(TRIM(k.raw_investor_name))
|
||||||
END AS holder_key,
|
END AS holder_key,
|
||||||
k.percentage_bps AS old_bps
|
SUM(k.percentage_bps) AS old_bps
|
||||||
FROM ksei_holdings k
|
FROM ksei_holdings k
|
||||||
JOIN tickers t ON t.id = k.ticker_id
|
JOIN tickers t ON t.id = k.ticker_id
|
||||||
LEFT JOIN entities e ON e.id = k.entity_id
|
LEFT JOIN entities e ON e.id = k.entity_id
|
||||||
WHERE k.report_date = ?1
|
WHERE k.release_sha256 = ?1
|
||||||
|
GROUP BY t.code, holder_key
|
||||||
),
|
),
|
||||||
to_snapshot AS (
|
to_snapshot AS (
|
||||||
SELECT
|
SELECT
|
||||||
t.code AS ticker_code,
|
t.code AS ticker_code,
|
||||||
COALESCE(e.canonical_name, k.raw_investor_name) AS entity_name,
|
MIN(COALESCE(e.canonical_name, k.raw_investor_name)) AS entity_name,
|
||||||
CASE
|
CASE
|
||||||
WHEN k.entity_id IS NOT NULL THEN 'id:' || k.entity_id
|
WHEN k.entity_id IS NOT NULL THEN 'id:' || k.entity_id
|
||||||
ELSE 'raw:' || UPPER(TRIM(k.raw_investor_name))
|
ELSE 'raw:' || UPPER(TRIM(k.raw_investor_name))
|
||||||
END AS holder_key,
|
END AS holder_key,
|
||||||
k.percentage_bps AS new_bps
|
SUM(k.percentage_bps) AS new_bps
|
||||||
FROM ksei_holdings k
|
FROM ksei_holdings k
|
||||||
JOIN tickers t ON t.id = k.ticker_id
|
JOIN tickers t ON t.id = k.ticker_id
|
||||||
LEFT JOIN entities e ON e.id = k.entity_id
|
LEFT JOIN entities e ON e.id = k.entity_id
|
||||||
WHERE k.report_date = ?2
|
WHERE k.release_sha256 = ?2
|
||||||
|
GROUP BY t.code, holder_key
|
||||||
),
|
),
|
||||||
keys AS (
|
keys AS (
|
||||||
SELECT ticker_code, holder_key FROM from_snapshot
|
SELECT ticker_code, holder_key FROM from_snapshot
|
||||||
|
|
@ -978,7 +999,7 @@ pub fn query_changes(
|
||||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||||
|
|
||||||
let rows = stmt
|
let rows = stmt
|
||||||
.query_map(params![from_date, to_date], |row| {
|
.query_map(params![from_release, to_release], |row| {
|
||||||
Ok((
|
Ok((
|
||||||
row.get::<_, String>(0)?,
|
row.get::<_, String>(0)?,
|
||||||
row.get::<_, String>(1)?,
|
row.get::<_, String>(1)?,
|
||||||
|
|
@ -1123,7 +1144,6 @@ pub fn manual_map(conn: &Connection, raw_name: &str, canonical_name: &str) -> Re
|
||||||
)
|
)
|
||||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||||
|
|
||||||
let _ = search::rebuild_fts(conn);
|
|
||||||
Ok(())
|
Ok(())
|
||||||
})();
|
})();
|
||||||
|
|
||||||
|
|
@ -1194,7 +1214,6 @@ pub fn merge_entities(conn: &Connection, keep_id: i64, merge_id: i64) -> Result<
|
||||||
conn.execute("DELETE FROM entities WHERE id = ?1", params![merge_id])
|
conn.execute("DELETE FROM entities WHERE id = ?1", params![merge_id])
|
||||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||||
|
|
||||||
let _ = search::rebuild_fts(conn);
|
|
||||||
Ok(())
|
Ok(())
|
||||||
})();
|
})();
|
||||||
|
|
||||||
|
|
@ -1383,13 +1402,15 @@ mod tests {
|
||||||
use chrono::NaiveDate;
|
use chrono::NaiveDate;
|
||||||
use rusqlite::{Connection, params};
|
use rusqlite::{Connection, params};
|
||||||
|
|
||||||
|
use crate::error::IdxError;
|
||||||
use crate::ownership::db::{
|
use crate::ownership::db::{
|
||||||
compute_concentration, ensure_schema, get_ticker_id, insert_bing_holdings,
|
compute_concentration, ensure_schema, get_ticker_id, insert_bing_holdings,
|
||||||
insert_ksei_holdings, insert_release, query_concentration, query_cross_holders,
|
insert_ksei_holdings, insert_release, query_changes, query_concentration,
|
||||||
query_ticker_holdings, release_exists, upsert_ticker, write_ksei_release,
|
query_cross_holders, query_ticker_holdings, release_exists, upsert_ticker,
|
||||||
|
write_ksei_release,
|
||||||
};
|
};
|
||||||
use crate::ownership::types::{
|
use crate::ownership::types::{
|
||||||
BingHolding, FlowSignal, KseiHolding, Locality, OwnershipRelease,
|
BingHolding, ChangeType, FlowSignal, KseiHolding, Locality, OwnershipRelease,
|
||||||
};
|
};
|
||||||
|
|
||||||
fn setup() -> Connection {
|
fn setup() -> Connection {
|
||||||
|
|
@ -1398,6 +1419,152 @@ mod tests {
|
||||||
conn
|
conn
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Writes one release of `(entity_id, raw_name, bps)` rows for ticker_id.
|
||||||
|
fn write_release(
|
||||||
|
conn: &Connection,
|
||||||
|
sha256: &str,
|
||||||
|
date: &str,
|
||||||
|
imported_at: i64,
|
||||||
|
ticker_id: i64,
|
||||||
|
rows: &[(Option<i64>, &str, i64)],
|
||||||
|
) {
|
||||||
|
let as_of_date = NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap();
|
||||||
|
let holdings: Vec<KseiHolding> = rows
|
||||||
|
.iter()
|
||||||
|
.map(|&(entity_id, raw, bps)| KseiHolding {
|
||||||
|
id: 0,
|
||||||
|
ticker_id,
|
||||||
|
entity_id,
|
||||||
|
raw_investor_name: raw.to_string(),
|
||||||
|
investor_type: None,
|
||||||
|
locality: None,
|
||||||
|
nationality: None,
|
||||||
|
domicile: None,
|
||||||
|
holdings_scripless: 1,
|
||||||
|
holdings_scrip: 0,
|
||||||
|
total_shares: 1,
|
||||||
|
percentage_bps: bps,
|
||||||
|
report_date: as_of_date,
|
||||||
|
release_sha256: sha256.to_string(),
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let release = OwnershipRelease {
|
||||||
|
id: 0,
|
||||||
|
source_url: None,
|
||||||
|
sha256: sha256.to_string(),
|
||||||
|
as_of_date,
|
||||||
|
row_count: holdings.len(),
|
||||||
|
imported_at,
|
||||||
|
};
|
||||||
|
write_ksei_release(conn, &release, &holdings, false).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn entity(conn: &Connection, name: &str) -> i64 {
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO entities (canonical_name, entity_type, country, created_at, updated_at)
|
||||||
|
VALUES (?1, NULL, NULL, 0, 0)",
|
||||||
|
params![name],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
conn.last_insert_rowid()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn changes_sum_an_investors_accounts_in_a_ticker() {
|
||||||
|
// KREN, 2026-09: a second account took KRESNA from 9.44% to 13.45%.
|
||||||
|
let conn = setup();
|
||||||
|
let kren = upsert_ticker(&conn, "KREN", None).unwrap();
|
||||||
|
let kresna = entity(&conn, "KRESNA PRIMA INVEST");
|
||||||
|
write_release(
|
||||||
|
&conn,
|
||||||
|
"aug",
|
||||||
|
"2026-08-31",
|
||||||
|
1,
|
||||||
|
kren,
|
||||||
|
&[(Some(kresna), "PT KRESNA PRIMA INVEST", 944)],
|
||||||
|
);
|
||||||
|
write_release(
|
||||||
|
&conn,
|
||||||
|
"sep",
|
||||||
|
"2026-09-30",
|
||||||
|
2,
|
||||||
|
kren,
|
||||||
|
&[
|
||||||
|
(Some(kresna), "PT KRESNA PRIMA INVEST", 944),
|
||||||
|
(Some(kresna), "PT. KRESNA PRIMA INVEST", 401),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
let rows = query_changes(&conn, "2026-08-31", "2026-09-30").unwrap();
|
||||||
|
|
||||||
|
assert_eq!(rows.len(), 1, "{rows:?}");
|
||||||
|
assert_eq!(rows[0].change_type, ChangeType::Increased);
|
||||||
|
assert_eq!(
|
||||||
|
(rows[0].old_bps, rows[0].new_bps, rows[0].delta_bps),
|
||||||
|
(Some(944), Some(1345), 401)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn changes_read_the_latest_import_for_a_date() {
|
||||||
|
// A corrected January file supersedes the original, so Jan -> Feb at
|
||||||
|
// the corrected 12% shows no change, not a spurious increase.
|
||||||
|
let conn = setup();
|
||||||
|
let bbca = upsert_ticker(&conn, "BBCA", None).unwrap();
|
||||||
|
let alpha = entity(&conn, "ALPHA");
|
||||||
|
write_release(
|
||||||
|
&conn,
|
||||||
|
"jan",
|
||||||
|
"2026-01-30",
|
||||||
|
1,
|
||||||
|
bbca,
|
||||||
|
&[(Some(alpha), "ALPHA", 1000)],
|
||||||
|
);
|
||||||
|
write_release(
|
||||||
|
&conn,
|
||||||
|
"jan-fix",
|
||||||
|
"2026-01-30",
|
||||||
|
2,
|
||||||
|
bbca,
|
||||||
|
&[(Some(alpha), "ALPHA", 1200)],
|
||||||
|
);
|
||||||
|
write_release(
|
||||||
|
&conn,
|
||||||
|
"feb",
|
||||||
|
"2026-02-27",
|
||||||
|
3,
|
||||||
|
bbca,
|
||||||
|
&[(Some(alpha), "ALPHA", 1200)],
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
query_changes(&conn, "2026-01-30", "2026-02-27")
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn changes_reject_a_date_without_a_release() {
|
||||||
|
let conn = setup();
|
||||||
|
let bbca = upsert_ticker(&conn, "BBCA", None).unwrap();
|
||||||
|
write_release(
|
||||||
|
&conn,
|
||||||
|
"feb",
|
||||||
|
"2026-02-27",
|
||||||
|
1,
|
||||||
|
bbca,
|
||||||
|
&[(None, "ALPHA", 1200)],
|
||||||
|
);
|
||||||
|
|
||||||
|
let err = query_changes(&conn, "2026-01-30", "2026-02-27").unwrap_err();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
matches!(err, IdxError::InvalidInput(ref m) if m.contains("2026-01-30")),
|
||||||
|
"{err:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_compute_concentration_pure_function() {
|
fn test_compute_concentration_pure_function() {
|
||||||
let metrics = compute_concentration(&[4000, 2500, 1000, 500]);
|
let metrics = compute_concentration(&[4000, 2500, 1000, 500]);
|
||||||
|
|
|
||||||
|
|
@ -109,34 +109,25 @@ pub fn normalize_name(raw: &str) -> String {
|
||||||
return upper;
|
return upper;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut tokens: Vec<&str> = upper.split_whitespace().collect();
|
let tokens: Vec<&str> = upper.split_whitespace().collect();
|
||||||
|
// Trim prefix and suffix tokens by narrowing a slice: linear, where
|
||||||
loop {
|
// removing from the front of the Vec was quadratic in hostile names.
|
||||||
let mut changed = false;
|
let mut kept = tokens.as_slice();
|
||||||
|
while let [first, rest @ ..] = kept
|
||||||
if let Some(first) = tokens.first().copied()
|
&& matches!(*first, "PT" | "PT.")
|
||||||
&& matches!(first, "PT" | "PT.")
|
|
||||||
{
|
{
|
||||||
tokens.remove(0);
|
kept = rest;
|
||||||
changed = true;
|
|
||||||
}
|
}
|
||||||
|
while let [rest @ .., last] = kept
|
||||||
if let Some(last) = tokens.last().copied()
|
|
||||||
&& matches!(
|
&& matches!(
|
||||||
last,
|
*last,
|
||||||
"TBK" | "Tbk" | "(PERSERO)" | "LIMITED" | "PTE" | "LTD"
|
"TBK" | "Tbk" | "(PERSERO)" | "LIMITED" | "PTE" | "LTD"
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
let _ = tokens.pop();
|
kept = rest;
|
||||||
changed = true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if !changed {
|
collapse_whitespace(&kept.join(" "))
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
collapse_whitespace(&tokens.join(" "))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Convert a KseiRawRow into a normalized KseiHoldingDraft.
|
/// Convert a KseiRawRow into a normalized KseiHoldingDraft.
|
||||||
|
|
@ -357,6 +348,14 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalize_name_strips_many_prefixes_in_linear_time() {
|
||||||
|
// Removing from the front of a Vec made this quadratic (~5e9 shifts).
|
||||||
|
let hostile = format!("{}ACME TBK", "PT ".repeat(100_000));
|
||||||
|
assert_eq!(normalize_name(&hostile), "ACME");
|
||||||
|
assert_eq!(normalize_name("PT TBK"), "");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_normalize_name() {
|
fn test_normalize_name() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|
|
||||||
|
|
@ -21,75 +21,31 @@ pub fn query_ownership_graph(
|
||||||
|
|
||||||
let root_node_id = detect_root_node(conn, root)?;
|
let root_node_id = detect_root_node(conn, root)?;
|
||||||
|
|
||||||
let mut visited: BTreeSet<String> = BTreeSet::new();
|
// Breadth-first over the latest edges with an exact visited set: linear
|
||||||
{
|
// in the graph size, unlike enumerating every path, and a node is never
|
||||||
let mut stmt = conn
|
// mistaken for one whose id merely contains it (`entity:1` in `entity:10`).
|
||||||
.prepare(
|
|
||||||
"WITH RECURSIVE
|
|
||||||
latest_ksei AS (
|
|
||||||
SELECT sha256
|
|
||||||
FROM ownership_releases
|
|
||||||
ORDER BY as_of_date DESC, imported_at DESC
|
|
||||||
LIMIT 1
|
|
||||||
),
|
|
||||||
latest_bing AS (
|
|
||||||
SELECT ticker_id, MAX(report_date) AS report_date
|
|
||||||
FROM bing_holdings
|
|
||||||
GROUP BY ticker_id
|
|
||||||
),
|
|
||||||
all_edges AS (
|
|
||||||
SELECT
|
|
||||||
'entity:' || k.entity_id AS from_id,
|
|
||||||
'ticker:' || t.code AS to_id
|
|
||||||
FROM ksei_holdings k
|
|
||||||
JOIN tickers t ON t.id = k.ticker_id
|
|
||||||
JOIN latest_ksei lk ON lk.sha256 = k.release_sha256
|
|
||||||
WHERE k.entity_id IS NOT NULL
|
|
||||||
|
|
||||||
UNION
|
|
||||||
|
|
||||||
SELECT
|
|
||||||
'entity:' || b.entity_id AS from_id,
|
|
||||||
'ticker:' || t.code AS to_id
|
|
||||||
FROM bing_holdings b
|
|
||||||
JOIN latest_bing lb
|
|
||||||
ON lb.ticker_id = b.ticker_id
|
|
||||||
AND lb.report_date = b.report_date
|
|
||||||
JOIN tickers t ON t.id = b.ticker_id
|
|
||||||
WHERE b.entity_id IS NOT NULL
|
|
||||||
AND b.signal = 'holder'
|
|
||||||
),
|
|
||||||
neighbors AS (
|
|
||||||
SELECT from_id AS a, to_id AS b FROM all_edges
|
|
||||||
UNION
|
|
||||||
SELECT to_id AS a, from_id AS b FROM all_edges
|
|
||||||
),
|
|
||||||
walk(node_id, depth, path) AS (
|
|
||||||
SELECT ?1 AS node_id, 0 AS depth, ?1 AS path
|
|
||||||
UNION ALL
|
|
||||||
SELECT n.b, w.depth + 1, w.path || '>' || n.b
|
|
||||||
FROM walk w
|
|
||||||
JOIN neighbors n ON n.a = w.node_id
|
|
||||||
WHERE w.depth < ?2
|
|
||||||
AND instr(w.path, n.b) = 0
|
|
||||||
)
|
|
||||||
SELECT DISTINCT node_id
|
|
||||||
FROM walk",
|
|
||||||
)
|
|
||||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
||||||
|
|
||||||
let rows = stmt
|
|
||||||
.query_map(params![root_node_id, depth as i64], |row| {
|
|
||||||
row.get::<_, String>(0)
|
|
||||||
})
|
|
||||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
||||||
|
|
||||||
for row in rows {
|
|
||||||
visited.insert(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut edges = query_all_edges(conn)?;
|
let mut edges = query_all_edges(conn)?;
|
||||||
|
let mut neighbors: HashMap<&str, Vec<&str>> = HashMap::new();
|
||||||
|
for edge in &edges {
|
||||||
|
neighbors.entry(&edge.from).or_default().push(&edge.to);
|
||||||
|
neighbors.entry(&edge.to).or_default().push(&edge.from);
|
||||||
|
}
|
||||||
|
let mut visited: BTreeSet<String> = BTreeSet::from([root_node_id.clone()]);
|
||||||
|
let mut frontier = vec![root_node_id.as_str()];
|
||||||
|
for _ in 0..depth {
|
||||||
|
let mut next = Vec::new();
|
||||||
|
for node in frontier {
|
||||||
|
for &neighbor in neighbors.get(node).into_iter().flatten() {
|
||||||
|
if visited.insert(neighbor.to_string()) {
|
||||||
|
next.push(neighbor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if next.is_empty() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
frontier = next;
|
||||||
|
}
|
||||||
edges.retain(|edge| visited.contains(&edge.from) && visited.contains(&edge.to));
|
edges.retain(|edge| visited.contains(&edge.from) && visited.contains(&edge.to));
|
||||||
|
|
||||||
let nodes = build_nodes(conn, &visited)?;
|
let nodes = build_nodes(conn, &visited)?;
|
||||||
|
|
@ -380,6 +336,58 @@ mod tests {
|
||||||
conn
|
conn
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn graph_reaches_holders_whose_id_is_a_prefix_of_a_visited_one() {
|
||||||
|
// From entity:10 via ticker:BB to entity:1; "entity:1" is a substring
|
||||||
|
// of "entity:10", which the old path check treated as visited.
|
||||||
|
let conn = setup();
|
||||||
|
let bb = upsert_ticker(&conn, "BB", None).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO entities (id, canonical_name, created_at, updated_at)
|
||||||
|
VALUES (10, 'HOLDER TEN', 0, 0), (1, 'HOLDER ONE', 0, 0)",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let release = OwnershipRelease {
|
||||||
|
id: 0,
|
||||||
|
source_url: None,
|
||||||
|
sha256: "prefix-graph".to_string(),
|
||||||
|
as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
||||||
|
row_count: 2,
|
||||||
|
imported_at: 1,
|
||||||
|
};
|
||||||
|
let row = |entity_id: i64, name: &str| KseiHolding {
|
||||||
|
id: 0,
|
||||||
|
ticker_id: bb,
|
||||||
|
entity_id: Some(entity_id),
|
||||||
|
raw_investor_name: name.to_string(),
|
||||||
|
investor_type: None,
|
||||||
|
locality: None,
|
||||||
|
nationality: None,
|
||||||
|
domicile: None,
|
||||||
|
holdings_scripless: 1,
|
||||||
|
holdings_scrip: 0,
|
||||||
|
total_shares: 1,
|
||||||
|
percentage_bps: 600,
|
||||||
|
report_date: release.as_of_date,
|
||||||
|
release_sha256: release.sha256.clone(),
|
||||||
|
};
|
||||||
|
write_ksei_release(
|
||||||
|
&conn,
|
||||||
|
&release,
|
||||||
|
&[row(10, "HOLDER TEN"), row(1, "HOLDER ONE")],
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let (nodes, edges) = query_ownership_graph(&conn, "HOLDER TEN", 2).unwrap();
|
||||||
|
|
||||||
|
let mut labels: Vec<_> = nodes.iter().map(|node| node.label.as_str()).collect();
|
||||||
|
labels.sort();
|
||||||
|
assert_eq!(labels, ["BB", "HOLDER ONE", "HOLDER TEN"]);
|
||||||
|
assert_eq!(edges.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn graph_uses_latest_ksei_release_only() {
|
fn graph_uses_latest_ksei_release_only() {
|
||||||
let conn = setup();
|
let conn = setup();
|
||||||
|
|
|
||||||
|
|
@ -3,109 +3,133 @@ use rusqlite::{Connection, params};
|
||||||
use crate::error::IdxError;
|
use crate::error::IdxError;
|
||||||
use crate::ownership::types::Entity;
|
use crate::ownership::types::Entity;
|
||||||
|
|
||||||
pub fn fts_search(conn: &Connection, query: &str, limit: usize) -> Result<Vec<Entity>, IdxError> {
|
/// Investors whose canonical name or any recorded alias (the raw names KSEI
|
||||||
|
/// prints, such as `PT ASTRA INTERNATIONAL TBK`) contains `query`, ignoring
|
||||||
|
/// ASCII case. Ranked by an exact name, then an exact alias, then a name
|
||||||
|
/// prefix, then shorter names.
|
||||||
|
pub fn search_entities(
|
||||||
|
conn: &Connection,
|
||||||
|
query: &str,
|
||||||
|
limit: usize,
|
||||||
|
) -> Result<Vec<Entity>, IdxError> {
|
||||||
let q = query.trim();
|
let q = query.trim();
|
||||||
if q.is_empty() {
|
if q.is_empty() {
|
||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
|
let limit =
|
||||||
let limit_i64 =
|
|
||||||
i64::try_from(limit).map_err(|e| IdxError::DatabaseError(format!("invalid limit: {e}")))?;
|
i64::try_from(limit).map_err(|e| IdxError::DatabaseError(format!("invalid limit: {e}")))?;
|
||||||
|
// `%` and `_` in the query are literal characters, not LIKE wildcards.
|
||||||
|
let escaped = q
|
||||||
|
.replace('\\', "\\\\")
|
||||||
|
.replace('%', "\\%")
|
||||||
|
.replace('_', "\\_");
|
||||||
|
let contains = format!("%{escaped}%");
|
||||||
|
let prefix = format!("{escaped}%");
|
||||||
|
|
||||||
// Ensure entity_fts has content. For external-content FTS table, populate manually.
|
|
||||||
let fts_count: i64 = conn
|
|
||||||
.query_row("SELECT COUNT(*) FROM entity_fts", [], |row| row.get(0))
|
|
||||||
.unwrap_or(0);
|
|
||||||
|
|
||||||
if fts_count == 0 {
|
|
||||||
let _ = rebuild_fts(conn);
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Ok(rows) = fts_query(conn, q, limit_i64)
|
|
||||||
&& !rows.is_empty()
|
|
||||||
{
|
|
||||||
return Ok(rows);
|
|
||||||
}
|
|
||||||
|
|
||||||
like_query(conn, q, limit_i64)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn fts_query(conn: &Connection, query: &str, limit: i64) -> Result<Vec<Entity>, IdxError> {
|
|
||||||
let mut stmt = conn
|
let mut stmt = conn
|
||||||
.prepare(
|
.prepare(
|
||||||
"SELECT e.id, e.canonical_name, e.entity_type, e.country
|
"SELECT e.id, e.canonical_name, e.entity_type, e.country,
|
||||||
FROM entity_fts f
|
MIN(CASE
|
||||||
JOIN entities e ON e.id = f.rowid
|
WHEN UPPER(e.canonical_name) = UPPER(?1) THEN 0
|
||||||
WHERE entity_fts MATCH ?1
|
WHEN UPPER(a.raw_name) = UPPER(?1) THEN 1
|
||||||
ORDER BY rank
|
WHEN e.canonical_name LIKE ?3 ESCAPE '\\' THEN 2
|
||||||
LIMIT ?2",
|
ELSE 3
|
||||||
)
|
END) AS match_rank
|
||||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
||||||
|
|
||||||
let rows = stmt
|
|
||||||
.query_map(params![query, limit], |row| {
|
|
||||||
Ok(Entity {
|
|
||||||
id: row.get(0)?,
|
|
||||||
canonical_name: row.get(1)?,
|
|
||||||
entity_type: row.get(2)?,
|
|
||||||
country: row.get(3)?,
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
||||||
|
|
||||||
let mut out = Vec::new();
|
|
||||||
for row in rows {
|
|
||||||
out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
|
||||||
}
|
|
||||||
Ok(out)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn like_query(conn: &Connection, query: &str, limit: i64) -> Result<Vec<Entity>, IdxError> {
|
|
||||||
let mut stmt = conn
|
|
||||||
.prepare(
|
|
||||||
"SELECT id, canonical_name, entity_type, country
|
|
||||||
FROM entities
|
|
||||||
WHERE canonical_name LIKE ?1 COLLATE NOCASE
|
|
||||||
ORDER BY canonical_name ASC
|
|
||||||
LIMIT ?2",
|
|
||||||
)
|
|
||||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
||||||
|
|
||||||
let like = format!("%{query}%");
|
|
||||||
let rows = stmt
|
|
||||||
.query_map(params![like, limit], |row| {
|
|
||||||
Ok(Entity {
|
|
||||||
id: row.get(0)?,
|
|
||||||
canonical_name: row.get(1)?,
|
|
||||||
entity_type: row.get(2)?,
|
|
||||||
country: row.get(3)?,
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
||||||
|
|
||||||
let mut out = Vec::new();
|
|
||||||
for row in rows {
|
|
||||||
out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
|
||||||
}
|
|
||||||
Ok(out)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn rebuild_fts(conn: &Connection) -> Result<(), IdxError> {
|
|
||||||
conn.execute("DELETE FROM entity_fts", [])
|
|
||||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
|
||||||
|
|
||||||
conn.execute(
|
|
||||||
"INSERT INTO entity_fts(rowid, canonical_name, aliases)
|
|
||||||
SELECT e.id, e.canonical_name, COALESCE(a.aliases, '')
|
|
||||||
FROM entities e
|
FROM entities e
|
||||||
LEFT JOIN (
|
LEFT JOIN entity_aliases a ON a.entity_id = e.id
|
||||||
SELECT entity_id, GROUP_CONCAT(raw_name, ' ') AS aliases
|
WHERE e.canonical_name LIKE ?2 ESCAPE '\\'
|
||||||
FROM entity_aliases
|
OR a.raw_name LIKE ?2 ESCAPE '\\'
|
||||||
GROUP BY entity_id
|
GROUP BY e.id
|
||||||
) a ON a.entity_id = e.id",
|
ORDER BY match_rank, LENGTH(e.canonical_name), e.canonical_name
|
||||||
[],
|
LIMIT ?4",
|
||||||
)
|
)
|
||||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||||
|
|
||||||
Ok(())
|
let rows = stmt
|
||||||
|
.query_map(params![q, contains, prefix, limit], |row| {
|
||||||
|
Ok(Entity {
|
||||||
|
id: row.get(0)?,
|
||||||
|
canonical_name: row.get(1)?,
|
||||||
|
entity_type: row.get(2)?,
|
||||||
|
country: row.get(3)?,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||||
|
|
||||||
|
rows.map(|row| row.map_err(|e| IdxError::DatabaseError(e.to_string())))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use rusqlite::{Connection, params};
|
||||||
|
|
||||||
|
use super::search_entities;
|
||||||
|
use crate::ownership::db::ensure_schema;
|
||||||
|
|
||||||
|
fn conn_with(entities: &[(&str, &[&str])]) -> Connection {
|
||||||
|
let conn = Connection::open_in_memory().unwrap();
|
||||||
|
ensure_schema(&conn).unwrap();
|
||||||
|
for (name, aliases) in entities {
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO entities (canonical_name, created_at, updated_at) VALUES (?1, 0, 0)",
|
||||||
|
params![name],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let id = conn.last_insert_rowid();
|
||||||
|
for alias in *aliases {
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO entity_aliases (entity_id, raw_name, source, method)
|
||||||
|
VALUES (?1, ?2, 'ksei', 'exact')",
|
||||||
|
params![id, alias],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
conn
|
||||||
|
}
|
||||||
|
|
||||||
|
fn names(conn: &Connection, query: &str) -> Vec<String> {
|
||||||
|
search_entities(conn, query, 10)
|
||||||
|
.unwrap()
|
||||||
|
.into_iter()
|
||||||
|
.map(|e| e.canonical_name)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn finds_investors_by_alias() {
|
||||||
|
let conn = conn_with(&[
|
||||||
|
("ASTRA INTERNATIONAL", &["PT ASTRA INTERNATIONAL TBK"]),
|
||||||
|
("ASTRA OTOPARTS", &[]),
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
names(&conn, "pt astra international tbk"),
|
||||||
|
["ASTRA INTERNATIONAL"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ranks_exact_then_prefix_then_shorter_and_lists_each_once() {
|
||||||
|
let conn = conn_with(&[
|
||||||
|
("SALIM GROUP HOLDINGS", &["PT SALIM", "SALIM"]),
|
||||||
|
("ANTHONI SALIM", &[]),
|
||||||
|
("SALIM", &[]),
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
names(&conn, "salim"),
|
||||||
|
["SALIM", "SALIM GROUP HOLDINGS", "ANTHONI SALIM"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn treats_like_wildcards_literally_and_empty_queries_as_no_match() {
|
||||||
|
let conn = conn_with(&[("ALPHA_FUND", &[]), ("ALPHAXFUND", &[]), ("BETA 100%", &[])]);
|
||||||
|
|
||||||
|
assert_eq!(names(&conn, "alpha_fund"), ["ALPHA_FUND"]);
|
||||||
|
assert_eq!(names(&conn, "100%"), ["BETA 100%"]);
|
||||||
|
assert!(names(&conn, " ").is_empty());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -394,6 +394,22 @@ fn ownership_tools_read_the_local_database() {
|
||||||
assert!(page["next_offset"].is_null(), "offset {past}: {page}");
|
assert!(page["next_offset"].is_null(), "offset {past}: {page}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Bad, unknown, or reversed dates are errors, not "everyone is new".
|
||||||
|
for (from, to) in [
|
||||||
|
("", "2026-02-27"),
|
||||||
|
("1999-01-01", "2026-02-27"),
|
||||||
|
("2026-02-27", "2026-01-30"),
|
||||||
|
("2026-01-30", "2026-01-30"),
|
||||||
|
] {
|
||||||
|
let (is_error, body) =
|
||||||
|
client.call("get_ownership_changes", json!({"from": from, "to": to}));
|
||||||
|
assert!(is_error, "{from}..{to}: {body}");
|
||||||
|
assert_eq!(body["code"], "INVALIDINPUT", "{from}..{to}: {body}");
|
||||||
|
}
|
||||||
|
let (is_error, body) = client.call("get_investor_holdings", json!({"entity": " "}));
|
||||||
|
assert!(is_error, "{body}");
|
||||||
|
assert_eq!(body["code"], "INVALIDINPUT", "{body}");
|
||||||
|
|
||||||
let (is_error, body) = client.call(
|
let (is_error, body) = client.call(
|
||||||
"get_ownership_changes",
|
"get_ownership_changes",
|
||||||
json!({"from": "2026-01-30", "to": "2026-02-27", "limit": 301}),
|
json!({"from": "2026-01-30", "to": "2026-02-27", "limit": 301}),
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue