mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-08-07 09:43:53 +00:00
feat(ownership): complete ownership intelligence module
Sprints 0-8: Types, schema, KSEI parser, entity resolution, DB CRUD, Bing API client, import pipeline, query commands, graph traversal, changes diff, entity resolution CLI, FTS5 search. - KSEI PDF parser (mutool stext + quick-xml, 99.2% accuracy) - 7 query commands: ticker, entity, search, cross-holders, concentration, flow, releases - Ownership graph with recursive CTE (ASCII tree + Graphviz DOT) - Release diff/changes between KSEI snapshots - Entity resolution CLI: unresolved, map, merge - FTS5 trigram search on entity names - Feature-gated under 'ownership' (default-on) - 82 tests passing
This commit is contained in:
parent
d49a8ed60e
commit
6671e22976
23 changed files with 5351 additions and 47 deletions
1447
src/ownership/db.rs
Normal file
1447
src/ownership/db.rs
Normal file
File diff suppressed because it is too large
Load diff
404
src/ownership/entities.rs
Normal file
404
src/ownership/entities.rs
Normal file
|
|
@ -0,0 +1,404 @@
|
|||
use chrono::NaiveDate;
|
||||
use rusqlite::{Connection, params};
|
||||
|
||||
use crate::error::IdxError;
|
||||
use crate::ownership::types::{
|
||||
InvestorTypeCode, KseiHoldingDraft, KseiRawRow, Locality, OwnershipSource,
|
||||
};
|
||||
|
||||
/// Parse Indonesian locale number string to i64.
|
||||
/// "1.533.682.440" → 1533682440, "0" → 0
|
||||
pub fn parse_id_number(s: &str) -> Result<i64, IdxError> {
|
||||
let trimmed = s.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"empty Indonesian number string".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let normalized = trimmed.replace(['.', ' '], "");
|
||||
if normalized.is_empty() || !normalized.chars().all(|c| c.is_ascii_digit()) {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"invalid Indonesian number format: {s}"
|
||||
)));
|
||||
}
|
||||
|
||||
normalized
|
||||
.parse::<i64>()
|
||||
.map_err(|e| IdxError::ParseError(format!("failed to parse Indonesian number '{s}': {e}")))
|
||||
}
|
||||
|
||||
/// Parse Indonesian locale percentage to basis points (i64).
|
||||
/// "54,94" → 5494, "0,00" → 0, "100,00" → 10000
|
||||
pub fn parse_id_percentage(s: &str) -> Result<i64, IdxError> {
|
||||
let trimmed = s.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"empty Indonesian percentage string".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut parts = trimmed.split(',');
|
||||
let whole = parts.next().unwrap_or_default();
|
||||
let frac = parts.next().unwrap_or("00");
|
||||
|
||||
if parts.next().is_some() {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"invalid Indonesian percentage format: {s}"
|
||||
)));
|
||||
}
|
||||
|
||||
if whole.is_empty() || !whole.chars().all(|c| c.is_ascii_digit()) {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"invalid Indonesian percentage whole part: {s}"
|
||||
)));
|
||||
}
|
||||
|
||||
if !frac.chars().all(|c| c.is_ascii_digit()) || frac.len() > 2 {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"invalid Indonesian percentage fractional part: {s}"
|
||||
)));
|
||||
}
|
||||
|
||||
let whole_i = whole.parse::<i64>().map_err(|e| {
|
||||
IdxError::ParseError(format!("failed to parse Indonesian percentage '{s}': {e}"))
|
||||
})?;
|
||||
|
||||
let frac_padded = if frac.len() == 1 {
|
||||
format!("{frac}0")
|
||||
} else {
|
||||
frac.to_string()
|
||||
};
|
||||
let frac_i = frac_padded.parse::<i64>().map_err(|e| {
|
||||
IdxError::ParseError(format!("failed to parse Indonesian percentage '{s}': {e}"))
|
||||
})?;
|
||||
|
||||
Ok((whole_i * 100) + frac_i)
|
||||
}
|
||||
|
||||
/// Parse KSEI date format to NaiveDate.
|
||||
/// "27-Feb-2026" → NaiveDate(2026, 2, 27)
|
||||
pub fn parse_ksei_date(s: &str) -> Result<NaiveDate, IdxError> {
|
||||
let trimmed = s.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(IdxError::ParseError("empty KSEI date string".to_string()));
|
||||
}
|
||||
|
||||
NaiveDate::parse_from_str(trimmed, "%d-%b-%Y")
|
||||
.map_err(|e| IdxError::ParseError(format!("invalid KSEI date '{s}': {e}")))
|
||||
}
|
||||
|
||||
/// Normalize an investor name for entity matching.
|
||||
/// Strips: "PT.", "PT ", "TBK", "Tbk", "(PERSERO)", "LIMITED", "PTE", "LTD"
|
||||
/// Collapses whitespace, trims, uppercases.
|
||||
pub fn normalize_name(raw: &str) -> String {
|
||||
let upper = collapse_whitespace(raw).to_uppercase();
|
||||
if upper.is_empty() {
|
||||
return upper;
|
||||
}
|
||||
|
||||
let mut tokens: Vec<&str> = upper.split_whitespace().collect();
|
||||
|
||||
loop {
|
||||
let mut changed = false;
|
||||
|
||||
if let Some(first) = tokens.first().copied()
|
||||
&& matches!(first, "PT" | "PT.")
|
||||
{
|
||||
tokens.remove(0);
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if let Some(last) = tokens.last().copied()
|
||||
&& matches!(
|
||||
last,
|
||||
"TBK" | "Tbk" | "(PERSERO)" | "LIMITED" | "PTE" | "LTD"
|
||||
)
|
||||
{
|
||||
let _ = tokens.pop();
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if !changed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
collapse_whitespace(&tokens.join(" "))
|
||||
}
|
||||
|
||||
/// Convert a KseiRawRow into a normalized KseiHoldingDraft.
|
||||
/// Applies all parsing functions above.
|
||||
pub fn normalize_ksei_row(raw: &KseiRawRow) -> Result<KseiHoldingDraft, IdxError> {
|
||||
let investor_type = normalize_investor_type(&raw.investor_type);
|
||||
let locality = normalize_locality(&raw.local_foreign);
|
||||
|
||||
Ok(KseiHoldingDraft {
|
||||
ticker_code: raw.share_code.trim().to_string(),
|
||||
issuer_name: optional_string(&raw.issuer_name),
|
||||
raw_investor_name: raw.investor_name.trim().to_string(),
|
||||
investor_type,
|
||||
locality,
|
||||
nationality: optional_string(&raw.nationality),
|
||||
domicile: optional_string(&raw.domicile),
|
||||
holdings_scripless: parse_id_number(&raw.holdings_scripless)?,
|
||||
holdings_scrip: parse_id_number(&raw.holdings_scrip)?,
|
||||
total_shares: parse_id_number(&raw.total_holding_shares)?,
|
||||
percentage_bps: parse_id_percentage(&raw.percentage)?,
|
||||
report_date: parse_ksei_date(&raw.date)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Find or create a canonical entity for a raw investor name.
|
||||
/// Strategy: exact match on normalized name → rule-based normalization → create new.
|
||||
/// Returns entity_id.
|
||||
pub fn resolve_entity(
|
||||
conn: &Connection,
|
||||
raw_name: &str,
|
||||
source: OwnershipSource,
|
||||
) -> Result<i64, IdxError> {
|
||||
let source_db = source_to_db(source);
|
||||
let raw_trimmed = raw_name.trim();
|
||||
|
||||
if raw_trimmed.is_empty() {
|
||||
return Err(IdxError::ParseError("empty raw entity name".to_string()));
|
||||
}
|
||||
|
||||
let exact_normalized = collapse_whitespace(raw_trimmed).to_uppercase();
|
||||
let rule_normalized = normalize_name(raw_trimmed);
|
||||
|
||||
if let Some(entity_id) = find_entity_by_alias(conn, raw_trimmed, source_db)? {
|
||||
return Ok(entity_id);
|
||||
}
|
||||
|
||||
if let Some(entity_id) = find_entity_by_canonical(conn, &exact_normalized)? {
|
||||
insert_alias(conn, entity_id, raw_trimmed, source_db, "exact")?;
|
||||
return Ok(entity_id);
|
||||
}
|
||||
|
||||
if rule_normalized != exact_normalized
|
||||
&& let Some(entity_id) = find_entity_by_canonical(conn, &rule_normalized)?
|
||||
{
|
||||
insert_alias(conn, entity_id, raw_trimmed, source_db, "rule")?;
|
||||
return Ok(entity_id);
|
||||
}
|
||||
|
||||
let now: i64 = chrono::Utc::now().timestamp();
|
||||
conn.execute(
|
||||
"INSERT INTO entities (canonical_name, entity_type, country, created_at, updated_at)
|
||||
VALUES (?1, NULL, NULL, ?2, ?2)",
|
||||
params![
|
||||
if rule_normalized.is_empty() {
|
||||
&exact_normalized
|
||||
} else {
|
||||
&rule_normalized
|
||||
},
|
||||
now
|
||||
],
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(format!("insert entity failed: {e}")))?;
|
||||
|
||||
let entity_id = conn.last_insert_rowid();
|
||||
insert_alias(conn, entity_id, raw_trimmed, source_db, "exact")?;
|
||||
|
||||
Ok(entity_id)
|
||||
}
|
||||
|
||||
fn normalize_investor_type(raw: &str) -> Option<InvestorTypeCode> {
|
||||
let value = raw.trim();
|
||||
if value.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(InvestorTypeCode(value.to_uppercase()))
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_locality(raw: &str) -> Option<Locality> {
|
||||
match raw.trim().to_uppercase().as_str() {
|
||||
"L" => Some(Locality::Local),
|
||||
"F" | "A" => Some(Locality::Foreign),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn optional_string(raw: &str) -> Option<String> {
|
||||
let value = raw.trim();
|
||||
if value.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(value.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn collapse_whitespace(input: &str) -> String {
|
||||
input.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
fn source_to_db(source: OwnershipSource) -> &'static str {
|
||||
match source {
|
||||
OwnershipSource::Ksei => "ksei",
|
||||
OwnershipSource::Bing => "bing",
|
||||
}
|
||||
}
|
||||
|
||||
fn find_entity_by_alias(
|
||||
conn: &Connection,
|
||||
raw_name: &str,
|
||||
source: &str,
|
||||
) -> Result<Option<i64>, IdxError> {
|
||||
conn.query_row(
|
||||
"SELECT entity_id FROM entity_aliases WHERE raw_name = ?1 AND source = ?2",
|
||||
params![raw_name, source],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.map(Some)
|
||||
.or_else(|e| match e {
|
||||
rusqlite::Error::QueryReturnedNoRows => Ok(None),
|
||||
_ => Err(IdxError::DatabaseError(e.to_string())),
|
||||
})
|
||||
}
|
||||
|
||||
fn find_entity_by_canonical(
|
||||
conn: &Connection,
|
||||
canonical_name: &str,
|
||||
) -> Result<Option<i64>, IdxError> {
|
||||
conn.query_row(
|
||||
"SELECT id FROM entities WHERE canonical_name = ?1",
|
||||
params![canonical_name],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.map(Some)
|
||||
.or_else(|e| match e {
|
||||
rusqlite::Error::QueryReturnedNoRows => Ok(None),
|
||||
_ => Err(IdxError::DatabaseError(e.to_string())),
|
||||
})
|
||||
}
|
||||
|
||||
fn insert_alias(
|
||||
conn: &Connection,
|
||||
entity_id: i64,
|
||||
raw_name: &str,
|
||||
source: &str,
|
||||
method: &str,
|
||||
) -> Result<(), IdxError> {
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO entity_aliases (entity_id, raw_name, source, confidence, method)
|
||||
VALUES (?1, ?2, ?3, 1.0, ?4)",
|
||||
params![entity_id, raw_name, source, method],
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(format!("insert alias failed: {e}")))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use chrono::NaiveDate;
|
||||
use rusqlite::Connection;
|
||||
|
||||
use crate::ownership::entities::{
|
||||
normalize_name, parse_id_number, parse_id_percentage, parse_ksei_date, resolve_entity,
|
||||
};
|
||||
use crate::ownership::types::OwnershipSource;
|
||||
|
||||
#[test]
|
||||
fn test_parse_id_number() {
|
||||
assert_eq!(parse_id_number("1.533.682.440").unwrap(), 1_533_682_440);
|
||||
assert_eq!(parse_id_number("0").unwrap(), 0);
|
||||
assert_eq!(parse_id_number("3.200.142.830").unwrap(), 3_200_142_830);
|
||||
assert!(parse_id_number("").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_id_percentage() {
|
||||
assert_eq!(parse_id_percentage("54,94").unwrap(), 5494);
|
||||
assert_eq!(parse_id_percentage("0,00").unwrap(), 0);
|
||||
assert_eq!(parse_id_percentage("100,00").unwrap(), 10000);
|
||||
assert_eq!(parse_id_percentage("41,10").unwrap(), 4110);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_ksei_date_various_months() {
|
||||
let samples = [
|
||||
("01-Jan-2026", NaiveDate::from_ymd_opt(2026, 1, 1).unwrap()),
|
||||
("01-Feb-2026", NaiveDate::from_ymd_opt(2026, 2, 1).unwrap()),
|
||||
("01-Mar-2026", NaiveDate::from_ymd_opt(2026, 3, 1).unwrap()),
|
||||
("01-Apr-2026", NaiveDate::from_ymd_opt(2026, 4, 1).unwrap()),
|
||||
("01-May-2026", NaiveDate::from_ymd_opt(2026, 5, 1).unwrap()),
|
||||
("01-Jun-2026", NaiveDate::from_ymd_opt(2026, 6, 1).unwrap()),
|
||||
("01-Jul-2026", NaiveDate::from_ymd_opt(2026, 7, 1).unwrap()),
|
||||
("01-Aug-2026", NaiveDate::from_ymd_opt(2026, 8, 1).unwrap()),
|
||||
("01-Sep-2026", NaiveDate::from_ymd_opt(2026, 9, 1).unwrap()),
|
||||
("01-Oct-2026", NaiveDate::from_ymd_opt(2026, 10, 1).unwrap()),
|
||||
("01-Nov-2026", NaiveDate::from_ymd_opt(2026, 11, 1).unwrap()),
|
||||
("01-Dec-2026", NaiveDate::from_ymd_opt(2026, 12, 1).unwrap()),
|
||||
];
|
||||
|
||||
for (input, expected) in samples {
|
||||
assert_eq!(parse_ksei_date(input).unwrap(), expected);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
parse_ksei_date("27-Feb-2026").unwrap(),
|
||||
NaiveDate::from_ymd_opt(2026, 2, 27).unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_normalize_name() {
|
||||
assert_eq!(
|
||||
normalize_name("PT. ASTRA INTERNATIONAL TBK"),
|
||||
"ASTRA INTERNATIONAL"
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_name("PT BANK CENTRAL ASIA Tbk"),
|
||||
"BANK CENTRAL ASIA"
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_name("UOB KAY HIAN PRIVATE LIMITED"),
|
||||
"UOB KAY HIAN PRIVATE"
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_name("DJS Ketenagakerjaan (JHT)"),
|
||||
"DJS KETENAGAKERJAAN (JHT)"
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_name("BPJS KETENAGAKERJAAN"),
|
||||
"BPJS KETENAGAKERJAAN"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_entity_create_then_reuse() {
|
||||
let conn = Connection::open_in_memory().unwrap();
|
||||
conn.execute_batch(
|
||||
r#"
|
||||
CREATE TABLE entities (
|
||||
id INTEGER PRIMARY KEY,
|
||||
canonical_name TEXT NOT NULL,
|
||||
entity_type TEXT,
|
||||
country TEXT,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE entity_aliases (
|
||||
id INTEGER PRIMARY KEY,
|
||||
entity_id INTEGER NOT NULL,
|
||||
raw_name TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
confidence REAL NOT NULL DEFAULT 1.0,
|
||||
method TEXT NOT NULL,
|
||||
UNIQUE(raw_name, source)
|
||||
);
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let id1 =
|
||||
resolve_entity(&conn, "PT. ASTRA INTERNATIONAL TBK", OwnershipSource::Ksei).unwrap();
|
||||
let id2 =
|
||||
resolve_entity(&conn, "PT. ASTRA INTERNATIONAL TBK", OwnershipSource::Ksei).unwrap();
|
||||
|
||||
assert_eq!(id1, id2);
|
||||
}
|
||||
}
|
||||
332
src/ownership/graph.rs
Normal file
332
src/ownership/graph.rs
Normal file
|
|
@ -0,0 +1,332 @@
|
|||
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
|
||||
|
||||
use rusqlite::{Connection, params};
|
||||
|
||||
use crate::error::IdxError;
|
||||
use crate::ownership::types::{GraphEdge, GraphNode, GraphNodeType, OwnershipSource};
|
||||
|
||||
/// Query ownership graph starting from a ticker or entity, traversing N hops.
|
||||
/// Root can be ticker code (`BBCA`) or entity name — auto-detect.
|
||||
pub fn query_ownership_graph(
|
||||
conn: &Connection,
|
||||
root: &str,
|
||||
depth: usize,
|
||||
) -> Result<(Vec<GraphNode>, Vec<GraphEdge>), IdxError> {
|
||||
let root = root.trim();
|
||||
if root.is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"graph root cannot be empty".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let root_node_id = detect_root_node(conn, root)?;
|
||||
|
||||
let mut visited: BTreeSet<String> = BTreeSet::new();
|
||||
{
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"WITH RECURSIVE
|
||||
all_edges AS (
|
||||
SELECT
|
||||
'entity:' || k.entity_id AS from_id,
|
||||
'ticker:' || t.code AS to_id
|
||||
FROM ksei_holdings k
|
||||
JOIN tickers t ON t.id = k.ticker_id
|
||||
WHERE k.entity_id IS NOT NULL
|
||||
|
||||
UNION
|
||||
|
||||
SELECT
|
||||
'entity:' || b.entity_id AS from_id,
|
||||
'ticker:' || t.code AS to_id
|
||||
FROM bing_holdings b
|
||||
JOIN tickers t ON t.id = b.ticker_id
|
||||
WHERE b.entity_id IS NOT NULL
|
||||
),
|
||||
neighbors AS (
|
||||
SELECT from_id AS a, to_id AS b FROM all_edges
|
||||
UNION
|
||||
SELECT to_id AS a, from_id AS b FROM all_edges
|
||||
),
|
||||
walk(node_id, depth, path) AS (
|
||||
SELECT ?1 AS node_id, 0 AS depth, ?1 AS path
|
||||
UNION ALL
|
||||
SELECT n.b, w.depth + 1, w.path || '>' || n.b
|
||||
FROM walk w
|
||||
JOIN neighbors n ON n.a = w.node_id
|
||||
WHERE w.depth < ?2
|
||||
AND instr(w.path, n.b) = 0
|
||||
)
|
||||
SELECT DISTINCT node_id
|
||||
FROM walk",
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
let rows = stmt
|
||||
.query_map(params![root_node_id, depth as i64], |row| {
|
||||
row.get::<_, String>(0)
|
||||
})
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
for row in rows {
|
||||
visited.insert(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
||||
}
|
||||
}
|
||||
|
||||
let mut edges = query_all_edges(conn)?;
|
||||
edges.retain(|edge| visited.contains(&edge.from) && visited.contains(&edge.to));
|
||||
|
||||
let nodes = build_nodes(conn, &visited)?;
|
||||
|
||||
Ok((nodes, edges))
|
||||
}
|
||||
|
||||
/// Format graph as ASCII tree for terminal display.
|
||||
pub fn format_graph_text(nodes: &[GraphNode], edges: &[GraphEdge]) -> String {
|
||||
if nodes.is_empty() {
|
||||
return "(empty graph)".to_string();
|
||||
}
|
||||
|
||||
let labels: HashMap<&str, (&str, GraphNodeType)> = nodes
|
||||
.iter()
|
||||
.map(|n| (n.id.as_str(), (n.label.as_str(), n.node_type)))
|
||||
.collect();
|
||||
|
||||
let mut ticker_to_entities: BTreeMap<&str, Vec<&GraphEdge>> = BTreeMap::new();
|
||||
for edge in edges {
|
||||
if let Some((_, GraphNodeType::Ticker)) = labels.get(edge.to.as_str()) {
|
||||
ticker_to_entities
|
||||
.entry(edge.to.as_str())
|
||||
.or_default()
|
||||
.push(edge);
|
||||
}
|
||||
}
|
||||
|
||||
let mut out = String::new();
|
||||
out.push_str(&format!("nodes: {} edges: {}\n", nodes.len(), edges.len()));
|
||||
|
||||
for (ticker_id, mut rels) in ticker_to_entities {
|
||||
rels.sort_by(|a, b| b.percentage_bps.cmp(&a.percentage_bps));
|
||||
let ticker_label = labels.get(ticker_id).map(|v| v.0).unwrap_or(ticker_id);
|
||||
out.push_str(&format!("\n{ticker_label} [TICKER]\n"));
|
||||
|
||||
for (idx, edge) in rels.iter().enumerate() {
|
||||
let holder_label = labels.get(edge.from.as_str()).map(|v| v.0).unwrap_or("?");
|
||||
let branch = if idx + 1 == rels.len() {
|
||||
"└─"
|
||||
} else {
|
||||
"├─"
|
||||
};
|
||||
out.push_str(&format!(
|
||||
"{branch} {holder_label} ({:.2}%, {})\n",
|
||||
edge.percentage_bps as f64 / 100.0,
|
||||
source_label(edge.source)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if edges.is_empty() {
|
||||
out.push_str("\n(no ownership edges found)\n");
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
/// Format graph as Graphviz DOT for export.
|
||||
pub fn format_graph_dot(nodes: &[GraphNode], edges: &[GraphEdge]) -> String {
|
||||
let mut out = String::new();
|
||||
out.push_str("digraph ownership {\n");
|
||||
out.push_str(" rankdir=LR;\n");
|
||||
out.push_str(" graph [fontname=\"Helvetica\"];\n");
|
||||
out.push_str(" node [fontname=\"Helvetica\"];\n");
|
||||
out.push_str(" edge [fontname=\"Helvetica\"];\n\n");
|
||||
|
||||
for node in nodes {
|
||||
let shape = match node.node_type {
|
||||
GraphNodeType::Entity => "ellipse",
|
||||
GraphNodeType::Ticker => "box",
|
||||
};
|
||||
out.push_str(&format!(
|
||||
" \"{}\" [label=\"{}\", shape={}];\n",
|
||||
escape_dot(&node.id),
|
||||
escape_dot(&node.label),
|
||||
shape
|
||||
));
|
||||
}
|
||||
|
||||
out.push('\n');
|
||||
for edge in edges {
|
||||
out.push_str(&format!(
|
||||
" \"{}\" -> \"{}\" [label=\"{:.2}% ({})\"];\n",
|
||||
escape_dot(&edge.from),
|
||||
escape_dot(&edge.to),
|
||||
edge.percentage_bps as f64 / 100.0,
|
||||
source_label(edge.source)
|
||||
));
|
||||
}
|
||||
|
||||
out.push_str("}\n");
|
||||
out
|
||||
}
|
||||
|
||||
fn detect_root_node(conn: &Connection, root: &str) -> Result<String, IdxError> {
|
||||
let maybe_ticker = conn
|
||||
.query_row(
|
||||
"SELECT code FROM tickers WHERE UPPER(code) = UPPER(?1) LIMIT 1",
|
||||
params![root],
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.ok();
|
||||
|
||||
if let Some(code) = maybe_ticker {
|
||||
return Ok(format!("ticker:{code}"));
|
||||
}
|
||||
|
||||
let q = format!("%{}%", root);
|
||||
let maybe_entity_id = conn
|
||||
.query_row(
|
||||
"SELECT id
|
||||
FROM entities
|
||||
WHERE canonical_name LIKE ?1 COLLATE NOCASE
|
||||
ORDER BY LENGTH(canonical_name) ASC, canonical_name ASC
|
||||
LIMIT 1",
|
||||
params![q],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)
|
||||
.ok();
|
||||
|
||||
if let Some(entity_id) = maybe_entity_id {
|
||||
return Ok(format!("entity:{entity_id}"));
|
||||
}
|
||||
|
||||
Err(IdxError::ParseError(format!(
|
||||
"graph root not found as ticker or entity: {root}"
|
||||
)))
|
||||
}
|
||||
|
||||
fn query_all_edges(conn: &Connection) -> Result<Vec<GraphEdge>, IdxError> {
|
||||
let mut out = Vec::new();
|
||||
|
||||
{
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT 'entity:' || k.entity_id,
|
||||
'ticker:' || t.code,
|
||||
k.percentage_bps
|
||||
FROM ksei_holdings k
|
||||
JOIN tickers t ON t.id = k.ticker_id
|
||||
WHERE k.entity_id IS NOT NULL",
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
let rows = stmt
|
||||
.query_map([], |row| {
|
||||
Ok(GraphEdge {
|
||||
from: row.get(0)?,
|
||||
to: row.get(1)?,
|
||||
percentage_bps: row.get(2)?,
|
||||
source: OwnershipSource::Ksei,
|
||||
})
|
||||
})
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
for row in rows {
|
||||
out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT 'entity:' || b.entity_id,
|
||||
'ticker:' || t.code,
|
||||
COALESCE(b.pct_ownership_bps, 0)
|
||||
FROM bing_holdings b
|
||||
JOIN tickers t ON t.id = b.ticker_id
|
||||
WHERE b.entity_id IS NOT NULL",
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
let rows = stmt
|
||||
.query_map([], |row| {
|
||||
Ok(GraphEdge {
|
||||
from: row.get(0)?,
|
||||
to: row.get(1)?,
|
||||
percentage_bps: row.get(2)?,
|
||||
source: OwnershipSource::Bing,
|
||||
})
|
||||
})
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
for row in rows {
|
||||
out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
||||
}
|
||||
}
|
||||
|
||||
dedup_edges(out)
|
||||
}
|
||||
|
||||
fn dedup_edges(edges: Vec<GraphEdge>) -> Result<Vec<GraphEdge>, IdxError> {
|
||||
let mut seen: HashSet<(String, String, i64, &'static str)> = HashSet::new();
|
||||
let mut out = Vec::new();
|
||||
|
||||
for edge in edges {
|
||||
let key = (
|
||||
edge.from.clone(),
|
||||
edge.to.clone(),
|
||||
edge.percentage_bps,
|
||||
source_label(edge.source),
|
||||
);
|
||||
if seen.insert(key) {
|
||||
out.push(edge);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn build_nodes(conn: &Connection, node_ids: &BTreeSet<String>) -> Result<Vec<GraphNode>, IdxError> {
|
||||
let mut nodes = Vec::new();
|
||||
|
||||
for node_id in node_ids {
|
||||
if let Some(entity_id) = node_id.strip_prefix("entity:") {
|
||||
let entity_id_num = entity_id.parse::<i64>().map_err(|e| {
|
||||
IdxError::ParseError(format!("invalid entity node id '{node_id}': {e}"))
|
||||
})?;
|
||||
let label = conn
|
||||
.query_row(
|
||||
"SELECT canonical_name FROM entities WHERE id = ?1",
|
||||
params![entity_id_num],
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.unwrap_or_else(|_| format!("entity:{entity_id_num}"));
|
||||
nodes.push(GraphNode {
|
||||
id: node_id.clone(),
|
||||
label,
|
||||
node_type: GraphNodeType::Entity,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(code) = node_id.strip_prefix("ticker:") {
|
||||
nodes.push(GraphNode {
|
||||
id: node_id.clone(),
|
||||
label: code.to_string(),
|
||||
node_type: GraphNodeType::Ticker,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(nodes)
|
||||
}
|
||||
|
||||
fn source_label(source: OwnershipSource) -> &'static str {
|
||||
match source {
|
||||
OwnershipSource::Ksei => "ksei",
|
||||
OwnershipSource::Bing => "bing",
|
||||
}
|
||||
}
|
||||
|
||||
fn escape_dot(value: &str) -> String {
|
||||
value.replace('"', "\\\"")
|
||||
}
|
||||
6
src/ownership/mod.rs
Normal file
6
src/ownership/mod.rs
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
pub mod db;
|
||||
pub mod entities;
|
||||
pub mod graph;
|
||||
pub mod parser;
|
||||
pub mod search;
|
||||
pub mod types;
|
||||
355
src/ownership/parser.rs
Normal file
355
src/ownership/parser.rs
Normal file
|
|
@ -0,0 +1,355 @@
|
|||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
use quick_xml::Reader;
|
||||
use quick_xml::events::Event;
|
||||
|
||||
use crate::error::IdxError;
|
||||
use crate::ownership::types::KseiRawRow;
|
||||
|
||||
/// Character grid for a single PDF page: y-coord → column-index → sorted (x, char) pairs.
|
||||
type PageGrid = HashMap<i32, HashMap<usize, Vec<(i32, char)>>>;
|
||||
|
||||
const Y_TOLERANCE: f32 = 0.8;
|
||||
|
||||
/// Inclusive-left, exclusive-right X ranges for each KSEI data column.
|
||||
const COLUMN_BOUNDS: [(f32, f32); 12] = [
|
||||
(15.0, 52.0), // date
|
||||
(52.0, 70.0), // share_code
|
||||
(70.0, 167.0), // issuer_name
|
||||
(167.0, 432.0), // investor_name
|
||||
(432.0, 463.0), // investor_type
|
||||
(463.0, 497.0), // local_foreign
|
||||
(497.0, 558.0), // nationality
|
||||
(558.0, 615.0), // domicile
|
||||
(615.0, 653.0), // holdings_scripless
|
||||
(653.0, 692.0), // holdings_scrip
|
||||
(692.0, 745.0), // total_holding_shares
|
||||
(745.0, 800.0), // percentage
|
||||
];
|
||||
|
||||
/// Parse a KSEI ownership PDF into raw rows.
|
||||
/// Shells out to `mutool` for XML extraction, then parses with quick-xml.
|
||||
pub fn parse_ksei_pdf(path: &Path) -> Result<Vec<KseiRawRow>, IdxError> {
|
||||
check_mutool()?;
|
||||
|
||||
let output = Command::new("mutool")
|
||||
.arg("convert")
|
||||
.arg("-F")
|
||||
.arg("stext")
|
||||
.arg("-o")
|
||||
.arg("-")
|
||||
.arg(path)
|
||||
.output()
|
||||
.map_err(|e| IdxError::PdfParseError(format!("failed to run mutool: {e}")))?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
return Err(IdxError::PdfParseError(format!(
|
||||
"mutool convert failed: {}",
|
||||
stderr.trim()
|
||||
)));
|
||||
}
|
||||
|
||||
let xml = String::from_utf8(output.stdout)
|
||||
.map_err(|e| IdxError::PdfParseError(format!("invalid utf-8 stext output: {e}")))?;
|
||||
|
||||
parse_stext_xml(&xml)
|
||||
}
|
||||
|
||||
/// Parse mutool stext XML output into raw rows.
|
||||
/// Pure function — takes XML string, returns parsed rows.
|
||||
pub fn parse_stext_xml(xml: &str) -> Result<Vec<KseiRawRow>, IdxError> {
|
||||
let mut reader = Reader::from_str(xml);
|
||||
reader.config_mut().trim_text(false);
|
||||
|
||||
let mut rows: Vec<KseiRawRow> = Vec::new();
|
||||
let mut current_page: Option<PageGrid> = None;
|
||||
let mut buf = Vec::new();
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(e)) if e.name().as_ref() == b"page" => {
|
||||
current_page = Some(HashMap::new());
|
||||
}
|
||||
Ok(Event::Empty(e)) if e.name().as_ref() == b"char" => {
|
||||
if let Some(page) = current_page.as_mut() {
|
||||
let mut x: Option<f32> = None;
|
||||
let mut y: Option<f32> = None;
|
||||
let mut c: Option<char> = None;
|
||||
|
||||
for attr_result in e.attributes().with_checks(false) {
|
||||
let attr = attr_result.map_err(|err| {
|
||||
IdxError::PdfParseError(format!("invalid XML attribute: {err}"))
|
||||
})?;
|
||||
|
||||
match attr.key.as_ref() {
|
||||
b"x" => {
|
||||
let s = attr.decode_and_unescape_value(reader.decoder()).map_err(
|
||||
|err| {
|
||||
IdxError::PdfParseError(format!(
|
||||
"invalid XML x attribute: {err}"
|
||||
))
|
||||
},
|
||||
)?;
|
||||
x = s.parse::<f32>().ok();
|
||||
}
|
||||
b"y" => {
|
||||
let s = attr.decode_and_unescape_value(reader.decoder()).map_err(
|
||||
|err| {
|
||||
IdxError::PdfParseError(format!(
|
||||
"invalid XML y attribute: {err}"
|
||||
))
|
||||
},
|
||||
)?;
|
||||
y = s.parse::<f32>().ok();
|
||||
}
|
||||
b"c" => {
|
||||
let s = attr.decode_and_unescape_value(reader.decoder()).map_err(
|
||||
|err| {
|
||||
IdxError::PdfParseError(format!(
|
||||
"invalid XML char attribute: {err}"
|
||||
))
|
||||
},
|
||||
)?;
|
||||
c = s.chars().next();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if let (Some(x_val), Some(y_val), Some(ch)) = (x, y, c)
|
||||
&& let Some(col_idx) = x_to_column(x_val)
|
||||
{
|
||||
let yb = y_bucket(y_val);
|
||||
let xi = (x_val * 100.0).round() as i32;
|
||||
page.entry(yb)
|
||||
.or_default()
|
||||
.entry(col_idx)
|
||||
.or_default()
|
||||
.push((xi, ch));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Event::End(e)) if e.name().as_ref() == b"page" => {
|
||||
if let Some(page) = current_page.take() {
|
||||
rows.extend(extract_rows_from_page(page));
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
Err(err) => {
|
||||
return Err(IdxError::PdfParseError(format!(
|
||||
"failed to parse stext XML: {err}"
|
||||
)));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Check if mutool is available in PATH.
|
||||
pub fn check_mutool() -> Result<(), IdxError> {
|
||||
// mutool with no args prints usage to stderr and exits non-zero,
|
||||
// so we just check that the binary is found and executable.
|
||||
Command::new("mutool")
|
||||
.arg("--help")
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.map_err(|e| IdxError::PdfParseError(format!("mutool not found in PATH: {e}")))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn extract_rows_from_page(page: PageGrid) -> Vec<KseiRawRow> {
|
||||
let mut page_rows = Vec::new();
|
||||
|
||||
let mut y_keys: Vec<i32> = page.keys().copied().collect();
|
||||
y_keys.sort_unstable();
|
||||
|
||||
for y in y_keys {
|
||||
let mut row = KseiRawRow {
|
||||
date: String::new(),
|
||||
share_code: String::new(),
|
||||
issuer_name: String::new(),
|
||||
investor_name: String::new(),
|
||||
investor_type: String::new(),
|
||||
local_foreign: String::new(),
|
||||
nationality: String::new(),
|
||||
domicile: String::new(),
|
||||
holdings_scripless: String::new(),
|
||||
holdings_scrip: String::new(),
|
||||
total_holding_shares: String::new(),
|
||||
percentage: String::new(),
|
||||
};
|
||||
|
||||
if let Some(col_map) = page.get(&y) {
|
||||
for (col_idx, chars) in col_map {
|
||||
let mut sorted = chars.clone();
|
||||
sorted.sort_by_key(|(x, _)| *x);
|
||||
let text = normalize_spaces(&sorted.iter().map(|(_, c)| c).collect::<String>());
|
||||
assign_column(&mut row, *col_idx, text);
|
||||
}
|
||||
}
|
||||
|
||||
if is_data_row(&row) {
|
||||
page_rows.push(row);
|
||||
}
|
||||
}
|
||||
|
||||
page_rows
|
||||
}
|
||||
|
||||
fn assign_column(row: &mut KseiRawRow, col_idx: usize, value: String) {
|
||||
match col_idx {
|
||||
0 => row.date = value,
|
||||
1 => row.share_code = value,
|
||||
2 => row.issuer_name = value,
|
||||
3 => row.investor_name = value,
|
||||
4 => row.investor_type = value,
|
||||
5 => row.local_foreign = value,
|
||||
6 => row.nationality = value,
|
||||
7 => row.domicile = value,
|
||||
8 => row.holdings_scripless = value,
|
||||
9 => row.holdings_scrip = value,
|
||||
10 => row.total_holding_shares = value,
|
||||
11 => row.percentage = value,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn x_to_column(x: f32) -> Option<usize> {
|
||||
COLUMN_BOUNDS
|
||||
.iter()
|
||||
.position(|(left, right)| x >= *left && x < *right)
|
||||
}
|
||||
|
||||
fn y_bucket(y: f32) -> i32 {
|
||||
(y / Y_TOLERANCE).round() as i32
|
||||
}
|
||||
|
||||
fn normalize_spaces(input: &str) -> String {
|
||||
let mut out = String::with_capacity(input.len());
|
||||
let mut prev_space = false;
|
||||
|
||||
for ch in input.chars() {
|
||||
if ch == ' ' {
|
||||
if !prev_space {
|
||||
out.push(' ');
|
||||
}
|
||||
prev_space = true;
|
||||
} else {
|
||||
out.push(ch);
|
||||
prev_space = false;
|
||||
}
|
||||
}
|
||||
|
||||
out.trim().to_string()
|
||||
}
|
||||
|
||||
fn is_data_row(row: &KseiRawRow) -> bool {
|
||||
is_ksei_date(&row.date)
|
||||
&& is_percentage_like(&row.percentage)
|
||||
&& !row.share_code.trim().is_empty()
|
||||
&& !row.investor_name.trim().is_empty()
|
||||
}
|
||||
|
||||
fn is_ksei_date(s: &str) -> bool {
|
||||
if s.len() != 11 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut parts = s.split('-');
|
||||
let day = parts.next();
|
||||
let mon = parts.next();
|
||||
let year = parts.next();
|
||||
|
||||
if parts.next().is_some() {
|
||||
return false;
|
||||
}
|
||||
|
||||
match (day, mon, year) {
|
||||
(Some(d), Some(m), Some(y)) => {
|
||||
d.len() == 2
|
||||
&& d.chars().all(|c| c.is_ascii_digit())
|
||||
&& m.len() == 3
|
||||
&& m.chars().all(|c| c.is_ascii_alphabetic())
|
||||
&& y.len() == 4
|
||||
&& y.chars().all(|c| c.is_ascii_digit())
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_percentage_like(s: &str) -> bool {
|
||||
let cleaned = s.trim();
|
||||
if cleaned.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut parts = cleaned.split(',');
|
||||
let left = parts.next();
|
||||
let right = parts.next();
|
||||
|
||||
if parts.next().is_some() {
|
||||
return false;
|
||||
}
|
||||
|
||||
match (left, right) {
|
||||
(Some(l), Some(r)) => {
|
||||
!l.is_empty()
|
||||
&& l.chars().all(|c| c.is_ascii_digit())
|
||||
&& !r.is_empty()
|
||||
&& r.chars().all(|c| c.is_ascii_digit())
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use super::{check_mutool, parse_ksei_pdf, parse_stext_xml};
|
||||
|
||||
#[test]
|
||||
fn test_parse_stext_xml_fixture_extracts_rows() {
|
||||
let fixture_path = Path::new("tests/fixtures/ksei_stext_sample.xml");
|
||||
let xml = fs::read_to_string(fixture_path).expect("failed to read stext fixture");
|
||||
|
||||
let rows = parse_stext_xml(&xml).expect("failed to parse fixture XML");
|
||||
assert_eq!(rows.len(), 3);
|
||||
|
||||
let first = &rows[0];
|
||||
assert_eq!(first.date, "27-Feb-2026");
|
||||
assert_eq!(first.share_code, "BBCA");
|
||||
assert_eq!(first.investor_name, "PT DWIMURIA INVESTAMA ANDALAN");
|
||||
assert_eq!(first.percentage, "54,94");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_ksei_pdf_real_file_row_count() {
|
||||
if check_mutool().is_err() {
|
||||
eprintln!("skipping mutool-dependent test: mutool not available");
|
||||
return;
|
||||
}
|
||||
|
||||
let pdf_path =
|
||||
Path::new("/var/lib/openclaw/projects/idx-cli/research/ownership_202603.pdf");
|
||||
if !pdf_path.exists() {
|
||||
eprintln!("skipping mutool-dependent test: sample PDF not found");
|
||||
return;
|
||||
}
|
||||
|
||||
let rows = parse_ksei_pdf(pdf_path).expect("failed to parse real KSEI PDF");
|
||||
assert!(
|
||||
rows.len() >= 7_200,
|
||||
"expected at least 7200 rows, got {}",
|
||||
rows.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
111
src/ownership/search.rs
Normal file
111
src/ownership/search.rs
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
use rusqlite::{Connection, params};
|
||||
|
||||
use crate::error::IdxError;
|
||||
use crate::ownership::types::Entity;
|
||||
|
||||
pub fn fts_search(conn: &Connection, query: &str, limit: usize) -> Result<Vec<Entity>, IdxError> {
|
||||
let q = query.trim();
|
||||
if q.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let limit_i64 =
|
||||
i64::try_from(limit).map_err(|e| IdxError::DatabaseError(format!("invalid limit: {e}")))?;
|
||||
|
||||
// Ensure entity_fts has content. For external-content FTS table, populate manually.
|
||||
let fts_count: i64 = conn
|
||||
.query_row("SELECT COUNT(*) FROM entity_fts", [], |row| row.get(0))
|
||||
.unwrap_or(0);
|
||||
|
||||
if fts_count == 0 {
|
||||
let _ = rebuild_fts(conn);
|
||||
}
|
||||
|
||||
if let Ok(rows) = fts_query(conn, q, limit_i64)
|
||||
&& !rows.is_empty()
|
||||
{
|
||||
return Ok(rows);
|
||||
}
|
||||
|
||||
like_query(conn, q, limit_i64)
|
||||
}
|
||||
|
||||
fn fts_query(conn: &Connection, query: &str, limit: i64) -> Result<Vec<Entity>, IdxError> {
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT e.id, e.canonical_name, e.entity_type, e.country
|
||||
FROM entity_fts f
|
||||
JOIN entities e ON e.id = f.rowid
|
||||
WHERE entity_fts MATCH ?1
|
||||
ORDER BY rank
|
||||
LIMIT ?2",
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
let rows = stmt
|
||||
.query_map(params![query, limit], |row| {
|
||||
Ok(Entity {
|
||||
id: row.get(0)?,
|
||||
canonical_name: row.get(1)?,
|
||||
entity_type: row.get(2)?,
|
||||
country: row.get(3)?,
|
||||
})
|
||||
})
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
let mut out = Vec::new();
|
||||
for row in rows {
|
||||
out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn like_query(conn: &Connection, query: &str, limit: i64) -> Result<Vec<Entity>, IdxError> {
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT id, canonical_name, entity_type, country
|
||||
FROM entities
|
||||
WHERE canonical_name LIKE ?1 COLLATE NOCASE
|
||||
ORDER BY canonical_name ASC
|
||||
LIMIT ?2",
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
let like = format!("%{query}%");
|
||||
let rows = stmt
|
||||
.query_map(params![like, limit], |row| {
|
||||
Ok(Entity {
|
||||
id: row.get(0)?,
|
||||
canonical_name: row.get(1)?,
|
||||
entity_type: row.get(2)?,
|
||||
country: row.get(3)?,
|
||||
})
|
||||
})
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
let mut out = Vec::new();
|
||||
for row in rows {
|
||||
out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub fn rebuild_fts(conn: &Connection) -> Result<(), IdxError> {
|
||||
conn.execute("DELETE FROM entity_fts", [])
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO entity_fts(rowid, canonical_name, aliases)
|
||||
SELECT e.id, e.canonical_name, COALESCE(a.aliases, '')
|
||||
FROM entities e
|
||||
LEFT JOIN (
|
||||
SELECT entity_id, GROUP_CONCAT(raw_name, ' ') AS aliases
|
||||
FROM entity_aliases
|
||||
GROUP BY entity_id
|
||||
) a ON a.entity_id = e.id",
|
||||
[],
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
451
src/ownership/types.rs
Normal file
451
src/ownership/types.rs
Normal file
|
|
@ -0,0 +1,451 @@
|
|||
use chrono::NaiveDate;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Which data source a holding originates from.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum OwnershipSource {
|
||||
/// KSEI shareholder registry source.
|
||||
Ksei,
|
||||
/// Bing institutional ownership source.
|
||||
Bing,
|
||||
}
|
||||
|
||||
/// Investor locality classification from KSEI.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum Locality {
|
||||
/// Local Indonesian investor.
|
||||
Local,
|
||||
/// Foreign investor.
|
||||
Foreign,
|
||||
}
|
||||
|
||||
/// Bing institutional flow signal.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum FlowSignal {
|
||||
/// Existing top holder.
|
||||
Holder,
|
||||
/// Net buyer for the period.
|
||||
Buyer,
|
||||
/// Net seller for the period.
|
||||
Seller,
|
||||
/// New position opened this period.
|
||||
NewPosition,
|
||||
/// Fully exited this period.
|
||||
Exited,
|
||||
}
|
||||
|
||||
/// Method used to resolve an entity alias to a canonical entity.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ResolutionMethod {
|
||||
/// Exact string match after normalization.
|
||||
Exact,
|
||||
/// Rule-based normalization match.
|
||||
Rule,
|
||||
/// Fuzzy or similarity-based match.
|
||||
Fuzzy,
|
||||
/// Human-curated manual mapping.
|
||||
Manual,
|
||||
}
|
||||
|
||||
/// Graph node category used in ownership network output.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum GraphNodeType {
|
||||
/// Canonical investor entity node.
|
||||
Entity,
|
||||
/// Listed issuer ticker node.
|
||||
Ticker,
|
||||
}
|
||||
|
||||
/// KSEI investor type code (for example: `CP`, `ID`, `IB`, `MF`, `SC`, `IS`, `OT`).
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct InvestorTypeCode(pub String);
|
||||
|
||||
/// Canonical resolved entity (investor or shareholder).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Entity {
|
||||
/// Internal entity identifier.
|
||||
pub id: i64,
|
||||
/// Canonical normalized display name.
|
||||
pub canonical_name: String,
|
||||
/// Optional coarse entity type (`fund`, `bank`, `conglomerate`, `govt`, `individual`).
|
||||
pub entity_type: Option<String>,
|
||||
/// Optional ISO-like country tag.
|
||||
pub country: Option<String>,
|
||||
}
|
||||
|
||||
/// A raw name variant mapped to a canonical entity.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EntityAlias {
|
||||
/// Internal alias identifier.
|
||||
pub id: i64,
|
||||
/// Referenced canonical entity id.
|
||||
pub entity_id: i64,
|
||||
/// Raw alias text from source.
|
||||
pub raw_name: String,
|
||||
/// Source that produced this alias.
|
||||
pub source: OwnershipSource,
|
||||
/// Resolution confidence score in `0.0..=1.0`.
|
||||
pub confidence: f64,
|
||||
/// Matching method used for resolution.
|
||||
pub method: ResolutionMethod,
|
||||
}
|
||||
|
||||
/// Ticker (issuer) reference metadata.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Ticker {
|
||||
/// Internal ticker identifier.
|
||||
pub id: i64,
|
||||
/// Exchange ticker code, for example `BBCA`.
|
||||
pub code: String,
|
||||
/// Optional long issuer name.
|
||||
pub name: Option<String>,
|
||||
/// Optional sector classification.
|
||||
pub sector: Option<String>,
|
||||
}
|
||||
|
||||
/// Single KSEI ownership row for one holder in one ticker and release.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct KseiHolding {
|
||||
/// Internal holding row identifier.
|
||||
pub id: i64,
|
||||
/// Referenced ticker id.
|
||||
pub ticker_id: i64,
|
||||
/// Referenced entity id, unresolved when `None`.
|
||||
pub entity_id: Option<i64>,
|
||||
/// Raw investor name exactly as imported.
|
||||
pub raw_investor_name: String,
|
||||
/// Optional KSEI investor type code.
|
||||
pub investor_type: Option<InvestorTypeCode>,
|
||||
/// Optional local/foreign classification.
|
||||
pub locality: Option<Locality>,
|
||||
/// Optional nationality text.
|
||||
pub nationality: Option<String>,
|
||||
/// Optional domicile text.
|
||||
pub domicile: Option<String>,
|
||||
/// Scripless holdings in shares.
|
||||
pub holdings_scripless: i64,
|
||||
/// Script holdings in shares.
|
||||
pub holdings_scrip: i64,
|
||||
/// Total shares held.
|
||||
pub total_shares: i64,
|
||||
/// Ownership percentage in basis points (`41.10% -> 4110`).
|
||||
pub percentage_bps: i64,
|
||||
/// Snapshot as-of date.
|
||||
pub report_date: NaiveDate,
|
||||
/// SHA-256 hash of source release for deduplication.
|
||||
pub release_sha256: String,
|
||||
}
|
||||
|
||||
/// Draft holding before entity resolution (without persisted row id and entity id).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct KseiHoldingDraft {
|
||||
/// Exchange ticker code, for example `BBCA`.
|
||||
pub ticker_code: String,
|
||||
/// Optional issuer name as provided by source row.
|
||||
pub issuer_name: Option<String>,
|
||||
/// Raw investor name exactly as imported.
|
||||
pub raw_investor_name: String,
|
||||
/// Optional KSEI investor type code.
|
||||
pub investor_type: Option<InvestorTypeCode>,
|
||||
/// Optional local/foreign classification.
|
||||
pub locality: Option<Locality>,
|
||||
/// Optional nationality text.
|
||||
pub nationality: Option<String>,
|
||||
/// Optional domicile text.
|
||||
pub domicile: Option<String>,
|
||||
/// Scripless holdings in shares.
|
||||
pub holdings_scripless: i64,
|
||||
/// Script holdings in shares.
|
||||
pub holdings_scrip: i64,
|
||||
/// Total shares held.
|
||||
pub total_shares: i64,
|
||||
/// Ownership percentage in basis points (`41.10% -> 4110`).
|
||||
pub percentage_bps: i64,
|
||||
/// Snapshot as-of date.
|
||||
pub report_date: NaiveDate,
|
||||
}
|
||||
|
||||
/// Single Bing institutional ownership row.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BingHolding {
|
||||
/// Internal holding row identifier.
|
||||
pub id: i64,
|
||||
/// Referenced ticker id.
|
||||
pub ticker_id: i64,
|
||||
/// Referenced entity id, unresolved when `None`.
|
||||
pub entity_id: Option<i64>,
|
||||
/// Raw investor name exactly as imported.
|
||||
pub raw_investor_name: String,
|
||||
/// Optional Bing investor type text.
|
||||
pub investor_type: Option<String>,
|
||||
/// Shares currently held.
|
||||
pub shares_held: Option<i64>,
|
||||
/// Share delta over period (`+buy`, `-sell`).
|
||||
pub shares_changed: Option<i64>,
|
||||
/// Ownership percentage in basis points.
|
||||
pub pct_ownership_bps: Option<i64>,
|
||||
/// Position value in USD.
|
||||
pub value_usd: Option<i64>,
|
||||
/// Source report date.
|
||||
pub report_date: NaiveDate,
|
||||
/// Institutional flow signal for the row.
|
||||
pub signal: FlowSignal,
|
||||
/// Import timestamp (unix epoch seconds).
|
||||
pub fetched_at: i64,
|
||||
}
|
||||
|
||||
/// Raw row extracted from KSEI PDF before normalization.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct KseiRawRow {
|
||||
/// Date string as appears in PDF (for example `27-Feb-2026`).
|
||||
pub date: String,
|
||||
/// Share code string (for example `BBCA`).
|
||||
pub share_code: String,
|
||||
/// Issuer name as appears in source.
|
||||
pub issuer_name: String,
|
||||
/// Investor name as appears in source.
|
||||
pub investor_name: String,
|
||||
/// Investor type code string.
|
||||
pub investor_type: String,
|
||||
/// Local or foreign marker (`L` or `F`).
|
||||
pub local_foreign: String,
|
||||
/// Nationality text.
|
||||
pub nationality: String,
|
||||
/// Domicile text.
|
||||
pub domicile: String,
|
||||
/// Scripless holdings string in Indonesian locale format.
|
||||
pub holdings_scripless: String,
|
||||
/// Script holdings string in Indonesian locale format.
|
||||
pub holdings_scrip: String,
|
||||
/// Total holdings string in Indonesian locale format.
|
||||
pub total_holding_shares: String,
|
||||
/// Percentage string in Indonesian locale decimal format.
|
||||
pub percentage: String,
|
||||
}
|
||||
|
||||
/// Raw holder object returned by Bing ownership endpoints.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BingHolderRaw {
|
||||
/// Investor name field from Bing payload.
|
||||
#[serde(alias = "investorName", alias = "InvestorName")]
|
||||
pub investor_name: Option<String>,
|
||||
/// Investor type field from Bing payload.
|
||||
#[serde(alias = "investorType", alias = "InvestorType")]
|
||||
pub investor_type: Option<String>,
|
||||
/// Shares held field from Bing payload.
|
||||
#[serde(alias = "sharesHeld", alias = "SharesHeld")]
|
||||
pub shares_held: Option<f64>,
|
||||
/// Shares changed field from Bing payload.
|
||||
#[serde(alias = "sharesChanged", alias = "SharesChanged")]
|
||||
pub shares_changed: Option<f64>,
|
||||
/// Percentage of shares outstanding field from Bing payload.
|
||||
#[serde(
|
||||
alias = "percentageOfSharesOutstanding",
|
||||
alias = "PercentageOfSharesOutstanding"
|
||||
)]
|
||||
pub pct_outstanding: Option<f64>,
|
||||
/// Position value field from Bing payload.
|
||||
#[serde(alias = "value", alias = "Value")]
|
||||
pub value: Option<f64>,
|
||||
/// Report date field from Bing payload.
|
||||
#[serde(alias = "reportDate", alias = "ReportDate")]
|
||||
pub report_date: Option<String>,
|
||||
}
|
||||
|
||||
/// Combined ownership view for a ticker (KSEI and Bing merged).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TickerOwnership {
|
||||
/// Ticker metadata.
|
||||
pub ticker: Ticker,
|
||||
/// Latest KSEI as-of date.
|
||||
pub ksei_as_of: Option<NaiveDate>,
|
||||
/// Latest Bing reporting period label.
|
||||
pub bing_as_of: Option<String>,
|
||||
/// Combined holder rows for display.
|
||||
pub holders: Vec<HolderRow>,
|
||||
/// Concentration metrics calculated for the ticker.
|
||||
pub concentration: ConcentrationMetrics,
|
||||
/// Optional institutional flow breakdown.
|
||||
pub flow: Option<InstitutionalFlow>,
|
||||
}
|
||||
|
||||
/// Single row in a combined holders table.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HolderRow {
|
||||
/// Ranking position in result set.
|
||||
pub rank: usize,
|
||||
/// Source dataset for the row.
|
||||
pub source: OwnershipSource,
|
||||
/// Canonical holder name or unresolved raw name.
|
||||
pub name: String,
|
||||
/// Optional referenced canonical entity id.
|
||||
pub entity_id: Option<i64>,
|
||||
/// Optional investor type label.
|
||||
pub investor_type: Option<String>,
|
||||
/// Optional local/foreign marker.
|
||||
pub locality: Option<Locality>,
|
||||
/// Held shares quantity.
|
||||
pub shares: i64,
|
||||
/// Ownership percentage in basis points.
|
||||
pub percentage_bps: i64,
|
||||
/// Optional flow signal (Bing rows only).
|
||||
pub signal: Option<FlowSignal>,
|
||||
}
|
||||
|
||||
/// Concentration metrics for ownership distribution of a ticker.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ConcentrationMetrics {
|
||||
/// Largest single holder percentage in basis points.
|
||||
pub top1_bps: i64,
|
||||
/// Sum of top 3 holder percentages in basis points.
|
||||
pub top3_bps: i64,
|
||||
/// Herfindahl-Hirschman index value.
|
||||
pub hhi: i64,
|
||||
/// Estimated free float in basis points (`10000 - known holders`).
|
||||
pub free_float_bps: i64,
|
||||
/// Number of KSEI holders above or equal to 1%.
|
||||
pub holder_count: usize,
|
||||
}
|
||||
|
||||
/// Institutional flow summary from Bing for one ticker and period.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct InstitutionalFlow {
|
||||
/// Reporting period label.
|
||||
pub period: String,
|
||||
/// Top buyer rows.
|
||||
pub top_buyers: Vec<HolderRow>,
|
||||
/// Top seller rows.
|
||||
pub top_sellers: Vec<HolderRow>,
|
||||
/// New position rows.
|
||||
pub new_positions: Vec<HolderRow>,
|
||||
/// Exited position rows.
|
||||
pub exited: Vec<HolderRow>,
|
||||
}
|
||||
|
||||
/// Cross-holding summary for a single entity.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EntityHoldings {
|
||||
/// Canonical entity metadata.
|
||||
pub entity: Entity,
|
||||
/// Number of distinct tickers held.
|
||||
pub ticker_count: usize,
|
||||
/// Per-ticker ownership rows.
|
||||
pub holdings: Vec<EntityTickerRow>,
|
||||
}
|
||||
|
||||
/// One ticker row within an entity holdings summary.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EntityTickerRow {
|
||||
/// Ticker metadata.
|
||||
pub ticker: Ticker,
|
||||
/// Source dataset for the holding.
|
||||
pub source: OwnershipSource,
|
||||
/// Shares held quantity.
|
||||
pub shares: i64,
|
||||
/// Ownership percentage in basis points.
|
||||
pub percentage_bps: i64,
|
||||
/// Report as-of date.
|
||||
pub report_date: NaiveDate,
|
||||
}
|
||||
|
||||
/// Cross-holder leaderboard row across multiple tickers.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CrossHolderRow {
|
||||
/// Canonical entity metadata.
|
||||
pub entity: Entity,
|
||||
/// Number of distinct tickers held.
|
||||
pub ticker_count: usize,
|
||||
/// Summed ownership basis points across tickers.
|
||||
pub total_bps: i64,
|
||||
}
|
||||
|
||||
/// Graph node used for ownership network visualization.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GraphNode {
|
||||
/// Stable graph node id (`entity:42` or `ticker:BBCA`).
|
||||
pub id: String,
|
||||
/// Display label.
|
||||
pub label: String,
|
||||
/// Node category.
|
||||
pub node_type: GraphNodeType,
|
||||
}
|
||||
|
||||
/// Graph edge used for ownership network visualization.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GraphEdge {
|
||||
/// Source node id.
|
||||
pub from: String,
|
||||
/// Target node id.
|
||||
pub to: String,
|
||||
/// Edge weight in basis points.
|
||||
pub percentage_bps: i64,
|
||||
/// Source dataset of the edge.
|
||||
pub source: OwnershipSource,
|
||||
}
|
||||
|
||||
/// Imported ownership release metadata.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct OwnershipRelease {
|
||||
/// Internal release id.
|
||||
pub id: i64,
|
||||
/// Optional source URL where release was obtained.
|
||||
pub source_url: Option<String>,
|
||||
/// SHA-256 hash for release-level deduplication.
|
||||
pub sha256: String,
|
||||
/// Release as-of date.
|
||||
pub as_of_date: NaiveDate,
|
||||
/// Parsed row count imported.
|
||||
pub row_count: usize,
|
||||
/// Import timestamp (unix epoch seconds).
|
||||
pub imported_at: i64,
|
||||
}
|
||||
|
||||
/// Type of ownership change between two snapshots.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ChangeType {
|
||||
/// Holder appears in `to` snapshot but not in `from`.
|
||||
New,
|
||||
/// Holder appears in `from` snapshot but not in `to`.
|
||||
Exited,
|
||||
/// Holder exists in both snapshots and percentage increased.
|
||||
Increased,
|
||||
/// Holder exists in both snapshots and percentage decreased.
|
||||
Decreased,
|
||||
}
|
||||
|
||||
/// One ownership change row for a ticker-holder pair.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChangeRow {
|
||||
/// Exchange ticker code.
|
||||
pub ticker_code: String,
|
||||
/// Canonical or raw holder name.
|
||||
pub entity_name: String,
|
||||
/// Change classification.
|
||||
pub change_type: ChangeType,
|
||||
/// Old percentage in basis points (from snapshot).
|
||||
pub old_bps: Option<i64>,
|
||||
/// New percentage in basis points (to snapshot).
|
||||
pub new_bps: Option<i64>,
|
||||
/// Delta in basis points (`new - old`, missing treated as 0).
|
||||
pub delta_bps: i64,
|
||||
}
|
||||
|
||||
/// Row describing unresolved or low-confidence alias mappings.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct UnresolvedRow {
|
||||
/// Raw investor name captured from source data.
|
||||
pub raw_name: String,
|
||||
/// Source marker (`ksei` or `bing`).
|
||||
pub source: String,
|
||||
/// Ticker code where the unresolved alias appears.
|
||||
pub ticker_code: String,
|
||||
/// Current canonical entity name when available.
|
||||
pub current_entity: Option<String>,
|
||||
/// Resolution confidence score.
|
||||
pub confidence: Option<f64>,
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue