mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-11-07 09:45:36 +00:00
refactor: split into idx-core library and idx-cli binary workspace
Phase 2 of docs/DESIGN.md, no behavior change: - crates/idx-core (library): api, analysis, cache, config file handling and provider selection, curl_impersonate, error, ownership, runtime, plus the shared test fixtures. clap derives are behind an optional `clap` feature so the library does not require clap. - crates/idx-cli (binary `idx`): cli/, output/, and IdxConfig (output format and color resolution). Package and binary names are unchanged. - config.rs is split: file read/write and validation move to core; a CLI test keeps core's OUTPUT_VALUES in sync with OutputSetting. - ownership::db::open_db/db_path drop their unused IdxConfig argument. - One workspace version; shared dependency versions; no third-party version changes in Cargo.lock. LICENSE is symlinked into each crate. - flake: workspace version, new source paths, and a per-crate package check (idx-cli is built from its tarball against packaged idx-core, since idx-core is not on crates.io yet). CI cache key, release path filter, and the golden whitespace-hook exclusion follow the new layout. Parity: all 143 golden outputs match byte for byte without regeneration; 157 core + 17 CLI unit tests (173 before plus the new OUTPUT_VALUES sync test), 62 CLI tests pass. Live A/B against main: 16/16 stocks cases identical; ownership discover got IDX 403 on both (exit 4 vs 1 is the phase 1 exit-code change). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
22b6dabaeb
commit
15947de51a
238 changed files with 544 additions and 375 deletions
468
crates/idx-core/src/analysis/fundamental.rs
Normal file
468
crates/idx-core/src/analysis/fundamental.rs
Normal file
|
|
@ -0,0 +1,468 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::api::types::Fundamentals;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GrowthReport {
|
||||
pub revenue_growth: Option<f64>,
|
||||
pub earnings_growth: Option<f64>,
|
||||
pub revenue_growth_pct: Option<f64>,
|
||||
pub earnings_growth_pct: Option<f64>,
|
||||
pub revenue_signal: String,
|
||||
pub earnings_signal: String,
|
||||
pub overall_signal: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ValuationReport {
|
||||
pub pe_trailing: Option<f64>,
|
||||
pub pe_forward: Option<f64>,
|
||||
pub pb: Option<f64>,
|
||||
pub roe: Option<f64>,
|
||||
pub roe_pct: Option<f64>,
|
||||
pub net_margin: Option<f64>,
|
||||
pub net_margin_pct: Option<f64>,
|
||||
pub ev_ebitda: Option<f64>,
|
||||
pub pe_signal: String,
|
||||
pub pb_signal: String,
|
||||
pub roe_signal: String,
|
||||
pub margin_signal: String,
|
||||
pub ev_ebitda_signal: String,
|
||||
pub overall_signal: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RiskReport {
|
||||
pub debt_to_equity: Option<f64>,
|
||||
pub current_ratio: Option<f64>,
|
||||
pub roa: Option<f64>,
|
||||
pub roa_pct: Option<f64>,
|
||||
pub de_signal: String,
|
||||
pub current_ratio_signal: String,
|
||||
pub overall_signal: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct FundamentalReport {
|
||||
pub symbol: String,
|
||||
pub growth: GrowthReport,
|
||||
pub valuation: ValuationReport,
|
||||
pub risk: RiskReport,
|
||||
pub overall_signal: String,
|
||||
}
|
||||
|
||||
pub fn analyze_growth(fundamentals: &Fundamentals) -> GrowthReport {
|
||||
let revenue_signal = growth_signal(fundamentals.revenue_growth);
|
||||
let earnings_signal = growth_signal(fundamentals.earnings_growth);
|
||||
let positive = ["strong", "moderate"];
|
||||
let negative = ["declining", "contracting"];
|
||||
|
||||
let overall_signal = if revenue_signal == "no data" && earnings_signal == "no data" {
|
||||
"no data"
|
||||
} else if positive.contains(&revenue_signal) && positive.contains(&earnings_signal) {
|
||||
"growing"
|
||||
} else if negative.contains(&revenue_signal) && negative.contains(&earnings_signal) {
|
||||
"shrinking"
|
||||
} else if revenue_signal == "no data" || earnings_signal == "no data" {
|
||||
let other = if earnings_signal == "no data" {
|
||||
revenue_signal
|
||||
} else {
|
||||
earnings_signal
|
||||
};
|
||||
if negative.contains(&other) {
|
||||
"mixed"
|
||||
} else {
|
||||
"incomplete data"
|
||||
}
|
||||
} else {
|
||||
"mixed"
|
||||
};
|
||||
|
||||
GrowthReport {
|
||||
revenue_growth: fundamentals.revenue_growth,
|
||||
earnings_growth: fundamentals.earnings_growth,
|
||||
revenue_growth_pct: ratio_pct(fundamentals.revenue_growth),
|
||||
earnings_growth_pct: ratio_pct(fundamentals.earnings_growth),
|
||||
revenue_signal: revenue_signal.to_string(),
|
||||
earnings_signal: earnings_signal.to_string(),
|
||||
overall_signal: overall_signal.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn analyze_valuation(fundamentals: &Fundamentals) -> ValuationReport {
|
||||
let ev_ebitda = match (fundamentals.enterprise_value, fundamentals.ebitda) {
|
||||
(Some(ev), Some(ebitda)) if ebitda > 0 => Some(round2(ev as f64 / ebitda as f64)),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let pe_signal = pe_signal(fundamentals.trailing_pe);
|
||||
let pb_signal = pb_signal(fundamentals.price_to_book);
|
||||
let ev_ebitda_signal = ev_ebitda_signal(ev_ebitda);
|
||||
let cheap = ["deep value", "undervalued"];
|
||||
let rich = ["premium", "expensive"];
|
||||
let price_signals: Vec<&str> = [pe_signal, pb_signal, ev_ebitda_signal]
|
||||
.into_iter()
|
||||
.filter(|signal| *signal != "no data")
|
||||
.collect();
|
||||
|
||||
let overall_signal = if price_signals.is_empty() {
|
||||
"no data"
|
||||
} else {
|
||||
let cheap_count = price_signals
|
||||
.iter()
|
||||
.filter(|signal| cheap.contains(signal))
|
||||
.count();
|
||||
let rich_count = price_signals
|
||||
.iter()
|
||||
.filter(|signal| rich.contains(signal))
|
||||
.count();
|
||||
|
||||
if cheap_count > price_signals.len() / 2 {
|
||||
"undervalued"
|
||||
} else if rich_count > price_signals.len() / 2 {
|
||||
"expensive"
|
||||
} else {
|
||||
"fairly valued"
|
||||
}
|
||||
};
|
||||
|
||||
ValuationReport {
|
||||
pe_trailing: fundamentals.trailing_pe,
|
||||
pe_forward: fundamentals.forward_pe,
|
||||
pb: fundamentals.price_to_book,
|
||||
roe: fundamentals.return_on_equity,
|
||||
roe_pct: ratio_pct(fundamentals.return_on_equity),
|
||||
net_margin: fundamentals.profit_margins,
|
||||
net_margin_pct: ratio_pct(fundamentals.profit_margins),
|
||||
ev_ebitda,
|
||||
pe_signal: pe_signal.to_string(),
|
||||
pb_signal: pb_signal.to_string(),
|
||||
roe_signal: roe_signal(fundamentals.return_on_equity).to_string(),
|
||||
margin_signal: margin_signal(fundamentals.profit_margins).to_string(),
|
||||
ev_ebitda_signal: ev_ebitda_signal.to_string(),
|
||||
overall_signal: overall_signal.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn analyze_risk(fundamentals: &Fundamentals) -> RiskReport {
|
||||
let de_signal = de_signal(fundamentals.debt_to_equity);
|
||||
let current_ratio_signal = cr_signal(fundamentals.current_ratio);
|
||||
|
||||
let overall_signal = if de_signal == "no data" && current_ratio_signal == "no data" {
|
||||
"no data"
|
||||
} else if de_signal == "no data" || current_ratio_signal == "no data" {
|
||||
"incomplete data"
|
||||
} else if de_signal == "highly leveraged" || current_ratio_signal == "weak" {
|
||||
"high risk"
|
||||
} else if de_signal == "conservative" && matches!(current_ratio_signal, "strong" | "adequate") {
|
||||
"low risk"
|
||||
} else {
|
||||
"moderate risk"
|
||||
};
|
||||
|
||||
RiskReport {
|
||||
debt_to_equity: fundamentals.debt_to_equity,
|
||||
current_ratio: fundamentals.current_ratio,
|
||||
roa: fundamentals.return_on_assets,
|
||||
roa_pct: ratio_pct(fundamentals.return_on_assets),
|
||||
de_signal: de_signal.to_string(),
|
||||
current_ratio_signal: current_ratio_signal.to_string(),
|
||||
overall_signal: overall_signal.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn analyze_fundamental(symbol: &str, fundamentals: &Fundamentals) -> FundamentalReport {
|
||||
let growth = analyze_growth(fundamentals);
|
||||
let valuation = analyze_valuation(fundamentals);
|
||||
let risk = analyze_risk(fundamentals);
|
||||
|
||||
let overall_signal = if growth.overall_signal == "growing"
|
||||
&& valuation.overall_signal != "expensive"
|
||||
&& risk.overall_signal != "high risk"
|
||||
{
|
||||
"healthy"
|
||||
} else if growth.overall_signal == "shrinking" && risk.overall_signal == "high risk" {
|
||||
"weak"
|
||||
} else if [
|
||||
&growth.overall_signal,
|
||||
&valuation.overall_signal,
|
||||
&risk.overall_signal,
|
||||
]
|
||||
.into_iter()
|
||||
.all(|signal| *signal == "no data")
|
||||
{
|
||||
"no data"
|
||||
} else {
|
||||
"mixed"
|
||||
};
|
||||
|
||||
FundamentalReport {
|
||||
symbol: symbol.to_string(),
|
||||
growth,
|
||||
valuation,
|
||||
risk,
|
||||
overall_signal: overall_signal.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn growth_signal(value: Option<f64>) -> &'static str {
|
||||
let Some(value) = value else {
|
||||
return "no data";
|
||||
};
|
||||
if value >= 0.20 {
|
||||
"strong"
|
||||
} else if value >= 0.10 {
|
||||
"moderate"
|
||||
} else if value >= 0.0 {
|
||||
"slow"
|
||||
} else if value >= -0.10 {
|
||||
"declining"
|
||||
} else {
|
||||
"contracting"
|
||||
}
|
||||
}
|
||||
|
||||
fn pe_signal(value: Option<f64>) -> &'static str {
|
||||
let Some(value) = value else {
|
||||
return "no data";
|
||||
};
|
||||
if value <= 0.0 {
|
||||
"no data"
|
||||
} else if value < 8.0 {
|
||||
"deep value"
|
||||
} else if value < 15.0 {
|
||||
"undervalued"
|
||||
} else if value < 25.0 {
|
||||
"fairly valued"
|
||||
} else if value < 40.0 {
|
||||
"premium"
|
||||
} else {
|
||||
"expensive"
|
||||
}
|
||||
}
|
||||
|
||||
fn pb_signal(value: Option<f64>) -> &'static str {
|
||||
let Some(value) = value else {
|
||||
return "no data";
|
||||
};
|
||||
if value <= 0.0 {
|
||||
"no data"
|
||||
} else if value < 1.0 {
|
||||
"deep value"
|
||||
} else if value < 2.0 {
|
||||
"undervalued"
|
||||
} else if value < 4.0 {
|
||||
"fairly valued"
|
||||
} else {
|
||||
"expensive"
|
||||
}
|
||||
}
|
||||
|
||||
fn roe_signal(value: Option<f64>) -> &'static str {
|
||||
let Some(value) = value else {
|
||||
return "no data";
|
||||
};
|
||||
if value >= 0.20 {
|
||||
"excellent"
|
||||
} else if value >= 0.15 {
|
||||
"strong"
|
||||
} else if value >= 0.10 {
|
||||
"adequate"
|
||||
} else if value >= 0.0 {
|
||||
"weak"
|
||||
} else {
|
||||
"negative"
|
||||
}
|
||||
}
|
||||
|
||||
fn margin_signal(value: Option<f64>) -> &'static str {
|
||||
let Some(value) = value else {
|
||||
return "no data";
|
||||
};
|
||||
if value >= 0.20 {
|
||||
"excellent"
|
||||
} else if value >= 0.10 {
|
||||
"healthy"
|
||||
} else if value >= 0.05 {
|
||||
"adequate"
|
||||
} else if value >= 0.0 {
|
||||
"thin"
|
||||
} else {
|
||||
"negative"
|
||||
}
|
||||
}
|
||||
|
||||
fn ev_ebitda_signal(value: Option<f64>) -> &'static str {
|
||||
let Some(value) = value else {
|
||||
return "no data";
|
||||
};
|
||||
if value <= 0.0 {
|
||||
"no data"
|
||||
} else if value < 8.0 {
|
||||
"undervalued"
|
||||
} else if value < 14.0 {
|
||||
"fairly valued"
|
||||
} else if value < 20.0 {
|
||||
"premium"
|
||||
} else {
|
||||
"expensive"
|
||||
}
|
||||
}
|
||||
|
||||
fn de_signal(value: Option<f64>) -> &'static str {
|
||||
let Some(value) = value else {
|
||||
return "no data";
|
||||
};
|
||||
if value < 0.0 {
|
||||
"negative equity"
|
||||
} else if value < 50.0 {
|
||||
"conservative"
|
||||
} else if value < 100.0 {
|
||||
"moderate"
|
||||
} else if value < 200.0 {
|
||||
"leveraged"
|
||||
} else {
|
||||
"highly leveraged"
|
||||
}
|
||||
}
|
||||
|
||||
fn cr_signal(value: Option<f64>) -> &'static str {
|
||||
let Some(value) = value else {
|
||||
return "no data";
|
||||
};
|
||||
if value >= 2.0 {
|
||||
"strong"
|
||||
} else if value >= 1.5 {
|
||||
"adequate"
|
||||
} else if value >= 1.0 {
|
||||
"tight"
|
||||
} else {
|
||||
"weak"
|
||||
}
|
||||
}
|
||||
|
||||
fn ratio_pct(value: Option<f64>) -> Option<f64> {
|
||||
value.map(|value| round2(value * 100.0))
|
||||
}
|
||||
|
||||
fn round2(value: f64) -> f64 {
|
||||
(value * 100.0).round() / 100.0
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
Fundamentals, analyze_fundamental, analyze_growth, analyze_risk, analyze_valuation,
|
||||
cr_signal, de_signal, ev_ebitda_signal, growth_signal, margin_signal, pb_signal, pe_signal,
|
||||
roe_signal,
|
||||
};
|
||||
|
||||
fn sample_fundamentals() -> Fundamentals {
|
||||
Fundamentals {
|
||||
trailing_pe: Some(12.5),
|
||||
forward_pe: Some(11.0),
|
||||
price_to_book: Some(1.8),
|
||||
return_on_equity: Some(0.18),
|
||||
profit_margins: Some(0.12),
|
||||
return_on_assets: Some(0.06),
|
||||
revenue_growth: Some(0.12),
|
||||
earnings_growth: Some(0.22),
|
||||
debt_to_equity: Some(40.0),
|
||||
current_ratio: Some(1.6),
|
||||
enterprise_value: Some(120),
|
||||
ebitda: Some(15),
|
||||
market_cap: Some(100),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn growth_signal_thresholds_match_python() {
|
||||
assert_eq!(growth_signal(None), "no data");
|
||||
assert_eq!(growth_signal(Some(0.20)), "strong");
|
||||
assert_eq!(growth_signal(Some(0.10)), "moderate");
|
||||
assert_eq!(growth_signal(Some(0.0)), "slow");
|
||||
assert_eq!(growth_signal(Some(-0.10)), "declining");
|
||||
assert_eq!(growth_signal(Some(-0.11)), "contracting");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn valuation_signal_thresholds_match_python() {
|
||||
assert_eq!(pe_signal(None), "no data");
|
||||
assert_eq!(pe_signal(Some(7.9)), "deep value");
|
||||
assert_eq!(pe_signal(Some(14.9)), "undervalued");
|
||||
assert_eq!(pe_signal(Some(24.9)), "fairly valued");
|
||||
assert_eq!(pe_signal(Some(39.9)), "premium");
|
||||
assert_eq!(pe_signal(Some(40.0)), "expensive");
|
||||
|
||||
assert_eq!(pb_signal(Some(0.9)), "deep value");
|
||||
assert_eq!(pb_signal(Some(1.9)), "undervalued");
|
||||
assert_eq!(pb_signal(Some(3.9)), "fairly valued");
|
||||
assert_eq!(pb_signal(Some(4.0)), "expensive");
|
||||
|
||||
assert_eq!(roe_signal(Some(0.20)), "excellent");
|
||||
assert_eq!(roe_signal(Some(0.15)), "strong");
|
||||
assert_eq!(roe_signal(Some(0.10)), "adequate");
|
||||
assert_eq!(roe_signal(Some(0.0)), "weak");
|
||||
assert_eq!(roe_signal(Some(-0.01)), "negative");
|
||||
|
||||
assert_eq!(margin_signal(Some(0.20)), "excellent");
|
||||
assert_eq!(margin_signal(Some(0.10)), "healthy");
|
||||
assert_eq!(margin_signal(Some(0.05)), "adequate");
|
||||
assert_eq!(margin_signal(Some(0.0)), "thin");
|
||||
assert_eq!(margin_signal(Some(-0.01)), "negative");
|
||||
|
||||
assert_eq!(ev_ebitda_signal(Some(7.9)), "undervalued");
|
||||
assert_eq!(ev_ebitda_signal(Some(13.9)), "fairly valued");
|
||||
assert_eq!(ev_ebitda_signal(Some(19.9)), "premium");
|
||||
assert_eq!(ev_ebitda_signal(Some(20.0)), "expensive");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn risk_signal_thresholds_match_python() {
|
||||
assert_eq!(de_signal(None), "no data");
|
||||
assert_eq!(de_signal(Some(-1.0)), "negative equity");
|
||||
assert_eq!(de_signal(Some(49.9)), "conservative");
|
||||
assert_eq!(de_signal(Some(99.9)), "moderate");
|
||||
assert_eq!(de_signal(Some(199.9)), "leveraged");
|
||||
assert_eq!(de_signal(Some(200.0)), "highly leveraged");
|
||||
|
||||
assert_eq!(cr_signal(None), "no data");
|
||||
assert_eq!(cr_signal(Some(2.0)), "strong");
|
||||
assert_eq!(cr_signal(Some(1.5)), "adequate");
|
||||
assert_eq!(cr_signal(Some(1.0)), "tight");
|
||||
assert_eq!(cr_signal(Some(0.99)), "weak");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn report_overalls_match_python_logic() {
|
||||
let fundamentals = sample_fundamentals();
|
||||
|
||||
let growth = analyze_growth(&fundamentals);
|
||||
assert_eq!(growth.overall_signal, "growing");
|
||||
|
||||
let valuation = analyze_valuation(&fundamentals);
|
||||
assert_eq!(valuation.overall_signal, "undervalued");
|
||||
assert_eq!(valuation.ev_ebitda, Some(8.0));
|
||||
|
||||
let risk = analyze_risk(&fundamentals);
|
||||
assert_eq!(risk.overall_signal, "low risk");
|
||||
|
||||
let fundamental = analyze_fundamental("BBCA.JK", &fundamentals);
|
||||
assert_eq!(fundamental.overall_signal, "healthy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incomplete_growth_and_high_risk_paths_match_python_logic() {
|
||||
let mut fundamentals = sample_fundamentals();
|
||||
fundamentals.revenue_growth = None;
|
||||
fundamentals.earnings_growth = Some(0.05);
|
||||
fundamentals.debt_to_equity = Some(250.0);
|
||||
fundamentals.current_ratio = Some(0.9);
|
||||
|
||||
let growth = analyze_growth(&fundamentals);
|
||||
assert_eq!(growth.overall_signal, "incomplete data");
|
||||
|
||||
let risk = analyze_risk(&fundamentals);
|
||||
assert_eq!(risk.overall_signal, "high risk");
|
||||
}
|
||||
}
|
||||
3
crates/idx-core/src/analysis/mod.rs
Normal file
3
crates/idx-core/src/analysis/mod.rs
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
pub mod fundamental;
|
||||
pub mod signals;
|
||||
pub mod technical;
|
||||
102
crates/idx-core/src/analysis/signals.rs
Normal file
102
crates/idx-core/src/analysis/signals.rs
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum Signal {
|
||||
Bullish,
|
||||
Bearish,
|
||||
Neutral,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TechnicalSignal {
|
||||
pub rsi: Signal,
|
||||
pub macd: Signal,
|
||||
pub trend: Signal,
|
||||
pub overall: Signal,
|
||||
}
|
||||
|
||||
pub fn interpret_rsi(value: f64) -> Signal {
|
||||
if value > 70.0 {
|
||||
Signal::Bearish
|
||||
} else if value < 30.0 {
|
||||
Signal::Bullish
|
||||
} else {
|
||||
Signal::Neutral
|
||||
}
|
||||
}
|
||||
|
||||
pub fn interpret_macd(histogram: f64, prev_histogram: Option<f64>) -> Signal {
|
||||
if let Some(prev) = prev_histogram {
|
||||
if histogram > 0.0 && histogram > prev {
|
||||
Signal::Bullish
|
||||
} else if histogram < 0.0 && histogram < prev {
|
||||
Signal::Bearish
|
||||
} else {
|
||||
Signal::Neutral
|
||||
}
|
||||
} else {
|
||||
Signal::Neutral
|
||||
}
|
||||
}
|
||||
|
||||
pub fn interpret_trend(price: f64, sma50: Option<f64>, sma200: Option<f64>) -> Signal {
|
||||
match (sma50, sma200) {
|
||||
(Some(s50), Some(s200)) if price > s50 && price > s200 => Signal::Bullish,
|
||||
(Some(s50), Some(s200)) if price < s50 && price < s200 => Signal::Bearish,
|
||||
_ => Signal::Neutral,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn overall_signal(rsi: Signal, macd: Signal, trend: Signal) -> Signal {
|
||||
let signals = [rsi, macd, trend];
|
||||
let bullish = signals.iter().filter(|&&s| s == Signal::Bullish).count();
|
||||
let bearish = signals.iter().filter(|&&s| s == Signal::Bearish).count();
|
||||
|
||||
if bullish >= 2 {
|
||||
Signal::Bullish
|
||||
} else if bearish >= 2 {
|
||||
Signal::Bearish
|
||||
} else {
|
||||
Signal::Neutral
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn interpret_rsi_thresholds() {
|
||||
assert_eq!(interpret_rsi(75.0), Signal::Bearish);
|
||||
assert_eq!(interpret_rsi(25.0), Signal::Bullish);
|
||||
assert_eq!(interpret_rsi(50.0), Signal::Neutral);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interpret_trend_thresholds() {
|
||||
assert_eq!(
|
||||
interpret_trend(120.0, Some(100.0), Some(110.0)),
|
||||
Signal::Bullish
|
||||
);
|
||||
assert_eq!(
|
||||
interpret_trend(80.0, Some(100.0), Some(90.0)),
|
||||
Signal::Bearish
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overall_majority_vote() {
|
||||
assert_eq!(
|
||||
overall_signal(Signal::Bullish, Signal::Bullish, Signal::Neutral),
|
||||
Signal::Bullish
|
||||
);
|
||||
assert_eq!(
|
||||
overall_signal(Signal::Bearish, Signal::Neutral, Signal::Bearish),
|
||||
Signal::Bearish
|
||||
);
|
||||
assert_eq!(
|
||||
overall_signal(Signal::Bullish, Signal::Bearish, Signal::Neutral),
|
||||
Signal::Neutral
|
||||
);
|
||||
}
|
||||
}
|
||||
212
crates/idx-core/src/analysis/technical.rs
Normal file
212
crates/idx-core/src/analysis/technical.rs
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
use serde::Serialize;
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct MacdResult {
|
||||
pub macd_line: Vec<Option<f64>>,
|
||||
pub signal_line: Vec<Option<f64>>,
|
||||
pub histogram: Vec<Option<f64>>,
|
||||
}
|
||||
|
||||
pub fn sma(data: &[f64], period: usize) -> Vec<Option<f64>> {
|
||||
let mut result = vec![None; data.len()];
|
||||
if period == 0 || period > data.len() {
|
||||
return result;
|
||||
}
|
||||
|
||||
let mut window_sum: f64 = data[..period].iter().sum();
|
||||
result[period - 1] = Some(window_sum / period as f64);
|
||||
|
||||
for idx in period..data.len() {
|
||||
window_sum += data[idx] - data[idx - period];
|
||||
result[idx] = Some(window_sum / period as f64);
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
pub fn ema(data: &[f64], period: usize) -> Vec<Option<f64>> {
|
||||
let mut result = vec![None; data.len()];
|
||||
if period == 0 || period > data.len() {
|
||||
return result;
|
||||
}
|
||||
|
||||
let multiplier = 2.0 / (period as f64 + 1.0);
|
||||
let seed = data[..period].iter().sum::<f64>() / period as f64;
|
||||
result[period - 1] = Some(seed);
|
||||
|
||||
let mut prev = seed;
|
||||
for idx in period..data.len() {
|
||||
let current = ((data[idx] - prev) * multiplier) + prev;
|
||||
result[idx] = Some(current);
|
||||
prev = current;
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
pub fn rsi(closes: &[f64], period: usize) -> Vec<Option<f64>> {
|
||||
let mut result = vec![None; closes.len()];
|
||||
if period == 0 || closes.len() <= period {
|
||||
return result;
|
||||
}
|
||||
|
||||
let mut gains = 0.0;
|
||||
let mut losses = 0.0;
|
||||
|
||||
for idx in 1..=period {
|
||||
let change = closes[idx] - closes[idx - 1];
|
||||
if change >= 0.0 {
|
||||
gains += change;
|
||||
} else {
|
||||
losses += -change;
|
||||
}
|
||||
}
|
||||
|
||||
let mut avg_gain = gains / period as f64;
|
||||
let mut avg_loss = losses / period as f64;
|
||||
|
||||
result[period] = Some(rsi_from_averages(avg_gain, avg_loss));
|
||||
|
||||
for idx in (period + 1)..closes.len() {
|
||||
let change = closes[idx] - closes[idx - 1];
|
||||
let gain = if change > 0.0 { change } else { 0.0 };
|
||||
let loss = if change < 0.0 { -change } else { 0.0 };
|
||||
|
||||
avg_gain = ((avg_gain * (period as f64 - 1.0)) + gain) / period as f64;
|
||||
avg_loss = ((avg_loss * (period as f64 - 1.0)) + loss) / period as f64;
|
||||
|
||||
result[idx] = Some(rsi_from_averages(avg_gain, avg_loss));
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
fn rsi_from_averages(avg_gain: f64, avg_loss: f64) -> f64 {
|
||||
if avg_loss == 0.0 {
|
||||
return 100.0;
|
||||
}
|
||||
|
||||
let rs = avg_gain / avg_loss;
|
||||
100.0 - (100.0 / (1.0 + rs))
|
||||
}
|
||||
|
||||
pub fn macd(closes: &[f64], fast: usize, slow: usize, signal: usize) -> MacdResult {
|
||||
let len = closes.len();
|
||||
let mut macd_line = vec![None; len];
|
||||
let mut signal_line = vec![None; len];
|
||||
let mut histogram = vec![None; len];
|
||||
|
||||
if len == 0 || fast == 0 || slow == 0 || signal == 0 {
|
||||
return MacdResult {
|
||||
macd_line,
|
||||
signal_line,
|
||||
histogram,
|
||||
};
|
||||
}
|
||||
|
||||
let fast_ema = ema(closes, fast);
|
||||
let slow_ema = ema(closes, slow);
|
||||
|
||||
for idx in 0..len {
|
||||
if let (Some(f), Some(s)) = (fast_ema[idx], slow_ema[idx]) {
|
||||
macd_line[idx] = Some(f - s);
|
||||
}
|
||||
}
|
||||
|
||||
let mut signal_seed = Vec::new();
|
||||
let signal_multiplier = 2.0 / (signal as f64 + 1.0);
|
||||
let mut prev_signal = None;
|
||||
|
||||
for idx in 0..len {
|
||||
if let Some(value) = macd_line[idx] {
|
||||
if prev_signal.is_none() {
|
||||
signal_seed.push(value);
|
||||
if signal_seed.len() == signal {
|
||||
let seed = signal_seed.iter().sum::<f64>() / signal as f64;
|
||||
signal_line[idx] = Some(seed);
|
||||
prev_signal = Some(seed);
|
||||
histogram[idx] = Some(value - seed);
|
||||
}
|
||||
} else if let Some(prev) = prev_signal {
|
||||
let current = ((value - prev) * signal_multiplier) + prev;
|
||||
signal_line[idx] = Some(current);
|
||||
prev_signal = Some(current);
|
||||
histogram[idx] = Some(value - current);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
MacdResult {
|
||||
macd_line,
|
||||
signal_line,
|
||||
histogram,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn volume_ratio(volumes: &[f64], period: usize) -> Option<f64> {
|
||||
if period == 0 || volumes.len() < period {
|
||||
return None;
|
||||
}
|
||||
|
||||
let start = volumes.len() - period;
|
||||
let avg = volumes[start..].iter().sum::<f64>() / period as f64;
|
||||
if avg == 0.0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
volumes.last().map(|last| *last / avg)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn approx_eq(left: f64, right: f64, eps: f64) {
|
||||
assert!((left - right).abs() <= eps, "left={left}, right={right}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sma_returns_expected_values() {
|
||||
let data = [1.0, 2.0, 3.0, 4.0, 5.0];
|
||||
let values = sma(&data, 3);
|
||||
assert_eq!(values, vec![None, None, Some(2.0), Some(3.0), Some(4.0)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rsi_returns_seeded_none_and_known_value() {
|
||||
// Classic Wilder example dataset; RSI(14) first computed value ~= 70.46.
|
||||
let closes = [
|
||||
44.34, 44.09, 44.15, 43.61, 44.33, 44.83, 45.10, 45.42, 45.84, 46.08, 45.89, 46.03,
|
||||
45.61, 46.28, 46.28,
|
||||
];
|
||||
let period = 14;
|
||||
let values = rsi(&closes, period);
|
||||
|
||||
assert_eq!(values.len(), closes.len());
|
||||
assert!(values.iter().take(period).all(Option::is_none));
|
||||
|
||||
let rsi_14 = values[period].expect("expected first RSI value");
|
||||
approx_eq(rsi_14, 70.46, 0.05);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn macd_shapes_are_correct() {
|
||||
let closes: Vec<f64> = (1..=60).map(|n| n as f64).collect();
|
||||
let result = macd(&closes, 12, 26, 9);
|
||||
|
||||
assert_eq!(result.macd_line.len(), closes.len());
|
||||
|
||||
let macd_nones = result.macd_line.iter().filter(|v| v.is_none()).count();
|
||||
let signal_nones = result.signal_line.iter().filter(|v| v.is_none()).count();
|
||||
assert!(signal_nones > macd_nones);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn volume_ratio_checks() {
|
||||
let volumes = [100.0, 120.0, 130.0, 150.0];
|
||||
let ratio = volume_ratio(&volumes, 3).expect("ratio should exist");
|
||||
approx_eq(ratio, 150.0 / ((120.0 + 130.0 + 150.0) / 3.0), 1e-10);
|
||||
|
||||
assert_eq!(volume_ratio(&volumes, 5), None);
|
||||
}
|
||||
}
|
||||
455
crates/idx-core/src/api/mod.rs
Normal file
455
crates/idx-core/src/api/mod.rs
Normal file
|
|
@ -0,0 +1,455 @@
|
|||
pub mod msn;
|
||||
pub mod types;
|
||||
pub mod yahoo;
|
||||
|
||||
use chrono::FixedOffset;
|
||||
|
||||
use crate::config::{HistoryProviderKind, ProviderKind};
|
||||
use crate::error::IdxError;
|
||||
use types::{
|
||||
Bar, CompanyProfile, EarningsReport, FinancialStatements, Fundamentals, InsightData, Interval,
|
||||
NewsItem, Period, Quote, SentimentData,
|
||||
};
|
||||
|
||||
/// IDX trades in Western Indonesia Time (WIB, UTC+7, no DST). Providers stamp
|
||||
/// candles in UTC, so day boundaries must be taken in this zone.
|
||||
pub(crate) const IDX_UTC_OFFSET_SECS: i32 = 7 * 3600;
|
||||
|
||||
pub(crate) fn idx_offset() -> FixedOffset {
|
||||
FixedOffset::east_opt(IDX_UTC_OFFSET_SECS).expect("valid WIB offset")
|
||||
}
|
||||
|
||||
/// Parse a provider's RFC 3339 trade time and express it in WIB.
|
||||
pub(crate) fn parse_as_of(raw: Option<&str>) -> Option<chrono::DateTime<FixedOffset>> {
|
||||
chrono::DateTime::parse_from_rfc3339(raw?)
|
||||
.ok()
|
||||
.map(|t| t.with_timezone(&idx_offset()))
|
||||
}
|
||||
|
||||
pub trait QuoteProvider {
|
||||
fn quote(&self, symbol: &str) -> Result<Quote, IdxError>;
|
||||
}
|
||||
|
||||
pub trait HistoryProvider {
|
||||
fn history(
|
||||
&self,
|
||||
symbol: &str,
|
||||
period: &Period,
|
||||
interval: &Interval,
|
||||
) -> Result<Vec<Bar>, IdxError>;
|
||||
}
|
||||
|
||||
pub trait FundamentalsProvider {
|
||||
fn fundamentals(&self, symbol: &str) -> Result<Fundamentals, IdxError>;
|
||||
}
|
||||
|
||||
/// Core provider trait — quote + fundamentals only.
|
||||
/// History is a separate capability (`HistoryProvider`) not all providers support
|
||||
/// (e.g. MSN Finance/Charts returns 404 for IDX/XIDX stocks).
|
||||
pub trait MarketDataProvider: QuoteProvider + FundamentalsProvider {}
|
||||
impl<T> MarketDataProvider for T where T: QuoteProvider + FundamentalsProvider {}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub trait ProfileProvider {
|
||||
fn profile(&self, symbol: &str) -> Result<CompanyProfile, IdxError>;
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub trait EarningsProvider {
|
||||
fn earnings(&self, symbol: &str) -> Result<EarningsReport, IdxError>;
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub trait FinancialsProvider {
|
||||
fn financials(&self, symbol: &str) -> Result<FinancialStatements, IdxError>;
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub trait SentimentProvider {
|
||||
fn sentiment(&self, symbol: &str) -> Result<SentimentData, IdxError>;
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub trait InsightsProvider {
|
||||
fn insights(&self, symbol: &str) -> Result<InsightData, IdxError>;
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub trait NewsProvider {
|
||||
fn news(&self, symbol: &str, limit: usize) -> Result<Vec<NewsItem>, IdxError>;
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub trait ScreenerProvider {
|
||||
/// `None` requests the complete candidate list for client-side ranking.
|
||||
fn screener(
|
||||
&self,
|
||||
filter: &str,
|
||||
region: &str,
|
||||
limit: Option<usize>,
|
||||
) -> Result<Vec<Quote>, IdxError>;
|
||||
}
|
||||
|
||||
pub struct SelectedProvider {
|
||||
kind: ProviderKind,
|
||||
market: Box<dyn MarketDataProvider>,
|
||||
profile: Option<Box<dyn ProfileProvider>>,
|
||||
financials: Option<Box<dyn FinancialsProvider>>,
|
||||
earnings: Option<Box<dyn EarningsProvider>>,
|
||||
sentiment: Option<Box<dyn SentimentProvider>>,
|
||||
insights: Option<Box<dyn InsightsProvider>>,
|
||||
news: Option<Box<dyn NewsProvider>>,
|
||||
screener: Option<Box<dyn ScreenerProvider>>,
|
||||
}
|
||||
|
||||
impl SelectedProvider {
|
||||
pub fn kind(&self) -> ProviderKind {
|
||||
self.kind
|
||||
}
|
||||
|
||||
pub fn market(&self) -> &dyn MarketDataProvider {
|
||||
self.market.as_ref()
|
||||
}
|
||||
|
||||
pub fn profile_provider(&self, subject: &str) -> Result<&dyn ProfileProvider, IdxError> {
|
||||
self.profile
|
||||
.as_deref()
|
||||
.ok_or_else(|| msn_capability_error(subject))
|
||||
}
|
||||
|
||||
pub fn financials_provider(&self, subject: &str) -> Result<&dyn FinancialsProvider, IdxError> {
|
||||
self.financials
|
||||
.as_deref()
|
||||
.ok_or_else(|| msn_capability_error(subject))
|
||||
}
|
||||
|
||||
pub fn earnings_provider(&self, subject: &str) -> Result<&dyn EarningsProvider, IdxError> {
|
||||
self.earnings
|
||||
.as_deref()
|
||||
.ok_or_else(|| msn_capability_error(subject))
|
||||
}
|
||||
|
||||
pub fn sentiment_provider(&self, subject: &str) -> Result<&dyn SentimentProvider, IdxError> {
|
||||
self.sentiment
|
||||
.as_deref()
|
||||
.ok_or_else(|| msn_capability_error(subject))
|
||||
}
|
||||
|
||||
pub fn insights_provider(&self, subject: &str) -> Result<&dyn InsightsProvider, IdxError> {
|
||||
self.insights
|
||||
.as_deref()
|
||||
.ok_or_else(|| msn_capability_error(subject))
|
||||
}
|
||||
|
||||
pub fn news_provider(&self, subject: &str) -> Result<&dyn NewsProvider, IdxError> {
|
||||
self.news
|
||||
.as_deref()
|
||||
.ok_or_else(|| msn_capability_error(subject))
|
||||
}
|
||||
|
||||
pub fn screener_provider(&self, subject: &str) -> Result<&dyn ScreenerProvider, IdxError> {
|
||||
self.screener
|
||||
.as_deref()
|
||||
.ok_or_else(|| msn_capability_error(subject))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn resolve_symbol(symbol: &str, exchange: &str) -> Result<String, IdxError> {
|
||||
let trimmed = symbol.trim().to_uppercase();
|
||||
if trimmed.is_empty() {
|
||||
return Err(IdxError::InvalidInput(
|
||||
"ticker symbol cannot be empty".into(),
|
||||
));
|
||||
}
|
||||
if let Some((base, suffix)) = trimmed.rsplit_once('.')
|
||||
&& !base.is_empty()
|
||||
&& !suffix.is_empty()
|
||||
{
|
||||
return Ok(trimmed);
|
||||
}
|
||||
Ok(format!("{trimmed}.{}", exchange.trim().to_uppercase()))
|
||||
}
|
||||
|
||||
pub fn default_provider(provider: ProviderKind, verbose: bool) -> SelectedProvider {
|
||||
build_selected_provider(
|
||||
provider,
|
||||
verbose,
|
||||
std::env::var("IDX_USE_MOCK_PROVIDER").is_ok(),
|
||||
)
|
||||
}
|
||||
|
||||
fn build_selected_provider(
|
||||
provider: ProviderKind,
|
||||
verbose: bool,
|
||||
use_mock: bool,
|
||||
) -> SelectedProvider {
|
||||
match (provider, use_mock) {
|
||||
(ProviderKind::Yahoo, true) => SelectedProvider {
|
||||
kind: ProviderKind::Yahoo,
|
||||
market: Box::new(MockProvider::from_fixtures(ProviderKind::Yahoo)),
|
||||
profile: None,
|
||||
financials: None,
|
||||
earnings: None,
|
||||
sentiment: None,
|
||||
insights: None,
|
||||
news: None,
|
||||
screener: None,
|
||||
},
|
||||
(ProviderKind::Yahoo, false) => SelectedProvider {
|
||||
kind: ProviderKind::Yahoo,
|
||||
market: Box::new(yahoo::YahooProvider::new(verbose)),
|
||||
profile: None,
|
||||
financials: None,
|
||||
earnings: None,
|
||||
sentiment: None,
|
||||
insights: None,
|
||||
news: None,
|
||||
screener: None,
|
||||
},
|
||||
(ProviderKind::Msn, true) => SelectedProvider {
|
||||
kind: ProviderKind::Msn,
|
||||
market: Box::new(MockProvider::from_fixtures(ProviderKind::Msn)),
|
||||
profile: Some(Box::new(msn::MsnProvider::new(verbose))),
|
||||
financials: Some(Box::new(msn::MsnProvider::new(verbose))),
|
||||
earnings: Some(Box::new(msn::MsnProvider::new(verbose))),
|
||||
sentiment: Some(Box::new(msn::MsnProvider::new(verbose))),
|
||||
insights: Some(Box::new(msn::MsnProvider::new(verbose))),
|
||||
news: Some(Box::new(msn::MsnProvider::new(verbose))),
|
||||
screener: Some(Box::new(msn::MsnProvider::new(verbose))),
|
||||
},
|
||||
(ProviderKind::Msn, false) => SelectedProvider {
|
||||
kind: ProviderKind::Msn,
|
||||
market: Box::new(msn::MsnProvider::new(verbose)),
|
||||
profile: Some(Box::new(msn::MsnProvider::new(verbose))),
|
||||
financials: Some(Box::new(msn::MsnProvider::new(verbose))),
|
||||
earnings: Some(Box::new(msn::MsnProvider::new(verbose))),
|
||||
sentiment: Some(Box::new(msn::MsnProvider::new(verbose))),
|
||||
insights: Some(Box::new(msn::MsnProvider::new(verbose))),
|
||||
news: Some(Box::new(msn::MsnProvider::new(verbose))),
|
||||
screener: Some(Box::new(msn::MsnProvider::new(verbose))),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn msn_capability_error(subject: &str) -> IdxError {
|
||||
IdxError::Unsupported(format!("{subject}: command requires --provider msn"))
|
||||
}
|
||||
|
||||
/// Resolves a history provider based on the selected market data provider and
|
||||
/// history provider strategy.
|
||||
///
|
||||
/// `history_mode=auto` keeps using Yahoo for IDX history because Yahoo provides
|
||||
/// full OHLCV candles. Explicit `msn` opts into MSN's price-only chart feed.
|
||||
pub fn history_provider(
|
||||
provider: ProviderKind,
|
||||
history_mode: HistoryProviderKind,
|
||||
verbose: bool,
|
||||
) -> Result<(ProviderKind, Box<dyn HistoryProvider>), IdxError> {
|
||||
let resolved = match history_mode {
|
||||
HistoryProviderKind::Yahoo => ProviderKind::Yahoo,
|
||||
HistoryProviderKind::Msn => ProviderKind::Msn,
|
||||
HistoryProviderKind::Auto => match provider {
|
||||
ProviderKind::Yahoo => ProviderKind::Yahoo,
|
||||
ProviderKind::Msn => ProviderKind::Yahoo,
|
||||
},
|
||||
};
|
||||
|
||||
if std::env::var("IDX_USE_MOCK_PROVIDER").is_ok() {
|
||||
return Ok((
|
||||
resolved,
|
||||
Box::new(MockProvider::from_fixtures_with_history_verbose(
|
||||
resolved, verbose,
|
||||
)),
|
||||
));
|
||||
}
|
||||
|
||||
match resolved {
|
||||
ProviderKind::Yahoo => Ok((resolved, Box::new(yahoo::YahooProvider::new(verbose)))),
|
||||
ProviderKind::Msn => Ok((resolved, Box::new(msn::MsnProvider::new(verbose)))),
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MockProvider {
|
||||
quote: Result<Quote, IdxError>,
|
||||
fundamentals: Result<Fundamentals, IdxError>,
|
||||
history: Result<Vec<Bar>, IdxError>,
|
||||
}
|
||||
|
||||
impl MockProvider {
|
||||
pub fn from_fixtures(provider: ProviderKind) -> Self {
|
||||
Self::from_fixtures_with_history_verbose(provider, false)
|
||||
}
|
||||
|
||||
pub fn from_fixtures_with_history_verbose(
|
||||
provider: ProviderKind,
|
||||
history_verbose: bool,
|
||||
) -> Self {
|
||||
if std::env::var("IDX_MOCK_ERROR").is_ok() {
|
||||
return Self::with_error(IdxError::ProviderUnavailable);
|
||||
}
|
||||
|
||||
match provider {
|
||||
ProviderKind::Yahoo => Self::from_yahoo_fixtures(history_verbose),
|
||||
ProviderKind::Msn => Self::from_msn_fixtures(),
|
||||
}
|
||||
}
|
||||
|
||||
fn from_yahoo_fixtures(history_verbose: bool) -> Self {
|
||||
let quote_raw = std::fs::read_to_string("tests/fixtures/chart_bbca_1d.json")
|
||||
.unwrap_or_else(|_| "{}".to_string());
|
||||
let history_path = std::env::var("IDX_MOCK_YAHOO_HISTORY_FIXTURE")
|
||||
.unwrap_or_else(|_| "tests/fixtures/chart_bbca_3mo.json".to_string());
|
||||
let history_raw =
|
||||
std::fs::read_to_string(&history_path).unwrap_or_else(|_| "{}".to_string());
|
||||
let fundamentals_raw = std::fs::read_to_string("tests/fixtures/quotesummary_bbca.json")
|
||||
.unwrap_or_else(|_| "{}".to_string());
|
||||
|
||||
let quote = yahoo::parse_quote_from_str("BBCA.JK", "e_raw)
|
||||
.map_err(|e| IdxError::ParseError(e.to_string()));
|
||||
let fundamentals = yahoo::parse_fundamentals_from_str("BBCA.JK", &fundamentals_raw)
|
||||
.map_err(|e| IdxError::ParseError(e.to_string()));
|
||||
let history =
|
||||
yahoo::parse_history_from_str_with_verbose("BBCA.JK", &history_raw, history_verbose)
|
||||
.map_err(|e| IdxError::ParseError(e.to_string()));
|
||||
|
||||
Self {
|
||||
quote,
|
||||
fundamentals,
|
||||
history,
|
||||
}
|
||||
}
|
||||
|
||||
fn from_msn_fixtures() -> Self {
|
||||
let quote_raw = std::fs::read_to_string("tests/fixtures/msn_quote_bbca.json")
|
||||
.unwrap_or_else(|_| "[]".to_string());
|
||||
let fundamentals_path = std::env::var("IDX_MOCK_MSN_KEYRATIOS_FIXTURE")
|
||||
.unwrap_or_else(|_| "tests/fixtures/msn_keyratios_bbca.json".to_string());
|
||||
let fundamentals_raw =
|
||||
std::fs::read_to_string(&fundamentals_path).unwrap_or_else(|_| "[]".to_string());
|
||||
|
||||
let quote = msn::parse_quote_from_str("BBCA.JK", "e_raw)
|
||||
.map_err(|e| IdxError::ParseError(e.to_string()));
|
||||
let fundamentals = msn::parse_fundamentals_from_str(&fundamentals_raw, Some("e_raw))
|
||||
.map_err(|e| IdxError::ParseError(e.to_string()));
|
||||
let history_raw = std::fs::read_to_string("tests/fixtures/msn_chart_bbca_3m.json")
|
||||
.unwrap_or_else(|_| "[]".to_string());
|
||||
let history = msn::parse_history_from_str("BBCA.JK", &history_raw)
|
||||
.map_err(|e| IdxError::ParseError(e.to_string()));
|
||||
|
||||
Self {
|
||||
quote,
|
||||
fundamentals,
|
||||
history,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_error(err: IdxError) -> Self {
|
||||
Self {
|
||||
quote: Err(err.clone()),
|
||||
fundamentals: Err(err.clone()),
|
||||
history: Err(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl QuoteProvider for MockProvider {
|
||||
fn quote(&self, symbol: &str) -> Result<Quote, IdxError> {
|
||||
let mut q = self.quote.clone()?;
|
||||
q.symbol = symbol.to_string();
|
||||
Ok(q)
|
||||
}
|
||||
}
|
||||
|
||||
impl FundamentalsProvider for MockProvider {
|
||||
fn fundamentals(&self, _symbol: &str) -> Result<Fundamentals, IdxError> {
|
||||
self.fundamentals.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl HistoryProvider for MockProvider {
|
||||
fn history(
|
||||
&self,
|
||||
_symbol: &str,
|
||||
_period: &Period,
|
||||
_interval: &Interval,
|
||||
) -> Result<Vec<Bar>, IdxError> {
|
||||
self.history.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ProviderKind, build_selected_provider, resolve_symbol};
|
||||
use crate::error::IdxError;
|
||||
|
||||
#[test]
|
||||
fn resolves_symbol_variants() {
|
||||
assert_eq!(resolve_symbol("bbca", "JK").unwrap(), "BBCA.JK");
|
||||
assert_eq!(resolve_symbol("BBCA.JK", "JK").unwrap(), "BBCA.JK");
|
||||
assert_eq!(resolve_symbol("TLKM.us", "JK").unwrap(), "TLKM.US");
|
||||
assert_eq!(resolve_symbol("abcd.ef.gh", "JK").unwrap(), "ABCD.EF.GH");
|
||||
assert_eq!(resolve_symbol(" bbri ", "jk").unwrap(), "BBRI.JK");
|
||||
// Empty ticker should return error
|
||||
assert!(resolve_symbol("", "JK").is_err());
|
||||
// Whitespace-only ticker should also return error
|
||||
assert!(resolve_symbol(" ", "JK").is_err());
|
||||
// Valid ticker returns Ok
|
||||
assert_eq!(resolve_symbol("BBCA", "JK").unwrap(), "BBCA.JK");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selected_provider_exposes_expected_capabilities_for_msn() {
|
||||
let provider = build_selected_provider(ProviderKind::Msn, false, false);
|
||||
|
||||
assert!(provider.profile_provider("BBCA.JK").is_ok());
|
||||
assert!(provider.financials_provider("BBCA.JK").is_ok());
|
||||
assert!(provider.earnings_provider("BBCA.JK").is_ok());
|
||||
assert!(provider.sentiment_provider("BBCA.JK").is_ok());
|
||||
assert!(provider.insights_provider("BBCA.JK").is_ok());
|
||||
assert!(provider.news_provider("BBCA.JK").is_ok());
|
||||
assert!(provider.screener_provider("screen").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selected_provider_exposes_expected_capabilities_for_mock_msn() {
|
||||
let provider = build_selected_provider(ProviderKind::Msn, false, true);
|
||||
|
||||
assert!(provider.profile_provider("BBCA.JK").is_ok());
|
||||
assert!(provider.financials_provider("BBCA.JK").is_ok());
|
||||
assert!(provider.earnings_provider("BBCA.JK").is_ok());
|
||||
assert!(provider.sentiment_provider("BBCA.JK").is_ok());
|
||||
assert!(provider.insights_provider("BBCA.JK").is_ok());
|
||||
assert!(provider.news_provider("BBCA.JK").is_ok());
|
||||
assert!(provider.screener_provider("screen").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selected_provider_rejects_msn_only_capabilities_for_yahoo() {
|
||||
let provider = build_selected_provider(ProviderKind::Yahoo, false, false);
|
||||
let err = match provider.profile_provider("BBCA.JK") {
|
||||
Ok(_) => panic!("yahoo should not expose msn-only profile"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert!(matches!(err, IdxError::Unsupported(_)));
|
||||
assert_eq!(
|
||||
err.to_string(),
|
||||
"unsupported: BBCA.JK: command requires --provider msn"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selected_provider_rejects_msn_only_capabilities_for_mock_yahoo() {
|
||||
let provider = build_selected_provider(ProviderKind::Yahoo, false, true);
|
||||
let err = match provider.screener_provider("screen") {
|
||||
Ok(_) => panic!("mock yahoo should not expose msn-only screener"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert!(matches!(err, IdxError::Unsupported(_)));
|
||||
assert_eq!(
|
||||
err.to_string(),
|
||||
"unsupported: screen: command requires --provider msn"
|
||||
);
|
||||
}
|
||||
}
|
||||
406
crates/idx-core/src/api/msn/bing.rs
Normal file
406
crates/idx-core/src/api/msn/bing.rs
Normal file
|
|
@ -0,0 +1,406 @@
|
|||
//! Bing Finance institutional ownership HTTP client.
|
||||
//!
|
||||
//! Provides access to 5 ownership signal endpoints from the Bing hedge fund data
|
||||
//! provider API. Each endpoint returns a list of institutional holders grouped by
|
||||
//! signal type (holders, buyers, sellers, new positions, exits).
|
||||
//!
|
||||
//! # Usage
|
||||
//! ```rust,ignore
|
||||
//! use crate::api::msn::bing::{BingEndpoint, fetch_all_ownership};
|
||||
//!
|
||||
//! // Fetch all 5 signals for BBCA (instrument ID bn91jc)
|
||||
//! let results = fetch_all_ownership("bn91jc", false)?;
|
||||
//! for (signal, holders) in results {
|
||||
//! println!("{:?}: {} rows", signal, holders.len());
|
||||
//! }
|
||||
//! ```
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::error::IdxError;
|
||||
use crate::ownership::types::{BingHolderRaw, FlowSignal};
|
||||
|
||||
/// Base URL for the Bing hedge fund data provider API.
|
||||
const BING_OWNERSHIP_BASE: &str =
|
||||
"https://services.bingapis.com/contentservices-finance.hedgefunddataprovider/api/v1";
|
||||
|
||||
/// Browser-like User-Agent matching existing MSN client pattern.
|
||||
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \
|
||||
(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
|
||||
|
||||
// ── Endpoint enum ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Bing ownership API endpoint variants.
|
||||
///
|
||||
/// Each variant maps to one Bing hedge-fund endpoint path and the
|
||||
/// [`FlowSignal`] it represents in the domain model.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BingEndpoint {
|
||||
/// Existing top institutional holders.
|
||||
TopShareHolders,
|
||||
/// Net buyers over the most recent reporting period.
|
||||
TopBuyers,
|
||||
/// Net sellers over the most recent reporting period.
|
||||
TopSellers,
|
||||
/// Institutions that opened a new position this period.
|
||||
TopNewShareHolders,
|
||||
/// Institutions that fully exited their position this period.
|
||||
TopExitedShareHolders,
|
||||
}
|
||||
|
||||
impl BingEndpoint {
|
||||
/// URL path segment used in the Bing API request.
|
||||
///
|
||||
/// The full URL is constructed as:
|
||||
/// `{BING_OWNERSHIP_BASE}/{path}/{instrument_id}`
|
||||
pub fn path(&self) -> &'static str {
|
||||
match self {
|
||||
Self::TopShareHolders => "GetSecurityTopShareHolders",
|
||||
Self::TopBuyers => "GetSecurityTopBuyers",
|
||||
Self::TopSellers => "GetSecurityTopSellers",
|
||||
Self::TopNewShareHolders => "GetSecurityTopNewShareHolders",
|
||||
Self::TopExitedShareHolders => "GetSecurityTopExitedShareHolders",
|
||||
}
|
||||
}
|
||||
|
||||
/// The [`FlowSignal`] this endpoint represents.
|
||||
pub fn signal(&self) -> FlowSignal {
|
||||
match self {
|
||||
Self::TopShareHolders => FlowSignal::Holder,
|
||||
Self::TopBuyers => FlowSignal::Buyer,
|
||||
Self::TopSellers => FlowSignal::Seller,
|
||||
Self::TopNewShareHolders => FlowSignal::NewPosition,
|
||||
Self::TopExitedShareHolders => FlowSignal::Exited,
|
||||
}
|
||||
}
|
||||
|
||||
/// All endpoint variants in a canonical order.
|
||||
pub fn all() -> &'static [BingEndpoint] {
|
||||
&[
|
||||
Self::TopShareHolders,
|
||||
Self::TopBuyers,
|
||||
Self::TopSellers,
|
||||
Self::TopNewShareHolders,
|
||||
Self::TopExitedShareHolders,
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
// ── HTTP helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
/// Build a `ureq::Agent` with timeouts matching existing MSN client conventions.
|
||||
fn build_agent() -> ureq::Agent {
|
||||
ureq::Agent::config_builder()
|
||||
.timeout_connect(Some(Duration::from_secs(5)))
|
||||
.timeout_recv_body(Some(Duration::from_secs(10)))
|
||||
.build()
|
||||
.into()
|
||||
}
|
||||
|
||||
/// Wrapper type used to handle both bare-array and wrapped-object responses
|
||||
/// from the Bing API. Some endpoints may return `[...]` directly, others may
|
||||
/// wrap in `{ "value": [...] }` or similar. We first try bare Vec, then
|
||||
/// fall back to the wrapper.
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum BingResponse {
|
||||
/// Direct JSON array of holder objects.
|
||||
Array(Vec<BingHolderRaw>),
|
||||
/// Object wrapper with a nested array under a common key.
|
||||
Wrapped(BingWrappedResponse),
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
struct BingWrappedResponse {
|
||||
/// `value` key used by some Bing OData-style responses.
|
||||
#[serde(alias = "value", alias = "Value", alias = "data", alias = "Data")]
|
||||
items: Option<Vec<BingHolderRaw>>,
|
||||
}
|
||||
|
||||
impl BingResponse {
|
||||
fn into_holders(self) -> Vec<BingHolderRaw> {
|
||||
match self {
|
||||
BingResponse::Array(v) => v,
|
||||
BingResponse::Wrapped(w) => w.items.unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Public API ────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Fetch ownership data from a single Bing endpoint.
|
||||
///
|
||||
/// `instrument_id` is the MSN instrument identifier (for example `bn91jc` for BBCA).
|
||||
/// Use [`crate::api::msn::symbols::resolve_msn_id`] to obtain this from a ticker code.
|
||||
///
|
||||
/// Empty responses (valid HTTP 200 with empty array) are returned as `Ok(vec![])`.
|
||||
/// This is expected for some IDX stocks that Bing does not cover.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`IdxError::Http`] on network or non-retriable HTTP errors.
|
||||
/// Returns [`IdxError::RateLimited`] after exhausting retries on 429.
|
||||
/// Returns [`IdxError::ParseError`] if the response body cannot be deserialized.
|
||||
pub fn fetch_holders(
|
||||
instrument_id: &str,
|
||||
endpoint: &BingEndpoint,
|
||||
verbose: bool,
|
||||
) -> Result<Vec<BingHolderRaw>, IdxError> {
|
||||
let url = format!("{BING_OWNERSHIP_BASE}/{}/{instrument_id}", endpoint.path());
|
||||
|
||||
if verbose {
|
||||
eprintln!("[bing] GET {url}");
|
||||
}
|
||||
|
||||
let agent = build_agent();
|
||||
let endpoint_name = endpoint.path();
|
||||
let mut wait = Duration::from_millis(500);
|
||||
|
||||
for attempt in 0..3 {
|
||||
let response = agent
|
||||
.get(&url)
|
||||
.header("User-Agent", USER_AGENT)
|
||||
.header("Accept", "application/json")
|
||||
.header("Accept-Language", "en-US,en;q=0.9")
|
||||
.header("Origin", "https://www.bing.com")
|
||||
.header("Referer", "https://www.bing.com/")
|
||||
.call();
|
||||
|
||||
match response {
|
||||
Ok(ok) => {
|
||||
let body = ok
|
||||
.into_body()
|
||||
.read_to_string()
|
||||
.map_err(|e| IdxError::Http(format!("bing {endpoint_name}: read body: {e}")))?;
|
||||
|
||||
if body.trim().is_empty() || body.trim() == "null" {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
|
||||
let parsed: BingResponse = serde_json::from_str(&body)
|
||||
.map_err(|e| IdxError::ParseError(format!("bing {endpoint_name}: {e}")))?;
|
||||
|
||||
return Ok(parsed.into_holders());
|
||||
}
|
||||
Err(ureq::Error::StatusCode(404)) => {
|
||||
// Instrument not found on Bing — return empty, not an error.
|
||||
return Ok(vec![]);
|
||||
}
|
||||
Err(ureq::Error::StatusCode(429)) => {
|
||||
if attempt < 2 {
|
||||
std::thread::sleep(wait);
|
||||
wait *= 2;
|
||||
continue;
|
||||
}
|
||||
return Err(IdxError::RateLimited);
|
||||
}
|
||||
Err(ureq::Error::StatusCode(code)) if code >= 500 => {
|
||||
if attempt < 2 {
|
||||
std::thread::sleep(wait);
|
||||
wait *= 2;
|
||||
continue;
|
||||
}
|
||||
return Err(IdxError::Http(format!(
|
||||
"bing {endpoint_name}: status {code}"
|
||||
)));
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(IdxError::Http(format!("bing {endpoint_name}: {err}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(IdxError::RateLimited)
|
||||
}
|
||||
|
||||
/// Fetch all 5 ownership endpoints for a given MSN instrument ID.
|
||||
///
|
||||
/// Returns results grouped by [`FlowSignal`] in canonical endpoint order:
|
||||
/// `Holder`, `Buyer`, `Seller`, `NewPosition`, `Exited`.
|
||||
///
|
||||
/// Endpoints that return empty data are still included as `(signal, vec![])`.
|
||||
///
|
||||
/// # Errors
|
||||
/// Propagates any non-empty error from [`fetch_holders`].
|
||||
pub fn fetch_all_ownership(
|
||||
instrument_id: &str,
|
||||
verbose: bool,
|
||||
) -> Result<Vec<(FlowSignal, Vec<BingHolderRaw>)>, IdxError> {
|
||||
let mut results = Vec::with_capacity(5);
|
||||
for endpoint in BingEndpoint::all() {
|
||||
let holders = fetch_holders(instrument_id, endpoint, verbose)?;
|
||||
results.push((endpoint.signal(), holders));
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// ── BingEndpoint::path() ──────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_endpoint_path_top_share_holders() {
|
||||
assert_eq!(
|
||||
BingEndpoint::TopShareHolders.path(),
|
||||
"GetSecurityTopShareHolders"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_endpoint_path_top_buyers() {
|
||||
assert_eq!(BingEndpoint::TopBuyers.path(), "GetSecurityTopBuyers");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_endpoint_path_top_sellers() {
|
||||
assert_eq!(BingEndpoint::TopSellers.path(), "GetSecurityTopSellers");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_endpoint_path_top_new_share_holders() {
|
||||
assert_eq!(
|
||||
BingEndpoint::TopNewShareHolders.path(),
|
||||
"GetSecurityTopNewShareHolders"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_endpoint_path_top_exited_share_holders() {
|
||||
assert_eq!(
|
||||
BingEndpoint::TopExitedShareHolders.path(),
|
||||
"GetSecurityTopExitedShareHolders"
|
||||
);
|
||||
}
|
||||
|
||||
// ── BingEndpoint::signal() ────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_endpoint_signal_mapping() {
|
||||
assert_eq!(BingEndpoint::TopShareHolders.signal(), FlowSignal::Holder);
|
||||
assert_eq!(BingEndpoint::TopBuyers.signal(), FlowSignal::Buyer);
|
||||
assert_eq!(BingEndpoint::TopSellers.signal(), FlowSignal::Seller);
|
||||
assert_eq!(
|
||||
BingEndpoint::TopNewShareHolders.signal(),
|
||||
FlowSignal::NewPosition
|
||||
);
|
||||
assert_eq!(
|
||||
BingEndpoint::TopExitedShareHolders.signal(),
|
||||
FlowSignal::Exited
|
||||
);
|
||||
}
|
||||
|
||||
// ── BingEndpoint::all() ───────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_endpoint_all_has_five_variants() {
|
||||
assert_eq!(BingEndpoint::all().len(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_endpoint_all_covers_all_signals() {
|
||||
use std::collections::HashSet;
|
||||
let signals: HashSet<String> = BingEndpoint::all()
|
||||
.iter()
|
||||
.map(|e| format!("{:?}", e.signal()))
|
||||
.collect();
|
||||
assert!(signals.contains("Holder"));
|
||||
assert!(signals.contains("Buyer"));
|
||||
assert!(signals.contains("Seller"));
|
||||
assert!(signals.contains("NewPosition"));
|
||||
assert!(signals.contains("Exited"));
|
||||
}
|
||||
|
||||
// ── Fixture deserialization ───────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_deserialize_holders_fixture() {
|
||||
let json = include_str!("../../../tests/fixtures/bing_holders_bbca.json");
|
||||
let holders: Vec<BingHolderRaw> = serde_json::from_str(json)
|
||||
.expect("bing_holders_bbca.json should deserialize into Vec<BingHolderRaw>");
|
||||
assert!(
|
||||
!holders.is_empty(),
|
||||
"fixture should have at least one holder"
|
||||
);
|
||||
// First holder should have a name
|
||||
assert!(
|
||||
holders[0].investor_name.is_some(),
|
||||
"first holder should have investor_name"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deserialize_buyers_fixture() {
|
||||
let json = include_str!("../../../tests/fixtures/bing_buyers_bbca.json");
|
||||
let holders: Vec<BingHolderRaw> = serde_json::from_str(json)
|
||||
.expect("bing_buyers_bbca.json should deserialize into Vec<BingHolderRaw>");
|
||||
assert!(
|
||||
!holders.is_empty(),
|
||||
"fixture should have at least one buyer"
|
||||
);
|
||||
}
|
||||
|
||||
// ── Empty response handling ───────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_deserialize_empty_array_is_ok() {
|
||||
let json = "[]";
|
||||
let holders: Vec<BingHolderRaw> = serde_json::from_str(json).expect("empty array is valid");
|
||||
assert!(holders.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bing_response_bare_array() {
|
||||
let json = r#"[
|
||||
{
|
||||
"investorName": "Vanguard Group",
|
||||
"investorType": "Institutional",
|
||||
"sharesHeld": 1234567.0,
|
||||
"reportDate": "2024-12-31"
|
||||
}
|
||||
]"#;
|
||||
let resp: BingResponse = serde_json::from_str(json).unwrap();
|
||||
let holders = resp.into_holders();
|
||||
assert_eq!(holders.len(), 1);
|
||||
assert_eq!(holders[0].investor_name.as_deref(), Some("Vanguard Group"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bing_response_wrapped_value_key() {
|
||||
let json = r#"{
|
||||
"value": [
|
||||
{
|
||||
"InvestorName": "BlackRock",
|
||||
"InvestorType": "Institutional",
|
||||
"SharesHeld": 9876543.0
|
||||
}
|
||||
]
|
||||
}"#;
|
||||
let resp: BingResponse = serde_json::from_str(json).unwrap();
|
||||
let holders = resp.into_holders();
|
||||
assert_eq!(holders.len(), 1);
|
||||
assert_eq!(holders[0].investor_name.as_deref(), Some("BlackRock"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bing_response_wrapped_empty_value() {
|
||||
let json = r#"{ "value": [] }"#;
|
||||
let resp: BingResponse = serde_json::from_str(json).unwrap();
|
||||
let holders = resp.into_holders();
|
||||
assert!(holders.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_holder_optional_fields_are_none() {
|
||||
let json = r#"[{"investorName": "Some Fund"}]"#;
|
||||
let holders: Vec<BingHolderRaw> = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(holders.len(), 1);
|
||||
assert!(holders[0].shares_held.is_none());
|
||||
assert!(holders[0].shares_changed.is_none());
|
||||
assert!(holders[0].pct_outstanding.is_none());
|
||||
assert!(holders[0].value.is_none());
|
||||
assert!(holders[0].report_date.is_none());
|
||||
}
|
||||
}
|
||||
466
crates/idx-core/src/api/msn/client.rs
Normal file
466
crates/idx-core/src/api/msn/client.rs
Normal file
|
|
@ -0,0 +1,466 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use serde::Serialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
|
||||
use crate::error::IdxError;
|
||||
|
||||
use super::raw_types::{
|
||||
KeyRatios, MsnQuote, RawChartResponse, RawEarningsResponse, RawEquity, RawFinancialStatement,
|
||||
RawInsight, RawNewsFeed, RawScreenerResponse, RawSentiment, ScreenerFilter, ScreenerOrder,
|
||||
ScreenerRequest,
|
||||
};
|
||||
use super::symbols::resolve_msn_id;
|
||||
use crate::api::types::{Interval, Period};
|
||||
|
||||
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
|
||||
const MSN_ASSETS_BASE_URL: &str = "https://assets.msn.com/service/";
|
||||
const MSN_API_BASE_URL: &str = "https://api.msn.com/msn/v0/pages/finance/";
|
||||
// Public API key from MSN Money website (embedded in frontend JS)
|
||||
const MSN_API_KEY: &str = "0QfOX3Vn51YCzitbLaRkTTBadtWpgTN8NZLW0C1SEM";
|
||||
|
||||
pub(super) struct MsnClient {
|
||||
agent: ureq::Agent,
|
||||
}
|
||||
|
||||
impl MsnClient {
|
||||
pub(super) fn new() -> Self {
|
||||
let agent: ureq::Agent = ureq::Agent::config_builder()
|
||||
.timeout_connect(Some(Duration::from_secs(5)))
|
||||
.timeout_recv_body(Some(Duration::from_secs(10)))
|
||||
.build()
|
||||
.into();
|
||||
|
||||
Self { agent }
|
||||
}
|
||||
|
||||
fn mock_error() -> Option<IdxError> {
|
||||
if std::env::var("IDX_USE_MOCK_PROVIDER").is_ok() && std::env::var("IDX_MOCK_ERROR").is_ok()
|
||||
{
|
||||
Some(IdxError::ProviderUnavailable)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn mock_body(endpoint: &str) -> Option<&'static str> {
|
||||
if std::env::var("IDX_USE_MOCK_PROVIDER").is_err() {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(match endpoint {
|
||||
"quote" => include_str!("../../../tests/fixtures/msn_quote_bbca.json"),
|
||||
"keyratios" => include_str!("../../../tests/fixtures/msn_keyratios_bbca.json"),
|
||||
"equities" => include_str!("../../../tests/fixtures/msn_profile_bbca.json"),
|
||||
"financialstatements" => {
|
||||
include_str!("../../../tests/fixtures/msn_financials_bbca.json")
|
||||
}
|
||||
"earnings" => include_str!("../../../tests/fixtures/msn_earnings_bbca.json"),
|
||||
"sentiment" => include_str!("../../../tests/fixtures/msn_sentiment_bbca.json"),
|
||||
"insights" => include_str!("../../../tests/fixtures/msn_insights_bbca.json"),
|
||||
"news" => include_str!("../../../tests/fixtures/msn_news_bbca.json"),
|
||||
"screener" => include_str!("../../../tests/fixtures/msn_screener_id_topperfs.json"),
|
||||
"chart" => include_str!("../../../tests/fixtures/msn_chart_bbca_3m.json"),
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
fn mock_json<T: DeserializeOwned>(endpoint: &str) -> Result<Option<T>, IdxError> {
|
||||
let Some(body) = Self::mock_body(endpoint) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
serde_json::from_str(body)
|
||||
.map(Some)
|
||||
.map_err(|e| IdxError::ParseError(format!("msn {endpoint}: {e}")))
|
||||
}
|
||||
|
||||
fn get_json<T: DeserializeOwned>(
|
||||
&self,
|
||||
url: &str,
|
||||
symbol: &str,
|
||||
endpoint: &str,
|
||||
) -> Result<T, IdxError> {
|
||||
if let Some(err) = Self::mock_error() {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if let Some(mocked) = Self::mock_json(endpoint)? {
|
||||
return Ok(mocked);
|
||||
}
|
||||
|
||||
let mut wait = Duration::from_millis(500);
|
||||
for attempt in 0..3 {
|
||||
let response = self
|
||||
.agent
|
||||
.get(url)
|
||||
.header("User-Agent", USER_AGENT)
|
||||
.header("Accept", "application/json")
|
||||
.header("Accept-Language", "en-US,en;q=0.9,id;q=0.8")
|
||||
.header("Origin", "https://www.msn.com")
|
||||
.header("Referer", "https://www.msn.com/")
|
||||
.call();
|
||||
|
||||
match response {
|
||||
Ok(ok) => {
|
||||
return ok
|
||||
.into_body()
|
||||
.read_json::<T>()
|
||||
.map_err(|e| IdxError::ParseError(format!("msn {endpoint}: {e}")));
|
||||
}
|
||||
Err(ureq::Error::StatusCode(404)) => {
|
||||
return Err(IdxError::SymbolNotFound(symbol.to_string()));
|
||||
}
|
||||
Err(ureq::Error::StatusCode(429)) => {
|
||||
if attempt < 2 {
|
||||
std::thread::sleep(wait);
|
||||
wait *= 2;
|
||||
continue;
|
||||
}
|
||||
return Err(IdxError::RateLimited);
|
||||
}
|
||||
Err(ureq::Error::StatusCode(code)) if code >= 500 => {
|
||||
if attempt < 2 {
|
||||
std::thread::sleep(wait);
|
||||
wait *= 2;
|
||||
continue;
|
||||
}
|
||||
return Err(IdxError::Http(format!("msn {endpoint}: status {code}")));
|
||||
}
|
||||
Err(err) => return Err(IdxError::Http(format!("msn {endpoint}: {err}"))),
|
||||
}
|
||||
}
|
||||
Err(IdxError::RateLimited)
|
||||
}
|
||||
|
||||
fn post_json<B: Serialize, T: DeserializeOwned>(
|
||||
&self,
|
||||
url: &str,
|
||||
body: &B,
|
||||
symbol: &str,
|
||||
endpoint: &str,
|
||||
) -> Result<T, IdxError> {
|
||||
if let Some(err) = Self::mock_error() {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if let Some(mocked) = Self::mock_json(endpoint)? {
|
||||
return Ok(mocked);
|
||||
}
|
||||
|
||||
let mut wait = Duration::from_millis(500);
|
||||
for attempt in 0..3 {
|
||||
let response = self
|
||||
.agent
|
||||
.post(url)
|
||||
.header("User-Agent", USER_AGENT)
|
||||
.header("Accept", "application/json")
|
||||
.header("Accept-Language", "en-US,en;q=0.9,id;q=0.8")
|
||||
.header("Origin", "https://www.msn.com")
|
||||
.header("Referer", "https://www.msn.com/")
|
||||
.header("Content-Type", "text/plain;charset=UTF-8")
|
||||
.send_json(body);
|
||||
|
||||
match response {
|
||||
Ok(ok) => {
|
||||
return ok
|
||||
.into_body()
|
||||
.read_json::<T>()
|
||||
.map_err(|e| IdxError::ParseError(format!("msn {endpoint}: {e}")));
|
||||
}
|
||||
Err(ureq::Error::StatusCode(404)) => {
|
||||
return Err(IdxError::SymbolNotFound(symbol.to_string()));
|
||||
}
|
||||
Err(ureq::Error::StatusCode(429)) => {
|
||||
if attempt < 2 {
|
||||
std::thread::sleep(wait);
|
||||
wait *= 2;
|
||||
continue;
|
||||
}
|
||||
return Err(IdxError::RateLimited);
|
||||
}
|
||||
Err(ureq::Error::StatusCode(code)) if code >= 500 => {
|
||||
if attempt < 2 {
|
||||
std::thread::sleep(wait);
|
||||
wait *= 2;
|
||||
continue;
|
||||
}
|
||||
return Err(IdxError::Http(format!("msn {endpoint}: status {code}")));
|
||||
}
|
||||
Err(err) => return Err(IdxError::Http(format!("msn {endpoint}: {err}"))),
|
||||
}
|
||||
}
|
||||
Err(IdxError::RateLimited)
|
||||
}
|
||||
|
||||
pub(super) fn fetch_quotes(&self, symbol: &str) -> Result<Vec<MsnQuote>, IdxError> {
|
||||
let id =
|
||||
resolve_msn_id(symbol).ok_or_else(|| IdxError::SymbolNotFound(symbol.to_string()))?;
|
||||
let url = format!(
|
||||
"{MSN_ASSETS_BASE_URL}Finance/Quotes?apikey={MSN_API_KEY}&ids={id}&wrapodata=false"
|
||||
);
|
||||
self.get_json(&url, symbol, "quote")
|
||||
}
|
||||
|
||||
pub(super) fn fetch_key_ratios(&self, symbol: &str) -> Result<Vec<KeyRatios>, IdxError> {
|
||||
let id =
|
||||
resolve_msn_id(symbol).ok_or_else(|| IdxError::SymbolNotFound(symbol.to_string()))?;
|
||||
let url =
|
||||
format!("{MSN_API_BASE_URL}keyratios?apikey={MSN_API_KEY}&ids={id}&wrapodata=false");
|
||||
self.get_json(&url, symbol, "keyratios")
|
||||
}
|
||||
|
||||
pub(super) fn fetch_equities(&self, symbol: &str) -> Result<Vec<RawEquity>, IdxError> {
|
||||
let id =
|
||||
resolve_msn_id(symbol).ok_or_else(|| IdxError::SymbolNotFound(symbol.to_string()))?;
|
||||
let url = format!(
|
||||
"{MSN_ASSETS_BASE_URL}Finance/Equities?apikey={MSN_API_KEY}&ids={id}&wrapodata=false"
|
||||
);
|
||||
self.get_json(&url, symbol, "equities")
|
||||
}
|
||||
|
||||
pub(super) fn fetch_financial_statements(
|
||||
&self,
|
||||
symbol: &str,
|
||||
) -> Result<Vec<RawFinancialStatement>, IdxError> {
|
||||
let id =
|
||||
resolve_msn_id(symbol).ok_or_else(|| IdxError::SymbolNotFound(symbol.to_string()))?;
|
||||
let url = format!(
|
||||
"{MSN_ASSETS_BASE_URL}Finance/Equities/financialstatements?apikey={MSN_API_KEY}&ids={id}&wrapodata=false"
|
||||
);
|
||||
self.get_json(&url, symbol, "financialstatements")
|
||||
}
|
||||
|
||||
pub(super) fn fetch_earnings(&self, symbol: &str) -> Result<RawEarningsResponse, IdxError> {
|
||||
let id =
|
||||
resolve_msn_id(symbol).ok_or_else(|| IdxError::SymbolNotFound(symbol.to_string()))?;
|
||||
let url = format!(
|
||||
"{MSN_ASSETS_BASE_URL}Finance/Events/Earnings?apikey={MSN_API_KEY}&ids={id}&wrapodata=false"
|
||||
);
|
||||
self.get_json(&url, symbol, "earnings")
|
||||
}
|
||||
|
||||
pub(super) fn fetch_sentiment(&self, symbol: &str) -> Result<Vec<RawSentiment>, IdxError> {
|
||||
let id =
|
||||
resolve_msn_id(symbol).ok_or_else(|| IdxError::SymbolNotFound(symbol.to_string()))?;
|
||||
let url = format!(
|
||||
"{MSN_ASSETS_BASE_URL}Finance/SentimentBrowser?apikey={MSN_API_KEY}&cm=id-id&it=web&scn=ANON&ids={id}&wrapodata=false&flightId=INeedDau"
|
||||
);
|
||||
self.get_json(&url, symbol, "sentiment")
|
||||
}
|
||||
|
||||
pub(super) fn fetch_insights(&self, symbol: &str) -> Result<Vec<RawInsight>, IdxError> {
|
||||
let id =
|
||||
resolve_msn_id(symbol).ok_or_else(|| IdxError::SymbolNotFound(symbol.to_string()))?;
|
||||
let url =
|
||||
format!("{MSN_API_BASE_URL}insights?apikey={MSN_API_KEY}&ids={id}&wrapodata=false");
|
||||
self.get_json(&url, symbol, "insights")
|
||||
}
|
||||
|
||||
pub(super) fn fetch_news(&self, symbol: &str, limit: usize) -> Result<RawNewsFeed, IdxError> {
|
||||
let id =
|
||||
resolve_msn_id(symbol).ok_or_else(|| IdxError::SymbolNotFound(symbol.to_string()))?;
|
||||
let url = format!(
|
||||
"{MSN_ASSETS_BASE_URL}MSN/Feed/me?$top={limit}&apikey={MSN_API_KEY}&cm=id-id&contentType=article,video,slideshow&it=web&query=ef_stock_{id}&queryType=entityfeed&responseSchema=cardview&scn=ANON&wrapodata=false"
|
||||
);
|
||||
self.get_json(&url, symbol, "news")
|
||||
}
|
||||
|
||||
fn post_screener(
|
||||
&self,
|
||||
url: &str,
|
||||
request: &ScreenerRequest,
|
||||
) -> Result<RawScreenerResponse, IdxError> {
|
||||
if std::env::var("IDX_USE_MOCK_PROVIDER").is_ok() {
|
||||
if let Some(err) = Self::mock_error() {
|
||||
return Err(err);
|
||||
}
|
||||
if let Some(path) = std::env::var_os("IDX_MOCK_MSN_SCREENER_FIXTURE") {
|
||||
let body = std::fs::read_to_string(path)
|
||||
.map_err(|e| IdxError::ParseError(format!("msn screener fixture: {e}")))?;
|
||||
let mut response: RawScreenerResponse = serde_json::from_str(&body)
|
||||
.map_err(|e| IdxError::ParseError(format!("msn screener fixture: {e}")))?;
|
||||
// Model MSN's request-sized batches while preserving its reported
|
||||
// total, so complete-list fetching uses the same path as live data.
|
||||
if let Some(quotes) = response.quote.as_mut() {
|
||||
quotes.truncate(request.limit);
|
||||
}
|
||||
return Ok(response);
|
||||
}
|
||||
}
|
||||
self.post_json(url, request, "SCREENER", "screener")
|
||||
}
|
||||
|
||||
pub(super) fn fetch_screener(
|
||||
&self,
|
||||
filter: &str,
|
||||
region: &str,
|
||||
limit: Option<usize>,
|
||||
) -> Result<RawScreenerResponse, IdxError> {
|
||||
let url =
|
||||
format!("{MSN_ASSETS_BASE_URL}Finance/Screener?apikey={MSN_API_KEY}&wrapodata=false");
|
||||
let mut req = ScreenerRequest {
|
||||
filter: vec![
|
||||
ScreenerFilter {
|
||||
key: filter.to_string(),
|
||||
key_group: "st_list_".to_string(),
|
||||
is_range: false,
|
||||
},
|
||||
ScreenerFilter {
|
||||
key: region.to_string(),
|
||||
key_group: "st_reg_".to_string(),
|
||||
is_range: false,
|
||||
},
|
||||
],
|
||||
order: ScreenerOrder {
|
||||
key: "st_1yr_asc_order".to_string(),
|
||||
dir: "desc".to_string(),
|
||||
},
|
||||
return_value_type: vec!["quote".to_string(), "equity".to_string()],
|
||||
screener_type: "stock".to_string(),
|
||||
limit: limit.unwrap_or(500),
|
||||
page_index: 0,
|
||||
};
|
||||
|
||||
if limit.is_some() {
|
||||
return self.post_screener(&url, &req);
|
||||
}
|
||||
fetch_complete_screener(|limit| {
|
||||
req.limit = limit;
|
||||
self.post_screener(&url, &req)
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn fetch_chart(
|
||||
&self,
|
||||
symbol: &str,
|
||||
period: &Period,
|
||||
interval: &Interval,
|
||||
) -> Result<Vec<RawChartResponse>, IdxError> {
|
||||
let id =
|
||||
resolve_msn_id(symbol).ok_or_else(|| IdxError::SymbolNotFound(symbol.to_string()))?;
|
||||
let chart_type = msn_chart_type(period, interval)?;
|
||||
let url = format!(
|
||||
"{MSN_ASSETS_BASE_URL}Finance/Charts?apikey={MSN_API_KEY}&cm=id-id&ids={id}&type={chart_type}&wrapodata=false"
|
||||
);
|
||||
self.get_json(&url, symbol, "chart")
|
||||
}
|
||||
}
|
||||
|
||||
fn fetch_complete_screener(
|
||||
mut fetch: impl FnMut(usize) -> Result<RawScreenerResponse, IdxError>,
|
||||
) -> Result<RawScreenerResponse, IdxError> {
|
||||
let total_count = |response: &RawScreenerResponse| {
|
||||
response
|
||||
.count
|
||||
.and_then(|count| usize::try_from(count).ok())
|
||||
.ok_or_else(|| IdxError::ParseError("MSN screener omitted a valid total count".into()))
|
||||
};
|
||||
let mut response = fetch(500)?;
|
||||
let count = total_count(&response)?;
|
||||
if response.quote.as_ref().map_or(0, Vec::len) < count {
|
||||
// MSN ignores pageIndex for these lists. Re-request the reported total
|
||||
// instead of silently treating the initial batch as the whole market.
|
||||
response = fetch(count)?;
|
||||
}
|
||||
if response.quote.as_ref().map_or(0, Vec::len) < total_count(&response)? {
|
||||
return Err(IdxError::ParseError(
|
||||
"MSN screener returned an incomplete candidate list".into(),
|
||||
));
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn msn_chart_type(period: &Period, interval: &Interval) -> Result<&'static str, IdxError> {
|
||||
if !matches!(interval, Interval::Day) {
|
||||
return Err(IdxError::Unsupported(
|
||||
"MSN charts currently support only --interval 1d for IDX history".into(),
|
||||
));
|
||||
}
|
||||
|
||||
match period {
|
||||
Period::OneMonth => Ok("1M"),
|
||||
Period::ThreeMonths => Ok("3M"),
|
||||
Period::OneYear => Ok("1Y"),
|
||||
_ => Err(IdxError::Unsupported(
|
||||
"MSN charts currently support --period 1mo, 3mo, or 1y with --interval 1d".into(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{fetch_complete_screener, msn_chart_type};
|
||||
use crate::api::msn::raw_types::RawScreenerResponse;
|
||||
use crate::api::types::{Interval, Period};
|
||||
use crate::error::IdxError;
|
||||
|
||||
fn screener_response(count: Option<usize>, returned: usize) -> RawScreenerResponse {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"count": count,
|
||||
"quote": (0..returned).map(|index| serde_json::json!({
|
||||
"symbol": format!("STOCK{index}"),
|
||||
"price": 100
|
||||
})).collect::<Vec<_>>()
|
||||
}))
|
||||
.expect("valid screener response")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_screener_includes_candidates_beyond_initial_batch() {
|
||||
let response =
|
||||
fetch_complete_screener(|limit| Ok(screener_response(Some(600), limit.min(600))))
|
||||
.expect("complete list");
|
||||
let symbols: Vec<_> = response
|
||||
.quote
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|quote| quote.symbol.unwrap())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
symbols,
|
||||
(0..600)
|
||||
.map(|index| format!("STOCK{index}"))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_screener_rejects_a_truncated_total_response() {
|
||||
let err = fetch_complete_screener(|_| Ok(screener_response(Some(600), 500)))
|
||||
.expect_err("truncated list must not be ranked as complete");
|
||||
assert!(matches!(err, IdxError::ParseError(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_screener_requires_a_total_count() {
|
||||
let err = fetch_complete_screener(|_| Ok(screener_response(None, 500)))
|
||||
.expect_err("unknown completeness must not be treated as a full list");
|
||||
assert!(matches!(err, IdxError::ParseError(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_supported_msn_chart_types() {
|
||||
assert_eq!(
|
||||
msn_chart_type(&Period::OneMonth, &Interval::Day).unwrap(),
|
||||
"1M"
|
||||
);
|
||||
assert_eq!(
|
||||
msn_chart_type(&Period::ThreeMonths, &Interval::Day).unwrap(),
|
||||
"3M"
|
||||
);
|
||||
assert_eq!(
|
||||
msn_chart_type(&Period::OneYear, &Interval::Day).unwrap(),
|
||||
"1Y"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_unsupported_msn_chart_types() {
|
||||
let err = msn_chart_type(&Period::ThreeMonths, &Interval::Week).unwrap_err();
|
||||
assert!(matches!(err, IdxError::Unsupported(_)));
|
||||
|
||||
let err = msn_chart_type(&Period::SixMonths, &Interval::Day).unwrap_err();
|
||||
assert!(matches!(err, IdxError::Unsupported(_)));
|
||||
}
|
||||
}
|
||||
1008
crates/idx-core/src/api/msn/map.rs
Normal file
1008
crates/idx-core/src/api/msn/map.rs
Normal file
File diff suppressed because it is too large
Load diff
119
crates/idx-core/src/api/msn/mod.rs
Normal file
119
crates/idx-core/src/api/msn/mod.rs
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
#[allow(dead_code, clippy::enum_variant_names)]
|
||||
#[cfg(feature = "ownership")]
|
||||
pub mod bing;
|
||||
mod client;
|
||||
mod map;
|
||||
mod parse;
|
||||
mod raw_types;
|
||||
mod symbols;
|
||||
|
||||
use crate::api::types::{
|
||||
Bar, CompanyProfile, EarningsReport, FinancialStatements, Fundamentals, InsightData, Interval,
|
||||
NewsItem, Period, Quote, SentimentData,
|
||||
};
|
||||
use crate::api::{
|
||||
EarningsProvider, FinancialsProvider, FundamentalsProvider, HistoryProvider, InsightsProvider,
|
||||
NewsProvider, ProfileProvider, QuoteProvider, ScreenerProvider, SentimentProvider,
|
||||
};
|
||||
use crate::error::IdxError;
|
||||
|
||||
use client::MsnClient;
|
||||
use map::{
|
||||
parse_earnings, parse_financial_statements, parse_fundamentals, parse_history, parse_insights,
|
||||
parse_news, parse_profile, parse_quote, parse_screener_results, parse_sentiment,
|
||||
};
|
||||
|
||||
pub(crate) use parse::{parse_fundamentals_from_str, parse_history_from_str, parse_quote_from_str};
|
||||
|
||||
pub struct MsnProvider {
|
||||
client: MsnClient,
|
||||
}
|
||||
|
||||
impl MsnProvider {
|
||||
pub fn new(_verbose: bool) -> Self {
|
||||
Self {
|
||||
client: MsnClient::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl QuoteProvider for MsnProvider {
|
||||
fn quote(&self, symbol: &str) -> Result<Quote, IdxError> {
|
||||
let quotes = self.client.fetch_quotes(symbol)?;
|
||||
parse_quote(symbol, "es)
|
||||
}
|
||||
}
|
||||
|
||||
impl FundamentalsProvider for MsnProvider {
|
||||
fn fundamentals(&self, symbol: &str) -> Result<Fundamentals, IdxError> {
|
||||
let ratios = self.client.fetch_key_ratios(symbol)?;
|
||||
let quote = self.client.fetch_quotes(symbol)?;
|
||||
parse_fundamentals(&ratios, quote.first())
|
||||
}
|
||||
}
|
||||
|
||||
impl HistoryProvider for MsnProvider {
|
||||
fn history(
|
||||
&self,
|
||||
symbol: &str,
|
||||
period: &Period,
|
||||
interval: &Interval,
|
||||
) -> Result<Vec<Bar>, IdxError> {
|
||||
let raw = self.client.fetch_chart(symbol, period, interval)?;
|
||||
parse_history(symbol, &raw)
|
||||
}
|
||||
}
|
||||
|
||||
impl ProfileProvider for MsnProvider {
|
||||
fn profile(&self, symbol: &str) -> Result<CompanyProfile, IdxError> {
|
||||
let raw = self.client.fetch_equities(symbol)?;
|
||||
parse_profile(symbol, &raw)
|
||||
}
|
||||
}
|
||||
|
||||
impl EarningsProvider for MsnProvider {
|
||||
fn earnings(&self, symbol: &str) -> Result<EarningsReport, IdxError> {
|
||||
let raw = self.client.fetch_earnings(symbol)?;
|
||||
parse_earnings(symbol, &raw)
|
||||
}
|
||||
}
|
||||
|
||||
impl FinancialsProvider for MsnProvider {
|
||||
fn financials(&self, symbol: &str) -> Result<FinancialStatements, IdxError> {
|
||||
let raw = self.client.fetch_financial_statements(symbol)?;
|
||||
parse_financial_statements(symbol, &raw)
|
||||
}
|
||||
}
|
||||
|
||||
impl SentimentProvider for MsnProvider {
|
||||
fn sentiment(&self, symbol: &str) -> Result<SentimentData, IdxError> {
|
||||
let raw = self.client.fetch_sentiment(symbol)?;
|
||||
parse_sentiment(symbol, &raw)
|
||||
}
|
||||
}
|
||||
|
||||
impl InsightsProvider for MsnProvider {
|
||||
fn insights(&self, symbol: &str) -> Result<InsightData, IdxError> {
|
||||
let raw = self.client.fetch_insights(symbol)?;
|
||||
parse_insights(symbol, &raw)
|
||||
}
|
||||
}
|
||||
|
||||
impl NewsProvider for MsnProvider {
|
||||
fn news(&self, symbol: &str, limit: usize) -> Result<Vec<NewsItem>, IdxError> {
|
||||
let raw = self.client.fetch_news(symbol, limit)?;
|
||||
parse_news(symbol, &raw)
|
||||
}
|
||||
}
|
||||
|
||||
impl ScreenerProvider for MsnProvider {
|
||||
fn screener(
|
||||
&self,
|
||||
filter: &str,
|
||||
region: &str,
|
||||
limit: Option<usize>,
|
||||
) -> Result<Vec<Quote>, IdxError> {
|
||||
let raw = self.client.fetch_screener(filter, region, limit)?;
|
||||
parse_screener_results(&raw)
|
||||
}
|
||||
}
|
||||
164
crates/idx-core/src/api/msn/parse.rs
Normal file
164
crates/idx-core/src/api/msn/parse.rs
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
use super::map::{parse_fundamentals, parse_history, parse_quote};
|
||||
use super::raw_types::{KeyRatios, MsnQuote, RawChartResponse};
|
||||
use crate::api::types::{Fundamentals, Ohlc, Quote};
|
||||
use crate::error::IdxError;
|
||||
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
pub(crate) fn parse_quote_from_str(symbol: &str, raw: &str) -> Result<Quote, IdxError> {
|
||||
let quotes: Vec<MsnQuote> =
|
||||
serde_json::from_str(raw).map_err(|e| IdxError::ParseError(e.to_string()))?;
|
||||
parse_quote(symbol, "es)
|
||||
}
|
||||
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
pub(crate) fn parse_fundamentals_from_str(
|
||||
raw: &str,
|
||||
quote_raw: Option<&str>,
|
||||
) -> Result<Fundamentals, IdxError> {
|
||||
let ratios: Vec<KeyRatios> =
|
||||
serde_json::from_str(raw).map_err(|e| IdxError::ParseError(e.to_string()))?;
|
||||
let quote = quote_raw
|
||||
.map(serde_json::from_str::<Vec<MsnQuote>>)
|
||||
.transpose()
|
||||
.map_err(|e| IdxError::ParseError(e.to_string()))?
|
||||
.and_then(|quotes| quotes.into_iter().next());
|
||||
parse_fundamentals(&ratios, quote.as_ref())
|
||||
}
|
||||
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
pub(crate) fn parse_history_from_str(symbol: &str, raw: &str) -> Result<Vec<Ohlc>, IdxError> {
|
||||
let charts: Vec<RawChartResponse> =
|
||||
serde_json::from_str(raw).map_err(|e| IdxError::ParseError(e.to_string()))?;
|
||||
parse_history(symbol, &charts)
|
||||
}
|
||||
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
#[allow(dead_code)]
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{parse_fundamentals_from_str, parse_history_from_str, parse_quote_from_str};
|
||||
|
||||
fn minimal_quote_raw() -> &'static str {
|
||||
r#"[{"symbol":"BBCA","marketCap":1215200000000000}]"#
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_quote_fixture_json() {
|
||||
let raw = std::fs::read_to_string("tests/fixtures/msn_quote_bbca.json")
|
||||
.expect("quote fixture exists");
|
||||
let quote = parse_quote_from_str("BBCA.JK", &raw).expect("quote parsed");
|
||||
assert_eq!(quote.symbol, "BBCA.JK");
|
||||
assert_eq!(quote.price, 9875);
|
||||
assert_eq!(quote.change, Some(117));
|
||||
assert_eq!(quote.volume, Some(12_300_000));
|
||||
assert_eq!(quote.market_cap, Some(1_215_200_000_000_000));
|
||||
assert_eq!(quote.avg_volume, Some(10_000_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_fundamentals_fixture_json() {
|
||||
let raw = std::fs::read_to_string("tests/fixtures/msn_keyratios_bbca.json")
|
||||
.expect("fundamentals fixture exists");
|
||||
let quote_raw = std::fs::read_to_string("tests/fixtures/msn_quote_bbca.json")
|
||||
.expect("quote fixture exists");
|
||||
let fundamentals =
|
||||
parse_fundamentals_from_str(&raw, Some("e_raw)).expect("fundamentals parsed");
|
||||
assert_eq!(fundamentals.trailing_pe, Some(25.4));
|
||||
assert_eq!(fundamentals.price_to_book, Some(4.6));
|
||||
assert_eq!(fundamentals.return_on_equity, Some(0.198));
|
||||
assert_eq!(fundamentals.revenue_growth, Some(0.081));
|
||||
assert_eq!(fundamentals.earnings_growth, Some(0.121));
|
||||
assert_eq!(fundamentals.market_cap, Some(1_215_200_000_000_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_history_fixture_json() {
|
||||
let raw = std::fs::read_to_string("tests/fixtures/msn_chart_bbca_3m.json")
|
||||
.expect("chart fixture exists");
|
||||
let history = parse_history_from_str("BBCA.JK", &raw).expect("chart fixture parsed");
|
||||
|
||||
assert_eq!(history.len(), 3);
|
||||
assert_eq!(history[0].date.to_string(), "2026-01-14");
|
||||
assert_eq!(history[0].close, 8000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_fundamentals_with_infinity_string_as_missing_data() {
|
||||
let raw = r#"[
|
||||
{
|
||||
"companyMetrics": [
|
||||
{
|
||||
"year": "2025",
|
||||
"fiscalPeriodType": "TTM",
|
||||
"priceToEarningsRatio": "Infinity",
|
||||
"priceToBookRatio": 4.6,
|
||||
"roe": 19.8,
|
||||
"profitMargin": 44.2,
|
||||
"debtToEquityRatio": 0.75,
|
||||
"currentRatio": 1.4
|
||||
}
|
||||
]
|
||||
}
|
||||
]"#;
|
||||
|
||||
let fundamentals = parse_fundamentals_from_str(raw, Some(minimal_quote_raw()))
|
||||
.expect("fundamentals parsed");
|
||||
|
||||
assert_eq!(fundamentals.trailing_pe, None);
|
||||
assert_eq!(fundamentals.price_to_book, Some(4.6));
|
||||
assert_eq!(fundamentals.return_on_equity, Some(0.198));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_fundamentals_with_negative_infinity_string_as_missing_data() {
|
||||
let raw = r#"[
|
||||
{
|
||||
"companyMetrics": [
|
||||
{
|
||||
"year": "2025",
|
||||
"fiscalPeriodType": "TTM",
|
||||
"priceToEarningsRatio": 12.5,
|
||||
"debtToEquityRatio": "-Infinity",
|
||||
"roe": 19.8,
|
||||
"profitMargin": 44.2,
|
||||
"currentRatio": 1.4
|
||||
}
|
||||
]
|
||||
}
|
||||
]"#;
|
||||
|
||||
let fundamentals = parse_fundamentals_from_str(raw, Some(minimal_quote_raw()))
|
||||
.expect("fundamentals parsed");
|
||||
|
||||
assert_eq!(fundamentals.trailing_pe, Some(12.5));
|
||||
assert_eq!(fundamentals.debt_to_equity, None);
|
||||
assert_eq!(fundamentals.return_on_equity, Some(0.198));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_fundamentals_with_nan_string_as_missing_data() {
|
||||
let raw = r#"[
|
||||
{
|
||||
"companyMetrics": [
|
||||
{
|
||||
"year": "2025",
|
||||
"fiscalPeriodType": "TTM",
|
||||
"revenueGrowthRate": "NaN",
|
||||
"earningsGrowthRate": 12.1,
|
||||
"roe": 19.8,
|
||||
"profitMargin": 44.2,
|
||||
"debtToEquityRatio": 0.75,
|
||||
"currentRatio": 1.4
|
||||
}
|
||||
]
|
||||
}
|
||||
]"#;
|
||||
|
||||
let fundamentals = parse_fundamentals_from_str(raw, Some(minimal_quote_raw()))
|
||||
.expect("fundamentals parsed");
|
||||
|
||||
assert_eq!(fundamentals.revenue_growth, None);
|
||||
assert_eq!(fundamentals.earnings_growth, Some(0.121));
|
||||
assert_eq!(fundamentals.return_on_equity, Some(0.198));
|
||||
}
|
||||
}
|
||||
393
crates/idx-core/src/api/msn/raw_types.rs
Normal file
393
crates/idx-core/src/api/msn/raw_types.rs
Normal file
|
|
@ -0,0 +1,393 @@
|
|||
use std::collections::HashMap;
|
||||
|
||||
use serde::de::Error as _;
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct MsnQuote {
|
||||
#[serde(default)]
|
||||
pub(crate) symbol: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) short_name: Option<String>,
|
||||
pub(crate) price: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub(crate) price_change: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub(crate) price_change_percent: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub(crate) price_previous_close: Option<f64>,
|
||||
#[serde(default, rename = "price52wHigh")]
|
||||
pub(crate) price_52w_high: Option<f64>,
|
||||
#[serde(default, rename = "price52wLow")]
|
||||
pub(crate) price_52w_low: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub(crate) accumulated_volume: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub(crate) average_volume: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub(crate) market_cap: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub(crate) return_ytd: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub(crate) time_last_traded: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct KeyRatios {
|
||||
#[allow(dead_code)]
|
||||
#[serde(default)]
|
||||
pub(crate) industry_metrics: Vec<IndustryMetric>,
|
||||
#[serde(default)]
|
||||
pub(crate) company_metrics: Vec<IndustryMetric>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct IndustryMetric {
|
||||
pub(crate) year: Option<String>,
|
||||
pub(crate) fiscal_period_type: Option<String>,
|
||||
#[serde(default, deserialize_with = "de_opt_f64_lenient")]
|
||||
pub(crate) revenue_growth_rate: Option<f64>,
|
||||
#[serde(default, deserialize_with = "de_opt_f64_lenient")]
|
||||
pub(crate) earnings_growth_rate: Option<f64>,
|
||||
#[serde(
|
||||
default,
|
||||
rename = "netIncomeYTDYTDGrowthRate",
|
||||
deserialize_with = "de_opt_f64_lenient"
|
||||
)]
|
||||
pub(crate) net_income_ytd_ytd_growth_rate: Option<f64>,
|
||||
#[serde(
|
||||
default,
|
||||
rename = "revenueYTDYTD",
|
||||
deserialize_with = "de_opt_f64_lenient"
|
||||
)]
|
||||
pub(crate) revenue_ytd_ytd: Option<f64>,
|
||||
#[serde(default, deserialize_with = "de_opt_f64_lenient")]
|
||||
pub(crate) net_margin: Option<f64>,
|
||||
#[serde(default, deserialize_with = "de_opt_f64_lenient")]
|
||||
pub(crate) profit_margin: Option<f64>,
|
||||
#[serde(default, deserialize_with = "de_opt_f64_lenient")]
|
||||
pub(crate) roe: Option<f64>,
|
||||
#[serde(default, rename = "roaTTM", deserialize_with = "de_opt_f64_lenient")]
|
||||
pub(crate) roa_ttm: Option<f64>,
|
||||
#[serde(default, deserialize_with = "de_opt_f64_lenient")]
|
||||
pub(crate) return_on_asset_current: Option<f64>,
|
||||
#[serde(default, deserialize_with = "de_opt_f64_lenient")]
|
||||
pub(crate) debt_to_equity_ratio: Option<f64>,
|
||||
#[serde(default, deserialize_with = "de_opt_f64_lenient")]
|
||||
pub(crate) current_ratio: Option<f64>,
|
||||
#[serde(default, deserialize_with = "de_opt_f64_lenient")]
|
||||
pub(crate) price_to_earnings_ratio: Option<f64>,
|
||||
#[serde(
|
||||
default,
|
||||
rename = "forwardPriceToEPS",
|
||||
deserialize_with = "de_opt_f64_lenient"
|
||||
)]
|
||||
pub(crate) forward_price_to_eps: Option<f64>,
|
||||
#[serde(default, deserialize_with = "de_opt_f64_lenient")]
|
||||
pub(crate) price_to_book_ratio: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawEquity {
|
||||
pub(super) id: Option<String>,
|
||||
pub(super) symbol: Option<String>,
|
||||
pub(super) short_name: Option<String>,
|
||||
pub(super) long_name: Option<String>,
|
||||
pub(super) display_name: Option<String>,
|
||||
pub(super) description: Option<String>,
|
||||
pub(super) sector: Option<String>,
|
||||
pub(super) industry: Option<String>,
|
||||
pub(super) website: Option<String>,
|
||||
pub(super) full_time_employees: Option<i64>,
|
||||
pub(super) address: Option<String>,
|
||||
pub(super) city: Option<String>,
|
||||
pub(super) country: Option<String>,
|
||||
pub(super) phone: Option<String>,
|
||||
pub(super) officers: Option<Vec<RawOfficer>>,
|
||||
pub(super) company: Option<RawCompany>,
|
||||
pub(super) localized_attributes: Option<HashMap<String, RawLocalizedAttribute>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawOfficer {
|
||||
pub(super) name: Option<String>,
|
||||
pub(super) title: Option<String>,
|
||||
pub(super) age: Option<i32>,
|
||||
pub(super) year_born: Option<i32>,
|
||||
pub(super) total_pay: Option<i64>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawCompany {
|
||||
pub(super) address: Option<RawCompanyAddress>,
|
||||
pub(super) description: Option<String>,
|
||||
pub(super) employees: Option<i64>,
|
||||
pub(super) industry: Option<String>,
|
||||
pub(super) sector: Option<String>,
|
||||
pub(super) website: Option<String>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawCompanyAddress {
|
||||
pub(super) street: Option<String>,
|
||||
pub(super) city: Option<String>,
|
||||
pub(super) country: Option<String>,
|
||||
pub(super) phone: Option<String>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawLocalizedAttribute {
|
||||
pub(super) display_name: Option<String>,
|
||||
pub(super) description: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawFinancialStatement {
|
||||
pub(super) underlying_instrument: Option<RawInstrumentInfo>,
|
||||
pub(super) balance_sheets: Option<RawStatementSection>,
|
||||
pub(super) cash_flow: Option<RawStatementSection>,
|
||||
#[serde(rename = "incomeStatement")]
|
||||
pub(super) income_statements: Option<RawStatementSection>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawInstrumentInfo {
|
||||
pub(super) instrument_id: Option<String>,
|
||||
pub(super) display_name: Option<String>,
|
||||
pub(super) short_name: Option<String>,
|
||||
pub(super) symbol: Option<String>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(super) struct RawStatementSection {
|
||||
#[serde(flatten)]
|
||||
pub(super) data: HashMap<String, serde_json::Value>,
|
||||
pub(super) currency: Option<String>,
|
||||
pub(super) source: Option<String>,
|
||||
#[serde(rename = "sourceDate")]
|
||||
pub(super) source_date: Option<String>,
|
||||
#[serde(rename = "reportDate")]
|
||||
pub(super) report_date: Option<String>,
|
||||
#[serde(rename = "endDate")]
|
||||
pub(super) end_date: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "PascalCase")]
|
||||
pub(super) struct RawEarningsResponse {
|
||||
pub(super) eps_last_year: Option<f64>,
|
||||
pub(super) revenue_last_year: Option<f64>,
|
||||
pub(super) forecast: Option<RawEarningsBucket>,
|
||||
pub(super) history: Option<RawEarningsBucket>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(super) struct RawEarningsBucket {
|
||||
pub(super) annual: Option<HashMap<String, RawEarningsData>>,
|
||||
pub(super) quarterly: Option<HashMap<String, RawEarningsData>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "PascalCase")]
|
||||
pub(super) struct RawEarningsData {
|
||||
pub(super) eps_actual: Option<f64>,
|
||||
pub(super) eps_surprise: Option<f64>,
|
||||
pub(super) eps_surprise_percent: Option<f64>,
|
||||
pub(super) eps_forecast: Option<f64>,
|
||||
pub(super) revenue_actual: Option<f64>,
|
||||
pub(super) revenue_surprise: Option<f64>,
|
||||
pub(super) revenue_forecast: Option<f64>,
|
||||
pub(super) earning_release_date: Option<String>,
|
||||
pub(super) ciq_fiscal_period_type: Option<String>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawSentiment {
|
||||
pub(super) symbol: Option<String>,
|
||||
pub(super) display_name: Option<String>,
|
||||
pub(super) sentiment_statistics: Option<Vec<RawSentimentStat>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawSentimentStat {
|
||||
pub(super) time_range_name: Option<String>,
|
||||
pub(super) bullish: Option<i32>,
|
||||
pub(super) bearish: Option<i32>,
|
||||
pub(super) neutral: Option<i32>,
|
||||
}
|
||||
|
||||
// Actual MSN insights API response: array of insight containers, each holding
|
||||
// individual insight items grouped by category (Valuation, Risk, etc.)
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawInsight {
|
||||
pub(super) instrument_id: Option<String>,
|
||||
pub(super) display_name: Option<String>,
|
||||
pub(super) insights: Option<Vec<RawInsightItem>>,
|
||||
pub(super) time_last_updated: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawInsightItem {
|
||||
pub(super) insight_name: Option<String>,
|
||||
pub(super) category: Option<String>,
|
||||
pub(super) insight_statement: Option<String>,
|
||||
pub(super) short_insight_statement: Option<String>,
|
||||
pub(super) details: Option<RawInsightDetails>,
|
||||
pub(super) time_last_updated: Option<String>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawInsightDetails {
|
||||
pub(super) evaluation_status: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(super) struct RawNewsFeed {
|
||||
pub(super) value: Option<Vec<RawNewsItem>>,
|
||||
#[serde(rename = "subCards")]
|
||||
pub(super) sub_cards: Option<Vec<RawNewsItem>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawNewsItem {
|
||||
pub(super) id: Option<String>,
|
||||
pub(super) title: Option<String>,
|
||||
pub(super) url: Option<String>,
|
||||
#[serde(rename = "abstract")]
|
||||
pub(super) description: Option<String>,
|
||||
pub(super) provider: Option<RawNewsProvider>,
|
||||
pub(super) published_date_time: Option<String>,
|
||||
pub(super) read_time_min: Option<i32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(super) struct RawNewsProvider {
|
||||
pub(super) name: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct ScreenerRequest {
|
||||
pub(super) filter: Vec<ScreenerFilter>,
|
||||
pub(super) order: ScreenerOrder,
|
||||
pub(super) return_value_type: Vec<String>,
|
||||
pub(super) screener_type: String,
|
||||
pub(super) limit: usize,
|
||||
pub(super) page_index: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct ScreenerFilter {
|
||||
pub(super) key: String,
|
||||
pub(super) key_group: String,
|
||||
pub(super) is_range: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(super) struct ScreenerOrder {
|
||||
pub(super) key: String,
|
||||
pub(super) dir: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawScreenerResponse {
|
||||
pub(super) count: Option<i32>,
|
||||
pub(super) quote: Option<Vec<MsnQuote>>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawChartResponse {
|
||||
#[serde(rename = "_p")]
|
||||
pub(super) id: Option<String>,
|
||||
pub(super) chart_type: Option<String>,
|
||||
pub(super) symbol: Option<String>,
|
||||
pub(super) series: Option<RawChartSeries>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct RawChartSeries {
|
||||
#[serde(default)]
|
||||
pub(super) time_stamps: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub(super) prices: Vec<Option<f64>>,
|
||||
#[serde(default)]
|
||||
pub(super) open_prices: Vec<Option<f64>>,
|
||||
#[serde(default)]
|
||||
pub(super) prices_high: Vec<Option<f64>>,
|
||||
#[serde(default)]
|
||||
pub(super) prices_low: Vec<Option<f64>>,
|
||||
#[serde(default)]
|
||||
pub(super) volumes: Vec<Option<f64>>,
|
||||
pub(super) start_time: Option<String>,
|
||||
pub(super) end_time: Option<String>,
|
||||
}
|
||||
|
||||
fn de_opt_f64_lenient<'de, D>(deserializer: D) -> Result<Option<f64>, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum NumberLike {
|
||||
F64(f64),
|
||||
String(String),
|
||||
}
|
||||
|
||||
let value = Option::<NumberLike>::deserialize(deserializer)?;
|
||||
match value {
|
||||
Some(NumberLike::F64(number)) if number.is_finite() => Ok(Some(number)),
|
||||
Some(NumberLike::F64(_)) => Ok(None),
|
||||
Some(NumberLike::String(raw)) => {
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty()
|
||||
|| trimmed.eq_ignore_ascii_case("nan")
|
||||
|| trimmed.eq_ignore_ascii_case("infinity")
|
||||
|| trimmed.eq_ignore_ascii_case("-infinity")
|
||||
{
|
||||
Ok(None)
|
||||
} else {
|
||||
trimmed
|
||||
.parse::<f64>()
|
||||
.map_err(D::Error::custom)
|
||||
.map(|number| {
|
||||
if number.is_finite() {
|
||||
Some(number)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
896
crates/idx-core/src/api/msn/symbol_ids.tsv
Normal file
896
crates/idx-core/src/api/msn/symbol_ids.tsv
Normal file
|
|
@ -0,0 +1,896 @@
|
|||
AADI cfatz2
|
||||
AALI bn8zk2
|
||||
ABBA bkf9ec
|
||||
ABDA bkf9h7
|
||||
ABMM bkf9k2
|
||||
ACES bn8zyc
|
||||
ACRO cc48gh
|
||||
ACST bkf9pr
|
||||
ADCP c59mf2
|
||||
ADES bkf9sm
|
||||
ADHI bn9127
|
||||
ADMF bkf9yc
|
||||
ADMG bn9152
|
||||
ADMR c4pl52
|
||||
ADRO bn917w
|
||||
AEGS carobh
|
||||
AGAR bsu9xm
|
||||
AGII bkfa7w
|
||||
AGRO bkfaar
|
||||
AGRS bkfadm
|
||||
AHAP bkfagh
|
||||
AIMS bkfajc
|
||||
AISA bkfam7
|
||||
AKKU bkfctc
|
||||
AKPI bkfcw7
|
||||
AKRA bkfcz2
|
||||
AKSI bkfd2w
|
||||
ALDO bkfd5r
|
||||
ALII ccanzr
|
||||
ALKA bkfd8m
|
||||
ALMI bkfdbh
|
||||
ALTO bkfdec
|
||||
AMAG bkfchw
|
||||
AMAN btysar
|
||||
AMAR btf6a2
|
||||
AMFG bkfckr
|
||||
AMIN bkfcnm
|
||||
AMMN ca993m
|
||||
AMMS c6yaw7
|
||||
AMOR btfbw7
|
||||
AMRT bkfcqh
|
||||
ANDI bgrwf2
|
||||
ANJT bkfem7
|
||||
ANTM bn8zmw
|
||||
APEX bkferw
|
||||
APIC bkfeur
|
||||
APII bkfexm
|
||||
APLI bkff1h
|
||||
APLN bkff4c
|
||||
ARCI c2il1h
|
||||
AREA ccsom7
|
||||
ARGO bkff77
|
||||
ARII bkffa2
|
||||
ARKA bqtwtc
|
||||
ARKO c6qaqh
|
||||
ARNA bkfffr
|
||||
ARTA bkffim
|
||||
ARTO bkffoc
|
||||
ASBI bkffzr
|
||||
ASDM bkfg3m
|
||||
ASGR bkfg6h
|
||||
ASHA c6a3yc
|
||||
ASII bn91gh
|
||||
ASJT bkfgc7
|
||||
ASLC c4xl52
|
||||
ASLI cc41ww
|
||||
ASMI bkfgf2
|
||||
ASPI btr8pr
|
||||
ASPR chbyjc
|
||||
ASRI bkfghw
|
||||
ASRM bkfgkr
|
||||
ASSA bkfgnm
|
||||
ATAP bwzup2
|
||||
ATIC bkffr7
|
||||
ATLA ccwjvh
|
||||
AUTO bkffu2
|
||||
AVIA c4bepr
|
||||
AWAN c9f9yc
|
||||
AXIO c6v4pr
|
||||
AYAM cbkfkr
|
||||
AYLS btox8m
|
||||
BABP bkfgqh
|
||||
BABY carnqh
|
||||
BACA bkfgtc
|
||||
BAIK ccbzar
|
||||
BAJA bkfgw7
|
||||
BALI bkfgz2
|
||||
BANK bxjrc7
|
||||
BAPA bkfh2w
|
||||
BAPI breeur
|
||||
BATA bkflyc
|
||||
BATR cddzvh
|
||||
BAUT c4xpm7
|
||||
BAYU bkfm27
|
||||
BBCA bn91jc
|
||||
BBHI bkfm7w
|
||||
BBKP bkfmar
|
||||
BBLD bkfmdm
|
||||
BBMD bkfmgh
|
||||
BBNI bn91m7
|
||||
BBRI bn91p2
|
||||
BBRM bkfmrw
|
||||
BBSI bvpgvh
|
||||
BBSS bu9cqh
|
||||
BBTN bkfmur
|
||||
BBYB bkfmxm
|
||||
BCAP bkfn1h
|
||||
BCIC bgt4rw
|
||||
BCIP bkfn4c
|
||||
BDKR c8zmyc
|
||||
BDMN bn91rw
|
||||
BEBS bzv7ar
|
||||
BEEF bokavh
|
||||
BEER c8if77
|
||||
BEKS bkfna2
|
||||
BELI c7svnm
|
||||
BELL bkfncw
|
||||
BESS btyakr
|
||||
BEST bkfh5r
|
||||
BFIN bkfh8m
|
||||
BGTG bkfhbh
|
||||
BHAT bua6k2
|
||||
BHIT bkfhec
|
||||
BIKE c5iucw
|
||||
BIMA bkfkz2
|
||||
BINA bn92u2
|
||||
BINO c45tp2
|
||||
BIPI bn92ww
|
||||
BIPP bkfl8m
|
||||
BIRD bkflbh
|
||||
BISI bn92zr
|
||||
BJBR bkflh7
|
||||
BJTM bn936h
|
||||
BKDP bkflmw
|
||||
BKSL bkflpr
|
||||
BKSW bkflsm
|
||||
BLES cdlvh7
|
||||
BLOG chc7m7
|
||||
BLTA bgt5u2
|
||||
BLTZ bkflvh
|
||||
BLUE bqrgar
|
||||
BMAS bkfoqh
|
||||
BMBL c8isqh
|
||||
BMHS c2md4c
|
||||
BMRI bn939c
|
||||
BMSR bkfow7
|
||||
BMTR bkfoz2
|
||||
BNBA bkfp2w
|
||||
BNBR bn93c7
|
||||
BNGA bkfp8m
|
||||
BNII bn93f2
|
||||
BNLI bn93hw
|
||||
BOAT cf4yu2
|
||||
BOBA c3wioc
|
||||
BOGA bkfph7
|
||||
BOLA bqe3p2
|
||||
BOLT bkfpk2
|
||||
BPFI bkfppr
|
||||
BPII bkfpsm
|
||||
BPTR bgth1h
|
||||
BRAM bkfpvh
|
||||
BREN cb1ra2
|
||||
BRIS bgtha2
|
||||
BRMS bn924c
|
||||
BRNA bkfq27
|
||||
BRPT bn9277
|
||||
BRRC cflfa2
|
||||
BSBK c7svhw
|
||||
BSDE bkfq7w
|
||||
BSIM bn92a2
|
||||
BSML c4is3m
|
||||
BSSR bkfqdm
|
||||
BSWD bkfqgh
|
||||
BTEK bkfqjc
|
||||
BTON bkfqp2
|
||||
BTPN bkfqrw
|
||||
BTPS bgtwjc
|
||||
BUAH c71dyc
|
||||
BUDI bkfqur
|
||||
BUKA c2wgkr
|
||||
BUKK bkfqxm
|
||||
BULL bkfnu2
|
||||
BUMI bn92fr
|
||||
BUVA bkfnzr
|
||||
BVIC bkfo3m
|
||||
BWPT bn92im
|
||||
BYAN bn92lh
|
||||
CAKK bne4c7
|
||||
CAMP bkfoc7
|
||||
CANI bkfof2
|
||||
CARE btysdm
|
||||
CARS bkfohw
|
||||
CASA bkfokr
|
||||
CASH bufc7w
|
||||
CASS bkfonm
|
||||
CBDK cflejc
|
||||
CBPE c8ipmw
|
||||
CBRE c8ifww
|
||||
CBUT c7sxgh
|
||||
CCSI bqh6ar
|
||||
CDIA chdaw7
|
||||
CEKA bkfkqh
|
||||
CENT bkfktc
|
||||
CFIN bkfkw7
|
||||
CGAS cc45zr
|
||||
CHEK chddzr
|
||||
CHEM c6qahw
|
||||
CHIP c8qlmw
|
||||
CINT bkfnfr
|
||||
CITA bkfnim
|
||||
CITY bjs6w7
|
||||
CLAY bona52
|
||||
CLEO bkfnoc
|
||||
CLPI bkfnr7
|
||||
CMNP bn93kr
|
||||
CMNT c3bxcw
|
||||
CMPP bkgku2
|
||||
CMRY c4b81h
|
||||
CNKO bkgkzr
|
||||
CNMA cagkkr
|
||||
COAL c78lp2
|
||||
COCO bpfy1h
|
||||
COIN chc7p2
|
||||
CPIN bn93nm
|
||||
CPRO bkgl9c
|
||||
CRAB c71ep2
|
||||
CRSN ca9dnm
|
||||
CSAP bn93z2
|
||||
CSIS bkglf2
|
||||
CSMI bua4a2
|
||||
CSRA btf6cw
|
||||
CTBN bkglhw
|
||||
CTRA bn942w
|
||||
CTTH bkglnm
|
||||
CUAN c8ztf2
|
||||
CYBR caj9jc
|
||||
DAAZ cf3dr7
|
||||
DADA btov1h
|
||||
DART bkglqh
|
||||
DATA cd3e27
|
||||
DAYA bkgltc
|
||||
DCII bxbr3m
|
||||
DEFI bkglw7
|
||||
DEPO c461k2
|
||||
DEWA bn93qh
|
||||
DEWI c6t4oc
|
||||
DFAM bkgm2w
|
||||
DGIK bkgm5r
|
||||
DGNS bxbs2w
|
||||
DGWG cfl46h
|
||||
DIGI bguqr7
|
||||
DILD bn93tc
|
||||
DIVA bny8f2
|
||||
DKFT bkgmbh
|
||||
DKHH cgo8rw
|
||||
DLTA bkgn52
|
||||
DMAS bkgn7w
|
||||
DMMX bsh6z2
|
||||
DMND bti2qh
|
||||
DNAR bkgnar
|
||||
DNET bkgndm
|
||||
DOID bkgngh
|
||||
DOOH c9pg52
|
||||
DOSS cecd8m
|
||||
DPNS bkgnjc
|
||||
DPUM bkgnm7
|
||||
DRMA c4is9c
|
||||
DSFI bkgnp2
|
||||
DSNG bkgnrw
|
||||
DSSA bn94ec
|
||||
DUTI bn94h7
|
||||
DVLA bkgmsm
|
||||
DWGL bkgmvh
|
||||
DYAN bkgmyc
|
||||
EAST bqrjmw
|
||||
ECII bkgn27
|
||||
EDGE bxmmkr
|
||||
EKAD bkgmec
|
||||
ELIT c8ifcw
|
||||
ELPI c71dh7
|
||||
ELSA bkgmh7
|
||||
ELTY bn94k2
|
||||
EMAS ci4x5r
|
||||
EMDE bkgo77
|
||||
EMTK bkgocw
|
||||
ENAK c53nww
|
||||
ENRG bkgofr
|
||||
ENZO bvqna2
|
||||
EPAC buzekr
|
||||
EPMT bkgoim
|
||||
ERAA bkgolh
|
||||
ERAL cakb4c
|
||||
ERTX bkgooc
|
||||
ESIP bsq1dm
|
||||
ESSA bkgnxm
|
||||
ESTA btxltc
|
||||
ESTI bkgo1h
|
||||
EURO c6zylh
|
||||
EXCL bn94pr
|
||||
FAPA bx7mww
|
||||
FAST bkgor7
|
||||
FASW bkgou2
|
||||
FILM bgva7w
|
||||
FIMP c1paqh
|
||||
FIRE bkgozr
|
||||
FISH bkgp3m
|
||||
FITT bqcw2w
|
||||
FLMC c2p22w
|
||||
FMII bkgp9c
|
||||
FOLK caj9ar
|
||||
FOOD bohgur
|
||||
FORE cghp7w
|
||||
FORU bkgpc7
|
||||
FPNI bkgphw
|
||||
FUJI bqrhww
|
||||
FUTR c8xc2w
|
||||
FWCT c8no4c
|
||||
GDST bkgpnm
|
||||
GDYR bkgpqh
|
||||
GEMA bkgptc
|
||||
GEMS bkgpw7
|
||||
GGRM bn94vh
|
||||
GGRP brivww
|
||||
GHON bkgq2w
|
||||
GIAA bkgq5r
|
||||
GJTL bn9527
|
||||
GLOB bgveur
|
||||
GLVA bt94fr
|
||||
GMFI bkgqbh
|
||||
GMTD bkgqk2
|
||||
GOLD bkgqmw
|
||||
GOLF cdml52
|
||||
GOOD bkgqsm
|
||||
GOTO c5tv8m
|
||||
GPRA bkgqvh
|
||||
GPSO c39qp2
|
||||
GRIA cakam7
|
||||
GRPH cc65vh
|
||||
GRPM ca9bar
|
||||
GSMF bkgqec
|
||||
GTBO bkgqh7
|
||||
GTRA c9a7c7
|
||||
GTSI c3atkr
|
||||
GULA c6ybbh
|
||||
GUNA cdm6oc
|
||||
GWSA bkgqyc
|
||||
GZCO bkgr27
|
||||
HADE bkgr52
|
||||
HAIS c3839c
|
||||
HAJJ c9c9z2
|
||||
HALO c8snbh
|
||||
HATM c6vxxm
|
||||
HBAT caj5ur
|
||||
HDFA bkgr7w
|
||||
HDIT bqsr9c
|
||||
HEAL bgvigh
|
||||
HELI bkgrdm
|
||||
HERO bkgrgh
|
||||
HEXA bn95ar
|
||||
HGII cfl3r7
|
||||
HILL c8xfhw
|
||||
HITS bkgrm7
|
||||
HMSP bn95jc
|
||||
HOKI bkgrur
|
||||
HOMI bvpl7w
|
||||
HOPE c24y52
|
||||
HRME bpplhw
|
||||
HRTA bkgs4c
|
||||
HRUM bn957w
|
||||
HUMI caj9p2
|
||||
HYGN ccbzjc
|
||||
IATA bkgsa2
|
||||
IBFN bkgscw
|
||||
IBOS c5z1vh
|
||||
IBST bkgsfr
|
||||
ICBP bkgsim
|
||||
ICON bkgslh
|
||||
IDEA c3at9c
|
||||
IDPR bkgsoc
|
||||
IFII bt26w7
|
||||
IFSH bsua4c
|
||||
IGAR bkgsr7
|
||||
IKAI bkgsww
|
||||
IKAN btov4c
|
||||
IKBI bkgszr
|
||||
IKPM cbetnm
|
||||
IMAS bkguk2
|
||||
IMJS bkgumw
|
||||
IMPC bkgupr
|
||||
INAI bkguvh
|
||||
INCF bkguyc
|
||||
INCI bkgv27
|
||||
INCO bn96cw
|
||||
INDF bn96fr
|
||||
INDO btf6ww
|
||||
INDR bkgvar
|
||||
INDS bkgvdm
|
||||
INDX bkgvgh
|
||||
INDY bkgvjc
|
||||
INET caehu2
|
||||
INKP bn96im
|
||||
INOV bqtw6h
|
||||
INPC bkgttc
|
||||
INPP bkgtw7
|
||||
INPS bkgtz2
|
||||
INRU bkgu2w
|
||||
INTA bkgu5r
|
||||
INTD bkgu8m
|
||||
INTP bn95m7
|
||||
IOTF cb1mar
|
||||
IPAC c2ip4c
|
||||
IPCC bgvlmw
|
||||
IPCM bkguec
|
||||
IPOL bkguh7
|
||||
IPTV bqrdim
|
||||
IRRA bs6n9c
|
||||
IRSX c8ptdm
|
||||
ISAP c87m5r
|
||||
ISAT bn95p2
|
||||
ISEA cdmkyc
|
||||
ISSP bn95rw
|
||||
ITIC bqptyc
|
||||
ITMA bkgt9c
|
||||
ITMG bn95ur
|
||||
JARR c6yb2w
|
||||
JAST bpztur
|
||||
JATI c9pikr
|
||||
JAWA bkgtf2
|
||||
JAYA bp3fkr
|
||||
JECC bkgthw
|
||||
JGLE bkgtkr
|
||||
JIHD bkgtnm
|
||||
JKON bn95xm
|
||||
JMAS bkgwcw
|
||||
JPFA bkgwfr
|
||||
JRPT bkgwim
|
||||
JSMR bn964c
|
||||
JSPT bkgwr7
|
||||
JTPE bkgwu2
|
||||
KAEF bkgvp2
|
||||
KAQI cg3ah7
|
||||
KARW bkgvrw
|
||||
KBAG bu8obh
|
||||
KBLI bkgvur
|
||||
KBLM bkgvxm
|
||||
KBLV bkgw1h
|
||||
KDSI bkgwww
|
||||
KDTN c7sxar
|
||||
KEEN brdma2
|
||||
KEJU bss8im
|
||||
KETR c7x3cw
|
||||
KIAS bgw8lh
|
||||
KICI bkgwzr
|
||||
KIJA bkgx3m
|
||||
KING c8v3jc
|
||||
KINO bkgx6h
|
||||
KIOS bkgx9c
|
||||
KJEN bqoa1h
|
||||
KKES c71dsm
|
||||
KKGI bkgxqh
|
||||
KLAS c9zg9c
|
||||
KLBF bn96u2
|
||||
KLIN c71ear
|
||||
KMDS bvpjz2
|
||||
KMTR bkgxw7
|
||||
KOBX bkgxz2
|
||||
KOCI cazma2
|
||||
KOIN bkgy2w
|
||||
KOKA cb2m5r
|
||||
KONI bkgy5r
|
||||
KOPI bkgy8m
|
||||
KOTA bqspm7
|
||||
KPIG bkgxf2
|
||||
KRAS bn96zr
|
||||
KREN bkgxnm
|
||||
KRYA c6vopr
|
||||
KSIX cfl4f2
|
||||
KUAS c3tgqh
|
||||
LABA c2ekur
|
||||
LABS cdml7w
|
||||
LAJU c8ni2w
|
||||
LAND bgwfxm
|
||||
LAPD bkgybh
|
||||
LCKM bkgyh7
|
||||
LEAD bkgyk2
|
||||
LFLO c1g5f2
|
||||
LIFE bqsrz2
|
||||
LINK bkgymw
|
||||
LION bkgypr
|
||||
LIVE ccbz52
|
||||
LMAX caj9gh
|
||||
LMPI bkgyyc
|
||||
LMSH bkgz27
|
||||
LOPI cb2a2w
|
||||
LPCK bkgz52
|
||||
LPGI bkgz7w
|
||||
LPIN bkgzar
|
||||
LPKR bn99ar
|
||||
LPLI bn99dm
|
||||
LPPF bkgzjc
|
||||
LPPS bn99gh
|
||||
LRNA bkgzp2
|
||||
LSIP bn99jc
|
||||
LTLS bkgzur
|
||||
LUCK bny8z2
|
||||
LUCY c1zlh7
|
||||
MAHA caem3m
|
||||
MAIN bkh177
|
||||
MANG cc463m
|
||||
MAPA bgwzm7
|
||||
MAPB bkh1cw
|
||||
MAPI bn98pr
|
||||
MARI bkh1im
|
||||
MARK bkh1lh
|
||||
MASB c2jw8m
|
||||
MAXI c9zd5r
|
||||
MAYA bkh1r7
|
||||
MBAP bkh1u2
|
||||
MBMA c9fe52
|
||||
MBSS bkh1ww
|
||||
MBTO bkh1zr
|
||||
MCAS bkh23m
|
||||
MCOL c3apnm
|
||||
MCOR bkh26h
|
||||
MDIA bkh2c7
|
||||
MDIY cff1lh
|
||||
MDKA bkh2f2
|
||||
MDKI bkh2hw
|
||||
MDLA cgbphw
|
||||
MDLN bkh2kr
|
||||
MDRN bkh2nm
|
||||
MEDC bn9a77
|
||||
MEDS c71idm
|
||||
MEGA bkh2tc
|
||||
MEJA ccbqh7
|
||||
MENN c9fbu2
|
||||
MERI che6cw
|
||||
MERK bkh2w7
|
||||
MFMI bkh35r
|
||||
MGLV c2cba2
|
||||
MGNA bkh38m
|
||||
MGRO bgxdcw
|
||||
MHKI ccwk27
|
||||
MICE bn9afr
|
||||
MIDI bn9aim
|
||||
MIKA bkh3h7
|
||||
MINA bkh427
|
||||
MINE cg3epr
|
||||
MIRA bkh452
|
||||
MITI bkh47w
|
||||
MKAP ccbrar
|
||||
MKPI bkh4dm
|
||||
MKTR c7sz6h
|
||||
MLBI bkh4gh
|
||||
MLIA bkh4jc
|
||||
MLPL bn9alh
|
||||
MLPT bkh4p2
|
||||
MMIX c87lz2
|
||||
MMLP bkh4rw
|
||||
MNCN bkh4ur
|
||||
MOLI b9fksm
|
||||
MORA c6zya2
|
||||
MPIX ccbqmw
|
||||
MPMX bkh3mw
|
||||
MPOW bkh3pr
|
||||
MPPA bn9azr
|
||||
MPRO bkh3vh
|
||||
MPXL c9ql7w
|
||||
MRAT bkh3yc
|
||||
MREI bkhcur
|
||||
MSIE cakqgh
|
||||
MSIN bgy7nm
|
||||
MSJA cc466h
|
||||
MSKY bkhcxm
|
||||
MSTI cbe9qh
|
||||
MTDL bkhd1h
|
||||
MTEL c45acw
|
||||
MTFN bgy82w
|
||||
MTLA bkhd4c
|
||||
MTMH c5w4lh
|
||||
MTPS bpmq9c
|
||||
MTSM bkhda2
|
||||
MTWI bkhdfr
|
||||
MUTU cajgyc
|
||||
MYOH bkhcgh
|
||||
MYOR bn9bkr
|
||||
MYTX bkhcp2
|
||||
NAIK cf6abh
|
||||
NANO c5es5r
|
||||
NASA bkhc52
|
||||
NASI c4earw
|
||||
NATO bor3r7
|
||||
NAYZ c8pvw7
|
||||
NCKL c9dmnm
|
||||
NELY bkhcar
|
||||
NEST cecdh7
|
||||
NETV c4zpnm
|
||||
NFCX bh2cm7
|
||||
NICE cc46f2
|
||||
NICK bkhdoc
|
||||
NICL c2og3m
|
||||
NIKL bkhdr7
|
||||
NINE c87ih7
|
||||
NIRO bkhdww
|
||||
NISP bkhdzr
|
||||
NOBU bkhe3m
|
||||
NPGF c1mpp2
|
||||
NRCA bkhe9c
|
||||
NSSS c92ssm
|
||||
NTBK c53nim
|
||||
NZIA brkv8m
|
||||
OASA bkhdim
|
||||
OBAT cflf77
|
||||
OBMD c4dnkr
|
||||
OILS c3a84c
|
||||
OKAS bkhdlh
|
||||
OLIV c64i27
|
||||
OMED c7t2jc
|
||||
OMRE bkhekr
|
||||
OPMS brkez2
|
||||
PACK c8sn2w
|
||||
PADA c87qyc
|
||||
PADI bkheqh
|
||||
PALM bkhetc
|
||||
PAMG bqrcxm
|
||||
PANI bh3952
|
||||
PANR bkhez2
|
||||
PANS bkhf2w
|
||||
PART cdlta2
|
||||
PBID bkhf5r
|
||||
PBRX bkhf8m
|
||||
PBSA bkhfbh
|
||||
PCAR bkhfec
|
||||
PDES bkhfh7
|
||||
PDPP c7wzkr
|
||||
PEGE bkhfk2
|
||||
PEHA boeyr7
|
||||
PEVE c8lqu2
|
||||
PGAS bn9ch7
|
||||
PGEO c8wmc7
|
||||
PGJO btf6im
|
||||
PGLI bkhfpr
|
||||
PGUN bv2tpr
|
||||
PICO bkhfvh
|
||||
PIPA c9cy5r
|
||||
PJAA bkhfyc
|
||||
PKPK bkhh1h
|
||||
PLAN bvsmnm
|
||||
PLIN bkhh77
|
||||
PMJS bt81mw
|
||||
PMMP bx3rtc
|
||||
PMUI che6fr
|
||||
PNBN bn9d27
|
||||
PNBS bn9d52
|
||||
PNGO bvilbh
|
||||
PNIN bkhhfr
|
||||
PNLF bn9d7w
|
||||
PNSE bkhhlh
|
||||
POLA bnw452
|
||||
POLI bokeh7
|
||||
POLU bqiq5r
|
||||
POLY bkhg27
|
||||
PORT bkhg7w
|
||||
POWR bn9dar
|
||||
PPGL bv7ka2
|
||||
PPRE bkhgdm
|
||||
PPRI caj5xm
|
||||
PPRO bkhggh
|
||||
PRAY c7svf2
|
||||
PRDA bkhgm7
|
||||
PRIM bh3bnm
|
||||
PSAB bkhgp2
|
||||
PSDN bkhgrw
|
||||
PSGO bssb7w
|
||||
PSKT bkhgur
|
||||
PSSI bkhgxm
|
||||
PTBA bn9i4c
|
||||
PTDU bwzbim
|
||||
PTIS bkhif2
|
||||
PTMP c8zh3m
|
||||
PTMR cese7w
|
||||
PTPP bkhihw
|
||||
PTPS cb26pr
|
||||
PTPW btowc7
|
||||
PTRO bkhinm
|
||||
PTSN bkhiqh
|
||||
PTSP bkhitc
|
||||
PUDP bkhiw7
|
||||
PURA btjqcw
|
||||
PURI bvmvlh
|
||||
PWON bkhiz2
|
||||
PYFA bkhj2w
|
||||
PZZA bh3dar
|
||||
RAAM c9pg7w
|
||||
RAFI c6zy1h
|
||||
RAJA bkhhu2
|
||||
RALS bn9iim
|
||||
RANC bkhhzr
|
||||
RATU cfl3oc
|
||||
RBMS bkhi3m
|
||||
RCCC c6y4ur
|
||||
RDTX bkhi6h
|
||||
REAL bsxdzr
|
||||
RELF ca43u2
|
||||
RELI bkhi9c
|
||||
RGAS cbe25r
|
||||
RICY bkhj5r
|
||||
RIGS bkhj8m
|
||||
RISE bh3par
|
||||
RMKE c4bf52
|
||||
RMKO cafyoc
|
||||
ROCK bvpnqh
|
||||
RODA bkhkur
|
||||
RONY bua477
|
||||
ROTI bkhkxm
|
||||
RSCH caoza2
|
||||
RSGK c3atf2
|
||||
RUIS bkhjh7
|
||||
RUNS c3athw
|
||||
SAFE bh4khw
|
||||
SAGE c8ztkr
|
||||
SAME bkhl1h
|
||||
SAMF bu77h7
|
||||
SAPX bkhl4c
|
||||
SATU bnmpyc
|
||||
SBMA c3apw7
|
||||
SCCO bkhl77
|
||||
SCMA bkhla2
|
||||
SCNP bvpljc
|
||||
SDMU bkhlcw
|
||||
SDPC bkhlfr
|
||||
SDRA bkhlim
|
||||
SEMA c4s94c
|
||||
SFAN bqigoc
|
||||
SGER bvdqa2
|
||||
SGRO bn9j9c
|
||||
SHID bkhlww
|
||||
SHIP bkhlzr
|
||||
SICO c5tatc
|
||||
SIDO bkhm6h
|
||||
SILO bn9jnm
|
||||
SIMP bkhmf2
|
||||
SINI bsn27w
|
||||
SIPD bkhpzr
|
||||
SKBM bkhq3m
|
||||
SKLT bkhq6h
|
||||
SKRN bkhqc7
|
||||
SLIS bs6kh7
|
||||
SMAR bkhrmw
|
||||
SMBR bn9lar
|
||||
SMCB bn9ldm
|
||||
SMDM bkhryc
|
||||
SMDR bkhs27
|
||||
SMGA cc93tc
|
||||
SMGR bn9lgh
|
||||
SMIL c9r7oc
|
||||
SMKL bqrfh7
|
||||
SMKM c5gqk2
|
||||
SMLE cc45oc
|
||||
SMMA bkhs7w
|
||||
SMMT bkhsar
|
||||
SMRA bn9ljc
|
||||
SMSM bkhsjc
|
||||
SNLK c1c3ur
|
||||
SOCI bkhmkr
|
||||
SOFA bv1prw
|
||||
SOHO bvpia2
|
||||
SOLA cd3j77
|
||||
SONA bkhmqh
|
||||
SOSS borblh
|
||||
SOTS bo5fk2
|
||||
SOUL c8hmu2
|
||||
SPMA bkhqhw
|
||||
SPRE cdllww
|
||||
SPTO bh4tpr
|
||||
SQMI bkhqkr
|
||||
SRAJ bkhqnm
|
||||
SRSN bkhqw7
|
||||
SRTG bkhqz2
|
||||
SSIA bkhr2w
|
||||
SSMS bkhr5r
|
||||
SSTM bkhr8m
|
||||
STAA c5grjc
|
||||
STAR bkhrbh
|
||||
STRK cb1rcw
|
||||
STTP bkhrec
|
||||
SULI bn9l52
|
||||
SUNI c8irzr
|
||||
SUPR bkhsp2
|
||||
SURE bkhsrw
|
||||
SURI cbriz2
|
||||
SWAT bh4xyc
|
||||
SWID c6q9ww
|
||||
TALF bkhsxm
|
||||
TAMA btnwec
|
||||
TAMU bkht1h
|
||||
TAPG c1pha2
|
||||
TARA bkht4c
|
||||
TAXI bkhta2
|
||||
TAYS c4bfdm
|
||||
TBIG bkhtcw
|
||||
TBLA bn9m77
|
||||
TBMS bkhtim
|
||||
TCID bkhtlh
|
||||
TCPI bh55nm
|
||||
TEBE bsqufr
|
||||
TELE bkhtr7
|
||||
TFAS brgc52
|
||||
TFCO bkhtzr
|
||||
TGKA bkhu6h
|
||||
TGRA bkhu9c
|
||||
TGUK ca99f2
|
||||
TIFA bkhuc7
|
||||
TINS bn9o2w
|
||||
TIRA bkhunm
|
||||
TIRT bkhuqh
|
||||
TKIM bn9o5r
|
||||
TLDN c5tiqh
|
||||
TLKM bn9o8m
|
||||
TMAS bn9oec
|
||||
TMPO bkhv8m
|
||||
TNCA bh5cim
|
||||
TOBA bkhvk2
|
||||
TOOL c71edm
|
||||
TOSK ccbqk2
|
||||
TOTL bn9ok2
|
||||
TOTO bkhvsm
|
||||
TOWR bn9omw
|
||||
TPIA bkhvyc
|
||||
TPMA bkhw27
|
||||
TRGU c6qatc
|
||||
TRIM bki2lh
|
||||
TRIN btfcbh
|
||||
TRIS bki2oc
|
||||
TRJA bviktc
|
||||
TRON c8zthw
|
||||
TRST bki2u2
|
||||
TRUE c2cfa2
|
||||
TRUK bh7r6h
|
||||
TRUS bki2zr
|
||||
TSPC bki33m
|
||||
TUGU bh72cw
|
||||
TYRE c9pfpr
|
||||
UANG bv1qlh
|
||||
UCID bt7nnm
|
||||
UDNG cbazxm
|
||||
UFOE bxjur7
|
||||
ULTJ bki36h
|
||||
UNIC bki39c
|
||||
UNIQ bzv6mw
|
||||
UNSP bn9pjc
|
||||
UNTD ccanww
|
||||
UNTR bn9pm7
|
||||
UNVR bn9pp2
|
||||
URBN bo5fsm
|
||||
UVCR c2ruww
|
||||
VAST c8sn8m
|
||||
VERN cesblh
|
||||
VICI bx3pp2
|
||||
VICO bki3nm
|
||||
VINS bki3qh
|
||||
VISI cchqzr
|
||||
VIVA bki45r
|
||||
VKTR ca2e9c
|
||||
VOKS bki4bh
|
||||
VRNA bki4k2
|
||||
VTNY c89txm
|
||||
WAPO bki4mw
|
||||
WEGE bki4pr
|
||||
WEHA bki4sm
|
||||
WGSH c4bevh
|
||||
WICO bki76h
|
||||
WIDI ca9dkr
|
||||
WIFI bx86dm
|
||||
WIIM bki79c
|
||||
WIKA bn9q77
|
||||
WINE c8isf2
|
||||
WINR c5yu77
|
||||
WINS bn9qa2
|
||||
WIRG c5o9z2
|
||||
WMUU bxjkec
|
||||
WOMF bki7hw
|
||||
WOOD bki7kr
|
||||
WOWS bsnwf2
|
||||
WSBP bki7z2
|
||||
WTON bn9r9c
|
||||
YELO bnehec
|
||||
YOII cfl49c
|
||||
YPAS bki8bh
|
||||
YULE bki7nm
|
||||
YUPI cgaarw
|
||||
ZATA c7x3im
|
||||
ZBRA bhacdm
|
||||
ZINC bki7qh
|
||||
ZONE bo5fyc
|
||||
ZYRX c1c1kr
|
||||
|
56
crates/idx-core/src/api/msn/symbols.rs
Normal file
56
crates/idx-core/src/api/msn/symbols.rs
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
use std::collections::HashMap;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
const SYMBOL_IDS_RAW: &str = include_str!("symbol_ids.tsv");
|
||||
|
||||
static SYMBOL_IDS: OnceLock<HashMap<&'static str, &'static str>> = OnceLock::new();
|
||||
|
||||
fn symbol_ids() -> &'static HashMap<&'static str, &'static str> {
|
||||
SYMBOL_IDS.get_or_init(|| {
|
||||
SYMBOL_IDS_RAW
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once('\t'))
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn ticker_from_symbol(symbol: &str) -> Option<String> {
|
||||
let trimmed = symbol.trim();
|
||||
if trimmed.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(
|
||||
trimmed
|
||||
.split('.')
|
||||
.next()
|
||||
.unwrap_or(trimmed)
|
||||
.trim()
|
||||
.to_uppercase(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn resolve_msn_id(symbol: &str) -> Option<&'static str> {
|
||||
let ticker = ticker_from_symbol(symbol)?;
|
||||
symbol_ids().get(ticker.as_str()).copied()
|
||||
}
|
||||
|
||||
pub(crate) fn normalized_symbol(requested: &str, fallback_ticker: &str) -> String {
|
||||
let trimmed = requested.trim().to_uppercase();
|
||||
if trimmed.contains('.') || fallback_ticker.is_empty() {
|
||||
trimmed
|
||||
} else {
|
||||
format!("{}.JK", fallback_ticker.trim().to_uppercase())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::resolve_msn_id;
|
||||
|
||||
#[test]
|
||||
fn resolves_symbol_variants() {
|
||||
assert_eq!(resolve_msn_id("BBCA"), Some("bn91jc"));
|
||||
assert_eq!(resolve_msn_id("bbca.jk"), Some("bn91jc"));
|
||||
assert_eq!(resolve_msn_id("INVALID"), None);
|
||||
}
|
||||
}
|
||||
330
crates/idx-core/src/api/types.rs
Normal file
330
crates/idx-core/src/api/types.rs
Normal file
|
|
@ -0,0 +1,330 @@
|
|||
use chrono::{DateTime, FixedOffset, NaiveDate};
|
||||
use serde::{Deserialize, Deserializer, Serialize, de::Error as _};
|
||||
|
||||
/// Snapshot quote data normalized from Yahoo Finance `/v8/finance/chart` response.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Quote {
|
||||
/// Trading symbol as returned by Yahoo `chart.result[0].meta.symbol`.
|
||||
pub symbol: String,
|
||||
/// Last traded regular market price in IDR (whole Rupiah), mapped from
|
||||
/// `chart.result[0].meta.regularMarketPrice` and rounded to nearest integer.
|
||||
#[serde(deserialize_with = "de_i64_from_number")]
|
||||
pub price: i64,
|
||||
/// Absolute day change in IDR (whole Rupiah), computed as
|
||||
/// `regularMarketPrice - previousClose` using rounded integer prices.
|
||||
/// `None` when the provider gives neither a previous close nor a change.
|
||||
#[serde(default, deserialize_with = "de_opt_i64_from_number")]
|
||||
pub change: Option<i64>,
|
||||
/// Percentage day change as decimal percent (`0-100` scale), computed from
|
||||
/// Yahoo `regularMarketPrice` and `previousClose` raw floats.
|
||||
/// `None` when the provider does not report it (never a fake `0.0`).
|
||||
#[serde(default)]
|
||||
pub change_pct: Option<f64>,
|
||||
/// Traded regular market volume (shares), from `regularMarketVolume`.
|
||||
/// `None` when the provider does not report it (never a fake `0`).
|
||||
#[serde(default, deserialize_with = "de_opt_u64_from_number")]
|
||||
pub volume: Option<u64>,
|
||||
/// Company market capitalization in IDR, from `marketCap`.
|
||||
#[serde(default, deserialize_with = "de_opt_u64_from_number")]
|
||||
pub market_cap: Option<u64>,
|
||||
/// 52-week high in IDR (whole Rupiah), from `fiftyTwoWeekHigh` rounded.
|
||||
#[serde(default, deserialize_with = "de_opt_i64_from_number")]
|
||||
pub week52_high: Option<i64>,
|
||||
/// 52-week low in IDR (whole Rupiah), from `fiftyTwoWeekLow` rounded.
|
||||
#[serde(default, deserialize_with = "de_opt_i64_from_number")]
|
||||
pub week52_low: Option<i64>,
|
||||
/// Relative position within 52-week range (`0.0..=1.0`), computed from raw
|
||||
/// Yahoo `fiftyTwoWeekLow` and `fiftyTwoWeekHigh`.
|
||||
pub week52_position: Option<f64>,
|
||||
/// Coarse 52-week range bucket derived from `week52_position`.
|
||||
pub range_signal: Option<String>,
|
||||
/// Previous close in IDR (whole Rupiah), from
|
||||
/// `previousClose` or `chartPreviousClose`, rounded.
|
||||
#[serde(default, deserialize_with = "de_opt_i64_from_number")]
|
||||
pub prev_close: Option<i64>,
|
||||
/// Average daily volume for the last 3 months (shares), from
|
||||
/// `averageDailyVolume3Month`.
|
||||
pub avg_volume: Option<u64>,
|
||||
/// Time of the last trade in WIB (UTC+7), from MSN `timeLastTraded` or
|
||||
/// Yahoo `regularMarketTime`. `None` when the provider does not say.
|
||||
/// Suspended stocks can carry a last trade months in the past.
|
||||
#[serde(default)]
|
||||
pub as_of: Option<DateTime<FixedOffset>>,
|
||||
}
|
||||
|
||||
/// OHLC candle data normalized from Yahoo Finance chart indicators.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Ohlc {
|
||||
/// Candle date (exchange-local day boundary from Yahoo timestamp).
|
||||
pub date: NaiveDate,
|
||||
/// Opening price in IDR (whole Rupiah), from `indicators.quote[0].open` rounded.
|
||||
#[serde(deserialize_with = "de_i64_from_number")]
|
||||
pub open: i64,
|
||||
/// Highest traded price in IDR (whole Rupiah), from `indicators.quote[0].high` rounded.
|
||||
#[serde(deserialize_with = "de_i64_from_number")]
|
||||
pub high: i64,
|
||||
/// Lowest traded price in IDR (whole Rupiah), from `indicators.quote[0].low` rounded.
|
||||
#[serde(deserialize_with = "de_i64_from_number")]
|
||||
pub low: i64,
|
||||
/// Closing price in IDR (whole Rupiah), from `indicators.quote[0].close` rounded.
|
||||
#[serde(deserialize_with = "de_i64_from_number")]
|
||||
pub close: i64,
|
||||
/// Traded volume (shares), from `indicators.quote[0].volume`.
|
||||
pub volume: u64,
|
||||
}
|
||||
|
||||
/// Fundamental metrics normalized from Yahoo Finance `/v10/finance/quoteSummary`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Fundamentals {
|
||||
pub trailing_pe: Option<f64>,
|
||||
pub forward_pe: Option<f64>,
|
||||
pub price_to_book: Option<f64>,
|
||||
pub return_on_equity: Option<f64>,
|
||||
pub profit_margins: Option<f64>,
|
||||
pub return_on_assets: Option<f64>,
|
||||
pub revenue_growth: Option<f64>,
|
||||
pub earnings_growth: Option<f64>,
|
||||
pub debt_to_equity: Option<f64>,
|
||||
pub current_ratio: Option<f64>,
|
||||
pub enterprise_value: Option<i64>,
|
||||
pub ebitda: Option<i64>,
|
||||
pub market_cap: Option<u64>,
|
||||
}
|
||||
|
||||
pub type Bar = Ohlc;
|
||||
|
||||
// Forward-looking types for planned MSN endpoints — used once capability traits are wired to CLI.
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CompanyProfile {
|
||||
pub id: String,
|
||||
pub symbol: String,
|
||||
pub short_name: String,
|
||||
pub long_name: String,
|
||||
pub description: String,
|
||||
pub sector: String,
|
||||
pub industry: String,
|
||||
pub website: String,
|
||||
pub employees: i64,
|
||||
pub address: String,
|
||||
pub city: String,
|
||||
pub country: String,
|
||||
pub phone: String,
|
||||
pub officers: Vec<Officer>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Officer {
|
||||
pub name: String,
|
||||
pub title: String,
|
||||
pub age: Option<i32>,
|
||||
pub year_born: Option<i32>,
|
||||
pub total_pay: Option<i64>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct FinancialStatements {
|
||||
pub instrument: InstrumentInfo,
|
||||
pub balance_sheet: Option<StatementSection>,
|
||||
pub cash_flow: Option<StatementSection>,
|
||||
pub income_statement: Option<StatementSection>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct InstrumentInfo {
|
||||
pub id: String,
|
||||
pub symbol: String,
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct StatementSection {
|
||||
pub values: std::collections::BTreeMap<String, f64>,
|
||||
pub currency: String,
|
||||
pub report_date: String,
|
||||
pub end_date: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EarningsReport {
|
||||
#[serde(default)]
|
||||
pub symbol: String,
|
||||
pub eps_last_year: f64,
|
||||
pub revenue_last_year: f64,
|
||||
pub forecast: Vec<EarningsData>,
|
||||
pub history: Vec<EarningsData>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EarningsData {
|
||||
pub eps_actual: Option<f64>,
|
||||
pub eps_forecast: Option<f64>,
|
||||
pub eps_surprise: Option<f64>,
|
||||
pub eps_surprise_pct: Option<f64>,
|
||||
pub revenue_actual: Option<f64>,
|
||||
pub revenue_forecast: Option<f64>,
|
||||
pub revenue_surprise: Option<f64>,
|
||||
pub earning_release_date: Option<String>,
|
||||
pub period_type: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SentimentData {
|
||||
pub symbol: String,
|
||||
pub statistics: Vec<SentimentPeriod>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SentimentPeriod {
|
||||
pub time_range: String,
|
||||
pub bullish: i32,
|
||||
pub bearish: i32,
|
||||
pub neutral: i32,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct InsightData {
|
||||
pub id: String,
|
||||
#[serde(default)]
|
||||
pub symbol: String,
|
||||
pub summary: String,
|
||||
pub highlights: Vec<String>,
|
||||
pub risks: Vec<String>,
|
||||
pub last_updated: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct NewsItem {
|
||||
pub id: String,
|
||||
#[serde(default)]
|
||||
pub symbol: String,
|
||||
pub title: String,
|
||||
pub url: String,
|
||||
pub description: String,
|
||||
pub provider: String,
|
||||
pub published_at: String,
|
||||
pub read_time_min: Option<i32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum NumberLike {
|
||||
I64(i64),
|
||||
U64(u64),
|
||||
F64(f64),
|
||||
}
|
||||
|
||||
fn de_i64_from_number<'de, D>(deserializer: D) -> Result<i64, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let value = NumberLike::deserialize(deserializer)?;
|
||||
Ok(match value {
|
||||
NumberLike::I64(v) => v,
|
||||
NumberLike::U64(v) => i64::try_from(v).map_err(D::Error::custom)?,
|
||||
NumberLike::F64(v) => v.round() as i64,
|
||||
})
|
||||
}
|
||||
|
||||
fn de_opt_i64_from_number<'de, D>(deserializer: D) -> Result<Option<i64>, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
Option::<NumberLike>::deserialize(deserializer).and_then(|v| {
|
||||
v.map(|n| match n {
|
||||
NumberLike::I64(x) => Ok(x),
|
||||
NumberLike::U64(x) => i64::try_from(x).map_err(D::Error::custom),
|
||||
NumberLike::F64(x) => Ok(x.round() as i64),
|
||||
})
|
||||
.transpose()
|
||||
})
|
||||
}
|
||||
|
||||
fn de_opt_u64_from_number<'de, D>(deserializer: D) -> Result<Option<u64>, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
Option::<NumberLike>::deserialize(deserializer).and_then(|v| {
|
||||
v.map(|n| match n {
|
||||
NumberLike::I64(x) => u64::try_from(x).map_err(D::Error::custom),
|
||||
NumberLike::U64(x) => Ok(x),
|
||||
NumberLike::F64(x) => {
|
||||
if x.is_sign_negative() {
|
||||
Err(D::Error::custom(
|
||||
"negative value cannot be converted to u64",
|
||||
))
|
||||
} else {
|
||||
Ok(x.round() as u64)
|
||||
}
|
||||
}
|
||||
})
|
||||
.transpose()
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "clap", derive(clap::ValueEnum))]
|
||||
pub enum Period {
|
||||
#[cfg_attr(feature = "clap", value(name = "1d"))]
|
||||
OneDay,
|
||||
#[cfg_attr(feature = "clap", value(name = "5d"))]
|
||||
FiveDays,
|
||||
#[cfg_attr(feature = "clap", value(name = "1mo"))]
|
||||
OneMonth,
|
||||
#[cfg_attr(feature = "clap", value(name = "3mo"))]
|
||||
ThreeMonths,
|
||||
#[cfg_attr(feature = "clap", value(name = "6mo"))]
|
||||
SixMonths,
|
||||
#[cfg_attr(feature = "clap", value(name = "1y"))]
|
||||
OneYear,
|
||||
#[cfg_attr(feature = "clap", value(name = "2y"))]
|
||||
TwoYears,
|
||||
#[cfg_attr(feature = "clap", value(name = "5y"))]
|
||||
FiveYears,
|
||||
}
|
||||
|
||||
impl Period {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::OneDay => "1d",
|
||||
Self::FiveDays => "5d",
|
||||
Self::OneMonth => "1mo",
|
||||
Self::ThreeMonths => "3mo",
|
||||
Self::SixMonths => "6mo",
|
||||
Self::OneYear => "1y",
|
||||
Self::TwoYears => "2y",
|
||||
Self::FiveYears => "5y",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "clap", derive(clap::ValueEnum))]
|
||||
pub enum Interval {
|
||||
#[cfg_attr(feature = "clap", value(name = "1d"))]
|
||||
Day,
|
||||
#[cfg_attr(feature = "clap", value(name = "1wk"))]
|
||||
Week,
|
||||
#[cfg_attr(feature = "clap", value(name = "1mo"))]
|
||||
Month,
|
||||
}
|
||||
|
||||
impl Interval {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Day => "1d",
|
||||
Self::Week => "1wk",
|
||||
Self::Month => "1mo",
|
||||
}
|
||||
}
|
||||
}
|
||||
359
crates/idx-core/src/api/yahoo/client.rs
Normal file
359
crates/idx-core/src/api/yahoo/client.rs
Normal file
|
|
@ -0,0 +1,359 @@
|
|||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::process::Stdio;
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::api::types::{Interval, Period};
|
||||
use crate::curl_impersonate;
|
||||
use crate::error::IdxError;
|
||||
use crate::runtime;
|
||||
|
||||
use super::raw_types::{ChartResponse, QuoteSummaryResponse};
|
||||
|
||||
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
|
||||
const BASE_URL: &str = "https://query2.finance.yahoo.com";
|
||||
const COOKIE_FETCH_URL: &str = "https://fc.yahoo.com";
|
||||
const CRUMB_FETCH_URL: &str = "https://query1.finance.yahoo.com/v1/test/getcrumb";
|
||||
|
||||
pub(super) struct YahooClient {
|
||||
agent: ureq::Agent,
|
||||
crumb: Mutex<Option<String>>,
|
||||
}
|
||||
|
||||
impl YahooClient {
|
||||
pub(super) fn new() -> Self {
|
||||
let agent: ureq::Agent = ureq::Agent::config_builder()
|
||||
.timeout_connect(Some(Duration::from_secs(5)))
|
||||
.timeout_recv_body(Some(Duration::from_secs(10)))
|
||||
.build()
|
||||
.into();
|
||||
|
||||
Self {
|
||||
agent,
|
||||
crumb: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn chart_url(symbol: &str, period: &Period, interval: &Interval) -> String {
|
||||
format!(
|
||||
"{BASE_URL}/v8/finance/chart/{symbol}?range={}&interval={}",
|
||||
period.as_str(),
|
||||
interval.as_str()
|
||||
)
|
||||
}
|
||||
|
||||
fn quote_summary_url(symbol: &str, crumb: &str) -> String {
|
||||
format!(
|
||||
"{BASE_URL}/v10/finance/quoteSummary/{symbol}?modules=summaryDetail,defaultKeyStatistics,financialData,assetProfile,incomeStatementHistory&crumb={crumb}"
|
||||
)
|
||||
}
|
||||
|
||||
fn cookie_jar_path() -> PathBuf {
|
||||
std::env::temp_dir().join(format!("idx_yf_{}.txt", std::process::id()))
|
||||
}
|
||||
|
||||
pub(super) fn parse_crumb_body(raw: &str) -> Result<String, IdxError> {
|
||||
let crumb = raw.trim();
|
||||
if crumb.is_empty() {
|
||||
return Err(IdxError::Http("received empty Yahoo crumb".to_string()));
|
||||
}
|
||||
|
||||
let normalized = crumb.to_ascii_lowercase();
|
||||
if normalized.contains("<html>") || normalized.contains("<!doctype html") {
|
||||
return Err(IdxError::Http(
|
||||
"received HTML instead of a Yahoo crumb".to_string(),
|
||||
));
|
||||
}
|
||||
if normalized.contains("too many requests") {
|
||||
return Err(IdxError::Http(
|
||||
"Yahoo crumb request was rate limited".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(crumb.to_string())
|
||||
}
|
||||
|
||||
fn cookie_header_from_jar(path: &Path) -> Result<String, IdxError> {
|
||||
let jar = std::fs::read_to_string(path).map_err(|e| {
|
||||
IdxError::Http(format!(
|
||||
"failed to read Yahoo cookie jar {}: {e}",
|
||||
path.display()
|
||||
))
|
||||
})?;
|
||||
|
||||
let cookies: Vec<String> = jar
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let candidate = trimmed.strip_prefix("#HttpOnly_").unwrap_or(trimmed);
|
||||
if candidate.starts_with('#') {
|
||||
return None;
|
||||
}
|
||||
|
||||
let fields: Vec<_> = candidate.split('\t').collect();
|
||||
if fields.len() < 7 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let name = fields[5].trim();
|
||||
let value = fields[6].trim();
|
||||
if name.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(format!("{name}={value}"))
|
||||
})
|
||||
.collect();
|
||||
|
||||
if cookies.is_empty() {
|
||||
return Err(IdxError::Http(format!(
|
||||
"Yahoo cookie jar {} did not contain any cookies",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(cookies.join("; "))
|
||||
}
|
||||
|
||||
fn fetch_crumb_via_curl(&self) -> Result<String, IdxError> {
|
||||
let binary = curl_impersonate::chrome_curl_binary()?;
|
||||
|
||||
let cookie_jar = Self::cookie_jar_path();
|
||||
let cookie_jar_str = cookie_jar.to_str().ok_or_else(|| {
|
||||
IdxError::Io(format!(
|
||||
"failed to encode Yahoo cookie jar path {}",
|
||||
cookie_jar.display()
|
||||
))
|
||||
})?;
|
||||
|
||||
// Step 1: fetch fc.yahoo.com to set A3 cookie (returns 404 but writes cookie jar).
|
||||
// We allow non-zero exit here since 404 still writes the cookie.
|
||||
let _ = Command::new(binary)
|
||||
.args(["--silent", "--cookie-jar", cookie_jar_str, COOKIE_FETCH_URL])
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.output();
|
||||
|
||||
// Step 2: fetch crumb with cookie jar (Chrome TLS fingerprint + A3 cookie).
|
||||
let output = curl_impersonate::run(
|
||||
"Yahoo crumb fetch",
|
||||
&["--silent", "--cookie", cookie_jar_str, CRUMB_FETCH_URL],
|
||||
)?;
|
||||
|
||||
let body = String::from_utf8_lossy(&output.stdout);
|
||||
Self::parse_crumb_body(&body)
|
||||
}
|
||||
|
||||
fn get_or_init_crumb(&self) -> Result<String, IdxError> {
|
||||
let mut guard = self
|
||||
.crumb
|
||||
.lock()
|
||||
.map_err(|e| IdxError::Io(format!("crumb lock poisoned: {e}")))?;
|
||||
|
||||
if let Some(crumb) = guard.as_ref() {
|
||||
return Ok(crumb.clone());
|
||||
}
|
||||
|
||||
let crumb = self.fetch_crumb_via_curl()?;
|
||||
*guard = Some(crumb.clone());
|
||||
Ok(crumb)
|
||||
}
|
||||
|
||||
fn clear_crumb(&self) -> Result<(), IdxError> {
|
||||
let mut guard = self
|
||||
.crumb
|
||||
.lock()
|
||||
.map_err(|e| IdxError::Io(format!("crumb lock poisoned: {e}")))?;
|
||||
*guard = None;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn retry_rate_limited<T, F>(&self, request: F) -> Result<Result<T, ureq::Error>, IdxError>
|
||||
where
|
||||
F: FnMut() -> Result<T, ureq::Error>,
|
||||
{
|
||||
Self::retry_rate_limited_with(request, std::thread::sleep, jitter)
|
||||
}
|
||||
|
||||
fn retry_rate_limited_with<T, F, S, J>(
|
||||
mut request: F,
|
||||
mut sleeper: S,
|
||||
mut jitter_fn: J,
|
||||
) -> Result<Result<T, ureq::Error>, IdxError>
|
||||
where
|
||||
F: FnMut() -> Result<T, ureq::Error>,
|
||||
S: FnMut(Duration),
|
||||
J: FnMut() -> Duration,
|
||||
{
|
||||
let mut wait = Duration::from_millis(250);
|
||||
|
||||
for attempt in 0..3 {
|
||||
match request() {
|
||||
Err(ureq::Error::StatusCode(429)) => {
|
||||
if attempt < 2 {
|
||||
sleeper(wait + jitter_fn());
|
||||
wait *= 2;
|
||||
continue;
|
||||
}
|
||||
return Err(IdxError::RateLimited);
|
||||
}
|
||||
other => return Ok(other),
|
||||
}
|
||||
}
|
||||
|
||||
Err(IdxError::RateLimited)
|
||||
}
|
||||
|
||||
pub(super) fn fetch_chart(
|
||||
&self,
|
||||
symbol: &str,
|
||||
period: &Period,
|
||||
interval: &Interval,
|
||||
) -> Result<ChartResponse, IdxError> {
|
||||
let url = Self::chart_url(symbol, period, interval);
|
||||
let response = self
|
||||
.retry_rate_limited(|| self.agent.get(&url).header("User-Agent", USER_AGENT).call())?;
|
||||
|
||||
match response {
|
||||
Ok(ok) => ok
|
||||
.into_body()
|
||||
.read_json::<ChartResponse>()
|
||||
.map_err(|e| IdxError::ParseError(e.to_string())),
|
||||
Err(ureq::Error::StatusCode(404)) => Err(IdxError::SymbolNotFound(symbol.to_string())),
|
||||
Err(e) => Err(IdxError::Http(e.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn fetch_quote_summary(
|
||||
&self,
|
||||
symbol: &str,
|
||||
) -> Result<QuoteSummaryResponse, IdxError> {
|
||||
for auth_attempt in 0..2 {
|
||||
let crumb = self.get_or_init_crumb()?;
|
||||
let cookie_header = match Self::cookie_header_from_jar(&Self::cookie_jar_path()) {
|
||||
Ok(header) => header,
|
||||
Err(err) => {
|
||||
runtime::warn(format!("failed to parse Yahoo cookie jar: {err}"));
|
||||
return Err(IdxError::AuthError(format!(
|
||||
"failed to parse Yahoo cookies: {err}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
let url = Self::quote_summary_url(symbol, &crumb);
|
||||
let response = self.retry_rate_limited(|| {
|
||||
let mut req = self.agent.get(&url).header("User-Agent", USER_AGENT);
|
||||
if !cookie_header.is_empty() {
|
||||
req = req.header("Cookie", &cookie_header);
|
||||
}
|
||||
req.call()
|
||||
})?;
|
||||
|
||||
match response {
|
||||
Ok(ok) => {
|
||||
return ok
|
||||
.into_body()
|
||||
.read_json::<QuoteSummaryResponse>()
|
||||
.map_err(|e| IdxError::ParseError(e.to_string()));
|
||||
}
|
||||
Err(ureq::Error::StatusCode(401)) => {
|
||||
if auth_attempt == 0 {
|
||||
self.clear_crumb()?;
|
||||
continue;
|
||||
}
|
||||
return Err(IdxError::Http(
|
||||
"yahoo quoteSummary returned unauthorized (401)".to_string(),
|
||||
));
|
||||
}
|
||||
Err(ureq::Error::StatusCode(404)) => {
|
||||
return Err(IdxError::SymbolNotFound(symbol.to_string()));
|
||||
}
|
||||
Err(e) => return Err(IdxError::Http(e.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
Err(IdxError::RateLimited)
|
||||
}
|
||||
}
|
||||
|
||||
fn jitter() -> Duration {
|
||||
Duration::from_millis(fastrand::u64(0..100))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::YahooClient;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
fn parses_crumb_body_trimmed() {
|
||||
let crumb = YahooClient::parse_crumb_body(" abc123xyz\n").expect("crumb should parse");
|
||||
assert_eq!(crumb, "abc123xyz");
|
||||
|
||||
let empty = YahooClient::parse_crumb_body(" \n").expect_err("empty crumb must fail");
|
||||
assert!(matches!(empty, crate::error::IdxError::Http(_)));
|
||||
|
||||
let html = YahooClient::parse_crumb_body("<html>blocked</html>")
|
||||
.expect_err("html crumb must fail");
|
||||
assert!(matches!(html, crate::error::IdxError::Http(_)));
|
||||
|
||||
let rate_limited = YahooClient::parse_crumb_body("Too Many Requests")
|
||||
.expect_err("rate limited crumb must fail");
|
||||
assert!(matches!(rate_limited, crate::error::IdxError::Http(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_rate_limited_succeeds_after_retry() {
|
||||
let mut attempts = 0;
|
||||
let mut sleeps = Vec::new();
|
||||
|
||||
let result = YahooClient::retry_rate_limited_with(
|
||||
|| {
|
||||
attempts += 1;
|
||||
if attempts < 3 {
|
||||
Err(ureq::Error::StatusCode(429))
|
||||
} else {
|
||||
Ok("ok")
|
||||
}
|
||||
},
|
||||
|duration| sleeps.push(duration),
|
||||
|| Duration::ZERO,
|
||||
)
|
||||
.expect("retry helper should not fail")
|
||||
.expect("third attempt should succeed");
|
||||
|
||||
assert_eq!(result, "ok");
|
||||
assert_eq!(attempts, 3);
|
||||
assert_eq!(
|
||||
sleeps,
|
||||
vec![Duration::from_millis(250), Duration::from_millis(500)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_rate_limited_returns_rate_limited_after_exhaustion() {
|
||||
let mut attempts = 0;
|
||||
let mut sleeps = Vec::new();
|
||||
|
||||
let err = YahooClient::retry_rate_limited_with(
|
||||
|| {
|
||||
attempts += 1;
|
||||
Err::<(), ureq::Error>(ureq::Error::StatusCode(429))
|
||||
},
|
||||
|duration| sleeps.push(duration),
|
||||
|| Duration::ZERO,
|
||||
)
|
||||
.expect_err("rate-limited helper should fail after three attempts");
|
||||
|
||||
assert!(matches!(err, crate::error::IdxError::RateLimited));
|
||||
assert_eq!(attempts, 3);
|
||||
assert_eq!(
|
||||
sleeps,
|
||||
vec![Duration::from_millis(250), Duration::from_millis(500)]
|
||||
);
|
||||
}
|
||||
}
|
||||
217
crates/idx-core/src/api/yahoo/map.rs
Normal file
217
crates/idx-core/src/api/yahoo/map.rs
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
use crate::api::types::{Fundamentals, Ohlc, Quote};
|
||||
use crate::error::IdxError;
|
||||
|
||||
use super::raw_types::{ChartError, ChartResponse, QuoteSummaryResponse};
|
||||
|
||||
pub(super) fn parse_quote(symbol: &str, chart: &ChartResponse) -> Result<Quote, IdxError> {
|
||||
if let Some(err) = chart.chart.error.as_ref() {
|
||||
return Err(map_yahoo_error(symbol, "chart", err));
|
||||
}
|
||||
|
||||
let result = chart
|
||||
.chart
|
||||
.result
|
||||
.as_ref()
|
||||
.and_then(|r| r.first())
|
||||
.ok_or(IdxError::ProviderUnavailable)?;
|
||||
let meta = result.meta.as_ref().ok_or(IdxError::ProviderUnavailable)?;
|
||||
let raw_price = meta
|
||||
.regular_market_price
|
||||
.ok_or_else(|| IdxError::SymbolNotFound(symbol.to_string()))?;
|
||||
let raw_prev_close = meta.previous_close.or(meta.chart_previous_close);
|
||||
|
||||
let price = round_price(raw_price);
|
||||
let prev_close = raw_prev_close.map(round_price);
|
||||
let change = prev_close.map(|p| price - p);
|
||||
let change_pct = raw_prev_close
|
||||
.filter(|p| *p != 0.0)
|
||||
.map(|p| ((raw_price - p) / p) * 100.0);
|
||||
|
||||
let (week52_position, range_signal) = match (meta.fifty_two_week_low, meta.fifty_two_week_high)
|
||||
{
|
||||
(Some(low), Some(high)) if high > low => {
|
||||
let pos = (raw_price - low) / (high - low);
|
||||
let signal = if pos > 0.66 {
|
||||
"upper"
|
||||
} else if pos < 0.33 {
|
||||
"lower"
|
||||
} else {
|
||||
"middle"
|
||||
};
|
||||
(Some(pos), Some(signal.to_string()))
|
||||
}
|
||||
_ => (None, None),
|
||||
};
|
||||
|
||||
Ok(Quote {
|
||||
symbol: meta.symbol.clone().unwrap_or_else(|| symbol.to_string()),
|
||||
price,
|
||||
change,
|
||||
change_pct,
|
||||
volume: meta.regular_market_volume,
|
||||
market_cap: meta.market_cap,
|
||||
week52_high: meta.fifty_two_week_high.map(round_price),
|
||||
week52_low: meta.fifty_two_week_low.map(round_price),
|
||||
week52_position,
|
||||
range_signal,
|
||||
prev_close,
|
||||
avg_volume: meta.average_daily_volume_3month,
|
||||
as_of: meta
|
||||
.regular_market_time
|
||||
.and_then(|ts| chrono::DateTime::from_timestamp(ts, 0))
|
||||
.map(|t| t.with_timezone(&crate::api::idx_offset())),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn parse_history(
|
||||
symbol: &str,
|
||||
chart: &ChartResponse,
|
||||
) -> Result<(Vec<Ohlc>, usize), IdxError> {
|
||||
if let Some(err) = chart.chart.error.as_ref() {
|
||||
return Err(map_yahoo_error(symbol, "chart", err));
|
||||
}
|
||||
|
||||
let result = chart
|
||||
.chart
|
||||
.result
|
||||
.as_ref()
|
||||
.and_then(|r| r.first())
|
||||
.ok_or(IdxError::ProviderUnavailable)?;
|
||||
let timestamps = result
|
||||
.timestamp
|
||||
.as_ref()
|
||||
.ok_or(IdxError::ProviderUnavailable)?;
|
||||
let quote = result
|
||||
.indicators
|
||||
.as_ref()
|
||||
.and_then(|i| i.quote.as_ref())
|
||||
.and_then(|q| q.first())
|
||||
.ok_or(IdxError::ProviderUnavailable)?;
|
||||
// Yahoo stamps weekly/monthly bars at local midnight (17:00Z the day before),
|
||||
// so take the date in the exchange's own offset, not UTC.
|
||||
let exchange_tz = result
|
||||
.meta
|
||||
.as_ref()
|
||||
.and_then(|m| m.gmtoffset)
|
||||
.and_then(chrono::FixedOffset::east_opt)
|
||||
.unwrap_or_else(crate::api::idx_offset);
|
||||
|
||||
let mut out = Vec::new();
|
||||
let mut dropped = 0usize;
|
||||
for (i, ts) in timestamps.iter().enumerate() {
|
||||
let open = quote
|
||||
.open
|
||||
.as_ref()
|
||||
.and_then(|v| v.get(i).copied().flatten())
|
||||
.map(round_price);
|
||||
let high = quote
|
||||
.high
|
||||
.as_ref()
|
||||
.and_then(|v| v.get(i).copied().flatten())
|
||||
.map(round_price);
|
||||
let low = quote
|
||||
.low
|
||||
.as_ref()
|
||||
.and_then(|v| v.get(i).copied().flatten())
|
||||
.map(round_price);
|
||||
let close = quote
|
||||
.close
|
||||
.as_ref()
|
||||
.and_then(|v| v.get(i).copied().flatten())
|
||||
.map(round_price);
|
||||
let volume = quote
|
||||
.volume
|
||||
.as_ref()
|
||||
.and_then(|v| v.get(i).copied().flatten());
|
||||
|
||||
if let (Some(open), Some(high), Some(low), Some(close), Some(volume)) =
|
||||
(open, high, low, close, volume)
|
||||
&& let Some(dt) = chrono::DateTime::from_timestamp(*ts, 0)
|
||||
{
|
||||
out.push(Ohlc {
|
||||
date: dt.with_timezone(&exchange_tz).date_naive(),
|
||||
open,
|
||||
high,
|
||||
low,
|
||||
close,
|
||||
volume,
|
||||
});
|
||||
} else {
|
||||
dropped += 1;
|
||||
}
|
||||
}
|
||||
|
||||
Ok((out, dropped))
|
||||
}
|
||||
|
||||
pub(super) fn parse_fundamentals(
|
||||
symbol: &str,
|
||||
quote_summary: &QuoteSummaryResponse,
|
||||
) -> Result<Fundamentals, IdxError> {
|
||||
if let Some(err) = quote_summary.quote_summary.error.as_ref() {
|
||||
return Err(map_yahoo_error(symbol, "quoteSummary", err));
|
||||
}
|
||||
|
||||
let result = quote_summary
|
||||
.quote_summary
|
||||
.result
|
||||
.as_ref()
|
||||
.and_then(|results| results.first())
|
||||
.ok_or(IdxError::ProviderUnavailable)?;
|
||||
|
||||
let stats = result.default_key_statistics.as_ref();
|
||||
let fin = result.financial_data.as_ref();
|
||||
let summary = result.summary_detail.as_ref();
|
||||
|
||||
Ok(Fundamentals {
|
||||
trailing_pe: stats
|
||||
.and_then(|s| s.trailing_pe.as_ref().and_then(|v| v.raw))
|
||||
.or_else(|| fin.and_then(|f| f.trailing_pe.as_ref().and_then(|v| v.raw)))
|
||||
.or_else(|| summary.and_then(|s| s.trailing_pe.as_ref().and_then(|v| v.raw))),
|
||||
forward_pe: stats
|
||||
.and_then(|s| s.forward_pe.as_ref().and_then(|v| v.raw))
|
||||
.or_else(|| fin.and_then(|f| f.forward_pe.as_ref().and_then(|v| v.raw)))
|
||||
.or_else(|| summary.and_then(|s| s.forward_pe.as_ref().and_then(|v| v.raw))),
|
||||
price_to_book: stats
|
||||
.and_then(|s| s.price_to_book.as_ref().and_then(|v| v.raw))
|
||||
.or_else(|| fin.and_then(|f| f.price_to_book.as_ref().and_then(|v| v.raw)))
|
||||
.or_else(|| summary.and_then(|s| s.price_to_book.as_ref().and_then(|v| v.raw))),
|
||||
return_on_equity: fin.and_then(|f| f.return_on_equity.as_ref().and_then(|v| v.raw)),
|
||||
profit_margins: fin.and_then(|f| f.profit_margins.as_ref().and_then(|v| v.raw)),
|
||||
return_on_assets: fin.and_then(|f| f.return_on_assets.as_ref().and_then(|v| v.raw)),
|
||||
revenue_growth: fin.and_then(|f| f.revenue_growth.as_ref().and_then(|v| v.raw)),
|
||||
earnings_growth: stats
|
||||
.and_then(|s| s.earnings_growth.as_ref().and_then(|v| v.raw))
|
||||
.or_else(|| fin.and_then(|f| f.earnings_growth.as_ref().and_then(|v| v.raw))),
|
||||
debt_to_equity: fin.and_then(|f| f.debt_to_equity.as_ref().and_then(|v| v.raw)),
|
||||
current_ratio: fin.and_then(|f| f.current_ratio.as_ref().and_then(|v| v.raw)),
|
||||
enterprise_value: stats
|
||||
.and_then(|s| s.enterprise_value.as_ref().and_then(|v| v.raw))
|
||||
.or_else(|| fin.and_then(|f| f.enterprise_value.as_ref().and_then(|v| v.raw))),
|
||||
ebitda: fin
|
||||
.and_then(|f| f.ebitda.as_ref().and_then(|v| v.raw))
|
||||
.or_else(|| stats.and_then(|s| s.ebitda.as_ref().and_then(|v| v.raw))),
|
||||
market_cap: fin
|
||||
.and_then(|f| f.market_cap.as_ref().and_then(|v| v.raw))
|
||||
.or_else(|| stats.and_then(|s| s.market_cap.as_ref().and_then(|v| v.raw)))
|
||||
.or_else(|| {
|
||||
summary
|
||||
.and_then(|s| s.market_cap.as_ref().and_then(|v| v.raw))
|
||||
.map(|n| n.round() as u64)
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
fn round_price(value: f64) -> i64 {
|
||||
value.round() as i64
|
||||
}
|
||||
|
||||
pub(super) fn map_yahoo_error(symbol: &str, endpoint: &str, err: &ChartError) -> IdxError {
|
||||
if err.code.eq_ignore_ascii_case("Not Found") {
|
||||
return IdxError::SymbolNotFound(symbol.to_string());
|
||||
}
|
||||
IdxError::Http(format!(
|
||||
"yahoo {endpoint} error {}: {}",
|
||||
err.code, err.description
|
||||
))
|
||||
}
|
||||
58
crates/idx-core/src/api/yahoo/mod.rs
Normal file
58
crates/idx-core/src/api/yahoo/mod.rs
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
mod client;
|
||||
mod map;
|
||||
mod parse;
|
||||
mod raw_types;
|
||||
|
||||
use crate::api::types::{Bar, Fundamentals, Interval, Period, Quote};
|
||||
use crate::api::{FundamentalsProvider, HistoryProvider, QuoteProvider};
|
||||
use crate::error::IdxError;
|
||||
|
||||
use client::YahooClient;
|
||||
use map::{parse_fundamentals, parse_quote};
|
||||
use parse::parse_history_with_verbose;
|
||||
|
||||
pub(crate) use parse::{
|
||||
parse_fundamentals_from_str, parse_history_from_str_with_verbose, parse_quote_from_str,
|
||||
};
|
||||
|
||||
pub struct YahooProvider {
|
||||
client: YahooClient,
|
||||
verbose: bool,
|
||||
}
|
||||
|
||||
impl YahooProvider {
|
||||
pub fn new(verbose: bool) -> Self {
|
||||
Self {
|
||||
client: YahooClient::new(),
|
||||
verbose,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl QuoteProvider for YahooProvider {
|
||||
fn quote(&self, symbol: &str) -> Result<Quote, IdxError> {
|
||||
let chart = self
|
||||
.client
|
||||
.fetch_chart(symbol, &Period::OneDay, &Interval::Day)?;
|
||||
parse_quote(symbol, &chart)
|
||||
}
|
||||
}
|
||||
|
||||
impl FundamentalsProvider for YahooProvider {
|
||||
fn fundamentals(&self, symbol: &str) -> Result<Fundamentals, IdxError> {
|
||||
let quote_summary = self.client.fetch_quote_summary(symbol)?;
|
||||
parse_fundamentals(symbol, "e_summary)
|
||||
}
|
||||
}
|
||||
|
||||
impl HistoryProvider for YahooProvider {
|
||||
fn history(
|
||||
&self,
|
||||
symbol: &str,
|
||||
period: &Period,
|
||||
interval: &Interval,
|
||||
) -> Result<Vec<Bar>, IdxError> {
|
||||
let chart = self.client.fetch_chart(symbol, period, interval)?;
|
||||
parse_history_with_verbose(symbol, &chart, self.verbose)
|
||||
}
|
||||
}
|
||||
151
crates/idx-core/src/api/yahoo/parse.rs
Normal file
151
crates/idx-core/src/api/yahoo/parse.rs
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
use crate::api::types::{Fundamentals, Ohlc, Quote};
|
||||
use crate::error::IdxError;
|
||||
|
||||
use super::map::{parse_fundamentals, parse_history, parse_quote};
|
||||
use super::raw_types::{ChartResponse, QuoteSummaryResponse};
|
||||
|
||||
pub(crate) fn parse_quote_from_str(symbol: &str, raw: &str) -> Result<Quote, IdxError> {
|
||||
let chart: ChartResponse =
|
||||
serde_json::from_str(raw).map_err(|e| IdxError::ParseError(e.to_string()))?;
|
||||
parse_quote(symbol, &chart)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn parse_history_from_str(symbol: &str, raw: &str) -> Result<Vec<Ohlc>, IdxError> {
|
||||
parse_history_from_str_with_verbose(symbol, raw, false)
|
||||
}
|
||||
|
||||
pub(crate) fn parse_history_from_str_with_verbose(
|
||||
symbol: &str,
|
||||
raw: &str,
|
||||
verbose: bool,
|
||||
) -> Result<Vec<Ohlc>, IdxError> {
|
||||
let chart: ChartResponse =
|
||||
serde_json::from_str(raw).map_err(|e| IdxError::ParseError(e.to_string()))?;
|
||||
parse_history_with_verbose(symbol, &chart, verbose)
|
||||
}
|
||||
|
||||
pub(crate) fn parse_fundamentals_from_str(
|
||||
symbol: &str,
|
||||
raw: &str,
|
||||
) -> Result<Fundamentals, IdxError> {
|
||||
let quote_summary: QuoteSummaryResponse =
|
||||
serde_json::from_str(raw).map_err(|e| IdxError::ParseError(e.to_string()))?;
|
||||
parse_fundamentals(symbol, "e_summary)
|
||||
}
|
||||
|
||||
pub(super) fn parse_history_with_verbose(
|
||||
symbol: &str,
|
||||
chart: &ChartResponse,
|
||||
verbose: bool,
|
||||
) -> Result<Vec<Ohlc>, IdxError> {
|
||||
let (history, dropped) = parse_history(symbol, chart)?;
|
||||
if dropped > 0 && verbose {
|
||||
crate::runtime::warn(format!(
|
||||
"dropped {dropped} OHLC row(s) from Yahoo response due to missing fields"
|
||||
));
|
||||
}
|
||||
Ok(history)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
ChartResponse, parse_fundamentals_from_str, parse_history_from_str,
|
||||
parse_history_from_str_with_verbose, parse_history_with_verbose, parse_quote,
|
||||
parse_quote_from_str,
|
||||
};
|
||||
|
||||
const SAMPLE: &str = r#"{
|
||||
"chart": {
|
||||
"result": [{
|
||||
"meta": {
|
||||
"symbol": "BBCA.JK",
|
||||
"regularMarketPrice": 9875.0,
|
||||
"previousClose": 9758.0,
|
||||
"regularMarketVolume": 12300000,
|
||||
"marketCap": 1215200000000000,
|
||||
"fiftyTwoWeekHigh": 10250.0,
|
||||
"fiftyTwoWeekLow": 7800.0,
|
||||
"averageDailyVolume3Month": 10000000
|
||||
},
|
||||
"timestamp": [1709251200,1709337600],
|
||||
"indicators": {"quote":[{
|
||||
"open":[9800.0,9850.0],
|
||||
"high":[9900.0,9900.0],
|
||||
"low":[9750.0,9800.0],
|
||||
"close":[9875.0,9880.0],
|
||||
"volume":[12300000,11000000]
|
||||
}]}
|
||||
}]
|
||||
}
|
||||
}"#;
|
||||
|
||||
#[test]
|
||||
fn parses_quote_and_history() {
|
||||
let chart: ChartResponse = serde_json::from_str(SAMPLE).expect("valid chart fixture");
|
||||
let quote = parse_quote("BBCA.JK", &chart).expect("quote parsed");
|
||||
assert_eq!(quote.symbol, "BBCA.JK");
|
||||
assert_eq!(quote.price, 9875);
|
||||
let history = parse_history_with_verbose("BBCA.JK", &chart, false).expect("history parsed");
|
||||
assert_eq!(history.len(), 2);
|
||||
assert_eq!(history[0].close, 9875);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn history_dates_use_exchange_offset_not_utc() {
|
||||
// Weekly bars are stamped at Monday 00:00 WIB, i.e. Sunday 17:00Z.
|
||||
let raw = r#"{"chart":{"error":null,"result":[{
|
||||
"meta":{"gmtoffset":25200},
|
||||
"timestamp":[1790528400],
|
||||
"indicators":{"quote":[{"open":[6000],"high":[6150],"low":[5975],"close":[6100],"volume":[1000]}]}
|
||||
}]}}"#;
|
||||
let history = parse_history_from_str("BBCA.JK", raw).expect("history parsed");
|
||||
assert_eq!(history[0].date.to_string(), "2026-09-28");
|
||||
|
||||
let without_offset = raw.replace(r#""gmtoffset":25200"#, r#""symbol":"BBCA.JK""#);
|
||||
let history = parse_history_from_str("BBCA.JK", &without_offset).expect("history parsed");
|
||||
assert_eq!(history[0].date.to_string(), "2026-09-28");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_realistic_fixture_json() {
|
||||
let quote_raw =
|
||||
std::fs::read_to_string("tests/fixtures/chart_bbca_1d.json").expect("fixture exists");
|
||||
let history_raw =
|
||||
std::fs::read_to_string("tests/fixtures/chart_bbca_3mo.json").expect("fixture exists");
|
||||
let fundamentals_raw = std::fs::read_to_string("tests/fixtures/quotesummary_bbca.json")
|
||||
.expect("fixture exists");
|
||||
|
||||
let quote = parse_quote_from_str("BBCA.JK", "e_raw).expect("fixture quote parsed");
|
||||
assert_eq!(quote.symbol, "BBCA.JK");
|
||||
assert_eq!(quote.market_cap, Some(1_215_200_000_000_000));
|
||||
assert_eq!(quote.avg_volume, Some(10_000_000));
|
||||
|
||||
let history =
|
||||
parse_history_from_str("BBCA.JK", &history_raw).expect("fixture history parsed");
|
||||
assert!(!history.is_empty());
|
||||
|
||||
let verbose_history = parse_history_from_str_with_verbose("BBCA.JK", &history_raw, true)
|
||||
.expect("fixture history parsed in verbose mode");
|
||||
assert_eq!(verbose_history.len(), history.len());
|
||||
assert_eq!(verbose_history[0].close, history[0].close);
|
||||
|
||||
let fundamentals = parse_fundamentals_from_str("BBCA.JK", &fundamentals_raw)
|
||||
.expect("fixture fundamentals parsed");
|
||||
assert_eq!(fundamentals.trailing_pe, Some(25.4));
|
||||
assert_eq!(fundamentals.forward_pe, Some(23.1));
|
||||
assert_eq!(fundamentals.price_to_book, Some(4.6));
|
||||
assert_eq!(fundamentals.earnings_growth, Some(0.121));
|
||||
assert_eq!(fundamentals.enterprise_value, Some(1_245_000_000_000_000));
|
||||
assert_eq!(fundamentals.ebitda, Some(58_500_000_000_000));
|
||||
assert_eq!(fundamentals.market_cap, Some(1_215_200_000_000_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_not_found_chart_error_to_symbol_not_found() {
|
||||
let raw = r#"{"chart":{"result":null,"error":{"code":"Not Found","description":"No data found"}}}"#;
|
||||
let err = parse_quote_from_str("INVALID.JK", raw).expect_err("expected symbol error");
|
||||
assert!(matches!(err, crate::error::IdxError::SymbolNotFound(_)));
|
||||
}
|
||||
}
|
||||
158
crates/idx-core/src/api/yahoo/raw_types.rs
Normal file
158
crates/idx-core/src/api/yahoo/raw_types.rs
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
// Raw serde structs for Yahoo API responses. Fields not yet consumed by map.rs are
|
||||
// retained for future fundamentals expansion; suppress dead_code for forward-compat.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(super) struct ChartResponse {
|
||||
pub(super) chart: ChartRoot,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct QuoteSummaryResponse {
|
||||
pub(super) quote_summary: QuoteSummaryRoot,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(super) struct QuoteSummaryRoot {
|
||||
pub(super) result: Option<Vec<QuoteSummaryResult>>,
|
||||
pub(super) error: Option<ChartError>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct QuoteSummaryResult {
|
||||
#[serde(default)]
|
||||
pub(super) summary_detail: Option<SummaryDetail>,
|
||||
#[serde(default)]
|
||||
pub(super) default_key_statistics: Option<DefaultKeyStatistics>,
|
||||
#[serde(default)]
|
||||
pub(super) financial_data: Option<FinancialData>,
|
||||
#[serde(default)]
|
||||
pub(super) asset_profile: Option<AssetProfile>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(super) struct ChartRoot {
|
||||
pub(super) result: Option<Vec<ChartResult>>,
|
||||
pub(super) error: Option<ChartError>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(super) struct ChartError {
|
||||
pub(super) code: String,
|
||||
pub(super) description: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(super) struct ChartResult {
|
||||
pub(super) meta: Option<ChartMeta>,
|
||||
pub(super) timestamp: Option<Vec<i64>>,
|
||||
pub(super) indicators: Option<Indicators>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[allow(dead_code)]
|
||||
pub(super) struct ChartMeta {
|
||||
pub(super) symbol: Option<String>,
|
||||
pub(super) regular_market_price: Option<f64>,
|
||||
pub(super) previous_close: Option<f64>,
|
||||
pub(super) chart_previous_close: Option<f64>,
|
||||
pub(super) regular_market_volume: Option<u64>,
|
||||
pub(super) regular_market_day_high: Option<f64>,
|
||||
pub(super) regular_market_day_low: Option<f64>,
|
||||
pub(super) market_cap: Option<u64>,
|
||||
pub(super) fifty_two_week_high: Option<f64>,
|
||||
pub(super) fifty_two_week_low: Option<f64>,
|
||||
#[serde(rename = "averageDailyVolume3Month")]
|
||||
pub(super) average_daily_volume_3month: Option<u64>,
|
||||
pub(super) gmtoffset: Option<i32>,
|
||||
pub(super) regular_market_time: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(super) struct Indicators {
|
||||
pub(super) quote: Option<Vec<IndicatorQuote>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(super) struct IndicatorQuote {
|
||||
pub(super) open: Option<Vec<Option<f64>>>,
|
||||
pub(super) high: Option<Vec<Option<f64>>>,
|
||||
pub(super) low: Option<Vec<Option<f64>>>,
|
||||
pub(super) close: Option<Vec<Option<f64>>>,
|
||||
pub(super) volume: Option<Vec<Option<u64>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SummaryDetail {
|
||||
#[serde(rename = "trailingPE")]
|
||||
pub trailing_pe: Option<FloatValue>,
|
||||
#[serde(rename = "forwardPE")]
|
||||
pub forward_pe: Option<FloatValue>,
|
||||
pub price_to_book: Option<FloatValue>,
|
||||
pub dividend_yield: Option<FloatValue>,
|
||||
pub market_cap: Option<FloatValue>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DefaultKeyStatistics {
|
||||
#[serde(rename = "trailingPE")]
|
||||
pub trailing_pe: Option<FloatValue>,
|
||||
#[serde(rename = "forwardPE")]
|
||||
pub forward_pe: Option<FloatValue>,
|
||||
pub price_to_book: Option<FloatValue>,
|
||||
pub earnings_growth: Option<FloatValue>,
|
||||
pub enterprise_value: Option<IntValue>,
|
||||
pub ebitda: Option<IntValue>,
|
||||
pub market_cap: Option<UIntValue>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct FinancialData {
|
||||
#[serde(rename = "trailingPE")]
|
||||
pub trailing_pe: Option<FloatValue>,
|
||||
#[serde(rename = "forwardPE")]
|
||||
pub forward_pe: Option<FloatValue>,
|
||||
pub price_to_book: Option<FloatValue>,
|
||||
pub return_on_equity: Option<FloatValue>,
|
||||
pub profit_margins: Option<FloatValue>,
|
||||
pub return_on_assets: Option<FloatValue>,
|
||||
pub revenue_growth: Option<FloatValue>,
|
||||
pub earnings_growth: Option<FloatValue>,
|
||||
pub debt_to_equity: Option<FloatValue>,
|
||||
pub current_ratio: Option<FloatValue>,
|
||||
pub enterprise_value: Option<IntValue>,
|
||||
pub ebitda: Option<IntValue>,
|
||||
pub market_cap: Option<UIntValue>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AssetProfile {
|
||||
pub sector: Option<String>,
|
||||
pub industry: Option<String>,
|
||||
pub long_business_summary: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct FloatValue {
|
||||
pub raw: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct IntValue {
|
||||
pub raw: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct UIntValue {
|
||||
pub raw: Option<u64>,
|
||||
}
|
||||
348
crates/idx-core/src/cache.rs
Normal file
348
crates/idx-core/src/cache.rs
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use directories::ProjectDirs;
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::IdxError;
|
||||
use crate::runtime;
|
||||
|
||||
const CURRENT_SCHEMA_VERSION: u32 = 1;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Cache {
|
||||
root: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct CacheEntry<T> {
|
||||
fetched_at: DateTime<Utc>,
|
||||
ttl_secs: u64,
|
||||
schema_version: u32,
|
||||
data: T,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct CacheInfo {
|
||||
pub path: PathBuf,
|
||||
pub files: usize,
|
||||
pub total_size: u64,
|
||||
pub oldest: Option<DateTime<Utc>>,
|
||||
pub newest: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
impl Cache {
|
||||
pub fn new() -> Result<Self, IdxError> {
|
||||
Ok(Self { root: cache_dir()? })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn with_root(root: PathBuf) -> Self {
|
||||
Self { root }
|
||||
}
|
||||
|
||||
pub fn get<T: DeserializeOwned>(
|
||||
&self,
|
||||
data_type: &str,
|
||||
symbol: &str,
|
||||
) -> Result<Option<T>, IdxError> {
|
||||
let Some(entry): Option<CacheEntry<T>> = self.read_entry(data_type, symbol)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
if is_expired(entry.fetched_at, entry.ttl_secs) {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(entry.data))
|
||||
}
|
||||
|
||||
pub fn get_stale<T: DeserializeOwned>(
|
||||
&self,
|
||||
data_type: &str,
|
||||
symbol: &str,
|
||||
) -> Result<Option<T>, IdxError> {
|
||||
Ok(self.read_entry::<T>(data_type, symbol)?.map(|e| e.data))
|
||||
}
|
||||
|
||||
/// Like [`Cache::get_stale`], but warns when the entry served is past its TTL.
|
||||
/// Used by `--offline`, which otherwise serves arbitrarily old data silently.
|
||||
pub fn get_offline<T: DeserializeOwned>(
|
||||
&self,
|
||||
data_type: &str,
|
||||
symbol: &str,
|
||||
) -> Result<Option<T>, IdxError> {
|
||||
let Some(entry) = self.read_entry::<T>(data_type, symbol)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
if is_expired(entry.fetched_at, entry.ttl_secs) {
|
||||
runtime::warn(format!(
|
||||
"offline: serving expired cache for {symbol} (fetched {})",
|
||||
entry.fetched_at.to_rfc3339()
|
||||
));
|
||||
}
|
||||
Ok(Some(entry.data))
|
||||
}
|
||||
|
||||
pub fn put<T: Serialize>(
|
||||
&self,
|
||||
data_type: &str,
|
||||
symbol: &str,
|
||||
data: &T,
|
||||
ttl_secs: u64,
|
||||
) -> Result<(), IdxError> {
|
||||
let path = self.entry_path(data_type, symbol);
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).map_err(|e| IdxError::Io(e.to_string()))?;
|
||||
}
|
||||
let entry = CacheEntry {
|
||||
fetched_at: Utc::now(),
|
||||
ttl_secs,
|
||||
schema_version: CURRENT_SCHEMA_VERSION,
|
||||
data,
|
||||
};
|
||||
let raw = serde_json::to_string_pretty(&entry)
|
||||
.map_err(|e| IdxError::ParseError(e.to_string()))?;
|
||||
fs::write(path, raw).map_err(|e| IdxError::Io(e.to_string()))
|
||||
}
|
||||
|
||||
pub fn info(&self) -> Result<CacheInfo, IdxError> {
|
||||
let mut files = 0usize;
|
||||
let mut total_size = 0u64;
|
||||
let mut oldest: Option<DateTime<Utc>> = None;
|
||||
let mut newest: Option<DateTime<Utc>> = None;
|
||||
|
||||
if self.root.exists() {
|
||||
Self::walk(&self.root, &mut |p| {
|
||||
if let Ok(meta) = fs::metadata(p)
|
||||
&& meta.is_file()
|
||||
{
|
||||
files += 1;
|
||||
total_size += meta.len();
|
||||
if let Ok(raw) = fs::read_to_string(p)
|
||||
&& let Ok(entry) =
|
||||
serde_json::from_str::<CacheEntry<serde_json::Value>>(&raw)
|
||||
{
|
||||
oldest = Some(oldest.map_or(entry.fetched_at, |o| o.min(entry.fetched_at)));
|
||||
newest = Some(newest.map_or(entry.fetched_at, |n| n.max(entry.fetched_at)));
|
||||
}
|
||||
}
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok(CacheInfo {
|
||||
path: self.root.clone(),
|
||||
files,
|
||||
total_size,
|
||||
oldest,
|
||||
newest,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn clear(&self) -> Result<(usize, Vec<PathBuf>), IdxError> {
|
||||
if !self.root.exists() {
|
||||
return Ok((0, Vec::new()));
|
||||
}
|
||||
let mut removed = 0usize;
|
||||
let mut failed = Vec::new();
|
||||
Self::walk(&self.root, &mut |p| {
|
||||
if p.is_file() {
|
||||
match fs::remove_file(p) {
|
||||
Ok(_) => removed += 1,
|
||||
Err(_) => failed.push(p.to_path_buf()),
|
||||
}
|
||||
}
|
||||
})?;
|
||||
Ok((removed, failed))
|
||||
}
|
||||
|
||||
fn walk<F: FnMut(&Path)>(dir: &Path, f: &mut F) -> Result<(), IdxError> {
|
||||
for entry in fs::read_dir(dir).map_err(|e| IdxError::Io(e.to_string()))? {
|
||||
let entry = entry.map_err(|e| IdxError::Io(e.to_string()))?;
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
Self::walk(&path, f)?;
|
||||
} else {
|
||||
f(&path);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_entry<T: DeserializeOwned>(
|
||||
&self,
|
||||
data_type: &str,
|
||||
symbol: &str,
|
||||
) -> Result<Option<CacheEntry<T>>, IdxError> {
|
||||
let path = self.entry_path(data_type, symbol);
|
||||
if !path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
let raw = match fs::read_to_string(&path) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
if e.kind() == std::io::ErrorKind::NotFound {
|
||||
return Ok(None);
|
||||
}
|
||||
return Err(IdxError::Io(e.to_string()));
|
||||
}
|
||||
};
|
||||
let entry: CacheEntry<T> = match serde_json::from_str(&raw) {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
runtime::warn(format!(
|
||||
"corrupted cache entry for {data_type}/{symbol}, treating as miss: {e}"
|
||||
));
|
||||
let _ = fs::remove_file(&path);
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
if entry.schema_version != CURRENT_SCHEMA_VERSION {
|
||||
runtime::info(format!(
|
||||
"cache schema mismatch for {} (got {}, expected {}), treating as miss",
|
||||
path.display(),
|
||||
entry.schema_version,
|
||||
CURRENT_SCHEMA_VERSION
|
||||
));
|
||||
let _ = fs::remove_file(&path);
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(entry))
|
||||
}
|
||||
|
||||
fn entry_path(&self, data_type: &str, symbol: &str) -> PathBuf {
|
||||
self.root.join(data_type).join(format!("{symbol}.json"))
|
||||
}
|
||||
}
|
||||
|
||||
fn is_expired(fetched_at: DateTime<Utc>, ttl_secs: u64) -> bool {
|
||||
let ttl =
|
||||
chrono::Duration::from_std(Duration::from_secs(ttl_secs)).unwrap_or(chrono::Duration::MAX);
|
||||
Utc::now().signed_duration_since(fetched_at) >= ttl
|
||||
}
|
||||
|
||||
pub fn cache_dir() -> Result<PathBuf, IdxError> {
|
||||
if let Ok(dir) = std::env::var("XDG_CACHE_HOME")
|
||||
&& !dir.is_empty()
|
||||
{
|
||||
let path = PathBuf::from(dir);
|
||||
if path.is_absolute() {
|
||||
return Ok(path.join("idx"));
|
||||
}
|
||||
}
|
||||
ProjectDirs::from("", "", "idx")
|
||||
.map(|d| d.cache_dir().to_path_buf())
|
||||
.ok_or_else(|| IdxError::ConfigError("unable to resolve cache dir".to_string()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::fs;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{Cache, CacheEntry};
|
||||
|
||||
static TMP_COUNTER: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq)]
|
||||
struct T {
|
||||
v: i32,
|
||||
}
|
||||
|
||||
fn tmp() -> std::path::PathBuf {
|
||||
let suffix = TMP_COUNTER.fetch_add(1, Ordering::Relaxed);
|
||||
let p =
|
||||
std::env::temp_dir().join(format!("idx-cache-test-{}-{suffix}", std::process::id()));
|
||||
let _ = fs::remove_dir_all(&p);
|
||||
fs::create_dir_all(&p).expect("create tmp cache dir");
|
||||
p
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_read_expire_and_stale() {
|
||||
let root = tmp();
|
||||
let cache = Cache::with_root(root.clone());
|
||||
|
||||
cache
|
||||
.put("quote", "BBCA.JK", &T { v: 7 }, 300)
|
||||
.expect("cache write");
|
||||
let fresh: Option<T> = cache.get("quote", "BBCA.JK").expect("cache read fresh");
|
||||
assert_eq!(fresh, Some(T { v: 7 }));
|
||||
|
||||
let path = root.join("quote/BBCA.JK.json");
|
||||
let mut entry: CacheEntry<T> =
|
||||
serde_json::from_str(&fs::read_to_string(&path).expect("read cache file"))
|
||||
.expect("parse cache entry");
|
||||
entry.fetched_at = chrono::Utc::now() - chrono::Duration::seconds(1000);
|
||||
fs::write(
|
||||
&path,
|
||||
serde_json::to_string(&entry).expect("serialize entry"),
|
||||
)
|
||||
.expect("write old entry");
|
||||
|
||||
let expired: Option<T> = cache.get("quote", "BBCA.JK").expect("cache read expired");
|
||||
assert_eq!(expired, None);
|
||||
|
||||
let stale: Option<T> = cache
|
||||
.get_stale("quote", "BBCA.JK")
|
||||
.expect("cache read stale");
|
||||
assert_eq!(stale, Some(T { v: 7 }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupted_cache_entry_returns_none_and_deletes_file() {
|
||||
let root = tmp();
|
||||
let cache = Cache::with_root(root.clone());
|
||||
|
||||
// Write invalid JSON to a cache file
|
||||
let path = root.join("quote/CORRUPT.JK.json");
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).expect("create parent dir");
|
||||
}
|
||||
fs::write(&path, "this is not valid json {{{").expect("write corrupted cache");
|
||||
|
||||
assert!(path.exists(), "corrupted file should exist before get()");
|
||||
|
||||
// get() should return Ok(None), not an error
|
||||
let result: Option<T> = cache
|
||||
.get("quote", "CORRUPT.JK")
|
||||
.expect("get should not error");
|
||||
assert_eq!(result, None, "corrupted entry should be treated as miss");
|
||||
|
||||
// The corrupted file should be deleted
|
||||
assert!(!path.exists(), "corrupted file should be deleted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupted_cache_entry_get_stale_returns_none_and_deletes_file() {
|
||||
let root = tmp();
|
||||
let cache = Cache::with_root(root.clone());
|
||||
|
||||
// Write invalid JSON to a cache file
|
||||
let path = root.join("quote/STALE_CORRUPT.JK.json");
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).expect("create parent dir");
|
||||
}
|
||||
fs::write(&path, "{ not valid json at all").expect("write corrupted cache");
|
||||
|
||||
assert!(
|
||||
path.exists(),
|
||||
"corrupted file should exist before get_stale()"
|
||||
);
|
||||
|
||||
// get_stale() should return Ok(None), not an error
|
||||
let result: Option<T> = cache
|
||||
.get_stale("quote", "STALE_CORRUPT.JK")
|
||||
.expect("get_stale should not error");
|
||||
assert_eq!(
|
||||
result, None,
|
||||
"corrupted entry should be treated as miss in get_stale"
|
||||
);
|
||||
|
||||
// The corrupted file should be deleted
|
||||
assert!(!path.exists(), "corrupted file should be deleted");
|
||||
}
|
||||
}
|
||||
497
crates/idx-core/src/config.rs
Normal file
497
crates/idx-core/src/config.rs
Normal file
|
|
@ -0,0 +1,497 @@
|
|||
//! Config file location, provider selection, and `config.toml` read/write.
|
||||
//! Presentation settings (output format, color) are resolved by frontends.
|
||||
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use directories::ProjectDirs;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::IdxError;
|
||||
|
||||
/// Values accepted for `general.output`; frontends decide what they mean.
|
||||
pub const OUTPUT_VALUES: &[&str] = &["auto", "table", "json", "jsonl"];
|
||||
|
||||
#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ProviderKind {
|
||||
Yahoo,
|
||||
Msn,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
|
||||
#[cfg_attr(feature = "clap", derive(clap::ValueEnum))]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
#[cfg_attr(feature = "clap", value(rename_all = "lower"))]
|
||||
pub enum HistoryProviderKind {
|
||||
Auto,
|
||||
Yahoo,
|
||||
Msn,
|
||||
}
|
||||
|
||||
impl ProviderKind {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Yahoo => "yahoo",
|
||||
Self::Msn => "msn",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Result<Self, IdxError> {
|
||||
if value.eq_ignore_ascii_case("yahoo") {
|
||||
Ok(Self::Yahoo)
|
||||
} else if value.eq_ignore_ascii_case("msn") {
|
||||
Ok(Self::Msn)
|
||||
} else {
|
||||
Err(IdxError::ConfigError(format!(
|
||||
"invalid provider '{value}' (expected yahoo or msn)"
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HistoryProviderKind {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Auto => "auto",
|
||||
Self::Yahoo => "yahoo",
|
||||
Self::Msn => "msn",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Result<Self, IdxError> {
|
||||
if value.eq_ignore_ascii_case("auto") {
|
||||
Ok(Self::Auto)
|
||||
} else if value.eq_ignore_ascii_case("yahoo") {
|
||||
Ok(Self::Yahoo)
|
||||
} else if value.eq_ignore_ascii_case("msn") {
|
||||
Ok(Self::Msn)
|
||||
} else {
|
||||
Err(IdxError::ConfigError(format!(
|
||||
"invalid history provider '{value}' (expected auto, yahoo, or msn)"
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_env_ttl(name: &str) -> Result<Option<u64>, IdxError> {
|
||||
let Ok(raw) = std::env::var(name) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(IdxError::ConfigError(format!(
|
||||
"invalid {name} value '{raw}': expected a non-negative integer"
|
||||
)));
|
||||
}
|
||||
|
||||
let parsed = trimmed.parse::<u64>().map_err(|_| {
|
||||
IdxError::ConfigError(format!(
|
||||
"invalid {name} value '{raw}': expected a non-negative integer"
|
||||
))
|
||||
})?;
|
||||
|
||||
Ok(Some(parsed))
|
||||
}
|
||||
|
||||
pub fn default_config_toml() -> String {
|
||||
"[general]\nprovider = \"msn\"\nhistory_provider = \"auto\"\nexchange = \"JK\"\noutput = \"auto\"\ncolor = true\n\n[cache]\nquote_ttl = 300\nfundamental_ttl = 3600\n".to_string()
|
||||
}
|
||||
|
||||
pub fn config_path() -> Result<PathBuf, IdxError> {
|
||||
if let Ok(dir) = std::env::var("XDG_CONFIG_HOME")
|
||||
&& !dir.is_empty()
|
||||
{
|
||||
let path = PathBuf::from(dir);
|
||||
if path.is_absolute() {
|
||||
return Ok(path.join("idx").join("config.toml"));
|
||||
}
|
||||
}
|
||||
ProjectDirs::from("", "", "idx")
|
||||
.map(|d| d.config_dir().join("config.toml"))
|
||||
.ok_or_else(|| IdxError::ConfigError("unable to resolve config dir".to_string()))
|
||||
}
|
||||
|
||||
pub fn ensure_default_config() -> Result<PathBuf, IdxError> {
|
||||
let path = config_path()?;
|
||||
if path.exists() {
|
||||
return Ok(path);
|
||||
}
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).map_err(|e| IdxError::Io(e.to_string()))?;
|
||||
}
|
||||
fs::write(&path, default_config_toml()).map_err(|e| IdxError::Io(e.to_string()))?;
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
pub fn get_config_value(key: &str) -> Result<Option<String>, IdxError> {
|
||||
let path = config_path()?;
|
||||
if !path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
let raw = fs::read_to_string(path).map_err(|e| IdxError::Io(e.to_string()))?;
|
||||
let value: toml::Value =
|
||||
toml::from_str(&raw).map_err(|e| IdxError::ConfigError(e.to_string()))?;
|
||||
|
||||
let mut cur = &value;
|
||||
for part in key.split('.') {
|
||||
let Some(next) = cur.get(part) else {
|
||||
return Ok(None);
|
||||
};
|
||||
cur = next;
|
||||
}
|
||||
Ok(Some(cur.to_string().trim_matches('"').to_string()))
|
||||
}
|
||||
|
||||
/// Known configuration keys and their valid values
|
||||
const KNOWN_CONFIG_KEYS: &[&str] = &[
|
||||
"general.provider",
|
||||
"general.history_provider",
|
||||
"general.exchange",
|
||||
"general.output",
|
||||
"general.color",
|
||||
"cache.quote_ttl",
|
||||
"cache.fundamental_ttl",
|
||||
"ownership.db_path",
|
||||
"ownership.snapshot_manifest",
|
||||
];
|
||||
|
||||
/// Validates a config key and value before writing
|
||||
fn validate_config_key_value(key: &str, value: &str) -> Result<(), IdxError> {
|
||||
normalize_config_value(key, value).map(|_| ())
|
||||
}
|
||||
|
||||
fn normalize_config_value(key: &str, value: &str) -> Result<toml::Value, IdxError> {
|
||||
match key {
|
||||
"general.provider" => Ok(toml::Value::String(
|
||||
ProviderKind::parse(value)?.as_str().to_string(),
|
||||
)),
|
||||
"general.history_provider" => Ok(toml::Value::String(
|
||||
HistoryProviderKind::parse(value)?.as_str().to_string(),
|
||||
)),
|
||||
"general.output" => {
|
||||
if !OUTPUT_VALUES.iter().any(|v| v.eq_ignore_ascii_case(value)) {
|
||||
return Err(IdxError::InvalidInput(format!(
|
||||
"invalid output format '{value}': expected 'auto', 'table', 'json', or 'jsonl'"
|
||||
)));
|
||||
}
|
||||
Ok(toml::Value::String(value.to_ascii_lowercase()))
|
||||
}
|
||||
"cache.quote_ttl" | "cache.fundamental_ttl" => {
|
||||
let parsed: i64 = value.parse().map_err(|_| {
|
||||
IdxError::InvalidInput(format!(
|
||||
"invalid TTL value '{value}': must be a non-negative integer"
|
||||
))
|
||||
})?;
|
||||
if parsed < 0 {
|
||||
return Err(IdxError::InvalidInput(format!(
|
||||
"invalid TTL value '{value}': must be non-negative"
|
||||
)));
|
||||
}
|
||||
Ok(toml::Value::Integer(parsed))
|
||||
}
|
||||
"general.exchange" => {
|
||||
// Exchange is free-form (e.g. JK, US, etc.)
|
||||
Ok(toml::Value::String(value.to_string()))
|
||||
}
|
||||
"ownership.db_path" => {
|
||||
// Ownership DB path is free-form and may be absolute or relative.
|
||||
Ok(toml::Value::String(value.to_string()))
|
||||
}
|
||||
"ownership.snapshot_manifest" => {
|
||||
// Snapshot manifest can be an absolute/relative path or a URL.
|
||||
Ok(toml::Value::String(value.to_string()))
|
||||
}
|
||||
"general.color" => {
|
||||
if !value.eq_ignore_ascii_case("true") && !value.eq_ignore_ascii_case("false") {
|
||||
return Err(IdxError::InvalidInput(format!(
|
||||
"invalid color value '{value}': expected 'true' or 'false'"
|
||||
)));
|
||||
}
|
||||
Ok(toml::Value::Boolean(value.eq_ignore_ascii_case("true")))
|
||||
}
|
||||
_ => Err(IdxError::InvalidInput(format!(
|
||||
"unknown config key '{}'. Valid keys: {}",
|
||||
key,
|
||||
KNOWN_CONFIG_KEYS.join(", ")
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_config_value(key: &str, value: &str) -> Result<(), IdxError> {
|
||||
validate_config_key_value(key, value)?;
|
||||
let normalized = normalize_config_value(key, value)?;
|
||||
|
||||
let path = ensure_default_config()?;
|
||||
let raw = fs::read_to_string(&path).map_err(|e| IdxError::Io(e.to_string()))?;
|
||||
let mut root: toml::Value =
|
||||
toml::from_str(&raw).map_err(|e| IdxError::ConfigError(e.to_string()))?;
|
||||
|
||||
let mut parts = key.split('.').peekable();
|
||||
let mut current = root
|
||||
.as_table_mut()
|
||||
.ok_or_else(|| IdxError::ConfigError("config root is not a table".to_string()))?;
|
||||
|
||||
while let Some(part) = parts.next() {
|
||||
if parts.peek().is_none() {
|
||||
current.insert(part.to_string(), normalized.clone());
|
||||
} else {
|
||||
let entry = current
|
||||
.entry(part.to_string())
|
||||
.or_insert_with(|| toml::Value::Table(toml::map::Map::new()));
|
||||
if !entry.is_table() {
|
||||
*entry = toml::Value::Table(toml::map::Map::new());
|
||||
}
|
||||
current = entry
|
||||
.as_table_mut()
|
||||
.ok_or_else(|| IdxError::ConfigError("invalid config path".to_string()))?;
|
||||
}
|
||||
}
|
||||
|
||||
fs::write(
|
||||
&path,
|
||||
toml::to_string_pretty(&root).map_err(|e| IdxError::ConfigError(e.to_string()))?,
|
||||
)
|
||||
.map_err(|e| IdxError::Io(e.to_string()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn parses_provider_values() {
|
||||
assert_eq!(
|
||||
super::ProviderKind::parse("yahoo").expect("yahoo provider"),
|
||||
super::ProviderKind::Yahoo
|
||||
);
|
||||
assert_eq!(
|
||||
super::ProviderKind::parse("MSN").expect("msn provider"),
|
||||
super::ProviderKind::Msn
|
||||
);
|
||||
assert!(super::ProviderKind::parse("unknown").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_history_provider_values() {
|
||||
assert_eq!(
|
||||
super::HistoryProviderKind::parse("auto").expect("auto history provider"),
|
||||
super::HistoryProviderKind::Auto
|
||||
);
|
||||
assert_eq!(
|
||||
super::HistoryProviderKind::parse("yahoo").expect("yahoo history provider"),
|
||||
super::HistoryProviderKind::Yahoo
|
||||
);
|
||||
assert_eq!(
|
||||
super::HistoryProviderKind::parse("MSN").expect("msn history provider"),
|
||||
super::HistoryProviderKind::Msn
|
||||
);
|
||||
assert!(super::HistoryProviderKind::parse("unknown").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_unknown_keys() {
|
||||
let result = super::validate_config_key_value("unknown.key", "value");
|
||||
assert!(result.is_err());
|
||||
let err = result.unwrap_err();
|
||||
assert!(matches!(err, super::IdxError::InvalidInput(_)));
|
||||
assert!(err.to_string().contains("unknown config key"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_invalid_provider() {
|
||||
let result = super::validate_config_key_value("general.provider", "bogus");
|
||||
assert!(result.is_err());
|
||||
assert!(result.unwrap_err().to_string().contains("invalid provider"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_valid_provider_yahoo() {
|
||||
assert!(super::validate_config_key_value("general.provider", "yahoo").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_valid_provider_msn() {
|
||||
assert!(super::validate_config_key_value("general.provider", "msn").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_valid_provider_case_insensitive() {
|
||||
assert!(super::validate_config_key_value("general.provider", "YAHOO").is_ok());
|
||||
assert!(super::validate_config_key_value("general.provider", "Msn").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_provider_lowercases_value() {
|
||||
assert_eq!(
|
||||
super::normalize_config_value("general.provider", "Msn").unwrap(),
|
||||
toml::Value::String("msn".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_invalid_history_provider() {
|
||||
let result = super::validate_config_key_value("general.history_provider", "bogus");
|
||||
assert!(result.is_err());
|
||||
assert!(
|
||||
result
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("invalid history provider")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_valid_history_provider_auto() {
|
||||
assert!(super::validate_config_key_value("general.history_provider", "auto").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_valid_history_provider_yahoo() {
|
||||
assert!(super::validate_config_key_value("general.history_provider", "yahoo").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_valid_history_provider_msn() {
|
||||
assert!(super::validate_config_key_value("general.history_provider", "msn").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_history_provider_lowercases_value() {
|
||||
assert_eq!(
|
||||
super::normalize_config_value("general.history_provider", "Auto").unwrap(),
|
||||
toml::Value::String("auto".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_invalid_output() {
|
||||
let result = super::validate_config_key_value("general.output", "bogus");
|
||||
assert!(result.is_err());
|
||||
assert!(
|
||||
result
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("invalid output format")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_valid_output_table() {
|
||||
assert!(super::validate_config_key_value("general.output", "table").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_valid_output_json() {
|
||||
assert!(super::validate_config_key_value("general.output", "json").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_valid_output_case_insensitive() {
|
||||
assert!(super::validate_config_key_value("general.output", "TABLE").is_ok());
|
||||
assert!(super::validate_config_key_value("general.output", "Json").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_output_lowercases_value() {
|
||||
assert_eq!(
|
||||
super::normalize_config_value("general.output", "Json").unwrap(),
|
||||
toml::Value::String("json".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_negative_ttl() {
|
||||
let result = super::validate_config_key_value("cache.quote_ttl", "-1");
|
||||
assert!(result.is_err());
|
||||
assert!(
|
||||
result
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("must be non-negative")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_non_numeric_ttl() {
|
||||
let result = super::validate_config_key_value("cache.quote_ttl", "not-a-number");
|
||||
assert!(result.is_err());
|
||||
assert!(
|
||||
result
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("must be a non-negative integer")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_valid_ttl() {
|
||||
assert!(super::validate_config_key_value("cache.quote_ttl", "300").is_ok());
|
||||
assert!(super::validate_config_key_value("cache.fundamental_ttl", "3600").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_zero_ttl() {
|
||||
assert!(super::validate_config_key_value("cache.quote_ttl", "0").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_exchange_any_value() {
|
||||
assert!(super::validate_config_key_value("general.exchange", "JK").is_ok());
|
||||
assert!(super::validate_config_key_value("general.exchange", "US").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_ownership_db_path() {
|
||||
assert!(super::validate_config_key_value("ownership.db_path", "/tmp/ownership.db").is_ok());
|
||||
assert!(super::validate_config_key_value("ownership.db_path", "data/ownership.db").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_ownership_snapshot_manifest() {
|
||||
assert!(
|
||||
super::validate_config_key_value(
|
||||
"ownership.snapshot_manifest",
|
||||
"https://example.com/latest.json"
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
assert!(
|
||||
super::validate_config_key_value(
|
||||
"ownership.snapshot_manifest",
|
||||
"snapshots/latest.json"
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_valid_color() {
|
||||
assert!(super::validate_config_key_value("general.color", "true").is_ok());
|
||||
assert!(super::validate_config_key_value("general.color", "false").is_ok());
|
||||
assert!(super::validate_config_key_value("general.color", "TRUE").is_ok());
|
||||
assert!(super::validate_config_key_value("general.color", "False").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_color_writes_boolean() {
|
||||
assert_eq!(
|
||||
super::normalize_config_value("general.color", "True").unwrap(),
|
||||
toml::Value::Boolean(true)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_invalid_color() {
|
||||
let result = super::validate_config_key_value("general.color", "foo");
|
||||
assert!(result.is_err());
|
||||
assert!(
|
||||
result
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("invalid color value")
|
||||
);
|
||||
|
||||
let result = super::validate_config_key_value("general.color", "123");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
125
crates/idx-core/src/curl_impersonate.rs
Normal file
125
crates/idx-core/src/curl_impersonate.rs
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
use std::process::{Command, Output};
|
||||
|
||||
use crate::error::IdxError;
|
||||
|
||||
const CANDIDATES: &[&str] = &[
|
||||
"curl_chrome142",
|
||||
"curl_chrome136",
|
||||
"curl_chrome133a",
|
||||
"curl_chrome131",
|
||||
"curl_chrome124",
|
||||
"curl_chrome120",
|
||||
"curl_chrome116",
|
||||
];
|
||||
const OVERRIDE_ENV: &str = "IDX_CURL_IMPERSONATE_BIN";
|
||||
|
||||
pub fn chrome_curl_binary() -> Result<String, IdxError> {
|
||||
if let Ok(value) = std::env::var(OVERRIDE_ENV) {
|
||||
let trimmed = value.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return Ok(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
CANDIDATES
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|bin| Command::new(bin).arg("--version").output().is_ok())
|
||||
.map(str::to_string)
|
||||
.ok_or_else(|| {
|
||||
IdxError::Http(format!(
|
||||
"no curl_chrome* binary found; set {OVERRIDE_ENV} or install nixpkgs#curl-impersonate-chrome"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn run(stage: &str, args: &[&str]) -> Result<Output, IdxError> {
|
||||
let owned_args: Vec<String> = args.iter().map(|arg| (*arg).to_string()).collect();
|
||||
run_owned(stage, &owned_args)
|
||||
}
|
||||
|
||||
pub fn run_owned(stage: &str, args: &[String]) -> Result<Output, IdxError> {
|
||||
let binary = chrome_curl_binary()?;
|
||||
run_binary(stage, &binary, args)
|
||||
}
|
||||
|
||||
/// Browser profiles tried, in order, when a site's bot protection rejects
|
||||
/// some TLS fingerprints but not others (IDX's Cloudflare rules block
|
||||
/// `curl_chrome142` on some pages while Firefox/Safari profiles pass).
|
||||
const FALLBACK_PROFILES: &[&str] = &[
|
||||
"curl_chrome142",
|
||||
"curl_firefox144",
|
||||
"curl_chrome136",
|
||||
"curl_firefox135",
|
||||
"curl_safari184",
|
||||
"curl_chrome131",
|
||||
"curl_chrome124",
|
||||
"curl_chrome120",
|
||||
"curl_chrome116",
|
||||
];
|
||||
const FALLBACK_ROUNDS: usize = 2;
|
||||
|
||||
/// Run curl with each available impersonation profile until `accept` approves
|
||||
/// the response body. `IDX_CURL_IMPERSONATE_BIN`, when set, is the only
|
||||
/// profile used. Returns the first accepted output, or the last error.
|
||||
pub fn run_owned_with_fallback(
|
||||
stage: &str,
|
||||
args: &[String],
|
||||
accept: &dyn Fn(&[u8]) -> Result<(), IdxError>,
|
||||
) -> Result<Output, IdxError> {
|
||||
let profiles: Vec<String> = match std::env::var(OVERRIDE_ENV) {
|
||||
Ok(value) if !value.trim().is_empty() => vec![value.trim().to_string()],
|
||||
_ => FALLBACK_PROFILES
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|bin| Command::new(bin).arg("--version").output().is_ok())
|
||||
.map(str::to_string)
|
||||
.collect(),
|
||||
};
|
||||
if profiles.is_empty() {
|
||||
return Err(IdxError::Http(format!(
|
||||
"no curl-impersonate binary found; set {OVERRIDE_ENV} or install nixpkgs#curl-impersonate"
|
||||
)));
|
||||
}
|
||||
|
||||
let mut last_error = None;
|
||||
for _ in 0..FALLBACK_ROUNDS {
|
||||
for binary in &profiles {
|
||||
match run_binary(stage, binary, args) {
|
||||
Ok(output) => match accept(&output.stdout) {
|
||||
Ok(()) => return Ok(output),
|
||||
Err(err) => last_error = Some(err),
|
||||
},
|
||||
Err(err) => last_error = Some(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(last_error.expect("at least one profile was tried"))
|
||||
}
|
||||
|
||||
fn run_binary(stage: &str, binary: &str, args: &[String]) -> Result<Output, IdxError> {
|
||||
let output = Command::new(binary).args(args).output().map_err(|e| {
|
||||
if e.kind() == std::io::ErrorKind::NotFound {
|
||||
return IdxError::Http(format!(
|
||||
"curl-impersonate binary '{binary}' not found; set {OVERRIDE_ENV} or install nixpkgs#curl-impersonate-chrome"
|
||||
));
|
||||
}
|
||||
IdxError::Http(format!("failed to run {binary} for {stage}: {e}"))
|
||||
})?;
|
||||
|
||||
if output.status.success() {
|
||||
return Ok(output);
|
||||
}
|
||||
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let detail = stderr.trim();
|
||||
Err(IdxError::Http(format!(
|
||||
"{stage} {binary} failed (status {}): {}",
|
||||
output.status,
|
||||
if detail.is_empty() {
|
||||
"no output"
|
||||
} else {
|
||||
detail
|
||||
}
|
||||
)))
|
||||
}
|
||||
115
crates/idx-core/src/error.rs
Normal file
115
crates/idx-core/src/error.rs
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
use serde::Serialize;
|
||||
use thiserror::Error;
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Error, Clone)]
|
||||
pub enum IdxError {
|
||||
#[error("symbol not found: {0}")]
|
||||
SymbolNotFound(String),
|
||||
#[error("provider rate limited")]
|
||||
RateLimited,
|
||||
#[error("provider unavailable")]
|
||||
ProviderUnavailable,
|
||||
#[error("unsupported: {0}")]
|
||||
Unsupported(String),
|
||||
#[error("parse error: {0}")]
|
||||
ParseError(String),
|
||||
#[error("cache miss: {0}")]
|
||||
CacheMiss(String),
|
||||
#[error("offline: {0}")]
|
||||
Offline(String),
|
||||
#[error("config error: {0}")]
|
||||
ConfigError(String),
|
||||
#[error("io error: {0}")]
|
||||
Io(String),
|
||||
#[error("http error: {0}")]
|
||||
Http(String),
|
||||
#[error("auth error: {0}")]
|
||||
AuthError(String),
|
||||
#[error("database error: {0}")]
|
||||
DatabaseError(String),
|
||||
#[error("PDF parse error: {0}")]
|
||||
PdfParseError(String),
|
||||
#[error("invalid input: {0}")]
|
||||
InvalidInput(String),
|
||||
#[error("no market data: {0}")]
|
||||
NoMarketData(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
|
||||
pub enum ErrorCode {
|
||||
SymbolNotFound,
|
||||
RateLimited,
|
||||
ProviderUnavailable,
|
||||
Unsupported,
|
||||
ParseError,
|
||||
CacheMiss,
|
||||
Offline,
|
||||
ConfigError,
|
||||
Io,
|
||||
Http,
|
||||
AuthError,
|
||||
DatabaseError,
|
||||
PdfParseError,
|
||||
InvalidInput,
|
||||
NoMarketData,
|
||||
}
|
||||
|
||||
impl IdxError {
|
||||
pub fn code(&self) -> ErrorCode {
|
||||
match self {
|
||||
Self::SymbolNotFound(_) => ErrorCode::SymbolNotFound,
|
||||
Self::RateLimited => ErrorCode::RateLimited,
|
||||
Self::ProviderUnavailable => ErrorCode::ProviderUnavailable,
|
||||
Self::Unsupported(_) => ErrorCode::Unsupported,
|
||||
Self::ParseError(_) => ErrorCode::ParseError,
|
||||
Self::CacheMiss(_) => ErrorCode::CacheMiss,
|
||||
Self::Offline(_) => ErrorCode::Offline,
|
||||
Self::ConfigError(_) => ErrorCode::ConfigError,
|
||||
Self::Io(_) => ErrorCode::Io,
|
||||
Self::Http(_) => ErrorCode::Http,
|
||||
Self::AuthError(_) => ErrorCode::AuthError,
|
||||
Self::DatabaseError(_) => ErrorCode::DatabaseError,
|
||||
Self::PdfParseError(_) => ErrorCode::PdfParseError,
|
||||
Self::InvalidInput(_) => ErrorCode::InvalidInput,
|
||||
Self::NoMarketData(_) => ErrorCode::NoMarketData,
|
||||
}
|
||||
}
|
||||
|
||||
/// Exit status by error class, so scripts can branch without parsing
|
||||
/// stderr. Clap usage errors also exit 2. See docs/DESIGN.md.
|
||||
pub fn exit_code(&self) -> i32 {
|
||||
match self.code() {
|
||||
ErrorCode::InvalidInput | ErrorCode::ConfigError => 2,
|
||||
ErrorCode::SymbolNotFound | ErrorCode::NoMarketData | ErrorCode::CacheMiss => 3,
|
||||
ErrorCode::Http
|
||||
| ErrorCode::RateLimited
|
||||
| ErrorCode::ProviderUnavailable
|
||||
| ErrorCode::Offline
|
||||
| ErrorCode::AuthError => 4,
|
||||
ErrorCode::Unsupported => 5,
|
||||
ErrorCode::Io
|
||||
| ErrorCode::ParseError
|
||||
| ErrorCode::DatabaseError
|
||||
| ErrorCode::PdfParseError => 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ErrorCode, IdxError};
|
||||
|
||||
#[test]
|
||||
fn display_and_code_work() {
|
||||
let err = IdxError::SymbolNotFound("BBCA".to_string());
|
||||
assert_eq!(err.to_string(), "symbol not found: BBCA");
|
||||
assert_eq!(err.code(), ErrorCode::SymbolNotFound);
|
||||
|
||||
let parse = IdxError::ParseError("bad json".to_string());
|
||||
assert_eq!(parse.code(), ErrorCode::ParseError);
|
||||
|
||||
let unsupported = IdxError::Unsupported("history unavailable".to_string());
|
||||
assert_eq!(unsupported.code(), ErrorCode::Unsupported);
|
||||
}
|
||||
}
|
||||
39
crates/idx-core/src/lib.rs
Normal file
39
crates/idx-core/src/lib.rs
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
//! idx-core: retrieval and analysis of Indonesian (IDX) market, financial, and
|
||||
//! investor data, shared by the `idx` CLI and other frontends.
|
||||
//! See docs/DESIGN.md.
|
||||
|
||||
pub mod analysis;
|
||||
pub mod api;
|
||||
pub mod cache;
|
||||
pub mod config;
|
||||
pub mod curl_impersonate;
|
||||
pub mod error;
|
||||
#[cfg(feature = "ownership")]
|
||||
pub mod ownership;
|
||||
|
||||
pub mod runtime {
|
||||
use std::fmt::Display;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
static QUIET: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
pub fn set_quiet(quiet: bool) {
|
||||
QUIET.store(quiet, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn is_quiet() -> bool {
|
||||
QUIET.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn info(message: impl Display) {
|
||||
if !is_quiet() {
|
||||
eprintln!("info: {message}");
|
||||
}
|
||||
}
|
||||
|
||||
pub fn warn(message: impl Display) {
|
||||
if !is_quiet() {
|
||||
eprintln!("warning: {message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
360
crates/idx-core/src/ownership/archive.rs
Normal file
360
crates/idx-core/src/ownership/archive.rs
Normal file
|
|
@ -0,0 +1,360 @@
|
|||
use std::fs;
|
||||
use std::io::{Cursor, Read};
|
||||
use std::path::Path;
|
||||
|
||||
use chrono::NaiveDate;
|
||||
|
||||
use crate::error::IdxError;
|
||||
use crate::ownership::types::{InvestorTypeCode, KseiHoldingDraft, Locality};
|
||||
|
||||
const EXPECTED_HEADER: &[&str] = &[
|
||||
"Date",
|
||||
"Code",
|
||||
"Type",
|
||||
"Sec. Num",
|
||||
"Price",
|
||||
"Local IS",
|
||||
"Local CP",
|
||||
"Local PF",
|
||||
"Local IB",
|
||||
"Local ID",
|
||||
"Local MF",
|
||||
"Local SC",
|
||||
"Local FD",
|
||||
"Local OT",
|
||||
"Total",
|
||||
"Foreign IS",
|
||||
"Foreign CP",
|
||||
"Foreign PF",
|
||||
"Foreign IB",
|
||||
"Foreign ID",
|
||||
"Foreign MF",
|
||||
"Foreign SC",
|
||||
"Foreign FD",
|
||||
"Foreign OT",
|
||||
"Total",
|
||||
];
|
||||
|
||||
const LOCAL_BUCKETS: &[(&str, usize)] = &[
|
||||
("IS", 5),
|
||||
("CP", 6),
|
||||
("PF", 7),
|
||||
("IB", 8),
|
||||
("ID", 9),
|
||||
("MF", 10),
|
||||
("SC", 11),
|
||||
("FD", 12),
|
||||
("OT", 13),
|
||||
];
|
||||
|
||||
const FOREIGN_BUCKETS: &[(&str, usize)] = &[
|
||||
("IS", 15),
|
||||
("CP", 16),
|
||||
("PF", 17),
|
||||
("IB", 18),
|
||||
("ID", 19),
|
||||
("MF", 20),
|
||||
("SC", 21),
|
||||
("FD", 22),
|
||||
("OT", 23),
|
||||
];
|
||||
|
||||
pub fn supports_local_archive_file(path: &Path) -> bool {
|
||||
path.extension()
|
||||
.and_then(|value| value.to_str())
|
||||
.map(|value| {
|
||||
let ext = value.trim().to_ascii_lowercase();
|
||||
ext == "txt" || ext == "zip"
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
pub fn parse_balancepos_file(path: &Path) -> Result<Vec<KseiHoldingDraft>, IdxError> {
|
||||
let raw = match path
|
||||
.extension()
|
||||
.and_then(|value| value.to_str())
|
||||
.map(|value| value.trim().to_ascii_lowercase())
|
||||
.as_deref()
|
||||
{
|
||||
Some("txt") => fs::read_to_string(path).map_err(|e| {
|
||||
IdxError::Io(format!(
|
||||
"failed to read KSEI archive TXT {}: {e}",
|
||||
path.display()
|
||||
))
|
||||
})?,
|
||||
Some("zip") => extract_txt_from_zip(path)?,
|
||||
_ => {
|
||||
return Err(IdxError::InvalidInput(format!(
|
||||
"unsupported local ownership archive file {}; expected .txt or .zip",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
||||
parse_balancepos_text(&raw)
|
||||
}
|
||||
|
||||
pub fn parse_balancepos_text(raw: &str) -> Result<Vec<KseiHoldingDraft>, IdxError> {
|
||||
let mut lines = raw.lines().filter(|line| !line.trim().is_empty());
|
||||
let header = lines
|
||||
.next()
|
||||
.ok_or_else(|| IdxError::ParseError("empty KSEI archive TXT input".to_string()))?;
|
||||
validate_header(header)?;
|
||||
|
||||
let mut drafts = Vec::new();
|
||||
for (line_number, line) in lines.enumerate() {
|
||||
let columns: Vec<&str> = line.split('|').map(str::trim).collect();
|
||||
if columns.len() != EXPECTED_HEADER.len() {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"invalid KSEI archive TXT row {}: expected {} columns, got {}",
|
||||
line_number + 2,
|
||||
EXPECTED_HEADER.len(),
|
||||
columns.len()
|
||||
)));
|
||||
}
|
||||
|
||||
if !columns[2].eq_ignore_ascii_case("EQUITY") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let report_date = parse_archive_date(columns[0])?;
|
||||
let ticker_code = columns[1].trim().to_uppercase();
|
||||
let sec_num = parse_archive_number(columns[3], line_number + 2, "Sec. Num")?;
|
||||
if sec_num <= 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
append_bucket_drafts(
|
||||
&mut drafts,
|
||||
&ticker_code,
|
||||
report_date,
|
||||
sec_num,
|
||||
columns.as_slice(),
|
||||
Locality::Local,
|
||||
LOCAL_BUCKETS,
|
||||
)?;
|
||||
append_bucket_drafts(
|
||||
&mut drafts,
|
||||
&ticker_code,
|
||||
report_date,
|
||||
sec_num,
|
||||
columns.as_slice(),
|
||||
Locality::Foreign,
|
||||
FOREIGN_BUCKETS,
|
||||
)?;
|
||||
}
|
||||
|
||||
if drafts.is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"no importable EQUITY rows found in KSEI archive TXT".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(drafts)
|
||||
}
|
||||
|
||||
fn extract_txt_from_zip(path: &Path) -> Result<String, IdxError> {
|
||||
let bytes = fs::read(path).map_err(|e| {
|
||||
IdxError::Io(format!(
|
||||
"failed to read KSEI archive ZIP {}: {e}",
|
||||
path.display()
|
||||
))
|
||||
})?;
|
||||
let cursor = Cursor::new(bytes);
|
||||
let mut zip = zip::ZipArchive::new(cursor)
|
||||
.map_err(|e| IdxError::ParseError(format!("failed to open KSEI archive ZIP: {e}")))?;
|
||||
|
||||
for index in 0..zip.len() {
|
||||
let mut file = zip.by_index(index).map_err(|e| {
|
||||
IdxError::ParseError(format!("failed to read KSEI archive ZIP entry: {e}"))
|
||||
})?;
|
||||
if file.is_dir() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = file.name().to_ascii_lowercase();
|
||||
if !name.ends_with(".txt") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut output = String::new();
|
||||
file.read_to_string(&mut output).map_err(|e| {
|
||||
IdxError::ParseError(format!("failed to decode KSEI archive TXT entry: {e}"))
|
||||
})?;
|
||||
return Ok(output);
|
||||
}
|
||||
|
||||
Err(IdxError::ParseError(
|
||||
"KSEI archive ZIP did not contain a TXT payload".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
fn validate_header(header: &str) -> Result<(), IdxError> {
|
||||
let columns: Vec<&str> = header.split('|').map(str::trim).collect();
|
||||
if columns != EXPECTED_HEADER {
|
||||
return Err(IdxError::ParseError(
|
||||
"KSEI archive TXT header did not match the expected balancepos layout".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_archive_date(raw: &str) -> Result<NaiveDate, IdxError> {
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.len() != 11 {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"invalid KSEI archive date `{trimmed}`"
|
||||
)));
|
||||
}
|
||||
|
||||
let canonical = format!(
|
||||
"{}-{}-{}",
|
||||
&trimmed[..2],
|
||||
titlecase_month(&trimmed[3..6]),
|
||||
&trimmed[7..11]
|
||||
);
|
||||
|
||||
NaiveDate::parse_from_str(&canonical, "%d-%b-%Y")
|
||||
.map_err(|e| IdxError::ParseError(format!("invalid KSEI archive date `{trimmed}`: {e}")))
|
||||
}
|
||||
|
||||
fn titlecase_month(raw: &str) -> String {
|
||||
let upper = raw.trim().to_ascii_uppercase();
|
||||
let mut chars = upper.chars();
|
||||
match chars.next() {
|
||||
Some(first) => {
|
||||
let mut output = String::new();
|
||||
output.push(first.to_ascii_uppercase());
|
||||
output.push_str(&chars.as_str().to_ascii_lowercase());
|
||||
output
|
||||
}
|
||||
None => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_archive_number(raw: &str, line_number: usize, field: &str) -> Result<i64, IdxError> {
|
||||
raw.trim().parse::<i64>().map_err(|e| {
|
||||
IdxError::ParseError(format!(
|
||||
"invalid KSEI archive TXT value in row {line_number} field `{field}`: {e}"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn append_bucket_drafts(
|
||||
drafts: &mut Vec<KseiHoldingDraft>,
|
||||
ticker_code: &str,
|
||||
report_date: NaiveDate,
|
||||
sec_num: i64,
|
||||
columns: &[&str],
|
||||
locality: Locality,
|
||||
buckets: &[(&str, usize)],
|
||||
) -> Result<(), IdxError> {
|
||||
for (investor_type, column_index) in buckets {
|
||||
let shares =
|
||||
parse_archive_number(columns[*column_index], 0, investor_type).map_err(|_| {
|
||||
IdxError::ParseError(format!(
|
||||
"invalid KSEI archive TXT share count for {ticker_code} {investor_type}"
|
||||
))
|
||||
})?;
|
||||
if shares <= 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
drafts.push(KseiHoldingDraft {
|
||||
ticker_code: ticker_code.to_string(),
|
||||
issuer_name: None,
|
||||
raw_investor_name: synthetic_holder_name(locality, investor_type),
|
||||
investor_type: Some(InvestorTypeCode((*investor_type).to_string())),
|
||||
locality: Some(locality),
|
||||
nationality: None,
|
||||
domicile: None,
|
||||
holdings_scripless: shares,
|
||||
holdings_scrip: 0,
|
||||
total_shares: shares,
|
||||
percentage_bps: compute_percentage_bps(shares, sec_num),
|
||||
report_date,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn synthetic_holder_name(locality: Locality, investor_type: &str) -> String {
|
||||
let prefix = match locality {
|
||||
Locality::Local => "LOCAL",
|
||||
Locality::Foreign => "FOREIGN",
|
||||
};
|
||||
format!("KSEI AGGREGATE {prefix} {investor_type}")
|
||||
}
|
||||
|
||||
fn compute_percentage_bps(shares: i64, sec_num: i64) -> i64 {
|
||||
let shares_i128 = i128::from(shares);
|
||||
let sec_num_i128 = i128::from(sec_num);
|
||||
let rounded = ((shares_i128 * 10_000) + (sec_num_i128 / 2)) / sec_num_i128;
|
||||
i64::try_from(rounded).unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::parse_balancepos_text;
|
||||
use crate::ownership::entities::normalize_ksei_row;
|
||||
use crate::ownership::parser::parse_stext_xml;
|
||||
use crate::ownership::types::Locality;
|
||||
|
||||
#[test]
|
||||
fn parses_balancepos_excerpt_into_bucket_holders() {
|
||||
let raw = include_str!("../../tests/fixtures/ksei_balancepos_20260227_excerpt.txt");
|
||||
let drafts = parse_balancepos_text(raw).expect("balancepos excerpt parses");
|
||||
|
||||
let local_cp = drafts
|
||||
.iter()
|
||||
.find(|draft| {
|
||||
draft.ticker_code == "AADI"
|
||||
&& draft.investor_type.as_ref().map(|code| code.0.as_str()) == Some("CP")
|
||||
&& draft.locality == Some(Locality::Local)
|
||||
})
|
||||
.expect("AADI local CP bucket");
|
||||
|
||||
assert_eq!(local_cp.raw_investor_name, "KSEI AGGREGATE LOCAL CP");
|
||||
assert_eq!(local_cp.total_shares, 5_035_745_466);
|
||||
assert_eq!(local_cp.percentage_bps, 6467);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn balancepos_cross_check_contains_pdf_holder_bucket() {
|
||||
let pdf_rows = parse_stext_xml(include_str!(
|
||||
"../../tests/fixtures/ksei_above1_stext_excerpt.xml"
|
||||
))
|
||||
.expect("pdf fixture rows");
|
||||
let pdf_drafts: Vec<_> = pdf_rows
|
||||
.iter()
|
||||
.map(normalize_ksei_row)
|
||||
.collect::<Result<_, _>>()
|
||||
.expect("normalized pdf drafts");
|
||||
let archive_drafts = parse_balancepos_text(include_str!(
|
||||
"../../tests/fixtures/ksei_balancepos_20260227_excerpt.txt"
|
||||
))
|
||||
.expect("archive drafts");
|
||||
|
||||
let pdf_local_cp = pdf_drafts
|
||||
.iter()
|
||||
.find(|draft| {
|
||||
draft.ticker_code == "AADI"
|
||||
&& draft.investor_type.as_ref().map(|code| code.0.as_str()) == Some("CP")
|
||||
&& draft.locality == Some(Locality::Local)
|
||||
})
|
||||
.expect("pdf local cp");
|
||||
let archive_local_cp = archive_drafts
|
||||
.iter()
|
||||
.find(|draft| {
|
||||
draft.ticker_code == "AADI"
|
||||
&& draft.investor_type.as_ref().map(|code| code.0.as_str()) == Some("CP")
|
||||
&& draft.locality == Some(Locality::Local)
|
||||
})
|
||||
.expect("archive local cp");
|
||||
|
||||
assert_eq!(pdf_local_cp.report_date, archive_local_cp.report_date);
|
||||
assert!(pdf_local_cp.total_shares <= archive_local_cp.total_shares);
|
||||
assert!(pdf_local_cp.percentage_bps <= archive_local_cp.percentage_bps);
|
||||
}
|
||||
}
|
||||
1962
crates/idx-core/src/ownership/db.rs
Normal file
1962
crates/idx-core/src/ownership/db.rs
Normal file
File diff suppressed because it is too large
Load diff
456
crates/idx-core/src/ownership/entities.rs
Normal file
456
crates/idx-core/src/ownership/entities.rs
Normal file
|
|
@ -0,0 +1,456 @@
|
|||
use chrono::NaiveDate;
|
||||
use rusqlite::{Connection, params};
|
||||
|
||||
use crate::error::IdxError;
|
||||
use crate::ownership::types::{
|
||||
InvestorTypeCode, KseiHoldingDraft, KseiRawRow, Locality, OwnershipSource,
|
||||
};
|
||||
|
||||
/// Parse Indonesian locale number string to i64.
|
||||
/// "1.533.682.440" → 1533682440, "0" → 0
|
||||
pub fn parse_id_number(s: &str) -> Result<i64, IdxError> {
|
||||
let trimmed = s.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"empty Indonesian number string".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let normalized = trimmed.replace(['.', ' '], "");
|
||||
if normalized.is_empty() || !normalized.chars().all(|c| c.is_ascii_digit()) {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"invalid Indonesian number format: {s}"
|
||||
)));
|
||||
}
|
||||
|
||||
normalized
|
||||
.parse::<i64>()
|
||||
.map_err(|e| IdxError::ParseError(format!("failed to parse Indonesian number '{s}': {e}")))
|
||||
}
|
||||
|
||||
fn parse_id_number_or_zero(s: &str) -> Result<i64, IdxError> {
|
||||
if s.trim().is_empty() {
|
||||
Ok(0)
|
||||
} else {
|
||||
parse_id_number(s)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse Indonesian locale percentage to basis points (i64).
|
||||
/// "54,94" → 5494, "0,00" → 0, "100,00" → 10000
|
||||
pub fn parse_id_percentage(s: &str) -> Result<i64, IdxError> {
|
||||
let trimmed = s.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"empty Indonesian percentage string".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut parts = trimmed.split(',');
|
||||
let whole = parts.next().unwrap_or_default();
|
||||
let frac = parts.next().unwrap_or("00");
|
||||
|
||||
if parts.next().is_some() {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"invalid Indonesian percentage format: {s}"
|
||||
)));
|
||||
}
|
||||
|
||||
if whole.is_empty() || !whole.chars().all(|c| c.is_ascii_digit()) {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"invalid Indonesian percentage whole part: {s}"
|
||||
)));
|
||||
}
|
||||
|
||||
if !frac.chars().all(|c| c.is_ascii_digit()) || frac.len() > 2 {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"invalid Indonesian percentage fractional part: {s}"
|
||||
)));
|
||||
}
|
||||
|
||||
let whole_i = whole.parse::<i64>().map_err(|e| {
|
||||
IdxError::ParseError(format!("failed to parse Indonesian percentage '{s}': {e}"))
|
||||
})?;
|
||||
|
||||
let frac_padded = if frac.len() == 1 {
|
||||
format!("{frac}0")
|
||||
} else {
|
||||
frac.to_string()
|
||||
};
|
||||
let frac_i = frac_padded.parse::<i64>().map_err(|e| {
|
||||
IdxError::ParseError(format!("failed to parse Indonesian percentage '{s}': {e}"))
|
||||
})?;
|
||||
|
||||
Ok((whole_i * 100) + frac_i)
|
||||
}
|
||||
|
||||
/// Parse KSEI date format to NaiveDate.
|
||||
/// "27-Feb-2026" → NaiveDate(2026, 2, 27)
|
||||
pub fn parse_ksei_date(s: &str) -> Result<NaiveDate, IdxError> {
|
||||
let trimmed = s.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(IdxError::ParseError("empty KSEI date string".to_string()));
|
||||
}
|
||||
|
||||
NaiveDate::parse_from_str(trimmed, "%d-%b-%Y")
|
||||
.map_err(|e| IdxError::ParseError(format!("invalid KSEI date '{s}': {e}")))
|
||||
}
|
||||
|
||||
/// Normalize an investor name for entity matching.
|
||||
/// Strips: "PT.", "PT ", "TBK", "Tbk", "(PERSERO)", "LIMITED", "PTE", "LTD"
|
||||
/// Collapses whitespace, trims, uppercases.
|
||||
pub fn normalize_name(raw: &str) -> String {
|
||||
let upper = collapse_whitespace(raw).to_uppercase();
|
||||
if upper.is_empty() {
|
||||
return upper;
|
||||
}
|
||||
|
||||
let mut tokens: Vec<&str> = upper.split_whitespace().collect();
|
||||
|
||||
loop {
|
||||
let mut changed = false;
|
||||
|
||||
if let Some(first) = tokens.first().copied()
|
||||
&& matches!(first, "PT" | "PT.")
|
||||
{
|
||||
tokens.remove(0);
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if let Some(last) = tokens.last().copied()
|
||||
&& matches!(
|
||||
last,
|
||||
"TBK" | "Tbk" | "(PERSERO)" | "LIMITED" | "PTE" | "LTD"
|
||||
)
|
||||
{
|
||||
let _ = tokens.pop();
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if !changed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
collapse_whitespace(&tokens.join(" "))
|
||||
}
|
||||
|
||||
/// Convert a KseiRawRow into a normalized KseiHoldingDraft.
|
||||
/// Applies all parsing functions above.
|
||||
pub fn normalize_ksei_row(raw: &KseiRawRow) -> Result<KseiHoldingDraft, IdxError> {
|
||||
let investor_type = normalize_investor_type(&raw.investor_type);
|
||||
let locality = normalize_locality(&raw.local_foreign);
|
||||
|
||||
Ok(KseiHoldingDraft {
|
||||
ticker_code: raw.share_code.trim().to_string(),
|
||||
issuer_name: optional_string(&raw.issuer_name),
|
||||
raw_investor_name: raw.investor_name.trim().to_string(),
|
||||
investor_type,
|
||||
locality,
|
||||
nationality: optional_string(&raw.nationality),
|
||||
domicile: optional_string(&raw.domicile),
|
||||
holdings_scripless: parse_id_number_or_zero(&raw.holdings_scripless)?,
|
||||
holdings_scrip: parse_id_number_or_zero(&raw.holdings_scrip)?,
|
||||
total_shares: parse_id_number(&raw.total_holding_shares)?,
|
||||
percentage_bps: parse_id_percentage(&raw.percentage)?,
|
||||
report_date: parse_ksei_date(&raw.date)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Find or create a canonical entity for a raw investor name.
|
||||
/// Strategy: exact match on normalized name → rule-based normalization → create new.
|
||||
/// Returns entity_id.
|
||||
pub fn resolve_entity(
|
||||
conn: &Connection,
|
||||
raw_name: &str,
|
||||
source: OwnershipSource,
|
||||
) -> Result<i64, IdxError> {
|
||||
let source_db = source_to_db(source);
|
||||
let raw_trimmed = raw_name.trim();
|
||||
|
||||
if raw_trimmed.is_empty() {
|
||||
return Err(IdxError::ParseError("empty raw entity name".to_string()));
|
||||
}
|
||||
|
||||
let exact_normalized = collapse_whitespace(raw_trimmed).to_uppercase();
|
||||
let rule_normalized = normalize_name(raw_trimmed);
|
||||
|
||||
if let Some(entity_id) = find_entity_by_alias(conn, raw_trimmed, source_db)? {
|
||||
return Ok(entity_id);
|
||||
}
|
||||
|
||||
if let Some(entity_id) = find_entity_by_canonical(conn, &exact_normalized)? {
|
||||
insert_alias(conn, entity_id, raw_trimmed, source_db, "exact")?;
|
||||
return Ok(entity_id);
|
||||
}
|
||||
|
||||
if rule_normalized != exact_normalized
|
||||
&& let Some(entity_id) = find_entity_by_canonical(conn, &rule_normalized)?
|
||||
{
|
||||
insert_alias(conn, entity_id, raw_trimmed, source_db, "rule")?;
|
||||
return Ok(entity_id);
|
||||
}
|
||||
|
||||
let now: i64 = chrono::Utc::now().timestamp();
|
||||
conn.execute(
|
||||
"INSERT INTO entities (canonical_name, entity_type, country, created_at, updated_at)
|
||||
VALUES (?1, NULL, NULL, ?2, ?2)",
|
||||
params![
|
||||
if rule_normalized.is_empty() {
|
||||
&exact_normalized
|
||||
} else {
|
||||
&rule_normalized
|
||||
},
|
||||
now
|
||||
],
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(format!("insert entity failed: {e}")))?;
|
||||
|
||||
let entity_id = conn.last_insert_rowid();
|
||||
insert_alias(conn, entity_id, raw_trimmed, source_db, "exact")?;
|
||||
|
||||
Ok(entity_id)
|
||||
}
|
||||
|
||||
pub(crate) fn normalize_investor_type(raw: &str) -> Option<InvestorTypeCode> {
|
||||
let value = raw.trim();
|
||||
if value.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(InvestorTypeCode(value.to_uppercase()))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn normalize_locality(raw: &str) -> Option<Locality> {
|
||||
match raw.trim().to_uppercase().as_str() {
|
||||
"L" | "D" => Some(Locality::Local),
|
||||
"F" | "A" => Some(Locality::Foreign),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn optional_string(raw: &str) -> Option<String> {
|
||||
let value = raw.trim();
|
||||
if value.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(value.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn collapse_whitespace(input: &str) -> String {
|
||||
input.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
fn source_to_db(source: OwnershipSource) -> &'static str {
|
||||
match source {
|
||||
OwnershipSource::Ksei => "ksei",
|
||||
OwnershipSource::Bing => "bing",
|
||||
}
|
||||
}
|
||||
|
||||
fn find_entity_by_alias(
|
||||
conn: &Connection,
|
||||
raw_name: &str,
|
||||
source: &str,
|
||||
) -> Result<Option<i64>, IdxError> {
|
||||
conn.query_row(
|
||||
"SELECT entity_id FROM entity_aliases WHERE raw_name = ?1 AND source = ?2",
|
||||
params![raw_name, source],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.map(Some)
|
||||
.or_else(|e| match e {
|
||||
rusqlite::Error::QueryReturnedNoRows => Ok(None),
|
||||
_ => Err(IdxError::DatabaseError(e.to_string())),
|
||||
})
|
||||
}
|
||||
|
||||
fn find_entity_by_canonical(
|
||||
conn: &Connection,
|
||||
canonical_name: &str,
|
||||
) -> Result<Option<i64>, IdxError> {
|
||||
conn.query_row(
|
||||
"SELECT id FROM entities WHERE canonical_name = ?1",
|
||||
params![canonical_name],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.map(Some)
|
||||
.or_else(|e| match e {
|
||||
rusqlite::Error::QueryReturnedNoRows => Ok(None),
|
||||
_ => Err(IdxError::DatabaseError(e.to_string())),
|
||||
})
|
||||
}
|
||||
|
||||
fn insert_alias(
|
||||
conn: &Connection,
|
||||
entity_id: i64,
|
||||
raw_name: &str,
|
||||
source: &str,
|
||||
method: &str,
|
||||
) -> Result<(), IdxError> {
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO entity_aliases (entity_id, raw_name, source, confidence, method)
|
||||
VALUES (?1, ?2, ?3, 1.0, ?4)",
|
||||
params![entity_id, raw_name, source, method],
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(format!("insert alias failed: {e}")))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use chrono::NaiveDate;
|
||||
use rusqlite::Connection;
|
||||
|
||||
use crate::ownership::entities::{
|
||||
normalize_ksei_row, normalize_name, parse_id_number, parse_id_percentage, parse_ksei_date,
|
||||
resolve_entity,
|
||||
};
|
||||
use crate::ownership::types::{KseiRawRow, OwnershipSource};
|
||||
|
||||
#[test]
|
||||
fn test_parse_id_number() {
|
||||
assert_eq!(parse_id_number("1.533.682.440").unwrap(), 1_533_682_440);
|
||||
assert_eq!(parse_id_number("0").unwrap(), 0);
|
||||
assert_eq!(parse_id_number("3.200.142.830").unwrap(), 3_200_142_830);
|
||||
assert!(parse_id_number("").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_id_percentage() {
|
||||
assert_eq!(parse_id_percentage("54,94").unwrap(), 5494);
|
||||
assert_eq!(parse_id_percentage("0,00").unwrap(), 0);
|
||||
assert_eq!(parse_id_percentage("100,00").unwrap(), 10000);
|
||||
assert_eq!(parse_id_percentage("41,10").unwrap(), 4110);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_ksei_date_various_months() {
|
||||
let samples = [
|
||||
("01-Jan-2026", NaiveDate::from_ymd_opt(2026, 1, 1).unwrap()),
|
||||
("01-Feb-2026", NaiveDate::from_ymd_opt(2026, 2, 1).unwrap()),
|
||||
("01-Mar-2026", NaiveDate::from_ymd_opt(2026, 3, 1).unwrap()),
|
||||
("01-Apr-2026", NaiveDate::from_ymd_opt(2026, 4, 1).unwrap()),
|
||||
("01-May-2026", NaiveDate::from_ymd_opt(2026, 5, 1).unwrap()),
|
||||
("01-Jun-2026", NaiveDate::from_ymd_opt(2026, 6, 1).unwrap()),
|
||||
("01-Jul-2026", NaiveDate::from_ymd_opt(2026, 7, 1).unwrap()),
|
||||
("01-Aug-2026", NaiveDate::from_ymd_opt(2026, 8, 1).unwrap()),
|
||||
("01-Sep-2026", NaiveDate::from_ymd_opt(2026, 9, 1).unwrap()),
|
||||
("01-Oct-2026", NaiveDate::from_ymd_opt(2026, 10, 1).unwrap()),
|
||||
("01-Nov-2026", NaiveDate::from_ymd_opt(2026, 11, 1).unwrap()),
|
||||
("01-Dec-2026", NaiveDate::from_ymd_opt(2026, 12, 1).unwrap()),
|
||||
];
|
||||
|
||||
for (input, expected) in samples {
|
||||
assert_eq!(parse_ksei_date(input).unwrap(), expected);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
parse_ksei_date("27-Feb-2026").unwrap(),
|
||||
NaiveDate::from_ymd_opt(2026, 2, 27).unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_normalize_name() {
|
||||
assert_eq!(
|
||||
normalize_name("PT. ASTRA INTERNATIONAL TBK"),
|
||||
"ASTRA INTERNATIONAL"
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_name("PT BANK CENTRAL ASIA Tbk"),
|
||||
"BANK CENTRAL ASIA"
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_name("UOB KAY HIAN PRIVATE LIMITED"),
|
||||
"UOB KAY HIAN PRIVATE"
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_name("DJS Ketenagakerjaan (JHT)"),
|
||||
"DJS KETENAGAKERJAAN (JHT)"
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_name("BPJS KETENAGAKERJAAN"),
|
||||
"BPJS KETENAGAKERJAAN"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_entity_create_then_reuse() {
|
||||
let conn = Connection::open_in_memory().unwrap();
|
||||
conn.execute_batch(
|
||||
r#"
|
||||
CREATE TABLE entities (
|
||||
id INTEGER PRIMARY KEY,
|
||||
canonical_name TEXT NOT NULL,
|
||||
entity_type TEXT,
|
||||
country TEXT,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE entity_aliases (
|
||||
id INTEGER PRIMARY KEY,
|
||||
entity_id INTEGER NOT NULL,
|
||||
raw_name TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
confidence REAL NOT NULL DEFAULT 1.0,
|
||||
method TEXT NOT NULL,
|
||||
UNIQUE(raw_name, source)
|
||||
);
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let id1 =
|
||||
resolve_entity(&conn, "PT. ASTRA INTERNATIONAL TBK", OwnershipSource::Ksei).unwrap();
|
||||
let id2 =
|
||||
resolve_entity(&conn, "PT. ASTRA INTERNATIONAL TBK", OwnershipSource::Ksei).unwrap();
|
||||
|
||||
assert_eq!(id1, id2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_normalize_ksei_row_defaults_empty_component_shares_to_zero() {
|
||||
let raw = KseiRawRow {
|
||||
date: "27-Apr-2026".to_string(),
|
||||
share_code: "AADI".to_string(),
|
||||
issuer_name: "ADARO ANDALAN INDONESIA Tbk".to_string(),
|
||||
investor_name: "ADARO STRATEGIC INVESTMENTS".to_string(),
|
||||
investor_type: "CP".to_string(),
|
||||
local_foreign: "D".to_string(),
|
||||
nationality: "INDONESIA".to_string(),
|
||||
domicile: String::new(),
|
||||
holdings_scripless: String::new(),
|
||||
holdings_scrip: String::new(),
|
||||
total_holding_shares: "3.200.142.830".to_string(),
|
||||
percentage: "66,18".to_string(),
|
||||
};
|
||||
|
||||
let draft = normalize_ksei_row(&raw).expect("empty component shares should normalize");
|
||||
assert_eq!(draft.holdings_scripless, 0);
|
||||
assert_eq!(draft.holdings_scrip, 0);
|
||||
assert_eq!(draft.total_shares, 3_200_142_830);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_normalize_ksei_row_still_requires_total_shares() {
|
||||
let raw = KseiRawRow {
|
||||
date: "27-Apr-2026".to_string(),
|
||||
share_code: "AADI".to_string(),
|
||||
issuer_name: "ADARO ANDALAN INDONESIA Tbk".to_string(),
|
||||
investor_name: "ADARO STRATEGIC INVESTMENTS".to_string(),
|
||||
investor_type: "CP".to_string(),
|
||||
local_foreign: "D".to_string(),
|
||||
nationality: "INDONESIA".to_string(),
|
||||
domicile: String::new(),
|
||||
holdings_scripless: String::new(),
|
||||
holdings_scrip: String::new(),
|
||||
total_holding_shares: String::new(),
|
||||
percentage: "66,18".to_string(),
|
||||
};
|
||||
|
||||
assert!(normalize_ksei_row(&raw).is_err());
|
||||
}
|
||||
}
|
||||
462
crates/idx-core/src/ownership/graph.rs
Normal file
462
crates/idx-core/src/ownership/graph.rs
Normal file
|
|
@ -0,0 +1,462 @@
|
|||
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
|
||||
|
||||
use rusqlite::{Connection, params};
|
||||
|
||||
use crate::error::IdxError;
|
||||
use crate::ownership::types::{GraphEdge, GraphNode, GraphNodeType, OwnershipSource};
|
||||
|
||||
/// Query ownership graph starting from a ticker or entity, traversing N hops.
|
||||
/// Root can be ticker code (`BBCA`) or entity name — auto-detect.
|
||||
pub fn query_ownership_graph(
|
||||
conn: &Connection,
|
||||
root: &str,
|
||||
depth: usize,
|
||||
) -> Result<(Vec<GraphNode>, Vec<GraphEdge>), IdxError> {
|
||||
let root = root.trim();
|
||||
if root.is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"graph root cannot be empty".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let root_node_id = detect_root_node(conn, root)?;
|
||||
|
||||
let mut visited: BTreeSet<String> = BTreeSet::new();
|
||||
{
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"WITH RECURSIVE
|
||||
latest_ksei AS (
|
||||
SELECT sha256
|
||||
FROM ownership_releases
|
||||
ORDER BY as_of_date DESC, imported_at DESC
|
||||
LIMIT 1
|
||||
),
|
||||
latest_bing AS (
|
||||
SELECT ticker_id, MAX(report_date) AS report_date
|
||||
FROM bing_holdings
|
||||
GROUP BY ticker_id
|
||||
),
|
||||
all_edges AS (
|
||||
SELECT
|
||||
'entity:' || k.entity_id AS from_id,
|
||||
'ticker:' || t.code AS to_id
|
||||
FROM ksei_holdings k
|
||||
JOIN tickers t ON t.id = k.ticker_id
|
||||
JOIN latest_ksei lk ON lk.sha256 = k.release_sha256
|
||||
WHERE k.entity_id IS NOT NULL
|
||||
|
||||
UNION
|
||||
|
||||
SELECT
|
||||
'entity:' || b.entity_id AS from_id,
|
||||
'ticker:' || t.code AS to_id
|
||||
FROM bing_holdings b
|
||||
JOIN latest_bing lb
|
||||
ON lb.ticker_id = b.ticker_id
|
||||
AND lb.report_date = b.report_date
|
||||
JOIN tickers t ON t.id = b.ticker_id
|
||||
WHERE b.entity_id IS NOT NULL
|
||||
AND b.signal = 'holder'
|
||||
),
|
||||
neighbors AS (
|
||||
SELECT from_id AS a, to_id AS b FROM all_edges
|
||||
UNION
|
||||
SELECT to_id AS a, from_id AS b FROM all_edges
|
||||
),
|
||||
walk(node_id, depth, path) AS (
|
||||
SELECT ?1 AS node_id, 0 AS depth, ?1 AS path
|
||||
UNION ALL
|
||||
SELECT n.b, w.depth + 1, w.path || '>' || n.b
|
||||
FROM walk w
|
||||
JOIN neighbors n ON n.a = w.node_id
|
||||
WHERE w.depth < ?2
|
||||
AND instr(w.path, n.b) = 0
|
||||
)
|
||||
SELECT DISTINCT node_id
|
||||
FROM walk",
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
let rows = stmt
|
||||
.query_map(params![root_node_id, depth as i64], |row| {
|
||||
row.get::<_, String>(0)
|
||||
})
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
for row in rows {
|
||||
visited.insert(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
||||
}
|
||||
}
|
||||
|
||||
let mut edges = query_all_edges(conn)?;
|
||||
edges.retain(|edge| visited.contains(&edge.from) && visited.contains(&edge.to));
|
||||
|
||||
let nodes = build_nodes(conn, &visited)?;
|
||||
|
||||
Ok((nodes, edges))
|
||||
}
|
||||
|
||||
/// Format graph as ASCII tree for terminal display.
|
||||
pub fn format_graph_text(nodes: &[GraphNode], edges: &[GraphEdge]) -> String {
|
||||
if nodes.is_empty() {
|
||||
return "(empty graph)".to_string();
|
||||
}
|
||||
|
||||
let labels: HashMap<&str, (&str, GraphNodeType)> = nodes
|
||||
.iter()
|
||||
.map(|n| (n.id.as_str(), (n.label.as_str(), n.node_type)))
|
||||
.collect();
|
||||
|
||||
let mut ticker_to_entities: BTreeMap<&str, Vec<&GraphEdge>> = BTreeMap::new();
|
||||
for edge in edges {
|
||||
if let Some((_, GraphNodeType::Ticker)) = labels.get(edge.to.as_str()) {
|
||||
ticker_to_entities
|
||||
.entry(edge.to.as_str())
|
||||
.or_default()
|
||||
.push(edge);
|
||||
}
|
||||
}
|
||||
|
||||
let mut out = String::new();
|
||||
out.push_str(&format!("nodes: {} edges: {}\n", nodes.len(), edges.len()));
|
||||
|
||||
for (ticker_id, mut rels) in ticker_to_entities {
|
||||
rels.sort_by_key(|edge| std::cmp::Reverse(edge.percentage_bps));
|
||||
let ticker_label = labels.get(ticker_id).map(|v| v.0).unwrap_or(ticker_id);
|
||||
out.push_str(&format!("\n{ticker_label} [TICKER]\n"));
|
||||
|
||||
for (idx, edge) in rels.iter().enumerate() {
|
||||
let holder_label = labels.get(edge.from.as_str()).map(|v| v.0).unwrap_or("?");
|
||||
let branch = if idx + 1 == rels.len() {
|
||||
"└─"
|
||||
} else {
|
||||
"├─"
|
||||
};
|
||||
out.push_str(&format!(
|
||||
"{branch} {holder_label} ({:.2}%, {})\n",
|
||||
edge.percentage_bps as f64 / 100.0,
|
||||
source_label(edge.source)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if edges.is_empty() {
|
||||
out.push_str("\n(no ownership edges found)\n");
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
/// Format graph as Graphviz DOT for export.
|
||||
pub fn format_graph_dot(nodes: &[GraphNode], edges: &[GraphEdge]) -> String {
|
||||
let mut out = String::new();
|
||||
out.push_str("digraph ownership {\n");
|
||||
out.push_str(" rankdir=LR;\n");
|
||||
out.push_str(" graph [fontname=\"Helvetica\"];\n");
|
||||
out.push_str(" node [fontname=\"Helvetica\"];\n");
|
||||
out.push_str(" edge [fontname=\"Helvetica\"];\n\n");
|
||||
|
||||
for node in nodes {
|
||||
let shape = match node.node_type {
|
||||
GraphNodeType::Entity => "ellipse",
|
||||
GraphNodeType::Ticker => "box",
|
||||
};
|
||||
out.push_str(&format!(
|
||||
" \"{}\" [label=\"{}\", shape={}];\n",
|
||||
escape_dot(&node.id),
|
||||
escape_dot(&node.label),
|
||||
shape
|
||||
));
|
||||
}
|
||||
|
||||
out.push('\n');
|
||||
for edge in edges {
|
||||
out.push_str(&format!(
|
||||
" \"{}\" -> \"{}\" [label=\"{:.2}% ({})\"];\n",
|
||||
escape_dot(&edge.from),
|
||||
escape_dot(&edge.to),
|
||||
edge.percentage_bps as f64 / 100.0,
|
||||
source_label(edge.source)
|
||||
));
|
||||
}
|
||||
|
||||
out.push_str("}\n");
|
||||
out
|
||||
}
|
||||
|
||||
fn detect_root_node(conn: &Connection, root: &str) -> Result<String, IdxError> {
|
||||
let maybe_ticker = conn
|
||||
.query_row(
|
||||
"SELECT code FROM tickers WHERE UPPER(code) = UPPER(?1) LIMIT 1",
|
||||
params![root],
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.ok();
|
||||
|
||||
if let Some(code) = maybe_ticker {
|
||||
return Ok(format!("ticker:{code}"));
|
||||
}
|
||||
|
||||
let q = format!("%{root}%");
|
||||
let maybe_entity_id = conn
|
||||
.query_row(
|
||||
"SELECT id
|
||||
FROM entities
|
||||
WHERE canonical_name LIKE ?1 COLLATE NOCASE
|
||||
ORDER BY LENGTH(canonical_name) ASC, canonical_name ASC
|
||||
LIMIT 1",
|
||||
params![q],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)
|
||||
.ok();
|
||||
|
||||
if let Some(entity_id) = maybe_entity_id {
|
||||
return Ok(format!("entity:{entity_id}"));
|
||||
}
|
||||
|
||||
Err(IdxError::ParseError(format!(
|
||||
"graph root not found as ticker or entity: {root}"
|
||||
)))
|
||||
}
|
||||
|
||||
fn query_all_edges(conn: &Connection) -> Result<Vec<GraphEdge>, IdxError> {
|
||||
let mut out = Vec::new();
|
||||
|
||||
if let Ok(release_sha) = conn.query_row(
|
||||
"SELECT sha256
|
||||
FROM ownership_releases
|
||||
ORDER BY as_of_date DESC, imported_at DESC
|
||||
LIMIT 1",
|
||||
[],
|
||||
|row| row.get::<_, String>(0),
|
||||
) {
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT 'entity:' || k.entity_id,
|
||||
'ticker:' || t.code,
|
||||
k.percentage_bps
|
||||
FROM ksei_holdings k
|
||||
JOIN tickers t ON t.id = k.ticker_id
|
||||
WHERE k.entity_id IS NOT NULL
|
||||
AND k.release_sha256 = ?1",
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
let rows = stmt
|
||||
.query_map(params![release_sha], |row| {
|
||||
Ok(GraphEdge {
|
||||
from: row.get(0)?,
|
||||
to: row.get(1)?,
|
||||
percentage_bps: row.get(2)?,
|
||||
source: OwnershipSource::Ksei,
|
||||
})
|
||||
})
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
for row in rows {
|
||||
out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"WITH latest_bing AS (
|
||||
SELECT ticker_id, MAX(report_date) AS report_date
|
||||
FROM bing_holdings
|
||||
GROUP BY ticker_id
|
||||
)
|
||||
SELECT 'entity:' || b.entity_id,
|
||||
'ticker:' || t.code,
|
||||
COALESCE(b.pct_ownership_bps, 0)
|
||||
FROM bing_holdings b
|
||||
JOIN latest_bing lb
|
||||
ON lb.ticker_id = b.ticker_id
|
||||
AND lb.report_date = b.report_date
|
||||
JOIN tickers t ON t.id = b.ticker_id
|
||||
WHERE b.entity_id IS NOT NULL
|
||||
AND b.signal = 'holder'",
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
let rows = stmt
|
||||
.query_map([], |row| {
|
||||
Ok(GraphEdge {
|
||||
from: row.get(0)?,
|
||||
to: row.get(1)?,
|
||||
percentage_bps: row.get(2)?,
|
||||
source: OwnershipSource::Bing,
|
||||
})
|
||||
})
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
for row in rows {
|
||||
out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
||||
}
|
||||
}
|
||||
|
||||
dedup_edges(out)
|
||||
}
|
||||
|
||||
fn dedup_edges(edges: Vec<GraphEdge>) -> Result<Vec<GraphEdge>, IdxError> {
|
||||
let mut seen: HashSet<(String, String, i64, &'static str)> = HashSet::new();
|
||||
let mut out = Vec::new();
|
||||
|
||||
for edge in edges {
|
||||
let key = (
|
||||
edge.from.clone(),
|
||||
edge.to.clone(),
|
||||
edge.percentage_bps,
|
||||
source_label(edge.source),
|
||||
);
|
||||
if seen.insert(key) {
|
||||
out.push(edge);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn build_nodes(conn: &Connection, node_ids: &BTreeSet<String>) -> Result<Vec<GraphNode>, IdxError> {
|
||||
let mut nodes = Vec::new();
|
||||
|
||||
for node_id in node_ids {
|
||||
if let Some(entity_id) = node_id.strip_prefix("entity:") {
|
||||
let entity_id_num = entity_id.parse::<i64>().map_err(|e| {
|
||||
IdxError::ParseError(format!("invalid entity node id '{node_id}': {e}"))
|
||||
})?;
|
||||
let label = conn
|
||||
.query_row(
|
||||
"SELECT canonical_name FROM entities WHERE id = ?1",
|
||||
params![entity_id_num],
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.unwrap_or_else(|_| format!("entity:{entity_id_num}"));
|
||||
nodes.push(GraphNode {
|
||||
id: node_id.clone(),
|
||||
label,
|
||||
node_type: GraphNodeType::Entity,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(code) = node_id.strip_prefix("ticker:") {
|
||||
nodes.push(GraphNode {
|
||||
id: node_id.clone(),
|
||||
label: code.to_string(),
|
||||
node_type: GraphNodeType::Ticker,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(nodes)
|
||||
}
|
||||
|
||||
fn source_label(source: OwnershipSource) -> &'static str {
|
||||
match source {
|
||||
OwnershipSource::Ksei => "ksei",
|
||||
OwnershipSource::Bing => "bing",
|
||||
}
|
||||
}
|
||||
|
||||
fn escape_dot(value: &str) -> String {
|
||||
value.replace('"', "\\\"")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use chrono::NaiveDate;
|
||||
use rusqlite::Connection;
|
||||
|
||||
use crate::ownership::db::{ensure_schema, upsert_ticker, write_ksei_release};
|
||||
use crate::ownership::types::{KseiHolding, OwnershipRelease};
|
||||
|
||||
use super::query_ownership_graph;
|
||||
|
||||
fn setup() -> Connection {
|
||||
let conn = Connection::open_in_memory().unwrap();
|
||||
ensure_schema(&conn).unwrap();
|
||||
conn
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn graph_uses_latest_ksei_release_only() {
|
||||
let conn = setup();
|
||||
let aa = upsert_ticker(&conn, "AA", None).unwrap();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO entities (canonical_name, entity_type, country, created_at, updated_at)
|
||||
VALUES ('OLDER HOLDER', NULL, NULL, 0, 0)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
let older_entity = conn.last_insert_rowid();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO entities (canonical_name, entity_type, country, created_at, updated_at)
|
||||
VALUES ('LATEST HOLDER', NULL, NULL, 0, 0)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
let latest_entity = conn.last_insert_rowid();
|
||||
|
||||
let old_release = OwnershipRelease {
|
||||
id: 0,
|
||||
source_url: None,
|
||||
sha256: "old-graph".to_string(),
|
||||
as_of_date: NaiveDate::from_ymd_opt(2026, 1, 30).unwrap(),
|
||||
row_count: 1,
|
||||
imported_at: 1,
|
||||
};
|
||||
let old_rows = vec![KseiHolding {
|
||||
id: 0,
|
||||
ticker_id: aa,
|
||||
entity_id: Some(older_entity),
|
||||
raw_investor_name: "OLDER HOLDER".to_string(),
|
||||
investor_type: None,
|
||||
locality: None,
|
||||
nationality: None,
|
||||
domicile: None,
|
||||
holdings_scripless: 1,
|
||||
holdings_scrip: 0,
|
||||
total_shares: 1,
|
||||
percentage_bps: 1000,
|
||||
report_date: old_release.as_of_date,
|
||||
release_sha256: old_release.sha256.clone(),
|
||||
}];
|
||||
write_ksei_release(&conn, &old_release, &old_rows, false).unwrap();
|
||||
|
||||
let latest_release = OwnershipRelease {
|
||||
id: 0,
|
||||
source_url: None,
|
||||
sha256: "latest-graph".to_string(),
|
||||
as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
||||
row_count: 1,
|
||||
imported_at: 2,
|
||||
};
|
||||
let latest_rows = vec![KseiHolding {
|
||||
id: 0,
|
||||
ticker_id: aa,
|
||||
entity_id: Some(latest_entity),
|
||||
raw_investor_name: "LATEST HOLDER".to_string(),
|
||||
investor_type: None,
|
||||
locality: None,
|
||||
nationality: None,
|
||||
domicile: None,
|
||||
holdings_scripless: 1,
|
||||
holdings_scrip: 0,
|
||||
total_shares: 1,
|
||||
percentage_bps: 1200,
|
||||
report_date: latest_release.as_of_date,
|
||||
release_sha256: latest_release.sha256.clone(),
|
||||
}];
|
||||
write_ksei_release(&conn, &latest_release, &latest_rows, false).unwrap();
|
||||
|
||||
let (nodes, edges) = query_ownership_graph(&conn, "AA", 1).unwrap();
|
||||
assert!(nodes.iter().any(|node| node.label == "LATEST HOLDER"));
|
||||
assert!(!nodes.iter().any(|node| node.label == "OLDER HOLDER"));
|
||||
assert_eq!(edges.len(), 1);
|
||||
assert_eq!(edges[0].percentage_bps, 1200);
|
||||
}
|
||||
}
|
||||
10
crates/idx-core/src/ownership/mod.rs
Normal file
10
crates/idx-core/src/ownership/mod.rs
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
pub mod archive;
|
||||
pub mod db;
|
||||
pub mod entities;
|
||||
pub mod graph;
|
||||
pub mod parser;
|
||||
pub mod remote;
|
||||
pub mod search;
|
||||
pub mod snapshot;
|
||||
pub mod types;
|
||||
pub mod xlsx;
|
||||
862
crates/idx-core/src/ownership/parser.rs
Normal file
862
crates/idx-core/src/ownership/parser.rs
Normal file
|
|
@ -0,0 +1,862 @@
|
|||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
use quick_xml::Reader;
|
||||
use quick_xml::events::{BytesStart, Event};
|
||||
|
||||
use crate::error::IdxError;
|
||||
use crate::ownership::types::KseiRawRow;
|
||||
|
||||
const Y_TOLERANCE: f32 = 0.8;
|
||||
const HEADER_MATCH_MIN: usize = 4;
|
||||
|
||||
const HEADER_LABELS: &[&str] = &[
|
||||
"DATE",
|
||||
"SHARECODE",
|
||||
"ISSUERNAME",
|
||||
"INVESTORNAME",
|
||||
"INVESTORTYPE",
|
||||
"INVESTORCLASSIFICATION",
|
||||
"LOCALFOREIGN",
|
||||
"NATIONALITY",
|
||||
"DOMICILE",
|
||||
"HOLDINGSSCRIPLESS",
|
||||
"HOLDINGSSCRIP",
|
||||
"TOTALHOLDINGSHARES",
|
||||
"PERCENTAGE",
|
||||
];
|
||||
|
||||
const HOLDER_REGISTER_SCHEMA_MARKERS: &[&str] = &[
|
||||
"TEXT=\"DATE\"",
|
||||
"TEXT=\"SHARE_CODE\"",
|
||||
"TEXT=\"INVESTOR_NAME\"",
|
||||
"TEXT=\"INVESTOR_TYPE\"",
|
||||
"TEXT=\"INVESTOR_CLASSIFICATION\"",
|
||||
"TEXT=\"LOCAL_FOREIGN\"",
|
||||
"TEXT=\"TOTAL_HOLDING_SHARES\"",
|
||||
"TEXT=\"PERCENTAGE\"",
|
||||
];
|
||||
const ANNOUNCEMENT_WRAPPER_SCHEMA_MARKERS: &[&str] =
|
||||
&["TEXT=\"PENGUMUMAN\"", "PT_BURSA_EFEK_INDONESIA_(BEI)"];
|
||||
const ABOVE_FIVE_SCHEMA_MARKERS: &[&str] =
|
||||
&["TEXT=\"INVS\"", "REKENING_TAMPUNGAN_KSEI", "CLOSED_MEMBER-"];
|
||||
const INVESTOR_TYPE_SCHEMA_MARKERS: &[&str] = &[
|
||||
"TEXT=\"STOCK_CODE\"",
|
||||
"TEXT=\"NUMBER_OF_SHARES\"",
|
||||
"TEXT=\"FOREIGN\"",
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum OwnershipPdfSchema {
|
||||
HolderRegister,
|
||||
AnnouncementWrapper,
|
||||
LegacyAboveFivePercent,
|
||||
LegacyInvestorType,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl OwnershipPdfSchema {
|
||||
pub fn label(self) -> &'static str {
|
||||
match self {
|
||||
Self::HolderRegister => "holder_register",
|
||||
Self::AnnouncementWrapper => "announcement_wrapper",
|
||||
Self::LegacyAboveFivePercent => "legacy_above5",
|
||||
Self::LegacyInvestorType => "legacy_investor_type",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct PageLine {
|
||||
x: f32,
|
||||
y: f32,
|
||||
text: String,
|
||||
}
|
||||
|
||||
/// Extract mutool stext XML from a PDF file.
|
||||
pub fn extract_pdf_stext(path: &Path) -> Result<String, IdxError> {
|
||||
check_mutool()?;
|
||||
|
||||
let output = Command::new("mutool")
|
||||
.arg("convert")
|
||||
.arg("-F")
|
||||
.arg("stext")
|
||||
.arg("-o")
|
||||
.arg("-")
|
||||
.arg(path)
|
||||
.output()
|
||||
.map_err(|e| IdxError::PdfParseError(format!("failed to run mutool: {e}")))?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
return Err(IdxError::PdfParseError(format!(
|
||||
"mutool convert failed: {}",
|
||||
stderr.trim()
|
||||
)));
|
||||
}
|
||||
|
||||
let xml = String::from_utf8(output.stdout)
|
||||
.map_err(|e| IdxError::PdfParseError(format!("invalid utf-8 stext output: {e}")))?;
|
||||
|
||||
Ok(xml)
|
||||
}
|
||||
|
||||
/// Classify a PDF schema from mutool stext XML before the row parser runs.
|
||||
pub fn classify_stext_xml(xml: &str) -> OwnershipPdfSchema {
|
||||
let normalized = normalize_stext_for_classification(xml);
|
||||
|
||||
if count_schema_markers(&normalized, HOLDER_REGISTER_SCHEMA_MARKERS) >= 5 {
|
||||
return OwnershipPdfSchema::HolderRegister;
|
||||
}
|
||||
if count_schema_markers(&normalized, ABOVE_FIVE_SCHEMA_MARKERS) >= 2 {
|
||||
return OwnershipPdfSchema::LegacyAboveFivePercent;
|
||||
}
|
||||
if count_schema_markers(&normalized, INVESTOR_TYPE_SCHEMA_MARKERS) >= 3 {
|
||||
return OwnershipPdfSchema::LegacyInvestorType;
|
||||
}
|
||||
if count_schema_markers(&normalized, ANNOUNCEMENT_WRAPPER_SCHEMA_MARKERS) >= 2 {
|
||||
return OwnershipPdfSchema::AnnouncementWrapper;
|
||||
}
|
||||
|
||||
OwnershipPdfSchema::Unknown
|
||||
}
|
||||
|
||||
/// Normalize stext XML for schema classification.
|
||||
///
|
||||
/// Newer KSEI PDFs use spaces in header text (e.g. `SHARE CODE` instead of
|
||||
/// `SHARE_CODE`, `INVESTOR CLASSIFICATION` instead of `INVESTOR_TYPE`). The
|
||||
/// schema markers use the underscored form, so we replace spaces with
|
||||
/// underscores inside `text="..."` attribute values to match both layouts.
|
||||
fn normalize_stext_for_classification(xml: &str) -> String {
|
||||
let upper = xml.to_ascii_uppercase();
|
||||
let mut result = String::with_capacity(upper.len());
|
||||
let bytes = upper.as_bytes();
|
||||
let mut i = 0;
|
||||
|
||||
while i < bytes.len() {
|
||||
if bytes[i..].starts_with(b"TEXT=\"") {
|
||||
result.push_str("TEXT=\"");
|
||||
i += 6; // skip past TEXT="
|
||||
// Collect everything until the closing quote.
|
||||
while i < bytes.len() && bytes[i] != b'"' {
|
||||
let ch = bytes[i] as char;
|
||||
result.push(if ch == ' ' { '_' } else { ch });
|
||||
i += 1;
|
||||
}
|
||||
if i < bytes.len() {
|
||||
result.push('"');
|
||||
i += 1; // skip closing quote
|
||||
}
|
||||
} else {
|
||||
result.push(bytes[i] as char);
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Parse a KSEI ownership PDF into raw rows.
|
||||
/// Shells out to `mutool` for XML extraction, classifies the schema,
|
||||
/// and only parses the supported holder-register layout.
|
||||
pub fn parse_ksei_pdf(path: &Path) -> Result<Vec<KseiRawRow>, IdxError> {
|
||||
let xml = extract_pdf_stext(path)?;
|
||||
ensure_supported_schema(classify_stext_xml(&xml))?;
|
||||
parse_stext_xml(&xml)
|
||||
}
|
||||
|
||||
/// Parse mutool stext XML output into raw rows.
|
||||
/// Pure function — takes XML string, returns parsed rows.
|
||||
pub fn parse_stext_xml(xml: &str) -> Result<Vec<KseiRawRow>, IdxError> {
|
||||
let mut reader = Reader::from_str(xml);
|
||||
reader.config_mut().trim_text(false);
|
||||
|
||||
let mut rows: Vec<KseiRawRow> = Vec::new();
|
||||
let mut current_page: Option<Vec<PageLine>> = None;
|
||||
let mut buf = Vec::new();
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(e)) if e.name().as_ref() == b"page" => {
|
||||
current_page = Some(Vec::new());
|
||||
}
|
||||
Ok(Event::Start(e)) if e.name().as_ref() == b"line" => {
|
||||
if let Some(page) = current_page.as_mut()
|
||||
&& let Some(line) = parse_line_attrs(&reader, &e)?
|
||||
{
|
||||
page.push(line);
|
||||
}
|
||||
}
|
||||
Ok(Event::End(e)) if e.name().as_ref() == b"page" => {
|
||||
if let Some(page) = current_page.take() {
|
||||
rows.extend(extract_rows_from_page(&page));
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
Err(err) => {
|
||||
return Err(IdxError::PdfParseError(format!(
|
||||
"failed to parse stext XML: {err}"
|
||||
)));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Check if mutool is available in PATH.
|
||||
pub fn check_mutool() -> Result<(), IdxError> {
|
||||
// mutool with no args prints usage to stderr and exits non-zero,
|
||||
// so we just check that the binary is found and executable.
|
||||
Command::new("mutool")
|
||||
.arg("--help")
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.map_err(|e| IdxError::PdfParseError(format!("mutool not found in PATH: {e}")))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_supported_schema(schema: OwnershipPdfSchema) -> Result<(), IdxError> {
|
||||
match schema {
|
||||
OwnershipPdfSchema::HolderRegister => Ok(()),
|
||||
OwnershipPdfSchema::AnnouncementWrapper => Err(IdxError::Unsupported(
|
||||
"IDX announcement wrapper PDFs are not importable; run `idx ownership discover` and use the `lamp1` attachment URL".to_string(),
|
||||
)),
|
||||
OwnershipPdfSchema::LegacyAboveFivePercent => Err(IdxError::Unsupported(
|
||||
"legacy IDX `above5` ownership PDFs are not supported for import; only the `above1` holder-register `lamp1` attachment is supported".to_string(),
|
||||
)),
|
||||
OwnershipPdfSchema::LegacyInvestorType => Err(IdxError::Unsupported(
|
||||
"legacy IDX `investor-type` ownership PDFs are not supported for import; only the `above1` holder-register `lamp1` attachment is supported".to_string(),
|
||||
)),
|
||||
OwnershipPdfSchema::Unknown => Err(IdxError::ParseError(
|
||||
"PDF did not match the supported KSEI holder-register layout".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn count_schema_markers(haystack: &str, markers: &[&str]) -> usize {
|
||||
markers
|
||||
.iter()
|
||||
.filter(|marker| haystack.contains(**marker))
|
||||
.count()
|
||||
}
|
||||
|
||||
fn parse_line_attrs(
|
||||
reader: &Reader<&[u8]>,
|
||||
event: &BytesStart<'_>,
|
||||
) -> Result<Option<PageLine>, IdxError> {
|
||||
let mut bbox: Option<String> = None;
|
||||
let mut text: Option<String> = None;
|
||||
|
||||
for attr_result in event.attributes().with_checks(false) {
|
||||
let attr = attr_result
|
||||
.map_err(|err| IdxError::PdfParseError(format!("invalid XML attribute: {err}")))?;
|
||||
|
||||
match attr.key.as_ref() {
|
||||
b"bbox" => {
|
||||
bbox = Some(
|
||||
attr.decode_and_unescape_value(reader.decoder())
|
||||
.map_err(|err| {
|
||||
IdxError::PdfParseError(format!("invalid XML bbox attribute: {err}"))
|
||||
})?
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
b"text" => {
|
||||
text = Some(
|
||||
attr.decode_and_unescape_value(reader.decoder())
|
||||
.map_err(|err| {
|
||||
IdxError::PdfParseError(format!("invalid XML text attribute: {err}"))
|
||||
})?
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let Some(text) = text.map(|value| normalize_spaces(&value)) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if text.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let Some(bbox) = bbox else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut parts = bbox.split_whitespace();
|
||||
let x = parts.next().and_then(|value| value.parse::<f32>().ok());
|
||||
let y = parts.next().and_then(|value| value.parse::<f32>().ok());
|
||||
|
||||
match (x, y) {
|
||||
(Some(x), Some(y)) => Ok(Some(PageLine { x, y, text })),
|
||||
_ => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_rows_from_page(page: &[PageLine]) -> Vec<KseiRawRow> {
|
||||
let mut rows_by_y: HashMap<i32, Vec<PageLine>> = HashMap::new();
|
||||
for line in page {
|
||||
rows_by_y
|
||||
.entry(y_bucket(line.y))
|
||||
.or_default()
|
||||
.push(line.clone());
|
||||
}
|
||||
|
||||
let mut y_keys: Vec<i32> = rows_by_y.keys().copied().collect();
|
||||
y_keys.sort_unstable();
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for y in y_keys {
|
||||
let Some(lines) = rows_by_y.get(&y) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let mut sorted = lines.clone();
|
||||
sorted.sort_by(|left, right| left.x.total_cmp(&right.x));
|
||||
|
||||
let texts: Vec<String> = sorted.into_iter().map(|line| line.text).collect();
|
||||
if is_header_row(&texts) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let row = parse_row_segments(&texts);
|
||||
if is_data_row(&row) {
|
||||
rows.push(row);
|
||||
}
|
||||
}
|
||||
|
||||
rows
|
||||
}
|
||||
|
||||
fn is_header_row(texts: &[String]) -> bool {
|
||||
texts
|
||||
.iter()
|
||||
.filter(|text| HEADER_LABELS.contains(&normalize_header_label(text).as_str()))
|
||||
.count()
|
||||
>= HEADER_MATCH_MIN
|
||||
}
|
||||
|
||||
fn parse_row_segments(texts: &[String]) -> KseiRawRow {
|
||||
let mut row = KseiRawRow {
|
||||
date: String::new(),
|
||||
share_code: String::new(),
|
||||
issuer_name: String::new(),
|
||||
investor_name: String::new(),
|
||||
investor_type: String::new(),
|
||||
local_foreign: String::new(),
|
||||
nationality: String::new(),
|
||||
domicile: String::new(),
|
||||
holdings_scripless: String::new(),
|
||||
holdings_scrip: String::new(),
|
||||
total_holding_shares: String::new(),
|
||||
percentage: String::new(),
|
||||
};
|
||||
|
||||
let mut remaining: Vec<String> = texts
|
||||
.iter()
|
||||
.map(|text| normalize_spaces(text))
|
||||
.filter(|text| !text.is_empty())
|
||||
.collect();
|
||||
if remaining.is_empty() {
|
||||
return row;
|
||||
}
|
||||
|
||||
if let Some((date, share_code)) = split_date_and_share(&remaining[0]) {
|
||||
row.date = date;
|
||||
row.share_code = share_code;
|
||||
let _ = remaining.remove(0);
|
||||
}
|
||||
|
||||
if row.date.is_empty() {
|
||||
return row;
|
||||
}
|
||||
|
||||
if row.share_code.is_empty()
|
||||
&& remaining
|
||||
.first()
|
||||
.is_some_and(|segment| is_share_code_like(segment))
|
||||
{
|
||||
row.share_code = remaining.remove(0);
|
||||
}
|
||||
|
||||
if remaining
|
||||
.last()
|
||||
.is_some_and(|segment| is_percentage_like(segment))
|
||||
{
|
||||
row.percentage = remaining.pop().unwrap_or_default();
|
||||
}
|
||||
let numeric_tail = pop_numeric_tail(&mut remaining);
|
||||
match numeric_tail.as_slice() {
|
||||
[scripless, scrip, total] => {
|
||||
row.holdings_scripless = scripless.clone();
|
||||
row.holdings_scrip = scrip.clone();
|
||||
row.total_holding_shares = total.clone();
|
||||
}
|
||||
// Some PDFs omit the zero-valued scrip column entirely.
|
||||
[scripless, total] => {
|
||||
row.holdings_scripless = scripless.clone();
|
||||
row.holdings_scrip = "0".to_string();
|
||||
row.total_holding_shares = total.clone();
|
||||
}
|
||||
[total] => {
|
||||
row.total_holding_shares = total.clone();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
let mut geo_fields = Vec::new();
|
||||
while remaining.len() > 2 {
|
||||
let Some(candidate) = remaining.last().cloned() else {
|
||||
break;
|
||||
};
|
||||
|
||||
if row.local_foreign.is_empty() && is_locality_marker(&candidate) {
|
||||
row.local_foreign = remaining.pop().unwrap_or_default();
|
||||
continue;
|
||||
}
|
||||
|
||||
if row.investor_type.is_empty() && is_investor_type_marker(&candidate) {
|
||||
row.investor_type = remaining.pop().unwrap_or_default();
|
||||
continue;
|
||||
}
|
||||
|
||||
if geo_fields.len() < 2 {
|
||||
geo_fields.push(remaining.pop().unwrap_or_default());
|
||||
continue;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
geo_fields.reverse();
|
||||
if let Some(first) = geo_fields.first() {
|
||||
row.nationality = first.clone();
|
||||
}
|
||||
if geo_fields.len() > 1 {
|
||||
row.domicile = geo_fields[1..].join(" ");
|
||||
}
|
||||
|
||||
if let Some(first) = remaining.first() {
|
||||
row.issuer_name = first.clone();
|
||||
}
|
||||
if remaining.len() > 1 {
|
||||
row.investor_name = remaining[1..].join(" ");
|
||||
}
|
||||
|
||||
row
|
||||
}
|
||||
|
||||
fn split_date_and_share(segment: &str) -> Option<(String, String)> {
|
||||
let trimmed = segment.trim();
|
||||
if trimmed.len() < 11 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let date = trimmed.get(..11)?.to_string();
|
||||
if !is_ksei_date(&date) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let share_code = trimmed.get(11..).unwrap_or_default().trim().to_string();
|
||||
Some((date, share_code))
|
||||
}
|
||||
|
||||
fn is_share_code_like(value: &str) -> bool {
|
||||
let trimmed = value.trim();
|
||||
(3..=8).contains(&trimmed.len())
|
||||
&& trimmed
|
||||
.chars()
|
||||
.all(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit())
|
||||
}
|
||||
|
||||
fn pop_numeric_tail(remaining: &mut Vec<String>) -> Vec<String> {
|
||||
let mut numeric_tail = Vec::new();
|
||||
while numeric_tail.len() < 3
|
||||
&& remaining
|
||||
.last()
|
||||
.is_some_and(|segment| is_id_number_like(segment))
|
||||
{
|
||||
numeric_tail.push(remaining.pop().unwrap_or_default());
|
||||
}
|
||||
numeric_tail.reverse();
|
||||
numeric_tail
|
||||
}
|
||||
|
||||
fn is_id_number_like(value: &str) -> bool {
|
||||
let trimmed = value.trim();
|
||||
!trimmed.is_empty()
|
||||
&& trimmed != "-"
|
||||
&& trimmed.chars().all(|ch| ch.is_ascii_digit() || ch == '.')
|
||||
}
|
||||
|
||||
fn is_investor_type_marker(value: &str) -> bool {
|
||||
let trimmed = value.trim();
|
||||
!trimmed.is_empty() && trimmed.len() <= 4 && trimmed.chars().all(|ch| ch.is_ascii_uppercase())
|
||||
}
|
||||
|
||||
fn is_locality_marker(value: &str) -> bool {
|
||||
matches!(
|
||||
value.trim().to_ascii_uppercase().as_str(),
|
||||
"L" | "F" | "A" | "D" | "LOCAL" | "FOREIGN" | "ASING" | "DOMESTIC"
|
||||
)
|
||||
}
|
||||
|
||||
fn normalize_header_label(text: &str) -> String {
|
||||
let mut normalized = String::new();
|
||||
for ch in text.chars() {
|
||||
if ch.is_ascii_alphanumeric() {
|
||||
normalized.push(ch.to_ascii_uppercase());
|
||||
}
|
||||
}
|
||||
normalized
|
||||
}
|
||||
|
||||
fn y_bucket(y: f32) -> i32 {
|
||||
(y / Y_TOLERANCE).round() as i32
|
||||
}
|
||||
|
||||
fn normalize_spaces(input: &str) -> String {
|
||||
let mut out = String::with_capacity(input.len());
|
||||
let mut prev_space = false;
|
||||
|
||||
for ch in input.chars() {
|
||||
if ch.is_whitespace() {
|
||||
if !prev_space {
|
||||
out.push(' ');
|
||||
}
|
||||
prev_space = true;
|
||||
} else {
|
||||
out.push(ch);
|
||||
prev_space = false;
|
||||
}
|
||||
}
|
||||
|
||||
out.trim().to_string()
|
||||
}
|
||||
|
||||
fn is_data_row(row: &KseiRawRow) -> bool {
|
||||
is_ksei_date(&row.date)
|
||||
&& is_percentage_like(&row.percentage)
|
||||
&& !row.share_code.trim().is_empty()
|
||||
&& !row.investor_name.trim().is_empty()
|
||||
}
|
||||
|
||||
fn is_ksei_date(s: &str) -> bool {
|
||||
if s.len() != 11 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut parts = s.split('-');
|
||||
let day = parts.next();
|
||||
let mon = parts.next();
|
||||
let year = parts.next();
|
||||
|
||||
if parts.next().is_some() {
|
||||
return false;
|
||||
}
|
||||
|
||||
match (day, mon, year) {
|
||||
(Some(d), Some(m), Some(y)) => {
|
||||
d.len() == 2
|
||||
&& d.chars().all(|c| c.is_ascii_digit())
|
||||
&& m.len() == 3
|
||||
&& m.chars().all(|c| c.is_ascii_alphabetic())
|
||||
&& y.len() == 4
|
||||
&& y.chars().all(|c| c.is_ascii_digit())
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_percentage_like(s: &str) -> bool {
|
||||
let cleaned = s.trim();
|
||||
if cleaned.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut parts = cleaned.split(',');
|
||||
let left = parts.next();
|
||||
let right = parts.next();
|
||||
|
||||
if parts.next().is_some() {
|
||||
return false;
|
||||
}
|
||||
|
||||
match (left, right) {
|
||||
(Some(l), Some(r)) => {
|
||||
!l.is_empty()
|
||||
&& l.chars().all(|c| c.is_ascii_digit())
|
||||
&& !r.is_empty()
|
||||
&& r.chars().all(|c| c.is_ascii_digit())
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::path::Path;
|
||||
|
||||
use crate::ownership::entities::normalize_ksei_row;
|
||||
|
||||
use super::{
|
||||
OwnershipPdfSchema, check_mutool, classify_stext_xml, parse_ksei_pdf, parse_row_segments,
|
||||
parse_stext_xml,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn test_parse_stext_xml_live_like_lines_extract_rows() {
|
||||
let xml = build_live_like_stext_xml();
|
||||
|
||||
let rows = parse_stext_xml(&xml).expect("failed to parse live-like fixture XML");
|
||||
assert_eq!(rows.len(), 3);
|
||||
|
||||
let first = &rows[0];
|
||||
assert_eq!(first.date, "27-Feb-2026");
|
||||
assert_eq!(first.share_code, "AADI");
|
||||
assert_eq!(first.issuer_name, "ADARO ANDALAN INDONESIA Tbk");
|
||||
assert_eq!(first.investor_name, "ADARO STRATEGIC INVESTMENTS");
|
||||
assert_eq!(first.investor_type, "CP");
|
||||
assert_eq!(first.local_foreign, "D");
|
||||
assert_eq!(first.nationality, "INDONESIA");
|
||||
assert_eq!(first.holdings_scripless, "3.200.142.830");
|
||||
assert_eq!(first.holdings_scrip, "0");
|
||||
assert_eq!(first.total_holding_shares, "3.200.142.830");
|
||||
assert_eq!(first.percentage, "66,18");
|
||||
|
||||
let last = &rows[2];
|
||||
assert_eq!(last.share_code, "BBRI");
|
||||
assert_eq!(last.investor_name, "PT NUSANTARA CAPITAL");
|
||||
assert_eq!(last.investor_type, "ID");
|
||||
assert_eq!(last.local_foreign, "A");
|
||||
assert_eq!(last.nationality, "SINGAPORE");
|
||||
assert_eq!(last.percentage, "15,00");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_stext_xml_real_above1_excerpt_extracts_row() {
|
||||
let xml = include_str!("../../tests/fixtures/ksei_above1_stext_excerpt.xml");
|
||||
|
||||
let rows = parse_stext_xml(xml).expect("failed to parse real above1 excerpt XML");
|
||||
assert_eq!(rows.len(), 1);
|
||||
|
||||
let row = &rows[0];
|
||||
assert_eq!(row.date, "27-Feb-2026");
|
||||
assert_eq!(row.share_code, "AADI");
|
||||
assert_eq!(row.issuer_name, "ADARO ANDALAN INDONESIA Tbk");
|
||||
assert_eq!(row.investor_name, "ADARO STRATEGIC INVESTMENTS");
|
||||
assert_eq!(row.investor_type, "CP");
|
||||
assert_eq!(row.local_foreign, "D");
|
||||
assert_eq!(row.nationality, "INDONESIA");
|
||||
assert_eq!(row.holdings_scripless, "3.200.142.830");
|
||||
assert_eq!(row.holdings_scrip, "0");
|
||||
assert_eq!(row.total_holding_shares, "3.200.142.830");
|
||||
assert_eq!(row.percentage, "41,10");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_row_segments_missing_scrip_defaults_to_zero() {
|
||||
let texts = vec![
|
||||
"27-Apr-2026 AADI".to_string(),
|
||||
"ADARO ANDALAN INDONESIA Tbk".to_string(),
|
||||
"ADARO STRATEGIC INVESTMENTS".to_string(),
|
||||
"CP".to_string(),
|
||||
"D".to_string(),
|
||||
"INDONESIA".to_string(),
|
||||
"3.200.142.830".to_string(),
|
||||
"3.200.142.830".to_string(),
|
||||
"66,18".to_string(),
|
||||
];
|
||||
|
||||
let row = parse_row_segments(&texts);
|
||||
assert_eq!(row.date, "27-Apr-2026");
|
||||
assert_eq!(row.share_code, "AADI");
|
||||
assert_eq!(row.holdings_scripless, "3.200.142.830");
|
||||
assert_eq!(row.holdings_scrip, "0");
|
||||
assert_eq!(row.total_holding_shares, "3.200.142.830");
|
||||
assert_eq!(row.percentage, "66,18");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_row_segments_missing_scrip_normalizes_without_error() {
|
||||
let texts = vec![
|
||||
"27-Apr-2026 AADI".to_string(),
|
||||
"ADARO ANDALAN INDONESIA Tbk".to_string(),
|
||||
"ADARO STRATEGIC INVESTMENTS".to_string(),
|
||||
"CP".to_string(),
|
||||
"D".to_string(),
|
||||
"INDONESIA".to_string(),
|
||||
"3.200.142.830".to_string(),
|
||||
"3.200.142.830".to_string(),
|
||||
"66,18".to_string(),
|
||||
];
|
||||
|
||||
let row = parse_row_segments(&texts);
|
||||
let draft = normalize_ksei_row(&row).expect("missing scrip column should normalize");
|
||||
|
||||
assert_eq!(draft.holdings_scripless, 3_200_142_830);
|
||||
assert_eq!(draft.holdings_scrip, 0);
|
||||
assert_eq!(draft.total_shares, 3_200_142_830);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_stext_xml_detects_supported_holder_register_schema() {
|
||||
let xml = include_str!("../../tests/fixtures/ksei_above1_stext_excerpt.xml");
|
||||
assert_eq!(classify_stext_xml(xml), OwnershipPdfSchema::HolderRegister);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_stext_xml_detects_announcement_wrapper_schema() {
|
||||
let xml = include_str!("../../tests/fixtures/ksei_announcement_wrapper_stext_excerpt.xml");
|
||||
assert_eq!(
|
||||
classify_stext_xml(xml),
|
||||
OwnershipPdfSchema::AnnouncementWrapper
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_stext_xml_detects_spaced_holder_register_schema() {
|
||||
let xml = include_str!("../../tests/fixtures/ksei_above1_spaced_stext_excerpt.xml");
|
||||
assert_eq!(classify_stext_xml(xml), OwnershipPdfSchema::HolderRegister);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_stext_xml_detects_legacy_above5_schema() {
|
||||
let xml = include_str!("../../tests/fixtures/ksei_above5_stext_excerpt.xml");
|
||||
assert_eq!(
|
||||
classify_stext_xml(xml),
|
||||
OwnershipPdfSchema::LegacyAboveFivePercent
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_stext_xml_detects_legacy_investor_type_schema() {
|
||||
let xml = include_str!("../../tests/fixtures/ksei_investor_type_stext_excerpt.xml");
|
||||
assert_eq!(
|
||||
classify_stext_xml(xml),
|
||||
OwnershipPdfSchema::LegacyInvestorType
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_ksei_pdf_real_file_row_count() {
|
||||
if check_mutool().is_err() {
|
||||
eprintln!("skipping mutool-dependent test: mutool not available");
|
||||
return;
|
||||
}
|
||||
|
||||
let pdf_path =
|
||||
Path::new("/var/lib/openclaw/projects/idx-cli/research/ownership_202603.pdf");
|
||||
if !pdf_path.exists() {
|
||||
eprintln!("skipping mutool-dependent test: sample PDF not found");
|
||||
return;
|
||||
}
|
||||
|
||||
let rows = parse_ksei_pdf(pdf_path).expect("failed to parse real KSEI PDF");
|
||||
assert!(
|
||||
rows.len() >= 7_200,
|
||||
"expected at least 7200 rows, got {}",
|
||||
rows.len()
|
||||
);
|
||||
}
|
||||
|
||||
fn build_live_like_stext_xml() -> String {
|
||||
let mut xml = String::from(r#"<?xml version="1.0"?><document>"#);
|
||||
|
||||
append_page(
|
||||
&mut xml,
|
||||
"page1",
|
||||
&[
|
||||
(31.56, 86.33, "DATE"),
|
||||
(56.64, 86.33, "SHARE_CODE"),
|
||||
(121.22, 86.33, "ISSUER_NAME"),
|
||||
(267.17, 86.33, "INVESTOR_NAME"),
|
||||
(390.89, 86.33, "INVESTOR_TYPE"),
|
||||
(434.11, 86.33, "LOCAL_FOREIGN"),
|
||||
(475.87, 86.33, "NATIONALITY"),
|
||||
(525.19, 86.33, "DOMICILE"),
|
||||
(574.63, 86.33, "HOLDINGS_SCRIPLESS"),
|
||||
(629.98, 86.33, "HOLDINGS_SCRIP"),
|
||||
(680.02, 86.33, "TOTAL_HOLDING_SHARES"),
|
||||
(741.22, 86.33, "PERCENTAGE"),
|
||||
(28.68, 91.01, "27-Feb-2026 AADI"),
|
||||
(85.10, 91.01, "ADARO ANDALAN INDONESIA Tbk"),
|
||||
(179.30, 91.01, "ADARO STRATEGIC INVESTMENTS"),
|
||||
(381.41, 91.01, "CP"),
|
||||
(424.75, 91.01, "D"),
|
||||
(504.07, 91.01, "INDONESIA"),
|
||||
(597.58, 91.01, "3.200.142.830"),
|
||||
(630.10, 91.01, "0"),
|
||||
(680.12, 91.01, "3.200.142.830"),
|
||||
(741.30, 91.01, "66,18"),
|
||||
(28.68, 96.01, "27-Feb-2026 AADI"),
|
||||
(85.10, 96.01, "ADARO ANDALAN INDONESIA Tbk"),
|
||||
(179.30, 96.01, "PUBLIC"),
|
||||
(381.41, 96.01, "OT"),
|
||||
(424.75, 96.01, "A"),
|
||||
(504.07, 96.01, "SINGAPORE"),
|
||||
(597.58, 96.01, "500.000.000"),
|
||||
(630.10, 96.01, "0"),
|
||||
(680.12, 96.01, "500.000.000"),
|
||||
(741.30, 96.01, "10,34"),
|
||||
],
|
||||
);
|
||||
|
||||
append_page(
|
||||
&mut xml,
|
||||
"page2",
|
||||
&[
|
||||
(28.68, 20.00, "27-Feb-2026 BBRI"),
|
||||
(85.10, 20.00, "BANK RAKYAT INDONESIA Tbk"),
|
||||
(179.30, 20.00, "PT NUSANTARA CAPITAL"),
|
||||
(381.41, 20.00, "ID"),
|
||||
(424.75, 20.00, "A"),
|
||||
(504.07, 20.00, "SINGAPORE"),
|
||||
(597.58, 20.00, "1.250.000.000"),
|
||||
(630.10, 20.00, "0"),
|
||||
(680.12, 20.00, "1.250.000.000"),
|
||||
(741.30, 20.00, "15,00"),
|
||||
],
|
||||
);
|
||||
|
||||
xml.push_str("</document>");
|
||||
xml
|
||||
}
|
||||
|
||||
fn append_page(xml: &mut String, id: &str, lines: &[(f32, f32, &str)]) {
|
||||
xml.push_str(&format!(r#"<page id="{id}" width="792" height="612">"#));
|
||||
for (x, y, text) in lines {
|
||||
append_line(xml, *x, *y, text);
|
||||
}
|
||||
xml.push_str("</page>");
|
||||
}
|
||||
|
||||
fn append_line(xml: &mut String, x: f32, y: f32, text: &str) {
|
||||
let width = x + (text.len() as f32 * 2.0);
|
||||
let height = y + 3.48;
|
||||
xml.push_str(&format!(
|
||||
r#"<line bbox="{x:.2} {y:.2} {width:.2} {height:.2}" text="{}"></line>"#,
|
||||
escape_xml_attr(text)
|
||||
));
|
||||
}
|
||||
|
||||
fn escape_xml_attr(text: &str) -> String {
|
||||
text.chars()
|
||||
.map(|ch| match ch {
|
||||
'&' => "&".to_string(),
|
||||
'<' => "<".to_string(),
|
||||
'>' => ">".to_string(),
|
||||
'"' => """.to_string(),
|
||||
'\'' => "'".to_string(),
|
||||
other => other.to_string(),
|
||||
})
|
||||
.collect::<String>()
|
||||
}
|
||||
}
|
||||
1214
crates/idx-core/src/ownership/remote.rs
Normal file
1214
crates/idx-core/src/ownership/remote.rs
Normal file
File diff suppressed because it is too large
Load diff
111
crates/idx-core/src/ownership/search.rs
Normal file
111
crates/idx-core/src/ownership/search.rs
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
use rusqlite::{Connection, params};
|
||||
|
||||
use crate::error::IdxError;
|
||||
use crate::ownership::types::Entity;
|
||||
|
||||
pub fn fts_search(conn: &Connection, query: &str, limit: usize) -> Result<Vec<Entity>, IdxError> {
|
||||
let q = query.trim();
|
||||
if q.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let limit_i64 =
|
||||
i64::try_from(limit).map_err(|e| IdxError::DatabaseError(format!("invalid limit: {e}")))?;
|
||||
|
||||
// Ensure entity_fts has content. For external-content FTS table, populate manually.
|
||||
let fts_count: i64 = conn
|
||||
.query_row("SELECT COUNT(*) FROM entity_fts", [], |row| row.get(0))
|
||||
.unwrap_or(0);
|
||||
|
||||
if fts_count == 0 {
|
||||
let _ = rebuild_fts(conn);
|
||||
}
|
||||
|
||||
if let Ok(rows) = fts_query(conn, q, limit_i64)
|
||||
&& !rows.is_empty()
|
||||
{
|
||||
return Ok(rows);
|
||||
}
|
||||
|
||||
like_query(conn, q, limit_i64)
|
||||
}
|
||||
|
||||
fn fts_query(conn: &Connection, query: &str, limit: i64) -> Result<Vec<Entity>, IdxError> {
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT e.id, e.canonical_name, e.entity_type, e.country
|
||||
FROM entity_fts f
|
||||
JOIN entities e ON e.id = f.rowid
|
||||
WHERE entity_fts MATCH ?1
|
||||
ORDER BY rank
|
||||
LIMIT ?2",
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
let rows = stmt
|
||||
.query_map(params![query, limit], |row| {
|
||||
Ok(Entity {
|
||||
id: row.get(0)?,
|
||||
canonical_name: row.get(1)?,
|
||||
entity_type: row.get(2)?,
|
||||
country: row.get(3)?,
|
||||
})
|
||||
})
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
let mut out = Vec::new();
|
||||
for row in rows {
|
||||
out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn like_query(conn: &Connection, query: &str, limit: i64) -> Result<Vec<Entity>, IdxError> {
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT id, canonical_name, entity_type, country
|
||||
FROM entities
|
||||
WHERE canonical_name LIKE ?1 COLLATE NOCASE
|
||||
ORDER BY canonical_name ASC
|
||||
LIMIT ?2",
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
let like = format!("%{query}%");
|
||||
let rows = stmt
|
||||
.query_map(params![like, limit], |row| {
|
||||
Ok(Entity {
|
||||
id: row.get(0)?,
|
||||
canonical_name: row.get(1)?,
|
||||
entity_type: row.get(2)?,
|
||||
country: row.get(3)?,
|
||||
})
|
||||
})
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
let mut out = Vec::new();
|
||||
for row in rows {
|
||||
out.push(row.map_err(|e| IdxError::DatabaseError(e.to_string()))?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub fn rebuild_fts(conn: &Connection) -> Result<(), IdxError> {
|
||||
conn.execute("DELETE FROM entity_fts", [])
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO entity_fts(rowid, canonical_name, aliases)
|
||||
SELECT e.id, e.canonical_name, COALESCE(a.aliases, '')
|
||||
FROM entities e
|
||||
LEFT JOIN (
|
||||
SELECT entity_id, GROUP_CONCAT(raw_name, ' ') AS aliases
|
||||
FROM entity_aliases
|
||||
GROUP BY entity_id
|
||||
) a ON a.entity_id = e.id",
|
||||
[],
|
||||
)
|
||||
.map_err(|e| IdxError::DatabaseError(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
894
crates/idx-core/src/ownership/snapshot.rs
Normal file
894
crates/idx-core/src/ownership/snapshot.rs
Normal file
|
|
@ -0,0 +1,894 @@
|
|||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use chrono::NaiveDate;
|
||||
use rusqlite::{Connection, OpenFlags};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::config::get_config_value;
|
||||
use crate::error::IdxError;
|
||||
use crate::ownership::db;
|
||||
use crate::ownership::types::OwnershipRelease;
|
||||
|
||||
pub const SNAPSHOT_MANIFEST_CONFIG_KEY: &str = "ownership.snapshot_manifest";
|
||||
pub const SNAPSHOT_MANIFEST_ENV: &str = "IDX_OWNERSHIP_SNAPSHOT_MANIFEST";
|
||||
pub const SNAPSHOT_MANIFEST_SCHEMA_VERSION: u32 = 1;
|
||||
pub const DEFAULT_SNAPSHOT_MANIFEST_URL: &str = "https://github.com/0xrsydn/idx-cli/releases/download/ownership-snapshot-current/ownership-snapshot-manifest.json";
|
||||
|
||||
/// ureq caps response bodies at 10 MiB by default. Snapshots with several
|
||||
/// months of history exceed that (history 3 is ~10.5 MB), so the download is
|
||||
/// bounded by the manifest's `size_bytes` instead, never above 1 GiB. The body
|
||||
/// is still verified against that size and the SHA-256 afterwards.
|
||||
pub(crate) const SNAPSHOT_DOWNLOAD_LIMIT_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
|
||||
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct OwnershipSnapshotManifest {
|
||||
pub schema_version: u32,
|
||||
pub generated_at: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub source: Option<OwnershipSnapshotSource>,
|
||||
pub snapshot: OwnershipSnapshotArtifact,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct OwnershipSnapshotSource {
|
||||
pub family: String,
|
||||
pub listing_page_url: String,
|
||||
pub query_url: String,
|
||||
pub pdf_url: String,
|
||||
pub title: String,
|
||||
pub publish_date: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub original_filename: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct OwnershipSnapshotArtifact {
|
||||
pub kind: String,
|
||||
pub compression: String,
|
||||
pub version: String,
|
||||
pub download_url: String,
|
||||
pub sqlite_sha256: String,
|
||||
pub size_bytes: u64,
|
||||
pub release_count: usize,
|
||||
pub latest_as_of_date: NaiveDate,
|
||||
pub latest_release_sha256: String,
|
||||
pub latest_row_count: usize,
|
||||
pub ticker_count: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum OwnershipSyncAction {
|
||||
Installed,
|
||||
Updated,
|
||||
Refreshed,
|
||||
NoChange,
|
||||
SkippedNewer,
|
||||
SkippedDiverged,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
|
||||
pub struct OwnershipSyncResult {
|
||||
pub action: OwnershipSyncAction,
|
||||
pub manifest: String,
|
||||
pub db_path: String,
|
||||
pub snapshot_version: String,
|
||||
pub latest_as_of_date: NaiveDate,
|
||||
pub release_count: usize,
|
||||
pub ticker_count: usize,
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct LocalSnapshotState {
|
||||
latest_release: Option<OwnershipRelease>,
|
||||
release_count: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct SyncDecision {
|
||||
action: OwnershipSyncAction,
|
||||
should_download: bool,
|
||||
reason: String,
|
||||
}
|
||||
|
||||
pub fn resolve_manifest_source(explicit: Option<&str>) -> Result<String, IdxError> {
|
||||
let env_value = std::env::var(SNAPSHOT_MANIFEST_ENV).ok();
|
||||
let config_value = get_config_value(SNAPSHOT_MANIFEST_CONFIG_KEY)?;
|
||||
|
||||
resolve_manifest_source_with(explicit, env_value.as_deref(), config_value.as_deref())
|
||||
}
|
||||
|
||||
fn resolve_manifest_source_with(
|
||||
explicit: Option<&str>,
|
||||
env_value: Option<&str>,
|
||||
config_value: Option<&str>,
|
||||
) -> Result<String, IdxError> {
|
||||
for value in [explicit, env_value, config_value].into_iter().flatten() {
|
||||
let trimmed = value.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return Ok(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(DEFAULT_SNAPSHOT_MANIFEST_URL.to_string())
|
||||
}
|
||||
|
||||
pub fn fetch_manifest(source: &str) -> Result<OwnershipSnapshotManifest, IdxError> {
|
||||
let raw = read_text(source, "ownership snapshot manifest")?;
|
||||
parse_manifest(&raw)
|
||||
}
|
||||
|
||||
pub fn parse_manifest(raw: &str) -> Result<OwnershipSnapshotManifest, IdxError> {
|
||||
let manifest: OwnershipSnapshotManifest = serde_json::from_str(raw).map_err(|e| {
|
||||
IdxError::ParseError(format!(
|
||||
"failed to parse ownership snapshot manifest JSON: {e}"
|
||||
))
|
||||
})?;
|
||||
validate_manifest(&manifest)?;
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
pub fn sync_snapshot(
|
||||
manifest_source: &str,
|
||||
db_path: &Path,
|
||||
force: bool,
|
||||
) -> Result<OwnershipSyncResult, IdxError> {
|
||||
let manifest = fetch_manifest(manifest_source)?;
|
||||
let local_state = inspect_local_db(db_path)?;
|
||||
let decision = build_sync_decision(local_state.as_ref(), &manifest.snapshot, force);
|
||||
|
||||
if !decision.should_download {
|
||||
return Ok(OwnershipSyncResult {
|
||||
action: decision.action,
|
||||
manifest: manifest_source.to_string(),
|
||||
db_path: db_path.display().to_string(),
|
||||
snapshot_version: manifest.snapshot.version.clone(),
|
||||
latest_as_of_date: manifest.snapshot.latest_as_of_date,
|
||||
release_count: manifest.snapshot.release_count,
|
||||
ticker_count: manifest.snapshot.ticker_count,
|
||||
reason: decision.reason,
|
||||
});
|
||||
}
|
||||
|
||||
if manifest.snapshot.size_bytes > SNAPSHOT_DOWNLOAD_LIMIT_BYTES {
|
||||
return Err(IdxError::InvalidInput(format!(
|
||||
"ownership snapshot manifest reports {} bytes, above the {} byte limit",
|
||||
manifest.snapshot.size_bytes, SNAPSHOT_DOWNLOAD_LIMIT_BYTES
|
||||
)));
|
||||
}
|
||||
// A body larger than the manifest says is rejected while downloading
|
||||
// (ureq fails once a body reaches its limit, hence the +1); the exact size
|
||||
// and SHA-256 are verified afterwards.
|
||||
let bytes = read_bytes(
|
||||
&manifest.snapshot.download_url,
|
||||
"ownership snapshot SQLite",
|
||||
manifest.snapshot.size_bytes + 1,
|
||||
)?;
|
||||
validate_downloaded_bytes(&bytes, &manifest.snapshot)?;
|
||||
|
||||
let temp_path = build_temp_path(db_path);
|
||||
if let Some(parent) = temp_path.parent()
|
||||
&& !parent.as_os_str().is_empty()
|
||||
{
|
||||
fs::create_dir_all(parent).map_err(|e| {
|
||||
IdxError::Io(format!(
|
||||
"failed to create snapshot temp directory {}: {e}",
|
||||
parent.display()
|
||||
))
|
||||
})?;
|
||||
}
|
||||
fs::write(&temp_path, &bytes).map_err(|e| {
|
||||
IdxError::Io(format!(
|
||||
"failed to write downloaded snapshot {}: {e}",
|
||||
temp_path.display()
|
||||
))
|
||||
})?;
|
||||
|
||||
let install_result = (|| -> Result<(), IdxError> {
|
||||
validate_snapshot_db(&temp_path, &manifest.snapshot)?;
|
||||
install_snapshot_file(&temp_path, db_path)
|
||||
})();
|
||||
|
||||
if install_result.is_err() {
|
||||
let _ = fs::remove_file(&temp_path);
|
||||
}
|
||||
install_result?;
|
||||
|
||||
Ok(OwnershipSyncResult {
|
||||
action: decision.action,
|
||||
manifest: manifest_source.to_string(),
|
||||
db_path: db_path.display().to_string(),
|
||||
snapshot_version: manifest.snapshot.version.clone(),
|
||||
latest_as_of_date: manifest.snapshot.latest_as_of_date,
|
||||
release_count: manifest.snapshot.release_count,
|
||||
ticker_count: manifest.snapshot.ticker_count,
|
||||
reason: decision.reason,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_manifest(manifest: &OwnershipSnapshotManifest) -> Result<(), IdxError> {
|
||||
if manifest.schema_version != SNAPSHOT_MANIFEST_SCHEMA_VERSION {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"unsupported ownership snapshot manifest schema_version {}; expected {}",
|
||||
manifest.schema_version, SNAPSHOT_MANIFEST_SCHEMA_VERSION
|
||||
)));
|
||||
}
|
||||
|
||||
if manifest.generated_at.trim().is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"ownership snapshot manifest is missing generated_at".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(source) = &manifest.source {
|
||||
if source.family.trim().is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"ownership snapshot manifest is missing source.family".to_string(),
|
||||
));
|
||||
}
|
||||
if source.listing_page_url.trim().is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"ownership snapshot manifest is missing source.listing_page_url".to_string(),
|
||||
));
|
||||
}
|
||||
if source.query_url.trim().is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"ownership snapshot manifest is missing source.query_url".to_string(),
|
||||
));
|
||||
}
|
||||
if source.pdf_url.trim().is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"ownership snapshot manifest is missing source.pdf_url".to_string(),
|
||||
));
|
||||
}
|
||||
if source.title.trim().is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"ownership snapshot manifest is missing source.title".to_string(),
|
||||
));
|
||||
}
|
||||
if source.publish_date.trim().is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"ownership snapshot manifest is missing source.publish_date".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let snapshot = &manifest.snapshot;
|
||||
if snapshot.kind.trim() != "sqlite" {
|
||||
return Err(IdxError::Unsupported(format!(
|
||||
"ownership snapshot kind `{}` is not supported; expected `sqlite`",
|
||||
snapshot.kind
|
||||
)));
|
||||
}
|
||||
if snapshot.compression.trim() != "none" {
|
||||
return Err(IdxError::Unsupported(format!(
|
||||
"ownership snapshot compression `{}` is not supported; expected `none`",
|
||||
snapshot.compression
|
||||
)));
|
||||
}
|
||||
if snapshot.version.trim().is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"ownership snapshot manifest is missing snapshot.version".to_string(),
|
||||
));
|
||||
}
|
||||
if snapshot.download_url.trim().is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"ownership snapshot manifest is missing snapshot.download_url".to_string(),
|
||||
));
|
||||
}
|
||||
if snapshot.size_bytes == 0 {
|
||||
return Err(IdxError::ParseError(
|
||||
"ownership snapshot manifest reported size_bytes=0".to_string(),
|
||||
));
|
||||
}
|
||||
if snapshot.release_count == 0 {
|
||||
return Err(IdxError::ParseError(
|
||||
"ownership snapshot manifest reported release_count=0".to_string(),
|
||||
));
|
||||
}
|
||||
if snapshot.latest_row_count == 0 {
|
||||
return Err(IdxError::ParseError(
|
||||
"ownership snapshot manifest reported latest_row_count=0".to_string(),
|
||||
));
|
||||
}
|
||||
if snapshot.ticker_count == 0 {
|
||||
return Err(IdxError::ParseError(
|
||||
"ownership snapshot manifest reported ticker_count=0".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
validate_sha256_hex(&snapshot.sqlite_sha256, "snapshot.sqlite_sha256")?;
|
||||
validate_sha256_hex(
|
||||
&snapshot.latest_release_sha256,
|
||||
"snapshot.latest_release_sha256",
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_sha256_hex(value: &str, field_name: &str) -> Result<(), IdxError> {
|
||||
let trimmed = value.trim();
|
||||
if trimmed.len() != 64 || !trimmed.chars().all(|ch| ch.is_ascii_hexdigit()) {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"ownership snapshot manifest field `{field_name}` must be a 64-character hex sha256"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn inspect_local_db(db_path: &Path) -> Result<Option<LocalSnapshotState>, IdxError> {
|
||||
if !db_path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let conn = Connection::open(db_path).map_err(|e| {
|
||||
IdxError::DatabaseError(format!("failed to open {}: {e}", db_path.display()))
|
||||
})?;
|
||||
db::ensure_schema(&conn)?;
|
||||
let releases = db::query_releases(&conn)?;
|
||||
|
||||
Ok(Some(LocalSnapshotState {
|
||||
latest_release: releases.first().cloned(),
|
||||
release_count: releases.len(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn build_sync_decision(
|
||||
local: Option<&LocalSnapshotState>,
|
||||
snapshot: &OwnershipSnapshotArtifact,
|
||||
force: bool,
|
||||
) -> SyncDecision {
|
||||
if force {
|
||||
return SyncDecision {
|
||||
action: if local.is_some() {
|
||||
OwnershipSyncAction::Refreshed
|
||||
} else {
|
||||
OwnershipSyncAction::Installed
|
||||
},
|
||||
should_download: true,
|
||||
reason: "force refresh requested".to_string(),
|
||||
};
|
||||
}
|
||||
|
||||
let Some(local) = local else {
|
||||
return SyncDecision {
|
||||
action: OwnershipSyncAction::Installed,
|
||||
should_download: true,
|
||||
reason: "local ownership database does not exist yet".to_string(),
|
||||
};
|
||||
};
|
||||
|
||||
let Some(latest) = &local.latest_release else {
|
||||
return SyncDecision {
|
||||
action: OwnershipSyncAction::Updated,
|
||||
should_download: true,
|
||||
reason: "local ownership database exists but has no imported releases".to_string(),
|
||||
};
|
||||
};
|
||||
|
||||
if latest.as_of_date > snapshot.latest_as_of_date {
|
||||
return SyncDecision {
|
||||
action: OwnershipSyncAction::SkippedNewer,
|
||||
should_download: false,
|
||||
reason: format!(
|
||||
"local ownership database is newer than the published snapshot (local {} > snapshot {})",
|
||||
latest.as_of_date.format("%Y-%m-%d"),
|
||||
snapshot.latest_as_of_date.format("%Y-%m-%d")
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if latest.as_of_date < snapshot.latest_as_of_date {
|
||||
return SyncDecision {
|
||||
action: OwnershipSyncAction::Updated,
|
||||
should_download: true,
|
||||
reason: format!(
|
||||
"published snapshot advances local ownership data from {} to {}",
|
||||
latest.as_of_date.format("%Y-%m-%d"),
|
||||
snapshot.latest_as_of_date.format("%Y-%m-%d")
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if latest.sha256 == snapshot.latest_release_sha256 {
|
||||
if latest.row_count != snapshot.latest_row_count {
|
||||
return SyncDecision {
|
||||
action: OwnershipSyncAction::Updated,
|
||||
should_download: true,
|
||||
reason: "local ownership database has the same latest release date but different row_count metadata".to_string(),
|
||||
};
|
||||
}
|
||||
|
||||
if local.release_count < snapshot.release_count {
|
||||
return SyncDecision {
|
||||
action: OwnershipSyncAction::Updated,
|
||||
should_download: true,
|
||||
reason: format!(
|
||||
"local ownership database is missing historical releases (local {} < snapshot {})",
|
||||
local.release_count, snapshot.release_count
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if local.release_count == snapshot.release_count {
|
||||
return SyncDecision {
|
||||
action: OwnershipSyncAction::NoChange,
|
||||
should_download: false,
|
||||
reason: format!(
|
||||
"ownership snapshot already current at {} with {} release(s)",
|
||||
snapshot.latest_as_of_date.format("%Y-%m-%d"),
|
||||
snapshot.release_count
|
||||
),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
SyncDecision {
|
||||
action: OwnershipSyncAction::SkippedDiverged,
|
||||
should_download: false,
|
||||
reason: format!(
|
||||
"local ownership database differs from the published snapshot for {}; use --force to replace it",
|
||||
snapshot.latest_as_of_date.format("%Y-%m-%d")
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_downloaded_bytes(
|
||||
bytes: &[u8],
|
||||
snapshot: &OwnershipSnapshotArtifact,
|
||||
) -> Result<(), IdxError> {
|
||||
let size_bytes = u64::try_from(bytes.len()).map_err(|e| {
|
||||
IdxError::ParseError(format!("snapshot download too large to validate: {e}"))
|
||||
})?;
|
||||
if size_bytes != snapshot.size_bytes {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"ownership snapshot size mismatch: manifest expected {} bytes, downloaded {} bytes",
|
||||
snapshot.size_bytes, size_bytes
|
||||
)));
|
||||
}
|
||||
|
||||
let actual_sha256 = sha256_hex(bytes);
|
||||
if actual_sha256 != snapshot.sqlite_sha256 {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"ownership snapshot checksum mismatch: manifest expected {}, downloaded {}",
|
||||
snapshot.sqlite_sha256, actual_sha256
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_snapshot_db(path: &Path, snapshot: &OwnershipSnapshotArtifact) -> Result<(), IdxError> {
|
||||
let conn =
|
||||
Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY).map_err(|e| {
|
||||
IdxError::DatabaseError(format!(
|
||||
"failed to open downloaded snapshot {}: {e}",
|
||||
path.display()
|
||||
))
|
||||
})?;
|
||||
|
||||
let quick_check: String = conn
|
||||
.query_row("PRAGMA quick_check(1)", [], |row| row.get(0))
|
||||
.map_err(|e| IdxError::DatabaseError(format!("snapshot quick_check failed: {e}")))?;
|
||||
if quick_check.trim() != "ok" {
|
||||
return Err(IdxError::DatabaseError(format!(
|
||||
"snapshot quick_check failed: {quick_check}"
|
||||
)));
|
||||
}
|
||||
|
||||
let releases = db::query_releases(&conn)?;
|
||||
if releases.len() != snapshot.release_count {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"downloaded snapshot release_count mismatch: manifest expected {}, sqlite has {}",
|
||||
snapshot.release_count,
|
||||
releases.len()
|
||||
)));
|
||||
}
|
||||
|
||||
let latest = releases.first().ok_or_else(|| {
|
||||
IdxError::ParseError("downloaded snapshot sqlite had no ownership releases".to_string())
|
||||
})?;
|
||||
|
||||
if latest.as_of_date != snapshot.latest_as_of_date {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"downloaded snapshot latest_as_of_date mismatch: manifest expected {}, sqlite has {}",
|
||||
snapshot.latest_as_of_date.format("%Y-%m-%d"),
|
||||
latest.as_of_date.format("%Y-%m-%d")
|
||||
)));
|
||||
}
|
||||
if latest.sha256 != snapshot.latest_release_sha256 {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"downloaded snapshot latest_release_sha256 mismatch: manifest expected {}, sqlite has {}",
|
||||
snapshot.latest_release_sha256, latest.sha256
|
||||
)));
|
||||
}
|
||||
if latest.row_count != snapshot.latest_row_count {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"downloaded snapshot latest_row_count mismatch: manifest expected {}, sqlite has {}",
|
||||
snapshot.latest_row_count, latest.row_count
|
||||
)));
|
||||
}
|
||||
|
||||
let ticker_count: i64 = conn
|
||||
.query_row("SELECT COUNT(*) FROM tickers", [], |row| row.get(0))
|
||||
.map_err(|e| IdxError::DatabaseError(format!("failed counting snapshot tickers: {e}")))?;
|
||||
let ticker_count = usize::try_from(ticker_count)
|
||||
.map_err(|e| IdxError::DatabaseError(format!("invalid snapshot ticker_count: {e}")))?;
|
||||
if ticker_count != snapshot.ticker_count {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"downloaded snapshot ticker_count mismatch: manifest expected {}, sqlite has {}",
|
||||
snapshot.ticker_count, ticker_count
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn build_temp_path(db_path: &Path) -> PathBuf {
|
||||
let file_name = db_path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("ownership.db");
|
||||
let temp_name = format!(".{file_name}.sync-{}.tmp", fastrand::u64(..));
|
||||
match db_path.parent() {
|
||||
Some(parent) if !parent.as_os_str().is_empty() => parent.join(temp_name),
|
||||
_ => PathBuf::from(temp_name),
|
||||
}
|
||||
}
|
||||
|
||||
fn install_snapshot_file(temp_path: &Path, db_path: &Path) -> Result<(), IdxError> {
|
||||
if let Some(parent) = db_path.parent()
|
||||
&& !parent.as_os_str().is_empty()
|
||||
{
|
||||
fs::create_dir_all(parent).map_err(|e| {
|
||||
IdxError::Io(format!(
|
||||
"failed to create ownership database directory {}: {e}",
|
||||
parent.display()
|
||||
))
|
||||
})?;
|
||||
}
|
||||
|
||||
let backup_path = build_backup_path(db_path);
|
||||
if backup_path.exists() {
|
||||
fs::remove_file(&backup_path).map_err(|e| {
|
||||
IdxError::Io(format!(
|
||||
"failed to remove stale backup {}: {e}",
|
||||
backup_path.display()
|
||||
))
|
||||
})?;
|
||||
}
|
||||
|
||||
let had_existing = db_path.exists();
|
||||
remove_sqlite_sidecars(db_path)?;
|
||||
|
||||
if had_existing {
|
||||
fs::rename(db_path, &backup_path).map_err(|e| {
|
||||
IdxError::Io(format!(
|
||||
"failed to move existing ownership database {} to backup {}: {e}",
|
||||
db_path.display(),
|
||||
backup_path.display()
|
||||
))
|
||||
})?;
|
||||
}
|
||||
|
||||
if let Err(err) = fs::rename(temp_path, db_path) {
|
||||
if had_existing {
|
||||
let _ = fs::rename(&backup_path, db_path);
|
||||
}
|
||||
return Err(IdxError::Io(format!(
|
||||
"failed to install ownership snapshot into {}: {err}",
|
||||
db_path.display()
|
||||
)));
|
||||
}
|
||||
|
||||
if had_existing && backup_path.exists() {
|
||||
fs::remove_file(&backup_path).map_err(|e| {
|
||||
IdxError::Io(format!(
|
||||
"failed to remove snapshot backup {}: {e}",
|
||||
backup_path.display()
|
||||
))
|
||||
})?;
|
||||
}
|
||||
|
||||
remove_sqlite_sidecars(db_path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn build_backup_path(db_path: &Path) -> PathBuf {
|
||||
let file_name = db_path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("ownership.db");
|
||||
let backup_name = format!(".{file_name}.sync-backup");
|
||||
match db_path.parent() {
|
||||
Some(parent) if !parent.as_os_str().is_empty() => parent.join(backup_name),
|
||||
_ => PathBuf::from(backup_name),
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_sqlite_sidecars(db_path: &Path) -> Result<(), IdxError> {
|
||||
for suffix in ["-wal", "-shm"] {
|
||||
let file_name = format!(
|
||||
"{}{suffix}",
|
||||
db_path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("ownership.db")
|
||||
);
|
||||
let sidecar = match db_path.parent() {
|
||||
Some(parent) if !parent.as_os_str().is_empty() => parent.join(&file_name),
|
||||
_ => PathBuf::from(&file_name),
|
||||
};
|
||||
|
||||
if sidecar.exists() {
|
||||
fs::remove_file(&sidecar).map_err(|e| {
|
||||
IdxError::Io(format!(
|
||||
"failed to remove SQLite sidecar {}: {e}",
|
||||
sidecar.display()
|
||||
))
|
||||
})?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sha256_hex(bytes: &[u8]) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(bytes);
|
||||
let digest = hasher.finalize();
|
||||
digest.iter().map(|byte| format!("{byte:02x}")).collect()
|
||||
}
|
||||
|
||||
fn read_text(source: &str, context: &str) -> Result<String, IdxError> {
|
||||
if is_http_source(source) {
|
||||
let response = ureq::get(source)
|
||||
.header("User-Agent", USER_AGENT)
|
||||
.header("Accept", "application/json,text/plain;q=0.9,*/*;q=0.8")
|
||||
.call()
|
||||
.map_err(|e| IdxError::Http(format!("failed to fetch {context}: {e}")))?;
|
||||
let mut body = response.into_body();
|
||||
return body
|
||||
.read_to_string()
|
||||
.map_err(|e| IdxError::Http(format!("failed reading {context} body: {e}")));
|
||||
}
|
||||
|
||||
let path = local_source_path(source);
|
||||
fs::read_to_string(&path)
|
||||
.map_err(|e| IdxError::Io(format!("failed to read {context} {}: {e}", path.display())))
|
||||
}
|
||||
|
||||
fn read_bytes(source: &str, context: &str, limit_bytes: u64) -> Result<Vec<u8>, IdxError> {
|
||||
if is_http_source(source) {
|
||||
let response = ureq::get(source)
|
||||
.header("User-Agent", USER_AGENT)
|
||||
.header(
|
||||
"Accept",
|
||||
"application/octet-stream,application/x-sqlite3,*/*;q=0.8",
|
||||
)
|
||||
.call()
|
||||
.map_err(|e| IdxError::Http(format!("failed to fetch {context}: {e}")))?;
|
||||
return response
|
||||
.into_body()
|
||||
.with_config()
|
||||
.limit(limit_bytes)
|
||||
.read_to_vec()
|
||||
.map_err(|e| {
|
||||
IdxError::Http(format!(
|
||||
"failed reading {context} body (larger than the manifest's size_bytes?): {e}"
|
||||
))
|
||||
});
|
||||
}
|
||||
|
||||
let path = local_source_path(source);
|
||||
fs::read(&path)
|
||||
.map_err(|e| IdxError::Io(format!("failed to read {context} {}: {e}", path.display())))
|
||||
}
|
||||
|
||||
fn is_http_source(source: &str) -> bool {
|
||||
let normalized = source.trim().to_ascii_lowercase();
|
||||
normalized.starts_with("http://") || normalized.starts_with("https://")
|
||||
}
|
||||
|
||||
fn local_source_path(source: &str) -> PathBuf {
|
||||
if let Some(path) = source.trim().strip_prefix("file://") {
|
||||
return PathBuf::from(path);
|
||||
}
|
||||
PathBuf::from(source.trim())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use chrono::NaiveDate;
|
||||
use rusqlite::Connection;
|
||||
|
||||
use super::{
|
||||
DEFAULT_SNAPSHOT_MANIFEST_URL, OwnershipSnapshotArtifact, OwnershipSnapshotManifest,
|
||||
OwnershipSnapshotSource, OwnershipSyncAction, SNAPSHOT_MANIFEST_SCHEMA_VERSION,
|
||||
build_sync_decision, parse_manifest, resolve_manifest_source_with,
|
||||
};
|
||||
use crate::ownership::db::{ensure_schema, insert_release};
|
||||
use crate::ownership::types::OwnershipRelease;
|
||||
|
||||
#[test]
|
||||
fn parse_manifest_rejects_unknown_schema_version() {
|
||||
let raw = r#"{
|
||||
"schema_version": 99,
|
||||
"generated_at": "2026-03-31T12:00:00Z",
|
||||
"snapshot": {
|
||||
"kind": "sqlite",
|
||||
"compression": "none",
|
||||
"version": "2026-02-27",
|
||||
"download_url": "/tmp/ownership.sqlite",
|
||||
"sqlite_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"size_bytes": 123,
|
||||
"release_count": 2,
|
||||
"latest_as_of_date": "2026-02-27",
|
||||
"latest_release_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
||||
"latest_row_count": 100,
|
||||
"ticker_count": 5
|
||||
}
|
||||
}"#;
|
||||
|
||||
let err = parse_manifest(raw).expect_err("schema_version should fail");
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("unsupported ownership snapshot manifest schema_version")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_sync_decision_updates_same_latest_release_when_history_is_incomplete() {
|
||||
let snapshot = OwnershipSnapshotArtifact {
|
||||
kind: "sqlite".to_string(),
|
||||
compression: "none".to_string(),
|
||||
version: "2026-02-27".to_string(),
|
||||
download_url: "/tmp/ownership.sqlite".to_string(),
|
||||
sqlite_sha256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
.to_string(),
|
||||
size_bytes: 123,
|
||||
release_count: 2,
|
||||
latest_as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
||||
latest_release_sha256:
|
||||
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(),
|
||||
latest_row_count: 7261,
|
||||
ticker_count: 955,
|
||||
};
|
||||
let local = super::LocalSnapshotState {
|
||||
latest_release: Some(OwnershipRelease {
|
||||
id: 1,
|
||||
source_url: None,
|
||||
sha256: snapshot.latest_release_sha256.clone(),
|
||||
as_of_date: snapshot.latest_as_of_date,
|
||||
row_count: snapshot.latest_row_count,
|
||||
imported_at: 0,
|
||||
}),
|
||||
release_count: 1,
|
||||
};
|
||||
|
||||
let decision = build_sync_decision(Some(&local), &snapshot, false);
|
||||
assert_eq!(decision.action, OwnershipSyncAction::Updated);
|
||||
assert!(decision.should_download);
|
||||
assert!(decision.reason.contains("missing historical releases"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_round_trip_parses_valid_payload() {
|
||||
let manifest = OwnershipSnapshotManifest {
|
||||
schema_version: SNAPSHOT_MANIFEST_SCHEMA_VERSION,
|
||||
generated_at: "2026-03-31T12:00:00Z".to_string(),
|
||||
source: Some(OwnershipSnapshotSource {
|
||||
family: "above1".to_string(),
|
||||
listing_page_url: "https://www.idx.co.id/id/berita/pengumuman/".to_string(),
|
||||
query_url: "https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%201&pageNumber=1&pageSize=10&lang=id".to_string(),
|
||||
pdf_url: "https://www.idx.co.id/StaticData/sample.pdf".to_string(),
|
||||
title: "Pemegang Saham di atas 1% (KSEI)".to_string(),
|
||||
publish_date: "2026-03-10T00:00:00".to_string(),
|
||||
original_filename: Some("sample.pdf".to_string()),
|
||||
}),
|
||||
snapshot: OwnershipSnapshotArtifact {
|
||||
kind: "sqlite".to_string(),
|
||||
compression: "none".to_string(),
|
||||
version: "2026-02-27".to_string(),
|
||||
download_url: "/tmp/ownership.sqlite".to_string(),
|
||||
sqlite_sha256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
.to_string(),
|
||||
size_bytes: 123,
|
||||
release_count: 1,
|
||||
latest_as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
||||
latest_release_sha256:
|
||||
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(),
|
||||
latest_row_count: 7261,
|
||||
ticker_count: 955,
|
||||
},
|
||||
};
|
||||
|
||||
let raw = serde_json::to_string(&manifest).unwrap();
|
||||
let parsed = parse_manifest(&raw).expect("valid manifest");
|
||||
assert_eq!(parsed, manifest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_without_source_metadata_still_parses() {
|
||||
let raw = r#"{
|
||||
"schema_version": 1,
|
||||
"generated_at": "2026-03-31T12:00:00Z",
|
||||
"snapshot": {
|
||||
"kind": "sqlite",
|
||||
"compression": "none",
|
||||
"version": "2026-02-27",
|
||||
"download_url": "/tmp/ownership.sqlite",
|
||||
"sqlite_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"size_bytes": 123,
|
||||
"release_count": 1,
|
||||
"latest_as_of_date": "2026-02-27",
|
||||
"latest_release_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
||||
"latest_row_count": 7261,
|
||||
"ticker_count": 955
|
||||
}
|
||||
}"#;
|
||||
|
||||
let parsed = parse_manifest(raw).expect("old manifest shape should still parse");
|
||||
assert!(parsed.source.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_manifest_source_uses_built_in_default_when_unset() {
|
||||
let resolved = resolve_manifest_source_with(None, None, None).expect("default manifest");
|
||||
assert_eq!(resolved, DEFAULT_SNAPSHOT_MANIFEST_URL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_manifest_source_prefers_explicit_then_env_then_config() {
|
||||
let explicit = resolve_manifest_source_with(
|
||||
Some("https://example.com/explicit.json"),
|
||||
Some("https://example.com/env.json"),
|
||||
Some("https://example.com/config.json"),
|
||||
)
|
||||
.expect("explicit manifest");
|
||||
assert_eq!(explicit, "https://example.com/explicit.json");
|
||||
|
||||
let env = resolve_manifest_source_with(
|
||||
Some(" "),
|
||||
Some("https://example.com/env.json"),
|
||||
Some("https://example.com/config.json"),
|
||||
)
|
||||
.expect("env manifest");
|
||||
assert_eq!(env, "https://example.com/env.json");
|
||||
|
||||
let config =
|
||||
resolve_manifest_source_with(None, Some(" "), Some("https://example.com/config.json"))
|
||||
.expect("config manifest");
|
||||
assert_eq!(config, "https://example.com/config.json");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ensure_schema_can_store_release_metadata_needed_for_snapshots() {
|
||||
let conn = Connection::open_in_memory().unwrap();
|
||||
ensure_schema(&conn).unwrap();
|
||||
insert_release(
|
||||
&conn,
|
||||
&OwnershipRelease {
|
||||
id: 0,
|
||||
source_url: Some("https://example.com/ownership.sqlite".to_string()),
|
||||
sha256: "abc".to_string(),
|
||||
as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(),
|
||||
row_count: 1,
|
||||
imported_at: 1,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let release_count: i64 = conn
|
||||
.query_row("SELECT COUNT(*) FROM ownership_releases", [], |row| {
|
||||
row.get(0)
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(release_count, 1);
|
||||
}
|
||||
}
|
||||
451
crates/idx-core/src/ownership/types.rs
Normal file
451
crates/idx-core/src/ownership/types.rs
Normal file
|
|
@ -0,0 +1,451 @@
|
|||
use chrono::NaiveDate;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Which data source a holding originates from.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum OwnershipSource {
|
||||
/// KSEI shareholder registry source.
|
||||
Ksei,
|
||||
/// Bing institutional ownership source.
|
||||
Bing,
|
||||
}
|
||||
|
||||
/// Investor locality classification from KSEI.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum Locality {
|
||||
/// Local Indonesian investor.
|
||||
Local,
|
||||
/// Foreign investor.
|
||||
Foreign,
|
||||
}
|
||||
|
||||
/// Bing institutional flow signal.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum FlowSignal {
|
||||
/// Existing top holder.
|
||||
Holder,
|
||||
/// Net buyer for the period.
|
||||
Buyer,
|
||||
/// Net seller for the period.
|
||||
Seller,
|
||||
/// New position opened this period.
|
||||
NewPosition,
|
||||
/// Fully exited this period.
|
||||
Exited,
|
||||
}
|
||||
|
||||
/// Method used to resolve an entity alias to a canonical entity.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ResolutionMethod {
|
||||
/// Exact string match after normalization.
|
||||
Exact,
|
||||
/// Rule-based normalization match.
|
||||
Rule,
|
||||
/// Fuzzy or similarity-based match.
|
||||
Fuzzy,
|
||||
/// Human-curated manual mapping.
|
||||
Manual,
|
||||
}
|
||||
|
||||
/// Graph node category used in ownership network output.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum GraphNodeType {
|
||||
/// Canonical investor entity node.
|
||||
Entity,
|
||||
/// Listed issuer ticker node.
|
||||
Ticker,
|
||||
}
|
||||
|
||||
/// KSEI investor type code (for example: `CP`, `ID`, `IB`, `MF`, `SC`, `IS`, `OT`).
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct InvestorTypeCode(pub String);
|
||||
|
||||
/// Canonical resolved entity (investor or shareholder).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Entity {
|
||||
/// Internal entity identifier.
|
||||
pub id: i64,
|
||||
/// Canonical normalized display name.
|
||||
pub canonical_name: String,
|
||||
/// Optional coarse entity type (`fund`, `bank`, `conglomerate`, `govt`, `individual`).
|
||||
pub entity_type: Option<String>,
|
||||
/// Optional ISO-like country tag.
|
||||
pub country: Option<String>,
|
||||
}
|
||||
|
||||
/// A raw name variant mapped to a canonical entity.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EntityAlias {
|
||||
/// Internal alias identifier.
|
||||
pub id: i64,
|
||||
/// Referenced canonical entity id.
|
||||
pub entity_id: i64,
|
||||
/// Raw alias text from source.
|
||||
pub raw_name: String,
|
||||
/// Source that produced this alias.
|
||||
pub source: OwnershipSource,
|
||||
/// Resolution confidence score in `0.0..=1.0`.
|
||||
pub confidence: f64,
|
||||
/// Matching method used for resolution.
|
||||
pub method: ResolutionMethod,
|
||||
}
|
||||
|
||||
/// Ticker (issuer) reference metadata.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Ticker {
|
||||
/// Internal ticker identifier.
|
||||
pub id: i64,
|
||||
/// Exchange ticker code, for example `BBCA`.
|
||||
pub code: String,
|
||||
/// Optional long issuer name.
|
||||
pub name: Option<String>,
|
||||
/// Optional sector classification.
|
||||
pub sector: Option<String>,
|
||||
}
|
||||
|
||||
/// Single KSEI ownership row for one holder in one ticker and release.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct KseiHolding {
|
||||
/// Internal holding row identifier.
|
||||
pub id: i64,
|
||||
/// Referenced ticker id.
|
||||
pub ticker_id: i64,
|
||||
/// Referenced entity id, unresolved when `None`.
|
||||
pub entity_id: Option<i64>,
|
||||
/// Raw investor name exactly as imported.
|
||||
pub raw_investor_name: String,
|
||||
/// Optional KSEI investor type code.
|
||||
pub investor_type: Option<InvestorTypeCode>,
|
||||
/// Optional local/foreign classification.
|
||||
pub locality: Option<Locality>,
|
||||
/// Optional nationality text.
|
||||
pub nationality: Option<String>,
|
||||
/// Optional domicile text.
|
||||
pub domicile: Option<String>,
|
||||
/// Scripless holdings in shares.
|
||||
pub holdings_scripless: i64,
|
||||
/// Script holdings in shares.
|
||||
pub holdings_scrip: i64,
|
||||
/// Total shares held.
|
||||
pub total_shares: i64,
|
||||
/// Ownership percentage in basis points (`41.10% -> 4110`).
|
||||
pub percentage_bps: i64,
|
||||
/// Snapshot as-of date.
|
||||
pub report_date: NaiveDate,
|
||||
/// SHA-256 hash of source release for deduplication.
|
||||
pub release_sha256: String,
|
||||
}
|
||||
|
||||
/// Draft holding before entity resolution (without persisted row id and entity id).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct KseiHoldingDraft {
|
||||
/// Exchange ticker code, for example `BBCA`.
|
||||
pub ticker_code: String,
|
||||
/// Optional issuer name as provided by source row.
|
||||
pub issuer_name: Option<String>,
|
||||
/// Raw investor name exactly as imported.
|
||||
pub raw_investor_name: String,
|
||||
/// Optional KSEI investor type code.
|
||||
pub investor_type: Option<InvestorTypeCode>,
|
||||
/// Optional local/foreign classification.
|
||||
pub locality: Option<Locality>,
|
||||
/// Optional nationality text.
|
||||
pub nationality: Option<String>,
|
||||
/// Optional domicile text.
|
||||
pub domicile: Option<String>,
|
||||
/// Scripless holdings in shares.
|
||||
pub holdings_scripless: i64,
|
||||
/// Script holdings in shares.
|
||||
pub holdings_scrip: i64,
|
||||
/// Total shares held.
|
||||
pub total_shares: i64,
|
||||
/// Ownership percentage in basis points (`41.10% -> 4110`).
|
||||
pub percentage_bps: i64,
|
||||
/// Snapshot as-of date.
|
||||
pub report_date: NaiveDate,
|
||||
}
|
||||
|
||||
/// Single Bing institutional ownership row.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BingHolding {
|
||||
/// Internal holding row identifier.
|
||||
pub id: i64,
|
||||
/// Referenced ticker id.
|
||||
pub ticker_id: i64,
|
||||
/// Referenced entity id, unresolved when `None`.
|
||||
pub entity_id: Option<i64>,
|
||||
/// Raw investor name exactly as imported.
|
||||
pub raw_investor_name: String,
|
||||
/// Optional Bing investor type text.
|
||||
pub investor_type: Option<String>,
|
||||
/// Shares currently held.
|
||||
pub shares_held: Option<i64>,
|
||||
/// Share delta over period (`+buy`, `-sell`).
|
||||
pub shares_changed: Option<i64>,
|
||||
/// Ownership percentage in basis points.
|
||||
pub pct_ownership_bps: Option<i64>,
|
||||
/// Position value in USD.
|
||||
pub value_usd: Option<i64>,
|
||||
/// Source report date.
|
||||
pub report_date: NaiveDate,
|
||||
/// Institutional flow signal for the row.
|
||||
pub signal: FlowSignal,
|
||||
/// Import timestamp (unix epoch seconds).
|
||||
pub fetched_at: i64,
|
||||
}
|
||||
|
||||
/// Raw row extracted from KSEI PDF before normalization.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct KseiRawRow {
|
||||
/// Date string as appears in PDF (for example `27-Feb-2026`).
|
||||
pub date: String,
|
||||
/// Share code string (for example `BBCA`).
|
||||
pub share_code: String,
|
||||
/// Issuer name as appears in source.
|
||||
pub issuer_name: String,
|
||||
/// Investor name as appears in source.
|
||||
pub investor_name: String,
|
||||
/// Investor type code string.
|
||||
pub investor_type: String,
|
||||
/// Local or foreign marker (`L` or `F`).
|
||||
pub local_foreign: String,
|
||||
/// Nationality text.
|
||||
pub nationality: String,
|
||||
/// Domicile text.
|
||||
pub domicile: String,
|
||||
/// Scripless holdings string in Indonesian locale format.
|
||||
pub holdings_scripless: String,
|
||||
/// Script holdings string in Indonesian locale format.
|
||||
pub holdings_scrip: String,
|
||||
/// Total holdings string in Indonesian locale format.
|
||||
pub total_holding_shares: String,
|
||||
/// Percentage string in Indonesian locale decimal format.
|
||||
pub percentage: String,
|
||||
}
|
||||
|
||||
/// Raw holder object returned by Bing ownership endpoints.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BingHolderRaw {
|
||||
/// Investor name field from Bing payload.
|
||||
#[serde(alias = "investorName", alias = "InvestorName")]
|
||||
pub investor_name: Option<String>,
|
||||
/// Investor type field from Bing payload.
|
||||
#[serde(alias = "investorType", alias = "InvestorType")]
|
||||
pub investor_type: Option<String>,
|
||||
/// Shares held field from Bing payload.
|
||||
#[serde(alias = "sharesHeld", alias = "SharesHeld")]
|
||||
pub shares_held: Option<f64>,
|
||||
/// Shares changed field from Bing payload.
|
||||
#[serde(alias = "sharesChanged", alias = "SharesChanged")]
|
||||
pub shares_changed: Option<f64>,
|
||||
/// Percentage of shares outstanding field from Bing payload.
|
||||
#[serde(
|
||||
alias = "percentageOfSharesOutstanding",
|
||||
alias = "PercentageOfSharesOutstanding"
|
||||
)]
|
||||
pub pct_outstanding: Option<f64>,
|
||||
/// Position value field from Bing payload.
|
||||
#[serde(alias = "value", alias = "Value")]
|
||||
pub value: Option<f64>,
|
||||
/// Report date field from Bing payload.
|
||||
#[serde(alias = "reportDate", alias = "ReportDate")]
|
||||
pub report_date: Option<String>,
|
||||
}
|
||||
|
||||
/// Combined ownership view for a ticker (KSEI and Bing merged).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TickerOwnership {
|
||||
/// Ticker metadata.
|
||||
pub ticker: Ticker,
|
||||
/// Latest KSEI as-of date.
|
||||
pub ksei_as_of: Option<NaiveDate>,
|
||||
/// Latest Bing reporting period label.
|
||||
pub bing_as_of: Option<String>,
|
||||
/// Combined holder rows for display.
|
||||
pub holders: Vec<HolderRow>,
|
||||
/// Concentration metrics calculated for the ticker.
|
||||
pub concentration: ConcentrationMetrics,
|
||||
/// Optional institutional flow breakdown.
|
||||
pub flow: Option<InstitutionalFlow>,
|
||||
}
|
||||
|
||||
/// Single row in a combined holders table.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HolderRow {
|
||||
/// Ranking position in result set.
|
||||
pub rank: usize,
|
||||
/// Source dataset for the row.
|
||||
pub source: OwnershipSource,
|
||||
/// Canonical holder name or unresolved raw name.
|
||||
pub name: String,
|
||||
/// Optional referenced canonical entity id.
|
||||
pub entity_id: Option<i64>,
|
||||
/// Optional investor type label.
|
||||
pub investor_type: Option<String>,
|
||||
/// Optional local/foreign marker.
|
||||
pub locality: Option<Locality>,
|
||||
/// Held shares quantity.
|
||||
pub shares: i64,
|
||||
/// Ownership percentage in basis points.
|
||||
pub percentage_bps: i64,
|
||||
/// Optional flow signal (Bing rows only).
|
||||
pub signal: Option<FlowSignal>,
|
||||
}
|
||||
|
||||
/// Concentration metrics for ownership distribution of a ticker.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ConcentrationMetrics {
|
||||
/// Largest single holder percentage in basis points.
|
||||
pub top1_bps: i64,
|
||||
/// Sum of top 3 holder percentages in basis points.
|
||||
pub top3_bps: i64,
|
||||
/// Herfindahl-Hirschman index value.
|
||||
pub hhi: i64,
|
||||
/// Estimated free float in basis points (`10000 - known holders`).
|
||||
pub free_float_bps: i64,
|
||||
/// Number of KSEI holders above or equal to 1%.
|
||||
pub holder_count: usize,
|
||||
}
|
||||
|
||||
/// Institutional flow summary from Bing for one ticker and period.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct InstitutionalFlow {
|
||||
/// Reporting period label.
|
||||
pub period: String,
|
||||
/// Top buyer rows.
|
||||
pub top_buyers: Vec<HolderRow>,
|
||||
/// Top seller rows.
|
||||
pub top_sellers: Vec<HolderRow>,
|
||||
/// New position rows.
|
||||
pub new_positions: Vec<HolderRow>,
|
||||
/// Exited position rows.
|
||||
pub exited: Vec<HolderRow>,
|
||||
}
|
||||
|
||||
/// Cross-holding summary for a single entity.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EntityHoldings {
|
||||
/// Canonical entity metadata.
|
||||
pub entity: Entity,
|
||||
/// Number of distinct tickers held.
|
||||
pub ticker_count: usize,
|
||||
/// Per-ticker ownership rows.
|
||||
pub holdings: Vec<EntityTickerRow>,
|
||||
}
|
||||
|
||||
/// One ticker row within an entity holdings summary.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EntityTickerRow {
|
||||
/// Ticker metadata.
|
||||
pub ticker: Ticker,
|
||||
/// Source dataset for the holding.
|
||||
pub source: OwnershipSource,
|
||||
/// Shares held quantity.
|
||||
pub shares: i64,
|
||||
/// Ownership percentage in basis points.
|
||||
pub percentage_bps: i64,
|
||||
/// Report as-of date.
|
||||
pub report_date: NaiveDate,
|
||||
}
|
||||
|
||||
/// Cross-holder leaderboard row across multiple tickers.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CrossHolderRow {
|
||||
/// Canonical entity metadata.
|
||||
pub entity: Entity,
|
||||
/// Number of distinct tickers held.
|
||||
pub ticker_count: usize,
|
||||
/// Summed ownership basis points across tickers.
|
||||
pub total_bps: i64,
|
||||
}
|
||||
|
||||
/// Graph node used for ownership network visualization.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GraphNode {
|
||||
/// Stable graph node id (`entity:42` or `ticker:BBCA`).
|
||||
pub id: String,
|
||||
/// Display label.
|
||||
pub label: String,
|
||||
/// Node category.
|
||||
pub node_type: GraphNodeType,
|
||||
}
|
||||
|
||||
/// Graph edge used for ownership network visualization.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GraphEdge {
|
||||
/// Source node id.
|
||||
pub from: String,
|
||||
/// Target node id.
|
||||
pub to: String,
|
||||
/// Edge weight in basis points.
|
||||
pub percentage_bps: i64,
|
||||
/// Source dataset of the edge.
|
||||
pub source: OwnershipSource,
|
||||
}
|
||||
|
||||
/// Imported ownership release metadata.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct OwnershipRelease {
|
||||
/// Internal release id.
|
||||
pub id: i64,
|
||||
/// Optional source URL where release was obtained.
|
||||
pub source_url: Option<String>,
|
||||
/// SHA-256 hash for release-level deduplication.
|
||||
pub sha256: String,
|
||||
/// Release as-of date.
|
||||
pub as_of_date: NaiveDate,
|
||||
/// Parsed row count imported.
|
||||
pub row_count: usize,
|
||||
/// Import timestamp (unix epoch seconds).
|
||||
pub imported_at: i64,
|
||||
}
|
||||
|
||||
/// Type of ownership change between two snapshots.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ChangeType {
|
||||
/// Holder appears in `to` snapshot but not in `from`.
|
||||
New,
|
||||
/// Holder appears in `from` snapshot but not in `to`.
|
||||
Exited,
|
||||
/// Holder exists in both snapshots and percentage increased.
|
||||
Increased,
|
||||
/// Holder exists in both snapshots and percentage decreased.
|
||||
Decreased,
|
||||
}
|
||||
|
||||
/// One ownership change row for a ticker-holder pair.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ChangeRow {
|
||||
/// Exchange ticker code.
|
||||
pub ticker_code: String,
|
||||
/// Canonical or raw holder name.
|
||||
pub entity_name: String,
|
||||
/// Change classification.
|
||||
pub change_type: ChangeType,
|
||||
/// Old percentage in basis points (from snapshot).
|
||||
pub old_bps: Option<i64>,
|
||||
/// New percentage in basis points (to snapshot).
|
||||
pub new_bps: Option<i64>,
|
||||
/// Delta in basis points (`new - old`, missing treated as 0).
|
||||
pub delta_bps: i64,
|
||||
}
|
||||
|
||||
/// Row describing unresolved or low-confidence alias mappings.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct UnresolvedRow {
|
||||
/// Raw investor name captured from source data.
|
||||
pub raw_name: String,
|
||||
/// Source marker (`ksei` or `bing`).
|
||||
pub source: String,
|
||||
/// Ticker code where the unresolved alias appears.
|
||||
pub ticker_code: String,
|
||||
/// Current canonical entity name when available.
|
||||
pub current_entity: Option<String>,
|
||||
/// Resolution confidence score.
|
||||
pub confidence: Option<f64>,
|
||||
}
|
||||
554
crates/idx-core/src/ownership/xlsx.rs
Normal file
554
crates/idx-core/src/ownership/xlsx.rs
Normal file
|
|
@ -0,0 +1,554 @@
|
|||
//! Parser for the IDX "Pemegang Saham di Atas 1%" XLSX workbook.
|
||||
//!
|
||||
//! Since June 2026 IDX publishes the monthly above-1% holder register as an
|
||||
//! XLSX file on the "Data Kepemilikan Saham" page instead of a PDF
|
||||
//! announcement. The workbook has one sheet: a disclaimer block, a `[PUBLIC]`
|
||||
//! marker, then a fixed 12-column table. Only that exact header is accepted,
|
||||
//! so the daily above-5% workbook and the investor-type / classification
|
||||
//! workbooks published on the same page are rejected instead of mis-imported.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::io::{Cursor, Read};
|
||||
use std::path::Path;
|
||||
|
||||
use chrono::{Duration, NaiveDate};
|
||||
use quick_xml::Reader;
|
||||
use quick_xml::events::{BytesStart, Event};
|
||||
|
||||
use crate::error::IdxError;
|
||||
use crate::ownership::entities::{normalize_investor_type, normalize_locality, parse_ksei_date};
|
||||
use crate::ownership::types::KseiHoldingDraft;
|
||||
|
||||
const EXPECTED_HEADER: [&str; 12] = [
|
||||
"DATE",
|
||||
"SHARE_CODE",
|
||||
"ISSUER_NAME",
|
||||
"INVESTOR_NAME",
|
||||
"INVESTOR_CLASSIFICATION",
|
||||
"LOCAL_FOREIGN",
|
||||
"NATIONALITY",
|
||||
"DOMICILE",
|
||||
"HOLDINGS_SCRIPLESS",
|
||||
"HOLDINGS_SCRIP",
|
||||
"TOTAL_HOLDING_SHARES",
|
||||
"PERCENTAGE",
|
||||
];
|
||||
|
||||
/// Rows above this line are disclaimer text; the header must appear before it.
|
||||
const MAX_HEADER_SEARCH_ROWS: usize = 50;
|
||||
|
||||
pub fn supports_xlsx_file(path: &Path) -> bool {
|
||||
path.extension()
|
||||
.and_then(|value| value.to_str())
|
||||
.is_some_and(|value| value.trim().eq_ignore_ascii_case("xlsx"))
|
||||
}
|
||||
|
||||
pub fn parse_above1_xlsx_file(path: &Path) -> Result<Vec<KseiHoldingDraft>, IdxError> {
|
||||
let bytes = fs::read(path).map_err(|e| {
|
||||
IdxError::Io(format!(
|
||||
"failed to read ownership XLSX {}: {e}",
|
||||
path.display()
|
||||
))
|
||||
})?;
|
||||
parse_above1_xlsx_bytes(&bytes)
|
||||
}
|
||||
|
||||
pub fn parse_above1_xlsx_bytes(bytes: &[u8]) -> Result<Vec<KseiHoldingDraft>, IdxError> {
|
||||
let mut zip = zip::ZipArchive::new(Cursor::new(bytes))
|
||||
.map_err(|e| IdxError::ParseError(format!("failed to open ownership XLSX: {e}")))?;
|
||||
|
||||
let shared_strings = match read_zip_entry(&mut zip, "xl/sharedStrings.xml")? {
|
||||
Some(xml) => parse_shared_strings(&xml)?,
|
||||
None => Vec::new(),
|
||||
};
|
||||
let sheet_name = first_worksheet_name(&mut zip)?;
|
||||
let sheet_xml = read_zip_entry(&mut zip, &sheet_name)?
|
||||
.ok_or_else(|| IdxError::ParseError(format!("ownership XLSX is missing {sheet_name}")))?;
|
||||
|
||||
let rows = parse_sheet_rows(&sheet_xml, &shared_strings)?;
|
||||
drafts_from_rows(&rows)
|
||||
}
|
||||
|
||||
fn read_zip_entry(
|
||||
zip: &mut zip::ZipArchive<Cursor<&[u8]>>,
|
||||
name: &str,
|
||||
) -> Result<Option<String>, IdxError> {
|
||||
let mut file = match zip.by_name(name) {
|
||||
Ok(file) => file,
|
||||
Err(zip::result::ZipError::FileNotFound) => return Ok(None),
|
||||
Err(e) => {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"failed to read ownership XLSX entry {name}: {e}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
let mut output = String::new();
|
||||
file.read_to_string(&mut output).map_err(|e| {
|
||||
IdxError::ParseError(format!("failed to decode ownership XLSX entry {name}: {e}"))
|
||||
})?;
|
||||
Ok(Some(output))
|
||||
}
|
||||
|
||||
fn first_worksheet_name(zip: &mut zip::ZipArchive<Cursor<&[u8]>>) -> Result<String, IdxError> {
|
||||
let mut sheets: Vec<String> = zip
|
||||
.file_names()
|
||||
.filter(|name| name.starts_with("xl/worksheets/") && name.ends_with(".xml"))
|
||||
.filter(|name| !name["xl/worksheets/".len()..].contains('/'))
|
||||
.map(str::to_string)
|
||||
.collect();
|
||||
// sheet1.xml, sheet2.xml, ... sort by their numeric suffix.
|
||||
sheets.sort_by_key(|name| {
|
||||
name.trim_start_matches("xl/worksheets/sheet")
|
||||
.trim_end_matches(".xml")
|
||||
.parse::<u32>()
|
||||
.unwrap_or(u32::MAX)
|
||||
});
|
||||
sheets
|
||||
.into_iter()
|
||||
.next()
|
||||
.ok_or_else(|| IdxError::ParseError("ownership XLSX has no worksheet".to_string()))
|
||||
}
|
||||
|
||||
fn xml_error(context: &str, err: impl std::fmt::Display) -> IdxError {
|
||||
IdxError::ParseError(format!("failed to parse ownership XLSX {context}: {err}"))
|
||||
}
|
||||
|
||||
/// Shared strings table: each `<si>` is one string, made of one `<t>` or
|
||||
/// several rich-text runs `<r><t>`. Phonetic hints (`<rPh>`) are skipped.
|
||||
fn parse_shared_strings(xml: &str) -> Result<Vec<String>, IdxError> {
|
||||
let mut reader = Reader::from_str(xml);
|
||||
let mut strings = Vec::new();
|
||||
let mut current: Option<String> = None;
|
||||
let mut in_text = false;
|
||||
let mut in_phonetic = false;
|
||||
|
||||
loop {
|
||||
match reader.read_event() {
|
||||
Ok(Event::Start(e)) => match e.local_name().as_ref() {
|
||||
b"si" => current = Some(String::new()),
|
||||
b"t" => in_text = true,
|
||||
b"rPh" => in_phonetic = true,
|
||||
_ => {}
|
||||
},
|
||||
Ok(Event::Empty(e)) if e.local_name().as_ref() == b"si" => strings.push(String::new()),
|
||||
Ok(Event::Text(text)) if in_text && !in_phonetic => {
|
||||
if let Some(value) = current.as_mut() {
|
||||
let unescaped = text
|
||||
.unescape()
|
||||
.map_err(|e| xml_error("shared strings", e))?;
|
||||
value.push_str(&unescaped);
|
||||
}
|
||||
}
|
||||
Ok(Event::End(e)) => match e.local_name().as_ref() {
|
||||
b"si" => strings.push(current.take().unwrap_or_default()),
|
||||
b"t" => in_text = false,
|
||||
b"rPh" => in_phonetic = false,
|
||||
_ => {}
|
||||
},
|
||||
Ok(Event::Eof) => break,
|
||||
Err(e) => return Err(xml_error("shared strings", e)),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(strings)
|
||||
}
|
||||
|
||||
/// One sheet row: its 1-based Excel row number and column letters
|
||||
/// (`A`, `B`, ...) mapped to cell text. Sheet XML omits blank rows, so the
|
||||
/// number comes from the row's `r` attribute, not its position.
|
||||
#[derive(Debug, Default)]
|
||||
struct SheetRow {
|
||||
number: usize,
|
||||
cells: HashMap<String, String>,
|
||||
}
|
||||
|
||||
fn row_number(element: &BytesStart<'_>, fallback: usize) -> Result<usize, IdxError> {
|
||||
for attribute in element.attributes() {
|
||||
let attribute = attribute.map_err(|e| xml_error("worksheet", e))?;
|
||||
if attribute.key.local_name().as_ref() == b"r" {
|
||||
if let Ok(number) = String::from_utf8_lossy(&attribute.value).trim().parse() {
|
||||
return Ok(number);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(fallback)
|
||||
}
|
||||
|
||||
fn parse_sheet_rows(xml: &str, shared_strings: &[String]) -> Result<Vec<SheetRow>, IdxError> {
|
||||
let mut reader = Reader::from_str(xml);
|
||||
let mut rows = Vec::new();
|
||||
let mut row: Option<SheetRow> = None;
|
||||
let mut cell: Option<(String, String)> = None;
|
||||
let mut value = String::new();
|
||||
let mut in_value = false;
|
||||
|
||||
loop {
|
||||
match reader.read_event() {
|
||||
Ok(Event::Start(e)) => match e.local_name().as_ref() {
|
||||
b"row" => {
|
||||
row = Some(SheetRow {
|
||||
number: row_number(&e, rows.len() + 1)?,
|
||||
cells: HashMap::new(),
|
||||
})
|
||||
}
|
||||
b"c" => {
|
||||
cell = Some(cell_ref_and_type(&e)?);
|
||||
value.clear();
|
||||
}
|
||||
b"v" | b"t" if cell.is_some() => in_value = true,
|
||||
_ => {}
|
||||
},
|
||||
Ok(Event::Empty(e)) if e.local_name().as_ref() == b"row" => rows.push(SheetRow {
|
||||
number: row_number(&e, rows.len() + 1)?,
|
||||
cells: HashMap::new(),
|
||||
}),
|
||||
Ok(Event::Text(text)) if in_value => {
|
||||
let unescaped = text.unescape().map_err(|e| xml_error("worksheet", e))?;
|
||||
value.push_str(&unescaped);
|
||||
}
|
||||
Ok(Event::End(e)) => match e.local_name().as_ref() {
|
||||
b"v" | b"t" => in_value = false,
|
||||
b"c" => {
|
||||
if let (Some((column, cell_type)), Some(current_row)) =
|
||||
(cell.take(), row.as_mut())
|
||||
{
|
||||
let text = if cell_type == "s" {
|
||||
let index = value.trim().parse::<usize>().map_err(|e| {
|
||||
xml_error("worksheet", format!("bad shared string index: {e}"))
|
||||
})?;
|
||||
shared_strings.get(index).cloned().ok_or_else(|| {
|
||||
xml_error(
|
||||
"worksheet",
|
||||
format!("shared string {index} out of range"),
|
||||
)
|
||||
})?
|
||||
} else {
|
||||
value.clone()
|
||||
};
|
||||
current_row.cells.insert(column, text);
|
||||
}
|
||||
}
|
||||
b"row" => {
|
||||
if let Some(current_row) = row.take() {
|
||||
rows.push(current_row);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
},
|
||||
Ok(Event::Eof) => break,
|
||||
Err(e) => return Err(xml_error("worksheet", e)),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
fn cell_ref_and_type(element: &BytesStart<'_>) -> Result<(String, String), IdxError> {
|
||||
let mut column = String::new();
|
||||
let mut cell_type = String::new();
|
||||
for attribute in element.attributes() {
|
||||
let attribute = attribute.map_err(|e| xml_error("worksheet", e))?;
|
||||
let raw = String::from_utf8_lossy(&attribute.value).to_string();
|
||||
match attribute.key.local_name().as_ref() {
|
||||
b"r" => column = raw.chars().take_while(char::is_ascii_alphabetic).collect(),
|
||||
b"t" => cell_type = raw,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if column.is_empty() {
|
||||
return Err(xml_error("worksheet", "cell without a reference"));
|
||||
}
|
||||
Ok((column, cell_type))
|
||||
}
|
||||
|
||||
fn drafts_from_rows(rows: &[SheetRow]) -> Result<Vec<KseiHoldingDraft>, IdxError> {
|
||||
let (header_index, columns) = rows
|
||||
.iter()
|
||||
.take(MAX_HEADER_SEARCH_ROWS)
|
||||
.enumerate()
|
||||
.find_map(|(index, row)| header_columns(row).map(|columns| (index, columns)))
|
||||
.ok_or_else(|| {
|
||||
IdxError::ParseError(
|
||||
"ownership XLSX header did not match the above-1% holder register layout \
|
||||
(DATE, SHARE_CODE, ... PERCENTAGE); the above-5% and investor-type \
|
||||
workbooks are not supported"
|
||||
.to_string(),
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut drafts = Vec::new();
|
||||
for row in &rows[header_index + 1..] {
|
||||
let line = row.number;
|
||||
let field = |name: &str| -> &str {
|
||||
columns
|
||||
.get(name)
|
||||
.and_then(|column| row.cells.get(column))
|
||||
.map(|value| value.trim())
|
||||
.unwrap_or("")
|
||||
};
|
||||
|
||||
let ticker_code = field("SHARE_CODE");
|
||||
let investor_name = field("INVESTOR_NAME");
|
||||
if ticker_code.is_empty() && investor_name.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if ticker_code.is_empty() || investor_name.is_empty() {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"ownership XLSX row {line} is missing SHARE_CODE or INVESTOR_NAME"
|
||||
)));
|
||||
}
|
||||
|
||||
drafts.push(KseiHoldingDraft {
|
||||
ticker_code: ticker_code.to_string(),
|
||||
issuer_name: optional(field("ISSUER_NAME")),
|
||||
raw_investor_name: investor_name.to_string(),
|
||||
investor_type: normalize_investor_type(field("INVESTOR_CLASSIFICATION")),
|
||||
locality: normalize_locality(field("LOCAL_FOREIGN")),
|
||||
nationality: optional(field("NATIONALITY")),
|
||||
domicile: optional(field("DOMICILE")),
|
||||
holdings_scripless: parse_shares(field("HOLDINGS_SCRIPLESS"), line, false)?,
|
||||
holdings_scrip: parse_shares(field("HOLDINGS_SCRIP"), line, false)?,
|
||||
total_shares: parse_shares(field("TOTAL_HOLDING_SHARES"), line, true)?,
|
||||
percentage_bps: parse_percentage_bps(field("PERCENTAGE"), line)?,
|
||||
report_date: parse_report_date(field("DATE"), line)?,
|
||||
});
|
||||
}
|
||||
|
||||
if drafts.is_empty() {
|
||||
return Err(IdxError::ParseError(
|
||||
"ownership XLSX contained a header but no holder rows".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(drafts)
|
||||
}
|
||||
|
||||
/// Map header labels to column letters if `row` is exactly the expected header.
|
||||
fn header_columns(row: &SheetRow) -> Option<HashMap<&'static str, String>> {
|
||||
let mut found: HashMap<&'static str, String> = HashMap::new();
|
||||
for (column, text) in &row.cells {
|
||||
let mut label = text.trim().to_ascii_uppercase().replace(' ', "_");
|
||||
if label.is_empty() {
|
||||
continue;
|
||||
}
|
||||
// Feb/Mar 2026 workbooks name the column INVESTOR_TYPE and hold codes
|
||||
// (CP, ID, ...); later ones use INVESTOR_CLASSIFICATION with words.
|
||||
if label == "INVESTOR_TYPE" {
|
||||
label = "INVESTOR_CLASSIFICATION".to_string();
|
||||
}
|
||||
let expected = EXPECTED_HEADER.iter().find(|name| **name == label)?;
|
||||
found.insert(expected, column.clone());
|
||||
}
|
||||
(found.len() == EXPECTED_HEADER.len()).then_some(found)
|
||||
}
|
||||
|
||||
fn optional(value: &str) -> Option<String> {
|
||||
(!value.is_empty()).then(|| value.to_string())
|
||||
}
|
||||
|
||||
fn parse_number(raw: &str, line: usize, name: &str) -> Result<f64, IdxError> {
|
||||
raw.replace(',', "")
|
||||
.parse::<f64>()
|
||||
.ok()
|
||||
.filter(|value| value.is_finite())
|
||||
.ok_or_else(|| {
|
||||
IdxError::ParseError(format!("ownership XLSX row {line}: invalid {name} '{raw}'"))
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_shares(raw: &str, line: usize, required: bool) -> Result<i64, IdxError> {
|
||||
if raw.is_empty() {
|
||||
return if required {
|
||||
Err(IdxError::ParseError(format!(
|
||||
"ownership XLSX row {line}: missing TOTAL_HOLDING_SHARES"
|
||||
)))
|
||||
} else {
|
||||
Ok(0)
|
||||
};
|
||||
}
|
||||
let value = parse_number(raw, line, "share count")?;
|
||||
if value < 0.0 || value.fract() != 0.0 || value > i64::MAX as f64 {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"ownership XLSX row {line}: share count '{raw}' is not a whole number"
|
||||
)));
|
||||
}
|
||||
Ok(value as i64)
|
||||
}
|
||||
|
||||
/// `54.94` (percent) -> 5494 basis points. Excel stores values such as
|
||||
/// `1.1499999999999999`, so round to the nearest basis point.
|
||||
fn parse_percentage_bps(raw: &str, line: usize) -> Result<i64, IdxError> {
|
||||
let value = parse_number(raw, line, "PERCENTAGE")?;
|
||||
if !(0.0..=100.0).contains(&value) {
|
||||
return Err(IdxError::ParseError(format!(
|
||||
"ownership XLSX row {line}: PERCENTAGE '{raw}' is out of range"
|
||||
)));
|
||||
}
|
||||
Ok((value * 100.0).round() as i64)
|
||||
}
|
||||
|
||||
/// DATE is an Excel serial day number (`46265` = 2026-08-31); accept the
|
||||
/// KSEI text form (`31-Aug-2026`) as well in case the cell is stored as text.
|
||||
fn parse_report_date(raw: &str, line: usize) -> Result<NaiveDate, IdxError> {
|
||||
if let Ok(serial) = raw.parse::<f64>() {
|
||||
let days = serial.trunc() as i64;
|
||||
let epoch = NaiveDate::from_ymd_opt(1899, 12, 30).expect("valid Excel epoch");
|
||||
if (1..=2_958_465).contains(&days) {
|
||||
return Ok(epoch + Duration::days(days));
|
||||
}
|
||||
}
|
||||
parse_ksei_date(raw).map_err(|_| {
|
||||
IdxError::ParseError(format!("ownership XLSX row {line}: invalid DATE '{raw}'"))
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn fixture() -> Vec<KseiHoldingDraft> {
|
||||
parse_above1_xlsx_bytes(include_bytes!(
|
||||
"../../tests/fixtures/ksei_above1_20260831_excerpt.xlsx"
|
||||
))
|
||||
.expect("above-1% XLSX excerpt parses")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_above1_xlsx_excerpt() {
|
||||
let drafts = fixture();
|
||||
// Pinned so a regression that silently drops rows fails loudly.
|
||||
assert_eq!(drafts.len(), 17);
|
||||
assert!(
|
||||
drafts
|
||||
.iter()
|
||||
.all(|d| d.report_date == NaiveDate::from_ymd_opt(2026, 8, 31).unwrap())
|
||||
);
|
||||
|
||||
let dwimuria = drafts
|
||||
.iter()
|
||||
.find(|d| {
|
||||
d.ticker_code == "BBCA" && d.raw_investor_name == "PT DWIMURIA INVESTAMA ANDALAN"
|
||||
})
|
||||
.expect("BBCA controlling holder present");
|
||||
assert_eq!(dwimuria.percentage_bps, 5494);
|
||||
assert_eq!(
|
||||
dwimuria.investor_type.as_ref().map(|t| t.0.as_str()),
|
||||
Some("CORPORATE")
|
||||
);
|
||||
assert_eq!(
|
||||
dwimuria.total_shares,
|
||||
dwimuria.holdings_scripless + dwimuria.holdings_scrip
|
||||
);
|
||||
}
|
||||
|
||||
fn header_row(labels: &[&str]) -> SheetRow {
|
||||
sheet_row(1, labels)
|
||||
}
|
||||
|
||||
fn sheet_row(number: usize, values: &[&str]) -> SheetRow {
|
||||
SheetRow {
|
||||
number,
|
||||
cells: values
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, value)| !value.is_empty())
|
||||
.map(|(index, value)| {
|
||||
(
|
||||
((b'A' + index as u8) as char).to_string(),
|
||||
value.to_string(),
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_legacy_investor_type_header() {
|
||||
let mut labels = EXPECTED_HEADER.to_vec();
|
||||
labels[4] = "INVESTOR_TYPE";
|
||||
let columns = header_columns(&header_row(&labels)).expect("legacy header accepted");
|
||||
assert_eq!(columns["INVESTOR_CLASSIFICATION"], "E");
|
||||
|
||||
let row = sheet_row(
|
||||
2,
|
||||
&[
|
||||
"46112",
|
||||
"AADI",
|
||||
"ADARO ANDALAN INDONESIA Tbk",
|
||||
"GARIBALDI THOHIR",
|
||||
"ID",
|
||||
"L ",
|
||||
"INDONESIAN",
|
||||
"INDONESIA",
|
||||
"454011607",
|
||||
"0",
|
||||
"454011607",
|
||||
"5.83",
|
||||
],
|
||||
);
|
||||
let drafts = drafts_from_rows(&[header_row(&labels), row]).expect("legacy rows parse");
|
||||
assert_eq!(
|
||||
drafts[0].investor_type.as_ref().map(|t| t.0.as_str()),
|
||||
Some("ID")
|
||||
);
|
||||
assert_eq!(
|
||||
drafts[0].report_date,
|
||||
NaiveDate::from_ymd_opt(2026, 3, 31).unwrap()
|
||||
);
|
||||
assert!(drafts[0].locality.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn row_errors_report_excel_row_numbers() {
|
||||
let missing_investor = sheet_row(
|
||||
709,
|
||||
&["46265", "BBCA", "BANK CENTRAL ASIA Tbk", "", "Corporate"],
|
||||
);
|
||||
let err = drafts_from_rows(&[header_row(&EXPECTED_HEADER), missing_investor])
|
||||
.expect_err("row without investor rejected");
|
||||
assert!(err.to_string().contains("row 709"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_header_with_unknown_or_missing_columns() {
|
||||
let mut labels = EXPECTED_HEADER.to_vec();
|
||||
labels[11] = "PERUBAHAN";
|
||||
assert!(header_columns(&header_row(&labels)).is_none());
|
||||
assert!(header_columns(&header_row(&EXPECTED_HEADER[..11])).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rounds_float_noise_in_percentages() {
|
||||
assert_eq!(parse_percentage_bps("1.1499999999999999", 1).unwrap(), 115);
|
||||
assert_eq!(parse_percentage_bps("41.1", 1).unwrap(), 4110);
|
||||
assert!(parse_percentage_bps("100.5", 1).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_excel_serial_and_text_dates() {
|
||||
let expected = NaiveDate::from_ymd_opt(2026, 8, 31).unwrap();
|
||||
assert_eq!(parse_report_date("46265", 1).unwrap(), expected);
|
||||
assert_eq!(parse_report_date("31-Aug-2026", 1).unwrap(), expected);
|
||||
assert!(parse_report_date("not a date", 1).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_above5_workbook() {
|
||||
let err = parse_above1_xlsx_bytes(include_bytes!(
|
||||
"../../tests/fixtures/ksei_above5_20260923_excerpt.xlsx"
|
||||
))
|
||||
.expect_err("above-5% workbook must be rejected");
|
||||
assert!(
|
||||
err.to_string().contains("above-1% holder register layout"),
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_non_xlsx_bytes() {
|
||||
assert!(parse_above1_xlsx_bytes(b"%PDF-1.7 not a workbook").is_err());
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue