refactor: schema-driven architecture, capability traits, hardened error paths

- Split parse.rs into raw_types.rs (serde structs) + map.rs (pure transforms) for MSN and Yahoo
- Replace Yahoo fundamentals dynamic HashMap with typed structs (SummaryDetail, DefaultKeyStatistics, etc.)
- Introduce capability-based provider traits: QuoteProvider, FundamentalsProvider, HistoryProvider
- Add future MSN capability traits: ProfileProvider, EarningsProvider, FinancialsProvider,
  SentimentProvider, InsightsProvider, NewsProvider (all dead_code until wired to CLI)
- Add shared domain types in src/api/types.rs (CompanyProfile, EarningsReport, FinancialStatements,
  SentimentData, InsightData, NewsItem)
- Harden error propagation: Yahoo cookie auth, MSN partial fundamentals, history symbol context,
  cache clear failures
- Strict config parsing: invalid IDX_OUTPUT returns ConfigError instead of silent fallback
- Cache schema version enforcement: version mismatch treated as cache miss
- Extract fetch_with_cache() helper in cli/stocks.rs
- Add MSN retry/backoff parity with Yahoo client
- Standardize Option<T> policy through parse/map layers
- All 56 tests passing, clippy clean
This commit is contained in:
Ciphercat 2026-03-06 19:17:50 +00:00
commit 3998e38ffc
17 changed files with 1185 additions and 969 deletions

View file

@ -1,10 +1,7 @@
use std::collections::BTreeMap;
use chrono::NaiveDate;
use chrono::{Datelike, NaiveDate};
use serde::de::Error as _;
use serde::{Deserialize, Deserializer};
use super::symbols::{normalized_symbol, ticker_from_symbol};
use super::map::{parse_fundamentals, parse_history, parse_history_with_drop_count, parse_quote};
use super::raw_types::{KeyRatios, MsnChart, MsnQuote};
use crate::api::types::{Fundamentals, Ohlc, Period, Quote};
use crate::error::IdxError;
@ -15,55 +12,6 @@ pub(crate) fn parse_quote_from_str(symbol: &str, raw: &str) -> Result<Quote, Idx
parse_quote(symbol, &quotes)
}
pub(super) fn parse_quote(symbol: &str, quotes: &[MsnQuote]) -> Result<Quote, IdxError> {
let quote = quotes.first().ok_or(IdxError::ProviderUnavailable)?;
let raw_price = quote
.price
.ok_or_else(|| IdxError::SymbolNotFound(symbol.to_string()))?;
let prev_close = quote.price_previous_close.map(round_price);
let price = round_price(raw_price);
let change = prev_close
.map(|previous| price - previous)
.or_else(|| quote.price_change.map(round_price))
.unwrap_or(0);
let ticker = quote
.symbol
.as_deref()
.and_then(ticker_from_symbol)
.unwrap_or_else(|| ticker_from_symbol(symbol).unwrap_or_default());
let (week52_position, range_signal) = match (quote.price_52w_low, quote.price_52w_high) {
(Some(low), Some(high)) if high > low => {
let position = (raw_price - low) / (high - low);
let signal = if position > 0.66 {
Some("upper".to_string())
} else if position < 0.33 {
Some("lower".to_string())
} else {
Some("middle".to_string())
};
(Some(position), signal)
}
_ => (None, None),
};
Ok(Quote {
symbol: normalized_symbol(symbol, &ticker),
price,
change,
change_pct: quote.price_change_percent.unwrap_or(0.0),
volume: round_u64(quote.accumulated_volume).unwrap_or(0),
market_cap: round_u64(quote.market_cap),
week52_high: quote.price_52w_high.map(round_price),
week52_low: quote.price_52w_low.map(round_price),
week52_position,
range_signal,
prev_close,
avg_volume: round_u64(quote.average_volume),
})
}
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) fn parse_fundamentals_from_str(
raw: &str,
@ -79,56 +27,24 @@ pub(crate) fn parse_fundamentals_from_str(
parse_fundamentals(&ratios, quote.as_ref())
}
pub(super) fn parse_fundamentals(
ratios: &[KeyRatios],
quote: Option<&MsnQuote>,
) -> Result<Fundamentals, IdxError> {
let ratios = ratios.first().ok_or(IdxError::ProviderUnavailable)?;
let metrics = if ratios.company_metrics.is_empty() {
&ratios.industry_metrics
} else {
&ratios.company_metrics
};
if preferred_metric(metrics).is_none() {
return Err(IdxError::ProviderUnavailable);
}
Ok(Fundamentals {
trailing_pe: best_metric_value(metrics, |metric| metric.price_to_earnings_ratio),
forward_pe: best_metric_value(metrics, |metric| metric.forward_price_to_eps),
price_to_book: best_metric_value(metrics, |metric| metric.price_to_book_ratio),
return_on_equity: best_metric_value(metrics, |metric| normalize_percentish(metric.roe)),
profit_margins: best_metric_value(metrics, |metric| {
normalize_percentish(metric.profit_margin.or(metric.net_margin))
}),
return_on_assets: best_metric_value(metrics, |metric| {
normalize_percentish(metric.roa_ttm.or(metric.return_on_asset_current))
}),
revenue_growth: best_metric_value(metrics, |metric| {
normalize_percentish(metric.revenue_ytd_ytd.or(metric.revenue_growth_rate))
}),
earnings_growth: best_metric_value(metrics, |metric| {
normalize_percentish(
metric
.net_income_ytd_ytd_growth_rate
.or(metric.earnings_growth_rate),
)
}),
debt_to_equity: best_metric_value(metrics, |metric| metric.debt_to_equity_ratio),
current_ratio: best_metric_value(metrics, |metric| {
sanitize_current_ratio(metric.current_ratio)
}),
enterprise_value: None,
ebitda: None,
market_cap: quote.and_then(|item| round_u64(item.market_cap)),
})
}
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) fn parse_history_from_str(period: &Period, raw: &str) -> Result<Vec<Ohlc>, IdxError> {
let charts: Vec<MsnChart> =
serde_json::from_str(raw).map_err(|e| IdxError::ParseError(e.to_string()))?;
parse_history_with_verbose(period, &charts, false)
parse_history(period, &charts)
}
#[allow(dead_code)] // retained for verbose history path, wired once MSN charts are exposed
pub(super) fn parse_history_with_verbose(
period: &Period,
charts: &[MsnChart],
verbose: bool,
) -> Result<Vec<Ohlc>, IdxError> {
let (history, dropped) = parse_history_with_drop_count(period, charts)?;
if dropped > 0 && verbose {
eprintln!("warning: dropped {dropped} OHLC row(s) from MSN response due to missing fields");
}
Ok(history)
}
#[allow(dead_code)]
@ -141,86 +57,27 @@ fn parse_close_only_history_from_str(
parse_close_only_history(period, &charts)
}
pub(super) fn parse_history_with_verbose(
period: &Period,
charts: &[MsnChart],
verbose: bool,
) -> Result<Vec<Ohlc>, IdxError> {
let chart = charts.first().ok_or(IdxError::ProviderUnavailable)?;
if !chart.series.has_real_ohlcv() {
return Err(IdxError::ParseError(
"msn does not expose real OHLC/volume for this history range".to_string(),
));
}
let timestamps = &chart.series.time_stamps;
let mut grouped: BTreeMap<NaiveDate, Ohlc> = BTreeMap::new();
let mut dropped = 0usize;
for (idx, raw_ts) in timestamps.iter().enumerate() {
let Some(date) = parse_chart_date(raw_ts) else {
dropped += 1;
continue;
};
let point = (
chart.series.open_prices.get(idx).copied(),
chart.series.prices_high.get(idx).copied(),
chart.series.prices_low.get(idx).copied(),
chart.series.prices.get(idx).copied(),
chart.series.volumes.get(idx).copied(),
);
let (Some(open), Some(high), Some(low), Some(close), Some(volume)) = point else {
dropped += 1;
continue;
};
let candle = Ohlc {
date,
open: round_price(open),
high: round_price(high),
low: round_price(low),
close: round_price(close),
volume: round_u64(Some(volume)).unwrap_or(0),
};
grouped
.entry(date)
.and_modify(|existing| {
existing.high = existing.high.max(candle.high);
existing.low = existing.low.min(candle.low);
existing.close = candle.close;
existing.volume = existing.volume.saturating_add(candle.volume);
})
.or_insert(candle);
}
let mut out: Vec<Ohlc> = grouped.into_values().collect();
trim_history_to_period(period, &mut out);
if dropped > 0 && verbose {
eprintln!("warning: dropped {dropped} OHLC row(s) from MSN response due to missing fields");
}
if out.is_empty() {
return Err(IdxError::ProviderUnavailable);
}
Ok(out)
}
fn parse_close_only_history(
period: &Period,
charts: &[MsnChart],
) -> Result<Vec<ClosePoint>, IdxError> {
let chart = charts.first().ok_or(IdxError::ProviderUnavailable)?;
let timestamps = &chart.series.time_stamps;
let mut grouped: BTreeMap<NaiveDate, ClosePoint> = BTreeMap::new();
let mut grouped: std::collections::BTreeMap<NaiveDate, ClosePoint> =
std::collections::BTreeMap::new();
for (idx, raw_ts) in timestamps.iter().enumerate() {
let Some(date) = parse_chart_date(raw_ts) else {
let Some(date) = chrono::DateTime::parse_from_rfc3339(raw_ts)
.map(|d| d.date_naive())
.ok()
.or_else(|| {
raw_ts
.parse::<i64>()
.ok()
.and_then(|ts| chrono::DateTime::from_timestamp(ts, 0).map(|d| d.date_naive()))
})
.or_else(|| NaiveDate::parse_from_str(raw_ts, "%Y-%m-%d").ok())
else {
continue;
};
let Some(close) = chart.series.prices.get(idx).copied() else {
@ -231,7 +88,7 @@ fn parse_close_only_history(
date,
ClosePoint {
date,
close: round_price(close),
close: close.round() as i64,
},
);
}
@ -246,25 +103,6 @@ fn parse_close_only_history(
Ok(out)
}
fn trim_history_to_period(period: &Period, rows: &mut Vec<Ohlc>) {
let days: i64 = match period {
Period::OneDay => return,
Period::FiveDays => 5,
Period::OneMonth => 31,
Period::ThreeMonths => 92,
Period::SixMonths => 183,
Period::OneYear => 366,
Period::TwoYears => 731,
Period::FiveYears => 1826,
};
let Some(last_date) = rows.last().map(|item| item.date) else {
return;
};
let cutoff = last_date - chrono::Duration::days(days.saturating_sub(1));
rows.retain(|item| item.date >= cutoff);
}
fn trim_close_history_to_period(period: &Period, rows: &mut Vec<ClosePoint>) {
let days: i64 = match period {
Period::OneDay => return,
@ -284,260 +122,6 @@ fn trim_close_history_to_period(period: &Period, rows: &mut Vec<ClosePoint>) {
rows.retain(|item| item.date >= cutoff);
}
fn preferred_metric(metrics: &[IndustryMetric]) -> Option<&IndustryMetric> {
metrics.iter().max_by_key(|metric| metric_rank(metric))
}
fn best_metric_value<T: Copy>(
metrics: &[IndustryMetric],
extractor: impl Fn(&IndustryMetric) -> Option<T>,
) -> Option<T> {
metrics
.iter()
.filter_map(|metric| extractor(metric).map(|value| (metric_rank(metric), value)))
.max_by_key(|(rank, _)| *rank)
.map(|(_, value)| value)
}
fn metric_rank(metric: &IndustryMetric) -> (i32, i32) {
(
metric
.year
.as_deref()
.and_then(|year| year.parse::<i32>().ok())
.unwrap_or(i32::MIN),
metric_period_priority(metric.fiscal_period_type.as_deref()),
)
}
fn metric_period_priority(period: Option<&str>) -> i32 {
match period.map(|value| value.trim()) {
Some(value) if value.eq_ignore_ascii_case("TTM") => 7,
Some(value)
if value.eq_ignore_ascii_case("ANNUAL")
|| value.eq_ignore_ascii_case("FY")
|| value.eq_ignore_ascii_case("YEAR") =>
{
6
}
Some(value) if value.eq_ignore_ascii_case("Q4") => 5,
Some(value) if value.eq_ignore_ascii_case("Q3") => 4,
Some(value) if value.eq_ignore_ascii_case("Q2") => 3,
Some(value) if value.eq_ignore_ascii_case("Q1") => 2,
Some(value) if value.eq_ignore_ascii_case("NTM") => 1,
_ => 0,
}
}
fn normalize_percentish(value: Option<f64>) -> Option<f64> {
value.and_then(|number| {
if !number.is_finite() {
None
} else if number.abs() > 1.0 {
Some(number / 100.0)
} else {
Some(number)
}
})
}
fn sanitize_current_ratio(value: Option<f64>) -> Option<f64> {
value.and_then(|number| {
if !number.is_finite() || number < 0.01 {
None
} else {
Some(number)
}
})
}
#[allow(dead_code)]
#[derive(Clone, Copy)]
pub(super) enum ResampleInterval {
Week,
Month,
}
#[allow(dead_code)]
pub(super) fn resample_history(rows: &[Ohlc], interval: ResampleInterval) -> Vec<Ohlc> {
let mut grouped: BTreeMap<(i32, u32), Ohlc> = BTreeMap::new();
for row in rows {
let key = match interval {
ResampleInterval::Week => {
let iso = row.date.iso_week();
(iso.year(), iso.week())
}
ResampleInterval::Month => (row.date.year(), row.date.month()),
};
grouped
.entry(key)
.and_modify(|existing| {
existing.high = existing.high.max(row.high);
existing.low = existing.low.min(row.low);
existing.close = row.close;
existing.volume = existing.volume.saturating_add(row.volume);
existing.date = row.date;
})
.or_insert_with(|| row.clone());
}
grouped.into_values().collect()
}
fn parse_chart_date(raw: &str) -> Option<NaiveDate> {
if let Ok(date) = chrono::DateTime::parse_from_rfc3339(raw) {
return Some(date.date_naive());
}
if let Ok(timestamp) = raw.parse::<i64>() {
return chrono::DateTime::from_timestamp(timestamp, 0).map(|dt| dt.date_naive());
}
NaiveDate::parse_from_str(raw, "%Y-%m-%d").ok()
}
fn round_price(value: f64) -> i64 {
value.round() as i64
}
fn round_u64(value: Option<f64>) -> Option<u64> {
value.and_then(|number| {
if !number.is_finite() || number.is_sign_negative() {
None
} else {
Some(number.round() as u64)
}
})
}
fn de_opt_f64_lenient<'de, D>(deserializer: D) -> Result<Option<f64>, D::Error>
where
D: Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(untagged)]
enum NumberLike {
F64(f64),
String(String),
}
let value = Option::<NumberLike>::deserialize(deserializer)?;
match value {
Some(NumberLike::F64(number)) if number.is_finite() => Ok(Some(number)),
Some(NumberLike::F64(_)) => Ok(None),
Some(NumberLike::String(raw)) => {
let trimmed = raw.trim();
if trimmed.is_empty() || trimmed.eq_ignore_ascii_case("nan") {
Ok(None)
} else {
trimmed.parse::<f64>().map(Some).map_err(D::Error::custom)
}
}
None => Ok(None),
}
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct MsnQuote {
#[serde(default)]
symbol: Option<String>,
price: Option<f64>,
#[serde(default)]
price_change: Option<f64>,
#[serde(default)]
price_change_percent: Option<f64>,
#[serde(default)]
price_previous_close: Option<f64>,
#[serde(default, rename = "price52wHigh")]
price_52w_high: Option<f64>,
#[serde(default, rename = "price52wLow")]
price_52w_low: Option<f64>,
#[serde(default)]
accumulated_volume: Option<f64>,
#[serde(default)]
average_volume: Option<f64>,
#[serde(default)]
market_cap: Option<f64>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct KeyRatios {
#[serde(default)]
industry_metrics: Vec<IndustryMetric>,
#[serde(default)]
company_metrics: Vec<IndustryMetric>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct IndustryMetric {
year: Option<String>,
fiscal_period_type: Option<String>,
#[serde(default)]
revenue_growth_rate: Option<f64>,
#[serde(default)]
earnings_growth_rate: Option<f64>,
#[serde(default, rename = "netIncomeYTDYTDGrowthRate")]
net_income_ytd_ytd_growth_rate: Option<f64>,
#[serde(default, rename = "revenueYTDYTD")]
revenue_ytd_ytd: Option<f64>,
#[serde(default)]
net_margin: Option<f64>,
#[serde(default)]
profit_margin: Option<f64>,
#[serde(default)]
roe: Option<f64>,
#[serde(default, rename = "roaTTM")]
roa_ttm: Option<f64>,
#[serde(default)]
return_on_asset_current: Option<f64>,
#[serde(default)]
debt_to_equity_ratio: Option<f64>,
#[serde(default, deserialize_with = "de_opt_f64_lenient")]
current_ratio: Option<f64>,
#[serde(default)]
price_to_earnings_ratio: Option<f64>,
#[serde(default, rename = "forwardPriceToEPS")]
forward_price_to_eps: Option<f64>,
#[serde(default)]
price_to_book_ratio: Option<f64>,
}
#[derive(Debug, Deserialize)]
pub(crate) struct MsnChart {
series: ChartSeries,
}
#[derive(Debug, Default, Deserialize)]
#[serde(rename_all = "camelCase")]
struct ChartSeries {
#[serde(default)]
time_stamps: Vec<String>,
#[serde(default)]
prices: Vec<f64>,
#[serde(default)]
open_prices: Vec<f64>,
#[serde(default)]
prices_high: Vec<f64>,
#[serde(default)]
prices_low: Vec<f64>,
#[serde(default)]
volumes: Vec<f64>,
}
impl ChartSeries {
fn has_real_ohlcv(&self) -> bool {
!self.time_stamps.is_empty()
&& self.open_prices.len() == self.time_stamps.len()
&& self.prices_high.len() == self.time_stamps.len()
&& self.prices_low.len() == self.time_stamps.len()
&& self.prices.len() == self.time_stamps.len()
&& self.volumes.len() == self.time_stamps.len()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct ClosePoint {
date: NaiveDate,
@ -547,8 +131,8 @@ struct ClosePoint {
#[cfg(test)]
mod tests {
use super::{
ResampleInterval, parse_close_only_history_from_str, parse_fundamentals_from_str,
parse_history_from_str, parse_quote_from_str, resample_history,
parse_close_only_history_from_str, parse_fundamentals_from_str, parse_history_from_str,
parse_quote_from_str,
};
use crate::api::types::{Ohlc, Period};
@ -657,83 +241,12 @@ mod tests {
},
];
let weekly = resample_history(&rows, ResampleInterval::Week);
let weekly =
super::super::map::resample_history(&rows, super::super::map::ResampleInterval::Week);
assert_eq!(weekly.len(), 2);
assert_eq!(weekly[0].open, 100);
assert_eq!(weekly[0].close, 109);
assert_eq!(weekly[0].volume, 21);
assert_eq!(weekly[1].close, 114);
}
#[test]
fn normalizes_live_style_percent_metrics() {
let raw = r#"[
{
"industryMetrics": [
{
"year": "2025",
"fiscalPeriodType": "Q1",
"revenueGrowthRate": 9.584679119559473,
"earningsGrowthRate": 28.793562408178182,
"netMargin": 35.05868669243578,
"roe": 16.27117054525313,
"returnOnAssetCurrent": 2.5707368150889867,
"debtToEquityRatio": 32.80253090283387,
"currentRatio": 9.38775908812586E-06,
"priceToEarningsRatio": 21.331183408517173,
"priceToBookRatio": 3.0625539678152234
},
{
"year": "2025",
"fiscalPeriodType": "TTM",
"revenueYTDYTD": 0.0481563350951302,
"netIncomeYTDYTDGrowthRate": 0.0492553610240516,
"profitMargin": 0.504190105842766,
"roe": 0.211493,
"roaTTM": 3.7919,
"priceToEarningsRatio": 17.296683642049683,
"priceToSalesRatio": 7.6652108104296985,
"priceToBookRatio": 3.107795874896335
},
{
"year": "2025",
"fiscalPeriodType": "NTM",
"forwardPriceToEPS": 14.723
}
],
"companyMetrics": [
{
"year": "2025",
"fiscalPeriodType": "TTM",
"revenueYTDYTD": 0.0481563350951302,
"netIncomeYTDYTDGrowthRate": 0.0492553610240516,
"profitMargin": 0.504190105842766,
"roe": 0.211493,
"roaTTM": 3.7919,
"priceToEarningsRatio": 17.296683642049683,
"priceToBookRatio": 3.107795874896335
},
{
"year": "2025",
"fiscalPeriodType": "NTM",
"forwardPriceToEPS": 14.723
}
]
}
]"#;
let quote_raw = r#"[{"symbol":"BBCA","marketCap":866500400000000.0}]"#;
let fundamentals =
parse_fundamentals_from_str(raw, Some(quote_raw)).expect("fundamentals parsed");
assert_eq!(fundamentals.trailing_pe, Some(17.296683642049683));
assert_eq!(fundamentals.forward_pe, Some(14.723));
assert_eq!(fundamentals.price_to_book, Some(3.107795874896335));
assert_eq!(fundamentals.return_on_equity, Some(0.211493));
assert_eq!(fundamentals.profit_margins, Some(0.504190105842766));
assert_eq!(fundamentals.return_on_assets, Some(0.037919));
assert_eq!(fundamentals.revenue_growth, Some(0.0481563350951302));
assert_eq!(fundamentals.earnings_growth, Some(0.0492553610240516));
assert_eq!(fundamentals.debt_to_equity, None);
assert_eq!(fundamentals.current_ratio, None);
}
}