Fix Yahoo schema handling, integer IDR prices, and resilience

This commit is contained in:
Ciphercat 2026-03-05 19:02:53 +00:00
commit f3cefaaaf4
7 changed files with 237 additions and 54 deletions

View file

@ -25,11 +25,11 @@ pub fn resolve_symbol(symbol: &str, exchange: &str) -> String {
format!("{trimmed}.{}", exchange.trim().to_uppercase())
}
pub fn default_provider() -> Box<dyn MarketDataProvider> {
pub fn default_provider(verbose: bool) -> Box<dyn MarketDataProvider> {
if std::env::var("IDX_USE_MOCK_PROVIDER").is_ok() {
Box::new(MockProvider::from_fixtures())
} else {
Box::new(yahoo::YahooProvider::new())
Box::new(yahoo::YahooProvider::new(verbose))
}
}

View file

@ -1,33 +1,125 @@
use chrono::NaiveDate;
use clap::ValueEnum;
use serde::{Deserialize, Serialize};
use serde::{Deserialize, Deserializer, Serialize, de::Error as _};
/// Snapshot quote data normalized from Yahoo Finance `/v8/finance/chart` response.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Quote {
/// Trading symbol as returned by Yahoo `chart.result[0].meta.symbol`.
pub symbol: String,
pub price: f64,
pub change: f64,
/// Last traded regular market price in IDR (whole Rupiah), mapped from
/// `chart.result[0].meta.regularMarketPrice` and rounded to nearest integer.
#[serde(deserialize_with = "de_i64_from_number")]
pub price: i64,
/// Absolute day change in IDR (whole Rupiah), computed as
/// `regularMarketPrice - previousClose` using rounded integer prices.
#[serde(deserialize_with = "de_i64_from_number")]
pub change: i64,
/// Percentage day change as decimal percent (`0-100` scale), computed from
/// Yahoo `regularMarketPrice` and `previousClose` raw floats.
pub change_pct: f64,
/// Traded regular market volume (shares), from `regularMarketVolume`.
pub volume: u64,
pub market_cap: Option<f64>,
pub week52_high: Option<f64>,
pub week52_low: Option<f64>,
/// Company market capitalization in IDR, from `marketCap`.
#[serde(default, deserialize_with = "de_opt_u64_from_number")]
pub market_cap: Option<u64>,
/// 52-week high in IDR (whole Rupiah), from `fiftyTwoWeekHigh` rounded.
#[serde(default, deserialize_with = "de_opt_i64_from_number")]
pub week52_high: Option<i64>,
/// 52-week low in IDR (whole Rupiah), from `fiftyTwoWeekLow` rounded.
#[serde(default, deserialize_with = "de_opt_i64_from_number")]
pub week52_low: Option<i64>,
/// Relative position within 52-week range (`0.0..=1.0`), computed from raw
/// Yahoo `fiftyTwoWeekLow` and `fiftyTwoWeekHigh`.
pub week52_position: Option<f64>,
/// Coarse 52-week range bucket derived from `week52_position`.
pub range_signal: Option<String>,
pub prev_close: Option<f64>,
/// Previous close in IDR (whole Rupiah), from
/// `previousClose` or `chartPreviousClose`, rounded.
#[serde(default, deserialize_with = "de_opt_i64_from_number")]
pub prev_close: Option<i64>,
/// Average daily volume for the last 3 months (shares), from
/// `averageDailyVolume3Month`.
pub avg_volume: Option<u64>,
}
/// OHLC candle data normalized from Yahoo Finance chart indicators.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Ohlc {
/// Candle date (exchange-local day boundary from Yahoo timestamp).
pub date: NaiveDate,
pub open: f64,
pub high: f64,
pub low: f64,
pub close: f64,
/// Opening price in IDR (whole Rupiah), from `indicators.quote[0].open` rounded.
#[serde(deserialize_with = "de_i64_from_number")]
pub open: i64,
/// Highest traded price in IDR (whole Rupiah), from `indicators.quote[0].high` rounded.
#[serde(deserialize_with = "de_i64_from_number")]
pub high: i64,
/// Lowest traded price in IDR (whole Rupiah), from `indicators.quote[0].low` rounded.
#[serde(deserialize_with = "de_i64_from_number")]
pub low: i64,
/// Closing price in IDR (whole Rupiah), from `indicators.quote[0].close` rounded.
#[serde(deserialize_with = "de_i64_from_number")]
pub close: i64,
/// Traded volume (shares), from `indicators.quote[0].volume`.
pub volume: u64,
}
#[derive(Debug, Deserialize)]
#[serde(untagged)]
enum NumberLike {
I64(i64),
U64(u64),
F64(f64),
}
fn de_i64_from_number<'de, D>(deserializer: D) -> Result<i64, D::Error>
where
D: Deserializer<'de>,
{
let value = NumberLike::deserialize(deserializer)?;
Ok(match value {
NumberLike::I64(v) => v,
NumberLike::U64(v) => i64::try_from(v).map_err(D::Error::custom)?,
NumberLike::F64(v) => v.round() as i64,
})
}
fn de_opt_i64_from_number<'de, D>(deserializer: D) -> Result<Option<i64>, D::Error>
where
D: Deserializer<'de>,
{
Option::<NumberLike>::deserialize(deserializer).and_then(|v| {
v.map(|n| match n {
NumberLike::I64(x) => Ok(x),
NumberLike::U64(x) => i64::try_from(x).map_err(D::Error::custom),
NumberLike::F64(x) => Ok(x.round() as i64),
})
.transpose()
})
}
fn de_opt_u64_from_number<'de, D>(deserializer: D) -> Result<Option<u64>, D::Error>
where
D: Deserializer<'de>,
{
Option::<NumberLike>::deserialize(deserializer).and_then(|v| {
v.map(|n| match n {
NumberLike::I64(x) => u64::try_from(x).map_err(D::Error::custom),
NumberLike::U64(x) => Ok(x),
NumberLike::F64(x) => {
if x.is_sign_negative() {
Err(D::Error::custom(
"negative value cannot be converted to u64",
))
} else {
Ok(x.round() as u64)
}
}
})
.transpose()
})
}
#[derive(Debug, Clone, Serialize, Deserialize, ValueEnum)]
pub enum Period {
#[value(name = "1d")]

View file

@ -12,13 +12,18 @@ const BASE_URL: &str = "https://query2.finance.yahoo.com";
pub struct YahooProvider {
agent: ureq::Agent,
verbose: bool,
}
impl YahooProvider {
pub fn new() -> Self {
Self {
agent: ureq::Agent::new_with_defaults(),
}
pub fn new(verbose: bool) -> Self {
let agent: ureq::Agent = ureq::Agent::config_builder()
.timeout_connect(Some(Duration::from_secs(5)))
.timeout_recv_body(Some(Duration::from_secs(10)))
.build()
.into();
Self { agent, verbose }
}
fn chart_url(symbol: &str, period: &Period, interval: &Interval) -> String {
@ -41,10 +46,14 @@ impl YahooProvider {
let response = self.agent.get(&url).header("User-Agent", USER_AGENT).call();
match response {
Ok(ok) => {
return ok
let chart = ok
.into_body()
.read_json::<ChartResponse>()
.map_err(|e| IdxError::ParseError(e.to_string()));
.map_err(|e| IdxError::ParseError(e.to_string()))?;
if let Some(err) = chart.chart.error.as_ref() {
return Err(map_chart_error(symbol, err));
}
return Ok(chart);
}
Err(ureq::Error::StatusCode(429)) => {
if attempt < 2 {
@ -52,6 +61,9 @@ impl YahooProvider {
wait *= 2;
}
}
Err(ureq::Error::StatusCode(404)) => {
return Err(IdxError::SymbolNotFound(symbol.to_string()));
}
Err(e) => return Err(IdxError::Http(e.to_string())),
}
}
@ -60,11 +72,23 @@ impl YahooProvider {
}
fn jitter() -> Duration {
let millis = (std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.subsec_millis() % 100)
.unwrap_or(42)) as u64;
Duration::from_millis(millis)
Duration::from_millis(fastrand::u64(0..100))
}
fn round_price(value: f64) -> i64 {
value.round() as i64
}
// verbose behavior is configured on YahooProvider and threaded into history parsing.
fn map_chart_error(symbol: &str, err: &ChartError) -> IdxError {
if err.code.eq_ignore_ascii_case("Not Found") {
return IdxError::SymbolNotFound(symbol.to_string());
}
IdxError::Http(format!(
"yahoo chart error {}: {}",
err.code, err.description
))
}
impl MarketDataProvider for YahooProvider {
@ -80,17 +104,24 @@ impl MarketDataProvider for YahooProvider {
interval: &Interval,
) -> Result<Vec<Ohlc>, IdxError> {
let chart = self.fetch_chart(symbol, period, interval)?;
parse_history(&chart)
parse_history_with_verbose(&chart, self.verbose)
}
}
pub(crate) fn parse_quote_from_str(symbol: &str, raw: &str) -> Result<Quote, IdxError> {
let chart: ChartResponse =
serde_json::from_str(raw).map_err(|e| IdxError::ParseError(e.to_string()))?;
if let Some(err) = chart.chart.error.as_ref() {
return Err(map_chart_error(symbol, err));
}
parse_quote(symbol, &chart)
}
fn parse_quote(symbol: &str, chart: &ChartResponse) -> Result<Quote, IdxError> {
if let Some(err) = chart.chart.error.as_ref() {
return Err(map_chart_error(symbol, err));
}
let result = chart
.chart
.result
@ -98,17 +129,26 @@ fn parse_quote(symbol: &str, chart: &ChartResponse) -> Result<Quote, IdxError> {
.and_then(|r| r.first())
.ok_or(IdxError::ProviderUnavailable)?;
let meta = result.meta.as_ref().ok_or(IdxError::ProviderUnavailable)?;
let price = meta
let raw_price = meta
.regular_market_price
.ok_or(IdxError::SymbolNotFound(symbol.to_string()))?;
let prev_close = meta.previous_close.or(meta.chart_previous_close);
let change = prev_close.map_or(0.0, |p| price - p);
let change_pct = prev_close.map_or(0.0, |p| if p != 0.0 { (change / p) * 100.0 } else { 0.0 });
let raw_prev_close = meta.previous_close.or(meta.chart_previous_close);
let price = round_price(raw_price);
let prev_close = raw_prev_close.map(round_price);
let change = prev_close.map_or(0, |p| price - p);
let change_pct = raw_prev_close.map_or(0.0, |p| {
if p != 0.0 {
((raw_price - p) / p) * 100.0
} else {
0.0
}
});
let (week52_position, range_signal) = match (meta.fifty_two_week_low, meta.fifty_two_week_high)
{
(Some(low), Some(high)) if high > low => {
let pos = (price - low) / (high - low);
let pos = (raw_price - low) / (high - low);
let signal = if pos > 0.66 {
"upper"
} else if pos < 0.33 {
@ -128,8 +168,8 @@ fn parse_quote(symbol: &str, chart: &ChartResponse) -> Result<Quote, IdxError> {
change_pct,
volume: meta.regular_market_volume.unwrap_or(0),
market_cap: meta.market_cap,
week52_high: meta.fifty_two_week_high,
week52_low: meta.fifty_two_week_low,
week52_high: meta.fifty_two_week_high.map(round_price),
week52_low: meta.fifty_two_week_low.map(round_price),
week52_position,
range_signal,
prev_close,
@ -140,10 +180,14 @@ fn parse_quote(symbol: &str, chart: &ChartResponse) -> Result<Quote, IdxError> {
pub(crate) fn parse_history_from_str(raw: &str) -> Result<Vec<Ohlc>, IdxError> {
let chart: ChartResponse =
serde_json::from_str(raw).map_err(|e| IdxError::ParseError(e.to_string()))?;
parse_history(&chart)
parse_history_with_verbose(&chart, false)
}
fn parse_history(chart: &ChartResponse) -> Result<Vec<Ohlc>, IdxError> {
fn parse_history_with_verbose(chart: &ChartResponse, verbose: bool) -> Result<Vec<Ohlc>, IdxError> {
if let Some(err) = chart.chart.error.as_ref() {
return Err(map_chart_error("unknown", err));
}
let result = chart
.chart
.result
@ -162,20 +206,28 @@ fn parse_history(chart: &ChartResponse) -> Result<Vec<Ohlc>, IdxError> {
.ok_or(IdxError::ProviderUnavailable)?;
let mut out = Vec::new();
let mut dropped = 0usize;
for (i, ts) in timestamps.iter().enumerate() {
let open = quote
.open
.as_ref()
.and_then(|v| v.get(i).copied().flatten());
.and_then(|v| v.get(i).copied().flatten())
.map(round_price);
let high = quote
.high
.as_ref()
.and_then(|v| v.get(i).copied().flatten());
let low = quote.low.as_ref().and_then(|v| v.get(i).copied().flatten());
.and_then(|v| v.get(i).copied().flatten())
.map(round_price);
let low = quote
.low
.as_ref()
.and_then(|v| v.get(i).copied().flatten())
.map(round_price);
let close = quote
.close
.as_ref()
.and_then(|v| v.get(i).copied().flatten());
.and_then(|v| v.get(i).copied().flatten())
.map(round_price);
let volume = quote
.volume
.as_ref()
@ -193,8 +245,17 @@ fn parse_history(chart: &ChartResponse) -> Result<Vec<Ohlc>, IdxError> {
close,
volume,
});
} else {
dropped += 1;
}
}
if dropped > 0 && verbose {
eprintln!(
"warning: dropped {dropped} OHLC row(s) from Yahoo response due to missing fields"
);
}
Ok(out)
}
@ -206,6 +267,14 @@ struct ChartResponse {
#[derive(Debug, Deserialize)]
struct ChartRoot {
result: Option<Vec<ChartResult>>,
error: Option<ChartError>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct ChartError {
code: String,
description: String,
}
#[derive(Debug, Deserialize)]
@ -226,9 +295,10 @@ struct ChartMeta {
regular_market_volume: Option<u64>,
regular_market_day_high: Option<f64>,
regular_market_day_low: Option<f64>,
market_cap: Option<f64>,
market_cap: Option<u64>,
fifty_two_week_high: Option<f64>,
fifty_two_week_low: Option<f64>,
#[serde(rename = "averageDailyVolume3Month")]
average_daily_volume_3month: Option<u64>,
}
@ -249,7 +319,8 @@ struct IndicatorQuote {
#[cfg(test)]
mod tests {
use super::{
ChartResponse, parse_history, parse_history_from_str, parse_quote, parse_quote_from_str,
ChartResponse, parse_history_from_str, parse_history_with_verbose, parse_quote,
parse_quote_from_str,
};
const SAMPLE: &str = r#"{
@ -282,10 +353,10 @@ mod tests {
let chart: ChartResponse = serde_json::from_str(SAMPLE).expect("valid chart fixture");
let quote = parse_quote("BBCA.JK", &chart).expect("quote parsed");
assert_eq!(quote.symbol, "BBCA.JK");
assert_eq!(quote.price, 9875.0);
let history = parse_history(&chart).expect("history parsed");
assert_eq!(quote.price, 9875);
let history = parse_history_with_verbose(&chart, false).expect("history parsed");
assert_eq!(history.len(), 2);
assert_eq!(history[0].close, 9875.0);
assert_eq!(history[0].close, 9875);
}
#[test]
@ -297,8 +368,17 @@ mod tests {
let quote = parse_quote_from_str("BBCA.JK", &quote_raw).expect("fixture quote parsed");
assert_eq!(quote.symbol, "BBCA.JK");
assert_eq!(quote.market_cap, Some(1_215_200_000_000_000));
assert_eq!(quote.avg_volume, Some(10_000_000));
let history = parse_history_from_str(&history_raw).expect("fixture history parsed");
assert!(!history.is_empty());
}
#[test]
fn maps_not_found_chart_error_to_symbol_not_found() {
let raw = r#"{"chart":{"result":null,"error":{"code":"Not Found","description":"No data found"}}}"#;
let err = parse_quote_from_str("INVALID.JK", raw).expect_err("expected symbol error");
assert!(matches!(err, crate::error::IdxError::SymbolNotFound(_)));
}
}

View file

@ -39,7 +39,7 @@ fn run() -> Result<(), IdxError> {
}
}
Commands::Stocks(stocks) => {
let provider = default_provider();
let provider = default_provider(cli.verbose > 0);
if let Err(err) = cli::stocks::handle(
stocks,
&config,

View file

@ -4,9 +4,8 @@ use owo_colors::OwoColorize;
use crate::api::types::{Ohlc, Quote};
use crate::error::IdxError;
pub fn format_idr(value: f64) -> String {
let rounded = value.round() as i64;
let chars: Vec<char> = rounded.to_string().chars().rev().collect();
pub fn format_idr(value: i64) -> String {
let chars: Vec<char> = value.to_string().chars().rev().collect();
let mut out = String::new();
for (i, ch) in chars.iter().enumerate() {
if i > 0 && i % 3 == 0 {
@ -17,6 +16,10 @@ pub fn format_idr(value: f64) -> String {
out.chars().rev().collect()
}
pub fn format_u64(value: u64) -> String {
format_idr(value as i64)
}
pub fn print_quotes(quotes: &[Quote], no_color: bool) -> Result<(), IdxError> {
let mut table = Table::new();
table
@ -36,12 +39,12 @@ pub fn print_quotes(quotes: &[Quote], no_color: bool) -> Result<(), IdxError> {
table.add_row(vec![
Cell::new(&q.symbol),
Cell::new(format_idr(q.price)),
Cell::new(format!("{:+.2}", q.change)),
Cell::new(format!("{:+}", q.change)),
pct_cell,
Cell::new(format_idr(q.volume as f64)),
Cell::new(format_u64(q.volume)),
Cell::new(
q.market_cap
.map(format_idr)
.map(format_u64)
.unwrap_or_else(|| "-".to_string()),
),
]);
@ -65,7 +68,7 @@ pub fn print_history(symbol: &str, history: &[Ohlc]) -> Result<(), IdxError> {
Cell::new(format_idr(item.high)),
Cell::new(format_idr(item.low)),
Cell::new(format_idr(item.close)),
Cell::new(format_idr(item.volume as f64)),
Cell::new(format_u64(item.volume)),
]);
}
println!("{table}");
@ -74,11 +77,11 @@ pub fn print_history(symbol: &str, history: &[Ohlc]) -> Result<(), IdxError> {
#[cfg(test)]
mod tests {
use super::format_idr;
use super::{format_idr, format_u64};
#[test]
fn formats_idr_numbers() {
assert_eq!(format_idr(9875.0), "9,875");
assert_eq!(format_idr(1_215_200_000_000_000.0), "1,215,200,000,000,000");
assert_eq!(format_idr(9875), "9,875");
assert_eq!(format_u64(1_215_200_000_000_000), "1,215,200,000,000,000");
}
}