0xrsydn/idx-cli
1
mirror of https://github.com/0xrsydn/idx-cli.git synced 2026-08-07 01:33:52 +00:00
Code Issues Projects Releases 1 Packages Wiki Activity

chore: prepare v0.2.1 release

Browse source
This commit is contained in:
Rasyidan Akbar F. 2026-04-06 17:47:20 +07:00
commit c94c826307
32 changed files with 2483 additions and 382 deletions
Download patch file Download diff file Expand all files Collapse all files

233
src/api/mod.rs
View file

@ -62,6 +62,75 @@ pub trait NewsProvider {
fn news(&self, symbol: &str, limit: usize) -> Result<Vec<NewsItem>, IdxError>;
}
#[allow(dead_code)]
pub trait ScreenerProvider {
fn screener(&self, filter: &str, region: &str, limit: usize) -> Result<Vec<Quote>, IdxError>;
}
pub struct SelectedProvider {
kind: ProviderKind,
market: Box<dyn MarketDataProvider>,
profile: Option<Box<dyn ProfileProvider>>,
financials: Option<Box<dyn FinancialsProvider>>,
earnings: Option<Box<dyn EarningsProvider>>,
sentiment: Option<Box<dyn SentimentProvider>>,
insights: Option<Box<dyn InsightsProvider>>,
news: Option<Box<dyn NewsProvider>>,
screener: Option<Box<dyn ScreenerProvider>>,
}
impl SelectedProvider {
pub fn kind(&self) -> ProviderKind {
self.kind
}
pub fn market(&self) -> &dyn MarketDataProvider {
self.market.as_ref()
}
pub fn profile_provider(&self, subject: &str) -> Result<&dyn ProfileProvider, IdxError> {
self.profile
.as_deref()
.ok_or_else(|| msn_capability_error(subject))
}
pub fn financials_provider(&self, subject: &str) -> Result<&dyn FinancialsProvider, IdxError> {
self.financials
.as_deref()
.ok_or_else(|| msn_capability_error(subject))
}
pub fn earnings_provider(&self, subject: &str) -> Result<&dyn EarningsProvider, IdxError> {
self.earnings
.as_deref()
.ok_or_else(|| msn_capability_error(subject))
}
pub fn sentiment_provider(&self, subject: &str) -> Result<&dyn SentimentProvider, IdxError> {
self.sentiment
.as_deref()
.ok_or_else(|| msn_capability_error(subject))
}
pub fn insights_provider(&self, subject: &str) -> Result<&dyn InsightsProvider, IdxError> {
self.insights
.as_deref()
.ok_or_else(|| msn_capability_error(subject))
}
pub fn news_provider(&self, subject: &str) -> Result<&dyn NewsProvider, IdxError> {
self.news
.as_deref()
.ok_or_else(|| msn_capability_error(subject))
}
pub fn screener_provider(&self, subject: &str) -> Result<&dyn ScreenerProvider, IdxError> {
self.screener
.as_deref()
.ok_or_else(|| msn_capability_error(subject))
}
}
pub fn resolve_symbol(symbol: &str, exchange: &str) -> Result<String, IdxError> {
let trimmed = symbol.trim().to_uppercase();
if trimmed.is_empty() {
@ -78,17 +147,71 @@ pub fn resolve_symbol(symbol: &str, exchange: &str) -> Result<String, IdxError>
Ok(format!("{trimmed}.{}", exchange.trim().to_uppercase()))
}
pub fn default_provider(provider: ProviderKind, verbose: bool) -> Box<dyn MarketDataProvider> {
if std::env::var("IDX_USE_MOCK_PROVIDER").is_ok() {
Box::new(MockProvider::from_fixtures(provider))
} else {
match provider {
ProviderKind::Yahoo => Box::new(yahoo::YahooProvider::new(verbose)),
ProviderKind::Msn => Box::new(msn::MsnProvider::new(verbose)),
}
pub fn default_provider(provider: ProviderKind, verbose: bool) -> SelectedProvider {
build_selected_provider(
provider,
verbose,
std::env::var("IDX_USE_MOCK_PROVIDER").is_ok(),
)
}
fn build_selected_provider(
provider: ProviderKind,
verbose: bool,
use_mock: bool,
) -> SelectedProvider {
match (provider, use_mock) {
(ProviderKind::Yahoo, true) => SelectedProvider {
kind: ProviderKind::Yahoo,
market: Box::new(MockProvider::from_fixtures(ProviderKind::Yahoo)),
profile: None,
financials: None,
earnings: None,
sentiment: None,
insights: None,
news: None,
screener: None,
},
(ProviderKind::Yahoo, false) => SelectedProvider {
kind: ProviderKind::Yahoo,
market: Box::new(yahoo::YahooProvider::new(verbose)),
profile: None,
financials: None,
earnings: None,
sentiment: None,
insights: None,
news: None,
screener: None,
},
(ProviderKind::Msn, true) => SelectedProvider {
kind: ProviderKind::Msn,
market: Box::new(MockProvider::from_fixtures(ProviderKind::Msn)),
profile: Some(Box::new(msn::MsnProvider::new(verbose))),
financials: Some(Box::new(msn::MsnProvider::new(verbose))),
earnings: Some(Box::new(msn::MsnProvider::new(verbose))),
sentiment: Some(Box::new(msn::MsnProvider::new(verbose))),
insights: Some(Box::new(msn::MsnProvider::new(verbose))),
news: Some(Box::new(msn::MsnProvider::new(verbose))),
screener: Some(Box::new(msn::MsnProvider::new(verbose))),
},
(ProviderKind::Msn, false) => SelectedProvider {
kind: ProviderKind::Msn,
market: Box::new(msn::MsnProvider::new(verbose)),
profile: Some(Box::new(msn::MsnProvider::new(verbose))),
financials: Some(Box::new(msn::MsnProvider::new(verbose))),
earnings: Some(Box::new(msn::MsnProvider::new(verbose))),
sentiment: Some(Box::new(msn::MsnProvider::new(verbose))),
insights: Some(Box::new(msn::MsnProvider::new(verbose))),
news: Some(Box::new(msn::MsnProvider::new(verbose))),
screener: Some(Box::new(msn::MsnProvider::new(verbose))),
},
}
}
fn msn_capability_error(subject: &str) -> IdxError {
IdxError::Unsupported(format!("{subject}: command requires --provider msn"))
}
/// Resolves a history provider based on the selected market data provider and
/// history provider strategy.
///
@ -115,7 +238,12 @@ pub fn history_provider(
.into(),
));
}
return Ok((resolved, Box::new(MockProvider::from_fixtures(resolved))));
return Ok((
resolved,
Box::new(MockProvider::from_fixtures_with_history_verbose(
resolved, verbose,
)),
));
}
match resolved {
@ -135,21 +263,30 @@ pub struct MockProvider {
impl MockProvider {
pub fn from_fixtures(provider: ProviderKind) -> Self {
Self::from_fixtures_with_history_verbose(provider, false)
}
pub fn from_fixtures_with_history_verbose(
provider: ProviderKind,
history_verbose: bool,
) -> Self {
if std::env::var("IDX_MOCK_ERROR").is_ok() {
return Self::with_error(IdxError::ProviderUnavailable);
}
match provider {
ProviderKind::Yahoo => Self::from_yahoo_fixtures(),
ProviderKind::Yahoo => Self::from_yahoo_fixtures(history_verbose),
ProviderKind::Msn => Self::from_msn_fixtures(),
}
}
fn from_yahoo_fixtures() -> Self {
fn from_yahoo_fixtures(history_verbose: bool) -> Self {
let quote_raw = std::fs::read_to_string("tests/fixtures/chart_bbca_1d.json")
.unwrap_or_else(|_| "{}".to_string());
let history_raw = std::fs::read_to_string("tests/fixtures/chart_bbca_3mo.json")
.unwrap_or_else(|_| "{}".to_string());
let history_path = std::env::var("IDX_MOCK_YAHOO_HISTORY_FIXTURE")
.unwrap_or_else(|_| "tests/fixtures/chart_bbca_3mo.json".to_string());
let history_raw =
std::fs::read_to_string(&history_path).unwrap_or_else(|_| "{}".to_string());
let fundamentals_raw = std::fs::read_to_string("tests/fixtures/quotesummary_bbca.json")
.unwrap_or_else(|_| "{}".to_string());
@ -157,8 +294,9 @@ impl MockProvider {
.map_err(|e| IdxError::ParseError(e.to_string()));
let fundamentals = yahoo::parse_fundamentals_from_str("BBCA.JK", &fundamentals_raw)
.map_err(|e| IdxError::ParseError(e.to_string()));
let history = yahoo::parse_history_from_str("BBCA.JK", &history_raw)
.map_err(|e| IdxError::ParseError(e.to_string()));
let history =
yahoo::parse_history_from_str_with_verbose("BBCA.JK", &history_raw, history_verbose)
.map_err(|e| IdxError::ParseError(e.to_string()));
Self {
quote,
@ -170,8 +308,10 @@ impl MockProvider {
fn from_msn_fixtures() -> Self {
let quote_raw = std::fs::read_to_string("tests/fixtures/msn_quote_bbca.json")
.unwrap_or_else(|_| "[]".to_string());
let fundamentals_raw = std::fs::read_to_string("tests/fixtures/msn_keyratios_bbca.json")
.unwrap_or_else(|_| "[]".to_string());
let fundamentals_path = std::env::var("IDX_MOCK_MSN_KEYRATIOS_FIXTURE")
.unwrap_or_else(|_| "tests/fixtures/msn_keyratios_bbca.json".to_string());
let fundamentals_raw =
std::fs::read_to_string(&fundamentals_path).unwrap_or_else(|_| "[]".to_string());
let quote = msn::parse_quote_from_str("BBCA.JK", &quote_raw)
.map_err(|e| IdxError::ParseError(e.to_string()));
@ -225,7 +365,8 @@ impl HistoryProvider for MockProvider {
#[cfg(test)]
mod tests {
use super::resolve_symbol;
use super::{ProviderKind, build_selected_provider, resolve_symbol};
use crate::error::IdxError;
#[test]
fn resolves_symbol_variants() {
@ -241,4 +382,60 @@ mod tests {
// Valid ticker returns Ok
assert_eq!(resolve_symbol("BBCA", "JK").unwrap(), "BBCA.JK");
}
#[test]
fn selected_provider_exposes_expected_capabilities_for_msn() {
let provider = build_selected_provider(ProviderKind::Msn, false, false);
assert!(provider.profile_provider("BBCA.JK").is_ok());
assert!(provider.financials_provider("BBCA.JK").is_ok());
assert!(provider.earnings_provider("BBCA.JK").is_ok());
assert!(provider.sentiment_provider("BBCA.JK").is_ok());
assert!(provider.insights_provider("BBCA.JK").is_ok());
assert!(provider.news_provider("BBCA.JK").is_ok());
assert!(provider.screener_provider("screen").is_ok());
}
#[test]
fn selected_provider_exposes_expected_capabilities_for_mock_msn() {
let provider = build_selected_provider(ProviderKind::Msn, false, true);
assert!(provider.profile_provider("BBCA.JK").is_ok());
assert!(provider.financials_provider("BBCA.JK").is_ok());
assert!(provider.earnings_provider("BBCA.JK").is_ok());
assert!(provider.sentiment_provider("BBCA.JK").is_ok());
assert!(provider.insights_provider("BBCA.JK").is_ok());
assert!(provider.news_provider("BBCA.JK").is_ok());
assert!(provider.screener_provider("screen").is_ok());
}
#[test]
fn selected_provider_rejects_msn_only_capabilities_for_yahoo() {
let provider = build_selected_provider(ProviderKind::Yahoo, false, false);
let err = match provider.profile_provider("BBCA.JK") {
Ok(_) => panic!("yahoo should not expose msn-only profile"),
Err(err) => err,
};
assert!(matches!(err, IdxError::Unsupported(_)));
assert_eq!(
err.to_string(),
"unsupported: BBCA.JK: command requires --provider msn"
);
}
#[test]
fn selected_provider_rejects_msn_only_capabilities_for_mock_yahoo() {
let provider = build_selected_provider(ProviderKind::Yahoo, false, true);
let err = match provider.screener_provider("screen") {
Ok(_) => panic!("mock yahoo should not expose msn-only screener"),
Err(err) => err,
};
assert!(matches!(err, IdxError::Unsupported(_)));
assert_eq!(
err.to_string(),
"unsupported: screen: command requires --provider msn"
);
}
}

167
src/api/msn/map.rs
View file

@ -65,13 +65,11 @@ pub(super) fn parse_fundamentals(
quote: Option<&MsnQuote>,
) -> Result<Fundamentals, IdxError> {
let ratios = ratios.first().ok_or(IdxError::ProviderUnavailable)?;
let metrics = if ratios.company_metrics.is_empty() {
&ratios.industry_metrics
} else {
&ratios.company_metrics
};
if preferred_metric(metrics).is_none() {
return Err(IdxError::ProviderUnavailable);
let metrics = &ratios.company_metrics;
if metrics.is_empty() || !metrics.iter().any(metric_has_supported_values) {
return Err(IdxError::Unsupported(
"company fundamentals unavailable from MSN; industry fallback is disabled".into(),
));
}
Ok(Fundamentals {
@ -105,8 +103,31 @@ pub(super) fn parse_fundamentals(
})
}
fn preferred_metric(metrics: &[IndustryMetric]) -> Option<&IndustryMetric> {
metrics.iter().max_by_key(|metric| metric_rank(metric))
fn metric_has_supported_values(metric: &IndustryMetric) -> bool {
[
metric
.price_to_earnings_ratio
.filter(|value| value.is_finite()),
metric
.forward_price_to_eps
.filter(|value| value.is_finite()),
metric.price_to_book_ratio.filter(|value| value.is_finite()),
normalize_percentish(metric.roe),
normalize_percentish(metric.profit_margin.or(metric.net_margin)),
normalize_percentish(metric.roa_ttm.or(metric.return_on_asset_current)),
normalize_percentish(metric.revenue_ytd_ytd.or(metric.revenue_growth_rate)),
normalize_percentish(
metric
.net_income_ytd_ytd_growth_rate
.or(metric.earnings_growth_rate),
),
metric
.debt_to_equity_ratio
.filter(|value| value.is_finite()),
sanitize_current_ratio(metric.current_ratio),
]
.into_iter()
.any(|value| value.is_some())
}
fn best_metric_value<T: Copy>(
@ -398,7 +419,8 @@ pub(super) fn parse_financial_statements(
.and_then(|v| v.instrument_id.clone())
.unwrap_or_default(),
symbol: instrument
.and_then(|v| v.symbol.clone())
.and_then(|v| v.symbol.as_deref().and_then(ticker_from_symbol))
.map(|ticker| normalized_symbol(symbol, &ticker))
.unwrap_or_else(|| symbol.to_string()),
name: instrument
.and_then(|v| v.display_name.clone().or_else(|| v.short_name.clone()))
@ -411,7 +433,7 @@ pub(super) fn parse_financial_statements(
}
pub(super) fn parse_earnings(
_symbol: &str,
symbol: &str,
raw: &RawEarningsResponse,
) -> Result<EarningsReport, IdxError> {
let mut forecast = Vec::new();
@ -430,6 +452,7 @@ pub(super) fn parse_earnings(
history.sort_by_key(|row| row.earning_release_date.clone().unwrap_or_default());
Ok(EarningsReport {
symbol: symbol.to_string(),
eps_last_year: raw.eps_last_year.unwrap_or_default(),
revenue_last_year: raw.revenue_last_year.unwrap_or_default(),
forecast,
@ -504,6 +527,7 @@ pub(super) fn parse_insights(symbol: &str, raw: &[RawInsight]) -> Result<Insight
.instrument_id
.clone()
.unwrap_or_else(|| symbol.to_string()),
symbol: symbol.to_string(),
summary: insight_overview(item.display_name.as_deref(), positive, negative, neutral),
highlights,
risks,
@ -518,7 +542,7 @@ pub(super) fn parse_insights(symbol: &str, raw: &[RawInsight]) -> Result<Insight
})
}
pub(super) fn parse_news(raw: &RawNewsFeed) -> Result<Vec<NewsItem>, IdxError> {
pub(super) fn parse_news(symbol: &str, raw: &RawNewsFeed) -> Result<Vec<NewsItem>, IdxError> {
let source = raw
.sub_cards
.as_ref()
@ -529,6 +553,7 @@ pub(super) fn parse_news(raw: &RawNewsFeed) -> Result<Vec<NewsItem>, IdxError> {
.iter()
.map(|item| NewsItem {
id: item.id.clone().unwrap_or_default(),
symbol: symbol.to_string(),
title: item.title.clone().unwrap_or_default(),
url: item.url.clone().unwrap_or_default(),
description: item.description.clone().unwrap_or_default(),
@ -549,12 +574,10 @@ pub(super) fn parse_screener_results(raw: &RawScreenerResponse) -> Result<Vec<Qu
.as_ref()
.ok_or_else(|| IdxError::ParseError("no screener data".into()))?;
// Build Quote directly from screener MsnQuote data; default price to 0 if missing
// (do not route through parse_quote which errors on missing price)
let results: Vec<Quote> = quotes
.iter()
.map(|q| {
let raw_price = q.price.unwrap_or(0.0);
.filter_map(|q| {
let raw_price = q.price.filter(|price| price.is_finite() && *price > 0.0)?;
let price = round_price(raw_price);
let prev_close = q.price_previous_close.map(round_price);
let change = prev_close
@ -565,7 +588,7 @@ pub(super) fn parse_screener_results(raw: &RawScreenerResponse) -> Result<Vec<Qu
.symbol
.as_deref()
.and_then(ticker_from_symbol)
.unwrap_or_default();
.filter(|ticker| !ticker.is_empty())?;
let (week52_position, range_signal) = match (q.price_52w_low, q.price_52w_high) {
(Some(low), Some(high)) if high > low => {
let pos = (raw_price - low) / (high - low);
@ -580,7 +603,7 @@ pub(super) fn parse_screener_results(raw: &RawScreenerResponse) -> Result<Vec<Qu
}
_ => (None, None),
};
Quote {
Some(Quote {
symbol: normalized_symbol(&ticker, &ticker),
price,
change,
@ -593,7 +616,7 @@ pub(super) fn parse_screener_results(raw: &RawScreenerResponse) -> Result<Vec<Qu
range_signal,
prev_close,
avg_volume: round_u64(q.average_volume),
}
})
})
.collect();
@ -682,9 +705,11 @@ fn collect_earnings(
#[cfg(test)]
mod tests {
use super::{
RawNewsFeed, RawScreenerResponse, RawSentiment, parse_news, parse_screener_results,
KeyRatios, RawFinancialStatement, RawNewsFeed, RawScreenerResponse, RawSentiment,
parse_financial_statements, parse_fundamentals, parse_news, parse_screener_results,
parse_sentiment,
};
use crate::error::IdxError;
#[test]
fn parses_sentiment_fixture_statistics() {
@ -709,16 +734,46 @@ mod tests {
serde_json::from_str(include_str!("../../../tests/fixtures/msn_news_bbca.json"))
.expect("news fixture should deserialize");
let items = parse_news(&raw).expect("news should parse");
let items = parse_news("BBCA.JK", &raw).expect("news should parse");
assert_eq!(items.len(), 1);
assert_eq!(items[0].id, "news-1");
assert_eq!(items[0].symbol, "BBCA.JK");
assert_eq!(items[0].title, "BCA reports steady growth");
assert_eq!(items[0].provider, "Contoso News");
assert_eq!(items[0].published_at, "2026-03-20T10:00:00Z");
assert_eq!(items[0].read_time_min, Some(3));
}
#[test]
fn parse_financial_statements_normalizes_instrument_symbol() {
let raw: Vec<RawFinancialStatement> = serde_json::from_str(
r#"[
{
"underlyingInstrument": {
"instrumentId": "bn91jc",
"displayName": "Bank Central Asia Tbk PT",
"symbol": "BBCA"
},
"balanceSheets": {
"currency": "IDR",
"reportDate": "2025-03-31T00:00:00Z",
"endDate": "2025-03-31T00:00:00Z",
"totalAssets": 1533763445000000.0
}
}
]"#,
)
.expect("financial statements should deserialize");
let financials =
parse_financial_statements("BBCA.JK", &raw).expect("financials should parse");
assert_eq!(financials.instrument.id, "bn91jc");
assert_eq!(financials.instrument.symbol, "BBCA.JK");
assert_eq!(financials.instrument.name, "Bank Central Asia Tbk PT");
}
#[test]
fn parses_screener_fixture_quotes() {
let raw: RawScreenerResponse = serde_json::from_str(include_str!(
@ -736,4 +791,74 @@ mod tests {
assert_eq!(quotes[0].range_signal.as_deref(), Some("upper"));
assert_eq!(quotes[0].avg_volume, Some(10_000_000));
}
#[test]
fn parse_screener_results_filters_invalid_rows() {
let raw: RawScreenerResponse = serde_json::from_str(
r#"{
"quote": [
{ "symbol": "BBCA", "price": 9875, "pricePreviousClose": 9758 },
{ "symbol": "BBRI", "price": 0 },
{ "symbol": "BMRI" },
{ "symbol": "", "price": 5150 }
]
}"#,
)
.expect("screener fixture should deserialize");
let quotes = parse_screener_results(&raw).expect("screener should parse");
assert_eq!(quotes.len(), 1);
assert_eq!(quotes[0].symbol, "BBCA.JK");
assert_eq!(quotes[0].price, 9_875);
}
#[test]
fn parse_screener_results_errors_when_all_rows_are_invalid() {
let raw: RawScreenerResponse = serde_json::from_str(
r#"{
"quote": [
{ "symbol": "BBRI", "price": 0 },
{ "symbol": "BMRI" }
]
}"#,
)
.expect("screener fixture should deserialize");
let err = parse_screener_results(&raw).expect_err("invalid screener rows should fail");
assert!(matches!(err, IdxError::ParseError(_)));
assert_eq!(
err.to_string(),
"parse error: screener returned no priced stocks"
);
}
#[test]
fn parse_fundamentals_rejects_industry_only_metrics() {
let raw: Vec<KeyRatios> = serde_json::from_str(
r#"[
{
"industryMetrics": [
{
"year": "2025",
"fiscalPeriodType": "TTM",
"priceToEarningsRatio": 12.5,
"priceToBookRatio": 1.7
}
],
"companyMetrics": []
}
]"#,
)
.expect("key ratios should deserialize");
let err = parse_fundamentals(&raw, None).expect_err("industry fallback should be rejected");
assert!(matches!(err, IdxError::Unsupported(_)));
assert_eq!(
err.to_string(),
"unsupported: company fundamentals unavailable from MSN; industry fallback is disabled"
);
}
}

21
src/api/msn/mod.rs
View file

@ -12,7 +12,7 @@ use crate::api::types::{
};
use crate::api::{
EarningsProvider, FinancialsProvider, FundamentalsProvider, InsightsProvider, NewsProvider,
ProfileProvider, QuoteProvider, SentimentProvider,
ProfileProvider, QuoteProvider, ScreenerProvider, SentimentProvider,
};
use crate::error::IdxError;
@ -34,16 +34,6 @@ impl MsnProvider {
client: MsnClient::new(),
}
}
pub fn screener(
&self,
filter: &str,
region: &str,
limit: usize,
) -> Result<Vec<Quote>, IdxError> {
let raw = self.client.fetch_screener(filter, region, limit)?;
parse_screener_results(&raw)
}
}
impl QuoteProvider for MsnProvider {
@ -99,6 +89,13 @@ impl InsightsProvider for MsnProvider {
impl NewsProvider for MsnProvider {
fn news(&self, symbol: &str, limit: usize) -> Result<Vec<NewsItem>, IdxError> {
let raw = self.client.fetch_news(symbol, limit)?;
parse_news(&raw)
parse_news(symbol, &raw)
}
}
impl ScreenerProvider for MsnProvider {
fn screener(&self, filter: &str, region: &str, limit: usize) -> Result<Vec<Quote>, IdxError> {
let raw = self.client.fetch_screener(filter, region, limit)?;
parse_screener_results(&raw)
}
}

1
src/api/msn/raw_types.rs
View file

@ -35,6 +35,7 @@ pub(crate) struct MsnQuote {
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct KeyRatios {
#[allow(dead_code)]
#[serde(default)]
pub(crate) industry_metrics: Vec<IndustryMetric>,
#[serde(default)]

6
src/api/types.rs
View file

@ -143,6 +143,8 @@ pub struct StatementSection {
#[allow(dead_code)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EarningsReport {
#[serde(default)]
pub symbol: String,
pub eps_last_year: f64,
pub revenue_last_year: f64,
pub forecast: Vec<EarningsData>,
@ -183,6 +185,8 @@ pub struct SentimentPeriod {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct InsightData {
pub id: String,
#[serde(default)]
pub symbol: String,
pub summary: String,
pub highlights: Vec<String>,
pub risks: Vec<String>,
@ -193,6 +197,8 @@ pub struct InsightData {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NewsItem {
pub id: String,
#[serde(default)]
pub symbol: String,
pub title: String,
pub url: String,
pub description: String,

192
src/api/yahoo/client.rs
View file

@ -1,11 +1,13 @@
use std::path::{Path, PathBuf};
use std::process::Command;
use std::process::Stdio;
use std::sync::Mutex;
use std::time::Duration;
use crate::api::types::{Interval, Period};
use crate::curl_impersonate;
use crate::error::IdxError;
use crate::runtime;
use super::raw_types::{ChartResponse, QuoteSummaryResponse};
@ -48,7 +50,7 @@ impl YahooClient {
}
fn cookie_jar_path() -> PathBuf {
PathBuf::from(format!("/tmp/idx_yf_{}.txt", std::process::id()))
std::env::temp_dir().join(format!("idx_yf_{}.txt", std::process::id()))
}
pub(super) fn parse_crumb_body(raw: &str) -> Result<String, IdxError> {
@ -132,14 +134,9 @@ impl YahooClient {
// Step 1: fetch fc.yahoo.com to set A3 cookie (returns 404 but writes cookie jar).
// We allow non-zero exit here since 404 still writes the cookie.
let _ = Command::new(binary)
.args([
"--silent",
"--cookie-jar",
cookie_jar_str,
COOKIE_FETCH_URL,
"--output",
"/dev/null",
])
.args(["--silent", "--cookie-jar", cookie_jar_str, COOKIE_FETCH_URL])
.stdout(Stdio::null())
.stderr(Stdio::null())
.output();
// Step 2: fetch crumb with cookie jar (Chrome TLS fingerprint + A3 cookie).
@ -176,36 +173,60 @@ impl YahooClient {
Ok(())
}
fn retry_rate_limited<T, F>(&self, request: F) -> Result<Result<T, ureq::Error>, IdxError>
where
F: FnMut() -> Result<T, ureq::Error>,
{
Self::retry_rate_limited_with(request, std::thread::sleep, jitter)
}
fn retry_rate_limited_with<T, F, S, J>(
mut request: F,
mut sleeper: S,
mut jitter_fn: J,
) -> Result<Result<T, ureq::Error>, IdxError>
where
F: FnMut() -> Result<T, ureq::Error>,
S: FnMut(Duration),
J: FnMut() -> Duration,
{
let mut wait = Duration::from_millis(250);
for attempt in 0..3 {
match request() {
Err(ureq::Error::StatusCode(429)) => {
if attempt < 2 {
sleeper(wait + jitter_fn());
wait *= 2;
continue;
}
return Err(IdxError::RateLimited);
}
other => return Ok(other),
}
}
Err(IdxError::RateLimited)
}
pub(super) fn fetch_chart(
&self,
symbol: &str,
period: &Period,
interval: &Interval,
) -> Result<ChartResponse, IdxError> {
let mut wait = Duration::from_millis(250);
for attempt in 0..3 {
let url = Self::chart_url(symbol, period, interval);
let response = self.agent.get(&url).header("User-Agent", USER_AGENT).call();
match response {
Ok(ok) => {
return ok
.into_body()
.read_json::<ChartResponse>()
.map_err(|e| IdxError::ParseError(e.to_string()));
}
Err(ureq::Error::StatusCode(429)) => {
if attempt < 2 {
std::thread::sleep(wait + jitter());
wait *= 2;
}
}
Err(ureq::Error::StatusCode(404)) => {
return Err(IdxError::SymbolNotFound(symbol.to_string()));
}
Err(e) => return Err(IdxError::Http(e.to_string())),
}
let url = Self::chart_url(symbol, period, interval);
let response = self
.retry_rate_limited(|| self.agent.get(&url).header("User-Agent", USER_AGENT).call())?;
match response {
Ok(ok) => ok
.into_body()
.read_json::<ChartResponse>()
.map_err(|e| IdxError::ParseError(e.to_string())),
Err(ureq::Error::StatusCode(404)) => Err(IdxError::SymbolNotFound(symbol.to_string())),
Err(e) => Err(IdxError::Http(e.to_string())),
}
Err(IdxError::RateLimited)
}
pub(super) fn fetch_quote_summary(
@ -217,48 +238,41 @@ impl YahooClient {
let cookie_header = match Self::cookie_header_from_jar(&Self::cookie_jar_path()) {
Ok(header) => header,
Err(err) => {
eprintln!("warning: failed to parse Yahoo cookie jar: {err}");
runtime::warn(format!("failed to parse Yahoo cookie jar: {err}"));
return Err(IdxError::AuthError(format!(
"failed to parse Yahoo cookies: {err}"
)));
}
};
let url = Self::quote_summary_url(symbol, &crumb);
let mut wait = Duration::from_millis(250);
for attempt in 0..3 {
let response = self.retry_rate_limited(|| {
let mut req = self.agent.get(&url).header("User-Agent", USER_AGENT);
if !cookie_header.is_empty() {
req = req.header("Cookie", &cookie_header);
}
let response = req.call();
match response {
Ok(ok) => {
return ok
.into_body()
.read_json::<QuoteSummaryResponse>()
.map_err(|e| IdxError::ParseError(e.to_string()));
}
Err(ureq::Error::StatusCode(401)) => {
if auth_attempt == 0 {
self.clear_crumb()?;
break;
}
return Err(IdxError::Http(
"yahoo quoteSummary returned unauthorized (401)".to_string(),
));
}
Err(ureq::Error::StatusCode(429)) => {
if attempt < 2 {
std::thread::sleep(wait + jitter());
wait *= 2;
}
}
Err(ureq::Error::StatusCode(404)) => {
return Err(IdxError::SymbolNotFound(symbol.to_string()));
}
Err(e) => return Err(IdxError::Http(e.to_string())),
req.call()
})?;
match response {
Ok(ok) => {
return ok
.into_body()
.read_json::<QuoteSummaryResponse>()
.map_err(|e| IdxError::ParseError(e.to_string()));
}
Err(ureq::Error::StatusCode(401)) => {
if auth_attempt == 0 {
self.clear_crumb()?;
continue;
}
return Err(IdxError::Http(
"yahoo quoteSummary returned unauthorized (401)".to_string(),
));
}
Err(ureq::Error::StatusCode(404)) => {
return Err(IdxError::SymbolNotFound(symbol.to_string()));
}
Err(e) => return Err(IdxError::Http(e.to_string())),
}
}
@ -273,6 +287,7 @@ fn jitter() -> Duration {
#[cfg(test)]
mod tests {
use super::YahooClient;
use std::time::Duration;
#[test]
fn parses_crumb_body_trimmed() {
@ -290,4 +305,55 @@ mod tests {
.expect_err("rate limited crumb must fail");
assert!(matches!(rate_limited, crate::error::IdxError::Http(_)));
}
#[test]
fn retry_rate_limited_succeeds_after_retry() {
let mut attempts = 0;
let mut sleeps = Vec::new();
let result = YahooClient::retry_rate_limited_with(
|| {
attempts += 1;
if attempts < 3 {
Err(ureq::Error::StatusCode(429))
} else {
Ok("ok")
}
},
|duration| sleeps.push(duration),
|| Duration::ZERO,
)
.expect("retry helper should not fail")
.expect("third attempt should succeed");
assert_eq!(result, "ok");
assert_eq!(attempts, 3);
assert_eq!(
sleeps,
vec![Duration::from_millis(250), Duration::from_millis(500)]
);
}
#[test]
fn retry_rate_limited_returns_rate_limited_after_exhaustion() {
let mut attempts = 0;
let mut sleeps = Vec::new();
let err = YahooClient::retry_rate_limited_with(
|| {
attempts += 1;
Err::<(), ureq::Error>(ureq::Error::StatusCode(429))
},
|duration| sleeps.push(duration),
|| Duration::ZERO,
)
.expect_err("rate-limited helper should fail after three attempts");
assert!(matches!(err, crate::error::IdxError::RateLimited));
assert_eq!(attempts, 3);
assert_eq!(
sleeps,
vec![Duration::from_millis(250), Duration::from_millis(500)]
);
}
}

4
src/api/yahoo/mod.rs
View file

@ -11,7 +11,9 @@ use client::YahooClient;
use map::{parse_fundamentals, parse_quote};
use parse::parse_history_with_verbose;
pub(crate) use parse::{parse_fundamentals_from_str, parse_history_from_str, parse_quote_from_str};
pub(crate) use parse::{
parse_fundamentals_from_str, parse_history_from_str_with_verbose, parse_quote_from_str,
};
pub struct YahooProvider {
client: YahooClient,

25
src/api/yahoo/parse.rs
View file

@ -10,10 +10,19 @@ pub(crate) fn parse_quote_from_str(symbol: &str, raw: &str) -> Result<Quote, Idx
parse_quote(symbol, &chart)
}
#[cfg(test)]
pub(crate) fn parse_history_from_str(symbol: &str, raw: &str) -> Result<Vec<Ohlc>, IdxError> {
parse_history_from_str_with_verbose(symbol, raw, false)
}
pub(crate) fn parse_history_from_str_with_verbose(
symbol: &str,
raw: &str,
verbose: bool,
) -> Result<Vec<Ohlc>, IdxError> {
let chart: ChartResponse =
serde_json::from_str(raw).map_err(|e| IdxError::ParseError(e.to_string()))?;
parse_history_with_verbose(symbol, &chart, false)
parse_history_with_verbose(symbol, &chart, verbose)
}
pub(crate) fn parse_fundamentals_from_str(
@ -32,9 +41,9 @@ pub(super) fn parse_history_with_verbose(
) -> Result<Vec<Ohlc>, IdxError> {
let (history, dropped) = parse_history(symbol, chart)?;
if dropped > 0 && verbose {
eprintln!(
"warning: dropped {dropped} OHLC row(s) from Yahoo response due to missing fields"
);
crate::runtime::warn(format!(
"dropped {dropped} OHLC row(s) from Yahoo response due to missing fields"
));
}
Ok(history)
}
@ -43,7 +52,8 @@ pub(super) fn parse_history_with_verbose(
mod tests {
use super::{
ChartResponse, parse_fundamentals_from_str, parse_history_from_str,
parse_history_with_verbose, parse_quote, parse_quote_from_str,
parse_history_from_str_with_verbose, parse_history_with_verbose, parse_quote,
parse_quote_from_str,
};
const SAMPLE: &str = r#"{
@ -100,6 +110,11 @@ mod tests {
parse_history_from_str("BBCA.JK", &history_raw).expect("fixture history parsed");
assert!(!history.is_empty());
let verbose_history = parse_history_from_str_with_verbose("BBCA.JK", &history_raw, true)
.expect("fixture history parsed in verbose mode");
assert_eq!(verbose_history.len(), history.len());
assert_eq!(verbose_history[0].close, history[0].close);
let fundamentals = parse_fundamentals_from_str("BBCA.JK", &fundamentals_raw)
.expect("fixture fundamentals parsed");
assert_eq!(fundamentals.trailing_pe, Some(25.4));