0xrsydn/idx-cli
1
mirror of https://github.com/0xrsydn/idx-cli.git synced 2026-08-07 01:33:52 +00:00
Code Issues Projects Releases 1 Packages Wiki Activity

fix(yahoo): use curl-impersonate for crumb auth (TLS fingerprinting)

Browse source 
Yahoo Finance blocks standard ureq/rustls via TLS fingerprinting (JA3/JA4).
Reverse-engineered from yfinance source: they use curl_cffi with Chrome impersonation.

Fix:
- Fetch cookie via curl-impersonate from fc.yahoo.com
- Fetch crumb via curl-impersonate from query1.finance.yahoo.com/v1/test/getcrumb
  (correct URL: getcrumb, not getCrumb or csrfToken)
- Parse Netscape cookie jar format, send cookies as header to quoteSummary
- 401 retry: clear crumb + cookie jar, re-auth on next attempt
- Crumb validation: reject HTML, empty, rate-limit responses
- Add curl-impersonate-chrome to flake.nix devShell
This commit is contained in:
Ciphercat 2026-03-06 07:31:11 +00:00
commit f89f4c90ce
2 changed files with 266 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

1
flake.nix
View file

@ -28,6 +28,7 @@
cargo-watch cargo-watch
cargo-nextest cargo-nextest
prek prek
curl-impersonate-chrome # required for Yahoo Finance auth (--impersonate chrome)
]; ];
env = { env = {

292
src/api/yahoo.rs
View file

@ -1,4 +1,7 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::process::{Command, Output};
use std::sync::Mutex;
use std::time::Duration; use std::time::Duration;
use serde::Deserialize; use serde::Deserialize;
@ -8,12 +11,14 @@ use crate::api::types::{Fundamentals, Interval, Ohlc, Period, Quote};
use crate::error::IdxError; use crate::error::IdxError;
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"; const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
// query2 works without crumb/cookie auth from datacenter IPs; query1 requires consent cookies
const BASE_URL: &str = "https://query2.finance.yahoo.com"; const BASE_URL: &str = "https://query2.finance.yahoo.com";
const COOKIE_FETCH_URL: &str = "https://fc.yahoo.com";
const CRUMB_FETCH_URL: &str = "https://query1.finance.yahoo.com/v1/test/getcrumb";
pub struct YahooProvider { pub struct YahooProvider {
agent: ureq::Agent, agent: ureq::Agent,
verbose: bool, verbose: bool,
crumb: Mutex<Option<String>>,
} }
impl YahooProvider { impl YahooProvider {
@ -24,7 +29,11 @@ impl YahooProvider {
.build() .build()
.into(); .into();
Self { agent, verbose } Self {
agent,
verbose,
crumb: Mutex::new(None),
}
} }
fn chart_url(symbol: &str, period: &Period, interval: &Interval) -> String { fn chart_url(symbol: &str, period: &Period, interval: &Interval) -> String {
@ -35,12 +44,197 @@ impl YahooProvider {
) )
} }
fn quote_summary_url(symbol: &str) -> String { fn quote_summary_url(symbol: &str, crumb: &str) -> String {
format!( format!(
"{BASE_URL}/v10/finance/quoteSummary/{symbol}?modules=defaultKeyStatistics,financialData,incomeStatementHistory" "{BASE_URL}/v10/finance/quoteSummary/{symbol}?modules=defaultKeyStatistics,financialData,incomeStatementHistory&crumb={crumb}"
) )
} }
fn cookie_jar_path() -> PathBuf {
PathBuf::from(format!("/tmp/idx_yf_{}.txt", std::process::id()))
}
fn parse_crumb_body(raw: &str) -> Result<String, IdxError> {
let crumb = raw.trim();
if crumb.is_empty() {
return Err(IdxError::Http("received empty Yahoo crumb".to_string()));
}
let normalized = crumb.to_ascii_lowercase();
if normalized.contains("<html>") || normalized.contains("<!doctype html") {
return Err(IdxError::Http(
"received HTML instead of a Yahoo crumb".to_string(),
));
}
if normalized.contains("too many requests") {
return Err(IdxError::Http(
"Yahoo crumb request was rate limited".to_string(),
));
}
Ok(crumb.to_string())
}
fn cookie_header_from_jar(path: &Path) -> Result<String, IdxError> {
let jar = std::fs::read_to_string(path).map_err(|e| {
IdxError::Http(format!(
"failed to read Yahoo cookie jar {}: {e}",
path.display()
))
})?;
let cookies: Vec<String> = jar
.lines()
.filter_map(|line| {
let trimmed = line.trim();
if trimmed.is_empty() {
return None;
}
let candidate = trimmed.strip_prefix("#HttpOnly_").unwrap_or(trimmed);
if candidate.starts_with('#') {
return None;
}
let fields: Vec<_> = candidate.split('\t').collect();
if fields.len() < 7 {
return None;
}
let name = fields[5].trim();
let value = fields[6].trim();
if name.is_empty() {
return None;
}
Some(format!("{name}={value}"))
})
.collect();
if cookies.is_empty() {
return Err(IdxError::Http(format!(
"Yahoo cookie jar {} did not contain any cookies",
path.display()
)));
}
Ok(cookies.join("; "))
}
fn run_curl(stage: &str, args: &[&str]) -> Result<Output, IdxError> {
// curl-impersonate is required for --impersonate chrome TLS fingerprinting.
// Install via: nix profile install nixpkgs#curl-impersonate-chrome
let output = Command::new("curl-impersonate")
.args(args)
.output()
.map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
return IdxError::Http(
"curl-impersonate not found; install curl-impersonate-chrome (nixpkgs#curl-impersonate-chrome)"
.to_string(),
);
}
IdxError::Http(format!("failed to run curl-impersonate for Yahoo {stage}: {e}"))
})?;
if output.status.success() {
return Ok(output);
}
let stderr = String::from_utf8_lossy(&output.stderr);
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr_trimmed = stderr.trim();
let stdout_trimmed = stdout.trim();
let detail = if stderr_trimmed.is_empty() {
stdout_trimmed
} else {
stderr_trimmed
};
if detail.contains("--impersonate")
|| detail.contains("unknown option")
|| detail.contains("unrecognized option")
{
return Err(IdxError::Http(
"curl does not support --impersonate; install a curl build with that feature"
.to_string(),
));
}
Err(IdxError::Http(format!(
"Yahoo {stage} curl failed (status {}): {}",
output.status,
if detail.is_empty() {
"no output"
} else {
detail
}
)))
}
fn fetch_crumb_via_curl(&self) -> Result<String, IdxError> {
let cookie_jar = Self::cookie_jar_path();
let cookie_jar_str = cookie_jar.to_str().ok_or_else(|| {
IdxError::Io(format!(
"failed to encode Yahoo cookie jar path {}",
cookie_jar.display()
))
})?;
Self::run_curl(
"cookie fetch",
&[
"--impersonate",
"chrome",
"--silent",
"--cookie-jar",
cookie_jar_str,
COOKIE_FETCH_URL,
"--output",
"/dev/null",
],
)?;
let output = Self::run_curl(
"crumb fetch",
&[
"--impersonate",
"chrome",
"--silent",
"--cookie",
cookie_jar_str,
CRUMB_FETCH_URL,
],
)?;
let body = String::from_utf8_lossy(&output.stdout);
Self::parse_crumb_body(&body)
}
fn get_or_init_crumb(&self) -> Result<String, IdxError> {
let mut guard = self
.crumb
.lock()
.map_err(|e| IdxError::Io(format!("crumb lock poisoned: {e}")))?;
if let Some(crumb) = guard.as_ref() {
return Ok(crumb.clone());
}
let crumb = self.fetch_crumb_via_curl()?;
*guard = Some(crumb.clone());
Ok(crumb)
}
fn clear_crumb(&self) -> Result<(), IdxError> {
let mut guard = self
.crumb
.lock()
.map_err(|e| IdxError::Io(format!("crumb lock poisoned: {e}")))?;
*guard = None;
Ok(())
}
fn fetch_chart( fn fetch_chart(
&self, &self,
symbol: &str, symbol: &str,
@ -78,33 +272,60 @@ impl YahooProvider {
} }
fn fetch_quote_summary(&self, symbol: &str) -> Result<QuoteSummaryResponse, IdxError> { fn fetch_quote_summary(&self, symbol: &str) -> Result<QuoteSummaryResponse, IdxError> {
let mut wait = Duration::from_millis(250); for auth_attempt in 0..2 {
for attempt in 0..3 { let crumb = self.get_or_init_crumb()?;
let url = Self::quote_summary_url(symbol); let cookie_header = match Self::cookie_header_from_jar(&Self::cookie_jar_path()) {
let response = self.agent.get(&url).header("User-Agent", USER_AGENT).call(); Ok(header) => header,
match response { Err(err) if auth_attempt == 0 => {
Ok(ok) => { self.clear_crumb()?;
let quote_summary = ok continue;
.into_body() }
.read_json::<QuoteSummaryResponse>() Err(err) => return Err(err),
.map_err(|e| IdxError::ParseError(e.to_string()))?; };
if let Some(err) = quote_summary.quote_summary.error.as_ref() { let url = Self::quote_summary_url(symbol, &crumb);
return Err(map_yahoo_error(symbol, "quoteSummary", err)); let mut wait = Duration::from_millis(250);
for attempt in 0..3 {
let response = self
.agent
.get(&url)
.header("User-Agent", USER_AGENT)
.header("Cookie", &cookie_header)
.call();
match response {
Ok(ok) => {
let quote_summary = ok
.into_body()
.read_json::<QuoteSummaryResponse>()
.map_err(|e| IdxError::ParseError(e.to_string()))?;
if let Some(err) = quote_summary.quote_summary.error.as_ref() {
return Err(map_yahoo_error(symbol, "quoteSummary", err));
}
return Ok(quote_summary);
} }
return Ok(quote_summary); Err(ureq::Error::StatusCode(401)) => {
} if auth_attempt == 0 {
Err(ureq::Error::StatusCode(429)) => { self.clear_crumb()?;
if attempt < 2 { break;
std::thread::sleep(wait + jitter()); }
wait *= 2; return Err(IdxError::Http(
"yahoo quoteSummary returned unauthorized (401)".to_string(),
));
} }
Err(ureq::Error::StatusCode(429)) => {
if attempt < 2 {
std::thread::sleep(wait + jitter());
wait *= 2;
}
}
Err(ureq::Error::StatusCode(404)) => {
return Err(IdxError::SymbolNotFound(symbol.to_string()));
}
Err(e) => return Err(IdxError::Http(e.to_string())),
} }
Err(ureq::Error::StatusCode(404)) => {
return Err(IdxError::SymbolNotFound(symbol.to_string()));
}
Err(e) => return Err(IdxError::Http(e.to_string())),
} }
} }
Err(IdxError::RateLimited) Err(IdxError::RateLimited)
} }
} }
@ -535,7 +756,7 @@ impl YahooNumber {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::{ use super::{
ChartResponse, parse_fundamentals_from_str, parse_history_from_str, ChartResponse, YahooProvider, parse_fundamentals_from_str, parse_history_from_str,
parse_history_with_verbose, parse_quote, parse_quote_from_str, parse_history_with_verbose, parse_quote, parse_quote_from_str,
}; };
@ -609,4 +830,21 @@ mod tests {
let err = parse_quote_from_str("INVALID.JK", raw).expect_err("expected symbol error"); let err = parse_quote_from_str("INVALID.JK", raw).expect_err("expected symbol error");
assert!(matches!(err, crate::error::IdxError::SymbolNotFound(_))); assert!(matches!(err, crate::error::IdxError::SymbolNotFound(_)));
} }
#[test]
fn parses_crumb_body_trimmed() {
let crumb = YahooProvider::parse_crumb_body(" abc123xyz\n").expect("crumb should parse");
assert_eq!(crumb, "abc123xyz");
let empty = YahooProvider::parse_crumb_body(" \n").expect_err("empty crumb must fail");
assert!(matches!(empty, crate::error::IdxError::Http(_)));
let html = YahooProvider::parse_crumb_body("<html>blocked</html>")
.expect_err("html crumb must fail");
assert!(matches!(html, crate::error::IdxError::Http(_)));
let rate_limited = YahooProvider::parse_crumb_body("Too Many Requests")
.expect_err("rate limited crumb must fail");
assert!(matches!(rate_limited, crate::error::IdxError::Http(_)));
}
} }