From 1796e581057418cfa93dfad6302269c3cdb551f8 Mon Sep 17 00:00:00 2001 From: hermes Date: Fri, 16 Oct 2026 07:58:57 +0000 Subject: [PATCH] feat(news): rank this company's news first and label each item MSN's feed for a stock is often mostly general market news: of 10 items, ADRO, BREN and CUAN had none about the company, BBRI one, yet they were presented as the company's news. Fetch at least 50 items, label each `relevance: company` (tagged by MSN with the stock's instrument id, or naming the ticker or the company) or `market`, and return company items first, newest first, trimmed to --limit. Company names come from the quote's short and display names, best effort: MSN tags Telkom articles with its NYSE ADR, not TLKM, so names catch what tags miss. Classification stays a pure function in parse_news. Output change, intentional: news JSON gains `relevance`, the table an ABOUT column (goldens regenerated), and the get_company tool description explains the field (tools.json and the website tools page regenerated). News cache keys change so entries without the field are not reused. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01V3syn2fN2LNo5pvTSqc6eh --- crates/idx-cli/src/output/table.rs | 3 +- .../tests/golden/stocks-news.json.golden | 2 +- .../tests/golden/stocks-news.table.golden | 10 +- crates/idx-core/src/api/msn/client.rs | 7 +- crates/idx-core/src/api/msn/map.rs | 199 ++++++++++++++++-- crates/idx-core/src/api/msn/mod.rs | 19 +- crates/idx-core/src/api/msn/raw_types.rs | 17 ++ crates/idx-core/src/api/types.rs | 21 ++ crates/idx-core/src/financial.rs | 3 +- crates/idx-mcp/src/server.rs | 5 +- crates/idx-mcp/tests/golden/tools.json | 2 +- web/src/content/docs/cli/stocks.md | 2 +- web/src/content/docs/mcp/tools.md | 2 +- 13 files changed, 254 insertions(+), 38 deletions(-) diff --git a/crates/idx-cli/src/output/table.rs b/crates/idx-cli/src/output/table.rs index cfa3a2e..fb5df0c 100644 --- a/crates/idx-cli/src/output/table.rs +++ b/crates/idx-cli/src/output/table.rs @@ -837,10 +837,11 @@ pub fn print_news(items: &[NewsItem]) -> Result<(), IdxError> { let mut table = Table::new(); table .load_preset(UTF8_FULL) - .set_header(vec!["TITLE", "PROVIDER", "DATE", "URL"]); + .set_header(vec!["TITLE", "ABOUT", "PROVIDER", "DATE", "URL"]); for item in items { table.add_row(vec![ Cell::new(&item.title), + Cell::new(item.relevance.as_str()), Cell::new(&item.provider), Cell::new(&item.published_at), Cell::new(truncate_url(&item.url)), diff --git a/crates/idx-cli/tests/golden/stocks-news.json.golden b/crates/idx-cli/tests/golden/stocks-news.json.golden index 647f530..d75ca62 100644 --- a/crates/idx-cli/tests/golden/stocks-news.json.golden +++ b/crates/idx-cli/tests/golden/stocks-news.json.golden @@ -1,4 +1,4 @@ exit: 0 --- stdout -[{"id":"news-1","symbol":"BBCA.JK","title":"BCA reports steady growth","url":"https://example.com/news/bca-growth","description":"BCA delivered steady quarterly growth across lending and deposits.","provider":"Contoso News","published_at":"2026-03-20T10:00:00Z","read_time_min":3}] +[{"id":"news-1","symbol":"BBCA.JK","title":"BCA reports steady growth","url":"https://example.com/news/bca-growth","description":"BCA delivered steady quarterly growth across lending and deposits.","provider":"Contoso News","published_at":"2026-03-20T10:00:00Z","read_time_min":3,"relevance":"market"}] --- stderr diff --git a/crates/idx-cli/tests/golden/stocks-news.table.golden b/crates/idx-cli/tests/golden/stocks-news.table.golden index f0bdf1e..f244ca1 100644 --- a/crates/idx-cli/tests/golden/stocks-news.table.golden +++ b/crates/idx-cli/tests/golden/stocks-news.table.golden @@ -1,8 +1,8 @@ exit: 0 --- stdout -┌───────────────────────────┬──────────────┬──────────────────────┬─────────────────────────────────────┐ -│ TITLE ┆ PROVIDER ┆ DATE ┆ URL │ -╞═══════════════════════════╪══════════════╪══════════════════════╪═════════════════════════════════════╡ -│ BCA reports steady growth ┆ Contoso News ┆ 2026-03-20T10:00:00Z ┆ https://example.com/news/bca-growth │ -└───────────────────────────┴──────────────┴──────────────────────┴─────────────────────────────────────┘ +┌───────────────────────────┬────────┬──────────────┬──────────────────────┬─────────────────────────────────────┐ +│ TITLE ┆ ABOUT ┆ PROVIDER ┆ DATE ┆ URL │ +╞═══════════════════════════╪════════╪══════════════╪══════════════════════╪═════════════════════════════════════╡ +│ BCA reports steady growth ┆ market ┆ Contoso News ┆ 2026-03-20T10:00:00Z ┆ https://example.com/news/bca-growth │ +└───────────────────────────┴────────┴──────────────┴──────────────────────┴─────────────────────────────────────┘ --- stderr diff --git a/crates/idx-core/src/api/msn/client.rs b/crates/idx-core/src/api/msn/client.rs index a62bff0..f61ef71 100644 --- a/crates/idx-core/src/api/msn/client.rs +++ b/crates/idx-core/src/api/msn/client.rs @@ -18,6 +18,8 @@ const MSN_ASSETS_BASE_URL: &str = "https://assets.msn.com/service/"; const MSN_API_BASE_URL: &str = "https://api.msn.com/msn/v0/pages/finance/"; // Public API key from MSN Money website (embedded in frontend JS) const MSN_API_KEY: &str = "0QfOX3Vn51YCzitbLaRkTTBadtWpgTN8NZLW0C1SEM"; +/// Items requested from a stock's news feed, at least; MSN returns ~40-50. +const NEWS_FETCH_MIN: usize = 50; pub(super) struct MsnClient { agent: ureq::Agent, @@ -272,8 +274,11 @@ impl MsnClient { pub(super) fn fetch_news(&self, symbol: &str, limit: usize) -> Result { let id = resolve_msn_id(symbol).ok_or_else(|| IdxError::SymbolNotFound(symbol.to_string()))?; + // Company items can sit below general market news in a stock's feed; + // fetch wider and let parse_news rank and trim to `limit`. + let top = limit.max(NEWS_FETCH_MIN); let url = format!( - "{MSN_ASSETS_BASE_URL}MSN/Feed/me?$top={limit}&apikey={MSN_API_KEY}&cm=id-id&contentType=article,video,slideshow&it=web&query=ef_stock_{id}&queryType=entityfeed&responseSchema=cardview&scn=ANON&wrapodata=false" + "{MSN_ASSETS_BASE_URL}MSN/Feed/me?$top={top}&apikey={MSN_API_KEY}&cm=id-id&contentType=article,video,slideshow&it=web&query=ef_stock_{id}&queryType=entityfeed&responseSchema=cardview&scn=ANON&wrapodata=false" ); self.get_json(&url, symbol, "news") } diff --git a/crates/idx-core/src/api/msn/map.rs b/crates/idx-core/src/api/msn/map.rs index 2889168..e94e594 100644 --- a/crates/idx-core/src/api/msn/map.rs +++ b/crates/idx-core/src/api/msn/map.rs @@ -6,7 +6,7 @@ use super::raw_types::{ use super::symbols::{normalized_symbol, ticker_from_symbol}; use crate::api::types::{ CompanyProfile, EarningsData, EarningsReport, FinancialStatements, Fundamentals, InsightData, - InstrumentInfo, NewsItem, Officer, Ohlc, Quote, SentimentData, SentimentPeriod, + InstrumentInfo, NewsItem, NewsRelevance, Officer, Ohlc, Quote, SentimentData, SentimentPeriod, StatementSection, }; use crate::error::IdxError; @@ -551,30 +551,108 @@ pub(super) fn parse_insights(symbol: &str, raw: &[RawInsight]) -> Result Result, IdxError> { +/// Parses a stock's news feed, ranks this company's items first and then the +/// rest (general market news), newest first within each, and keeps `limit`. +/// An item is about the company when MSN tags it with the stock's instrument +/// id (`msn_id`), or its title or description names the ticker or one of +/// `names`. +pub(super) fn parse_news( + symbol: &str, + msn_id: &str, + names: &[String], + raw: &RawNewsFeed, + limit: usize, +) -> Result, IdxError> { let source = raw .sub_cards .as_ref() .or(raw.value.as_ref()) .ok_or_else(|| IdxError::ParseError("no news data".into()))?; + let ticker = ticker_from_symbol(symbol).unwrap_or_default(); - Ok(source + let mut items: Vec = source .iter() - .map(|item| NewsItem { - id: item.id.clone().unwrap_or_default(), - symbol: symbol.to_string(), - title: item.title.clone().unwrap_or_default(), - url: item.url.clone().unwrap_or_default(), - description: item.description.clone().unwrap_or_default(), - provider: item - .provider - .as_ref() - .and_then(|p| p.name.clone()) - .unwrap_or_default(), - published_at: item.published_date_time.clone().unwrap_or_default(), - read_time_min: item.read_time_min, + .map(|item| { + let title = item.title.clone().unwrap_or_default(); + let description = item.description.clone().unwrap_or_default(); + let tagged = item.finance_metadata.as_ref().is_some_and(|meta| { + meta.stocks + .iter() + .any(|stock| stock.stock_id.as_deref() == Some(msn_id)) + }); + let named = [&title, &description] + .iter() + .any(|text| mentions_company(text, &ticker, names)); + NewsItem { + id: item.id.clone().unwrap_or_default(), + symbol: symbol.to_string(), + title, + url: item.url.clone().unwrap_or_default(), + description, + provider: item + .provider + .as_ref() + .and_then(|p| p.name.clone()) + .unwrap_or_default(), + published_at: item.published_date_time.clone().unwrap_or_default(), + read_time_min: item.read_time_min, + relevance: if tagged || named { + NewsRelevance::Company + } else { + NewsRelevance::Market + }, + } }) - .collect()) + .collect(); + // ISO 8601 UTC timestamps sort as strings; missing dates sort last. + items.sort_by(|a, b| { + a.relevance + .cmp(&b.relevance) + .then_with(|| b.published_at.cmp(&a.published_at)) + }); + items.truncate(limit); + Ok(items) +} + +/// Whether `text` names the ticker as a whole word (tickers are written in +/// capitals) or contains one of the company `names`, ignoring case. +fn mentions_company(text: &str, ticker: &str, names: &[String]) -> bool { + let whole_word = |at: usize| { + let before = text[..at].chars().next_back(); + let after = text[at + ticker.len()..].chars().next(); + !before.is_some_and(char::is_alphanumeric) && !after.is_some_and(char::is_alphanumeric) + }; + if !ticker.is_empty() && text.match_indices(ticker).any(|(at, _)| whole_word(at)) { + return true; + } + let text = text.to_lowercase(); + names.iter().any(|name| text.contains(&name.to_lowercase())) +} + +/// Names an article may use for a company, from MSN's quote names: the short +/// name ("Telkom Indonesia") and the display name without legal suffixes +/// ("Telkom Indonesia (Persero) Tbk PT" -> "Telkom Indonesia"). Abbreviated +/// short names ("Bank Cntrl Asia") never match and are harmless. +pub(super) fn company_names(short_name: Option<&str>, display_name: Option<&str>) -> Vec { + const SUFFIXES: [&str; 3] = ["(persero)", "tbk", "pt"]; + let display = display_name.map(|name| { + name.split_whitespace() + .filter(|word| !SUFFIXES.contains(&word.to_lowercase().trim_end_matches('.'))) + .collect::>() + .join(" ") + }); + let mut names: Vec = Vec::new(); + for name in [short_name.map(str::to_string), display] + .into_iter() + .flatten() + { + let name = name.trim().to_string(); + // Short fragments ("PT", "BRI") would match unrelated text. + if name.len() >= 6 && !names.iter().any(|n| n.eq_ignore_ascii_case(&name)) { + names.push(name); + } + } + names } pub(super) fn parse_screener_results(raw: &RawScreenerResponse) -> Result, IdxError> { @@ -785,9 +863,10 @@ fn collect_earnings( #[cfg(test)] mod tests { use super::{ - KeyRatios, MsnQuote, RawChartResponse, RawFinancialStatement, RawNewsFeed, - RawScreenerResponse, RawSentiment, parse_financial_statements, parse_fundamentals, - parse_history, parse_news, parse_quote, parse_screener_results, parse_sentiment, + KeyRatios, MsnQuote, NewsRelevance, RawChartResponse, RawFinancialStatement, RawNewsFeed, + RawScreenerResponse, RawSentiment, company_names, parse_financial_statements, + parse_fundamentals, parse_history, parse_news, parse_quote, parse_screener_results, + parse_sentiment, }; use crate::error::IdxError; @@ -814,7 +893,7 @@ mod tests { serde_json::from_str(include_str!("../../../tests/fixtures/msn_news_bbca.json")) .expect("news fixture should deserialize"); - let items = parse_news("BBCA.JK", &raw).expect("news should parse"); + let items = parse_news("BBCA.JK", "bn91jc", &[], &raw, 10).expect("news should parse"); assert_eq!(items.len(), 1); assert_eq!(items[0].id, "news-1"); @@ -823,6 +902,84 @@ mod tests { assert_eq!(items[0].provider, "Contoso News"); assert_eq!(items[0].published_at, "2026-03-20T10:00:00Z"); assert_eq!(items[0].read_time_min, Some(3)); + assert_eq!(items[0].relevance, NewsRelevance::Market); + } + + fn news_feed(items: serde_json::Value) -> RawNewsFeed { + serde_json::from_value(serde_json::json!({ "value": items })).expect("news feed") + } + + #[test] + fn parse_news_ranks_company_items_first_then_newest() { + let raw = news_feed(serde_json::json!([ + {"id": "market-new", "title": "IHSG anjlok usai pengumuman MSCI", + "publishedDateTime": "2026-10-16T08:00:00Z", + "financeMetadata": {"stocks": [{"stockId": "a1xz1h"}]}}, + {"id": "tagged", "title": "Laba bersih naik", + "publishedDateTime": "2026-10-14T08:00:00Z", + "financeMetadata": {"stocks": [{"stockId": "bn9o8m"}]}}, + {"id": "ticker", "title": "Saham TLKM menguat", + "publishedDateTime": "2026-10-15T08:00:00Z"}, + {"id": "name", "title": "Dirut baru", "abstract": "Telkom Indonesia menunjuk direksi", + "publishedDateTime": "2026-10-13T08:00:00Z"}, + {"id": "market-old", "title": "Rupiah melemah", + "publishedDateTime": "2026-10-12T08:00:00Z"} + ])); + let names = vec!["Telkom Indonesia".to_string()]; + + let items = parse_news("TLKM.JK", "bn9o8m", &names, &raw, 10).expect("news"); + + let order: Vec<_> = items.iter().map(|i| (i.id.as_str(), i.relevance)).collect(); + assert_eq!( + order, + [ + ("ticker", NewsRelevance::Company), + ("tagged", NewsRelevance::Company), + ("name", NewsRelevance::Company), + ("market-new", NewsRelevance::Market), + ("market-old", NewsRelevance::Market), + ] + ); + let limited = parse_news("TLKM.JK", "bn9o8m", &names, &raw, 2).expect("news"); + assert_eq!(limited.len(), 2); + assert_eq!(limited[1].id, "tagged"); + } + + #[test] + fn ticker_match_requires_a_whole_word() { + let raw = news_feed(serde_json::json!([ + {"id": "a", "title": "BBRI dan BMRI melemah"}, + {"id": "b", "title": "Saham XBBRI dan BBRIX"}, + {"id": "c", "title": "bbri lowercase is not a ticker"} + ])); + + let items = parse_news("BBRI.JK", "bn91p2", &[], &raw, 10).expect("news"); + + let company: Vec<_> = items + .iter() + .filter(|i| i.relevance == NewsRelevance::Company) + .map(|i| i.id.as_str()) + .collect(); + assert_eq!(company, ["a"]); + } + + #[test] + fn company_names_strip_legal_suffixes_and_skip_short_names() { + assert_eq!( + company_names( + Some("Telkom Indonesia"), + Some("Telkom Indonesia (Persero) Tbk PT") + ), + ["Telkom Indonesia"] + ); + assert_eq!( + company_names( + Some("Bank BRI"), + Some("Bank Rakyat Indonesia (Persero) Tbk PT") + ), + ["Bank BRI", "Bank Rakyat Indonesia"] + ); + assert!(company_names(Some("PT"), Some("Tbk PT")).is_empty()); } #[test] diff --git a/crates/idx-core/src/api/msn/mod.rs b/crates/idx-core/src/api/msn/mod.rs index 37bf2a3..021d5e1 100644 --- a/crates/idx-core/src/api/msn/mod.rs +++ b/crates/idx-core/src/api/msn/mod.rs @@ -19,9 +19,11 @@ use crate::error::IdxError; use client::MsnClient; use map::{ - parse_earnings, parse_financial_statements, parse_fundamentals, parse_history, parse_insights, - parse_news, parse_profile, parse_quote, parse_screener_results, parse_sentiment, + company_names, parse_earnings, parse_financial_statements, parse_fundamentals, parse_history, + parse_insights, parse_news, parse_profile, parse_quote, parse_screener_results, + parse_sentiment, }; +use symbols::resolve_msn_id; #[cfg(feature = "testing")] pub(crate) use parse::{parse_fundamentals_from_str, parse_history_from_str, parse_quote_from_str}; @@ -102,8 +104,19 @@ impl InsightsProvider for MsnProvider { impl NewsProvider for MsnProvider { fn news(&self, symbol: &str, limit: usize) -> Result, IdxError> { + let id = + resolve_msn_id(symbol).ok_or_else(|| IdxError::SymbolNotFound(symbol.to_string()))?; let raw = self.client.fetch_news(symbol, limit)?; - parse_news(symbol, &raw) + // Company names catch articles MSN did not tag; without them (quote + // unavailable) the ticker and tags still classify. + let names = self + .client + .fetch_quotes(symbol) + .ok() + .and_then(|quotes| quotes.into_iter().next()) + .map(|quote| company_names(quote.short_name.as_deref(), quote.display_name.as_deref())) + .unwrap_or_default(); + parse_news(symbol, id, &names, &raw, limit) } } diff --git a/crates/idx-core/src/api/msn/raw_types.rs b/crates/idx-core/src/api/msn/raw_types.rs index ab2381d..ee3452e 100644 --- a/crates/idx-core/src/api/msn/raw_types.rs +++ b/crates/idx-core/src/api/msn/raw_types.rs @@ -11,6 +11,8 @@ pub(crate) struct MsnQuote { pub(crate) symbol: Option, #[serde(default)] pub(crate) short_name: Option, + #[serde(default)] + pub(crate) display_name: Option, pub(crate) price: Option, #[serde(default)] pub(crate) price_change: Option, @@ -281,6 +283,21 @@ pub(super) struct RawNewsItem { pub(super) provider: Option, pub(super) published_date_time: Option, pub(super) read_time_min: Option, + #[serde(default)] + pub(super) finance_metadata: Option, +} + +#[derive(Debug, Deserialize)] +pub(super) struct RawNewsFinanceMetadata { + /// MSN instrument ids the article is tagged with. + #[serde(default)] + pub(super) stocks: Vec, +} + +#[derive(Debug, Deserialize)] +pub(super) struct RawNewsStock { + #[serde(rename = "stockId")] + pub(super) stock_id: Option, } #[derive(Debug, Deserialize)] diff --git a/crates/idx-core/src/api/types.rs b/crates/idx-core/src/api/types.rs index bf2552c..8949e8f 100644 --- a/crates/idx-core/src/api/types.rs +++ b/crates/idx-core/src/api/types.rs @@ -214,6 +214,27 @@ pub struct NewsItem { pub provider: String, pub published_at: String, pub read_time_min: Option, + /// Whether the item is about this company or general market news; the + /// provider's feed for a stock can be mostly the latter. + #[serde(default)] + pub relevance: NewsRelevance, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum NewsRelevance { + Company, + #[default] + Market, +} + +impl NewsRelevance { + pub fn as_str(self) -> &'static str { + match self { + Self::Company => "company", + Self::Market => "market", + } + } } #[derive(Debug, Deserialize)] diff --git a/crates/idx-core/src/financial.rs b/crates/idx-core/src/financial.rs index 19c2fb4..69486ea 100644 --- a/crates/idx-core/src/financial.rs +++ b/crates/idx-core/src/financial.rs @@ -122,7 +122,8 @@ impl Financial<'_> { policy: CachePolicy, ) -> Result>, IdxError> { let resolved = self.idx.resolve(symbol)?; - let key = format!("{resolved}-{limit}"); + // "ranked": entries cached before relevance ranking lack the field. + let key = format!("{resolved}-{limit}-ranked"); let provider = self.idx.provider.news_provider(&resolved)?; let ttl = self.idx.config.quote_ttl; let fetched = self.fetch("news", &key, &resolved, ttl, policy, || { diff --git a/crates/idx-mcp/src/server.rs b/crates/idx-mcp/src/server.rs index eba79fa..558d0ed 100644 --- a/crates/idx-mcp/src/server.rs +++ b/crates/idx-mcp/src/server.rs @@ -162,8 +162,9 @@ for one or more IDX tickers." #[tool( description = "Company data for one IDX ticker: profile, financial statements, \ -earnings, analyst sentiment, insights, and news. Sections that fail are listed under \ -`errors`; the call fails only if every section fails." +earnings, analyst sentiment, insights, and news. News items carry `relevance`: \ +`company` (about this ticker) first, then `market` (general market news). Sections \ +that fail are listed under `errors`; the call fails only if every section fails." )] async fn get_company(&self, params: Parameters) -> ToolResult { let CompanyParams { symbol, sections } = params.0; diff --git a/crates/idx-mcp/tests/golden/tools.json b/crates/idx-mcp/tests/golden/tools.json index ffd6b54..9287ed4 100644 --- a/crates/idx-mcp/tests/golden/tools.json +++ b/crates/idx-mcp/tests/golden/tools.json @@ -58,7 +58,7 @@ "name": "compare_stocks" }, { - "description": "Company data for one IDX ticker: profile, financial statements, earnings, analyst sentiment, insights, and news. Sections that fail are listed under `errors`; the call fails only if every section fails.", + "description": "Company data for one IDX ticker: profile, financial statements, earnings, analyst sentiment, insights, and news. News items carry `relevance`: `company` (about this ticker) first, then `market` (general market news). Sections that fail are listed under `errors`; the call fails only if every section fails.", "inputSchema": { "$defs": { "CompanySection": { diff --git a/web/src/content/docs/cli/stocks.md b/web/src/content/docs/cli/stocks.md index 88c2b8c..3f366c4 100644 --- a/web/src/content/docs/cli/stocks.md +++ b/web/src/content/docs/cli/stocks.md @@ -26,7 +26,7 @@ them from stdin. `history` takes `--period` (`1d` to `5y`) and `--interval` | `earnings` | reported and estimated earnings | | `sentiment` | crowd sentiment | | `insights` | AI insights from the provider | -| `news` | recent news | +| `news` | recent news, this company's first (`ABOUT` = `company`), then general market news (`market`) | ## Analysis diff --git a/web/src/content/docs/mcp/tools.md b/web/src/content/docs/mcp/tools.md index 6d85ce7..16e57fa 100644 --- a/web/src/content/docs/mcp/tools.md +++ b/web/src/content/docs/mcp/tools.md @@ -29,7 +29,7 @@ Compare fundamental reports (growth, valuation, risk) across IDX tickers. Ticker ## `get_company` -Company data for one IDX ticker: profile, financial statements, earnings, analyst sentiment, insights, and news. Sections that fail are listed under `errors`; the call fails only if every section fails. +Company data for one IDX ticker: profile, financial statements, earnings, analyst sentiment, insights, and news. News items carry `relevance`: `company` (about this ticker) first, then `market` (general market news). Sections that fail are listed under `errors`; the call fails only if every section fails. | Input | Type | Required | Description | | --- | --- | --- | --- |