diff --git a/.gitignore b/.gitignore index d7292fe..834cbcc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ target/ .direnv/ .claude/ result +result-* tmp/ # Internal docs (specs, research, business strategy) diff --git a/TODO.md b/TODO.md index 27c2ce1..93a3a1a 100644 --- a/TODO.md +++ b/TODO.md @@ -162,6 +162,9 @@ - [ ] Additional providers (Alpha Vantage, Twelve Data, IDX official) ## 🔬 Latest Smoke Findings (2026-04-02) +- [x] Combined PR verification on `2026-10-02`: JSON/offline-cache fixes, Nix/Crane CI, and market-data fixes pass `nix flake check -L` on x86_64-linux (173 unit tests, 89 CLI tests, 62 mock smoke cases), plus the publisher and `sh` installer regression scripts. +- [x] Fixed the integration review's screener truncation: full-list queries now re-request MSN's reported total rather than cap candidates at 500, reject incomplete totals, and use separate `all` cache keys. Live US top-performers returned 600 unique stocks; offline limits 3 and 25 matched the full result's prefixes. One upstream HTTP 499 occurred before the successful live check. +- [x] Removed the committed `result-1` Nix output symlink and ignored numbered result links. - [x] Publish hardening batch on `2026-09-25`: `scripts/publish-ownership-snapshot.sh` now uploads an immutable content-addressed SQLite asset (date + sha256) and verifies it before publishing the manifest as the final commit point, so a partial upload no longer leaves a manifest that points at a missing asset; the manifest is trusted for a no-op only when the referenced asset matches the manifest SHA-256 and size at the configured repo/tag, and GitHub auth/network errors never become "up-to-date" - [x] Publish integrity follow-up on `2026-09-25`: no-op and upload verification now check the full SHA-256, not just the filename and size; the publisher uses GitHub's `sha256:` asset digest when present and falls back to `gh release download` + local hashing for legacy assets, rejects a `download_url` that names a different repo/tag, and refuses to publish the manifest when the uploaded asset's SHA-256 does not match - [x] `--history` is bounded (maximum `1000`, leading zeros stripped before conversion) so an enormous value cannot wrap negative and look already satisfied diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 446413d..c02e846 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -42,6 +42,22 @@ When `history_provider = auto` (default): MSN Charts are price-only for IDX. The CLI normalizes them into `Ohlc` rows by using the chart price as open/high/low/close and `0` volume. +MSN candle dates use WIB (UTC+7); Yahoo candle dates use `meta.gmtoffset`, with +WIB as the fallback. + +### Quote and Screener Data +- Quotes expose nullable `as_of` last-trade timestamps in WIB. Table output flags + quotes whose last trade was more than seven days ago. +- `change`, `change_pct`, and `volume` are nullable when the provider omits them; + JSON consumers must not assume these fields are always numeric. +- Listed MSN instruments without a price return `NOMARKETDATA`; unknown symbols + still return `SYMBOLNOTFOUND`. Empty sentiment responses represent no votes. +- Performers, high-volume, and large-cap screens fetch the complete candidate + list before local ranking and `--limit`. MSN ignores `pageIndex` for these + lists, so the provider uses the response's `count` to request the full list + when the initial batch is incomplete, and rejects a still-truncated response. +- Full-list screen caches use an `all` key, separate from older limited results. + ### Capability Gating ``` @@ -121,3 +137,11 @@ CLI flags > environment variables > config file > defaults - Table mode: human-readable error on stderr - JSON mode: `{"error": true, "code": "...", "message": "..."}` - Exit code 0 on success, non-zero on failure + +## CLI Output and Offline Cache +- `version`, `cache info`, `cache clear`, and `ownership resolve map|merge` emit + JSON objects when JSON output is selected. +- `stocks quote` and `stocks compare` require at least one symbol. +- Offline reads warn on stderr when serving cache entries past their TTL, + including the cache fetch timestamp. `--quiet` suppresses these warnings. + Staleness is not yet represented by a versioned JSON result envelope. diff --git a/docs/SMOKE.md b/docs/SMOKE.md index 7bd1503..55755e4 100644 --- a/docs/SMOKE.md +++ b/docs/SMOKE.md @@ -22,6 +22,33 @@ scripts/live-smoke.sh --bin ./tmp/release-install/bin/idx --no-build --mode mock scripts/audit-msn-fundamentals.sh --tickers BUMI,ADRO,AIMS ``` +## Nix CI Checks + +`nix flake check -L` runs formatting, Clippy, Rust tests, crate packaging, the +application and ownership-publisher builds, and the offline mock smoke. Crane +shares dependency build artifacts between the Rust derivations. CI restores the +Nix store from the GitHub Actions cache, keyed on the flake and Cargo inputs. + +The publisher regression script remains a separate CI step: + +```bash +scripts/publish-ownership-snapshot-test.sh +``` + +For a live full-list screener check, use an isolated config/cache and run: + +```bash +idx -o json stocks screen --region us --filter top-performers --limit 600 +idx --offline -o json stocks screen --region us --filter top-performers --limit 3 +idx --offline -o json stocks screen --region us --filter top-performers --limit 25 +``` + +The first command should not stop at the old 500-row cap when MSN reports more +candidates. The two offline results must match prefixes of the same cached full +list. Provider counts and prices vary; live HTTP failures are not offline-test +failures. + + ## Modes - `live` runs the default real-network baseline: `general`, `live-table`, and `ownership` diff --git a/result-1 b/result-1 deleted file mode 120000 index 2d66130..0000000 --- a/result-1 +++ /dev/null @@ -1 +0,0 @@ -/nix/store/jh7j89lzmrvvhm44dv96znzlf522p2ac-idx-cli-test-0.2.4 \ No newline at end of file diff --git a/src/api/mod.rs b/src/api/mod.rs index 8e2258b..9fad9cb 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -81,7 +81,13 @@ pub trait NewsProvider { #[allow(dead_code)] pub trait ScreenerProvider { - fn screener(&self, filter: &str, region: &str, limit: usize) -> Result, IdxError>; + /// `None` requests the complete candidate list for client-side ranking. + fn screener( + &self, + filter: &str, + region: &str, + limit: Option, + ) -> Result, IdxError>; } pub struct SelectedProvider { diff --git a/src/api/msn/client.rs b/src/api/msn/client.rs index 089e423..3a6b033 100644 --- a/src/api/msn/client.rs +++ b/src/api/msn/client.rs @@ -270,11 +270,11 @@ impl MsnClient { &self, filter: &str, region: &str, - limit: usize, + limit: Option, ) -> Result { let url = format!("{MSN_ASSETS_BASE_URL}Finance/Screener?apikey={MSN_API_KEY}&wrapodata=false"); - let req = ScreenerRequest { + let mut req = ScreenerRequest { filter: vec![ ScreenerFilter { key: filter.to_string(), @@ -293,11 +293,17 @@ impl MsnClient { }, return_value_type: vec!["quote".to_string(), "equity".to_string()], screener_type: "stock".to_string(), - limit, + limit: limit.unwrap_or(500), page_index: 0, }; - self.post_json(&url, &req, "SCREENER", "screener") + if limit.is_some() { + return self.post_json(&url, &req, "SCREENER", "screener"); + } + fetch_complete_screener(|limit| { + req.limit = limit; + self.post_json(&url, &req, "SCREENER", "screener") + }) } pub(super) fn fetch_chart( @@ -316,6 +322,30 @@ impl MsnClient { } } +fn fetch_complete_screener( + mut fetch: impl FnMut(usize) -> Result, +) -> Result { + let total_count = |response: &RawScreenerResponse| { + response + .count + .and_then(|count| usize::try_from(count).ok()) + .ok_or_else(|| IdxError::ParseError("MSN screener omitted a valid total count".into())) + }; + let mut response = fetch(500)?; + let count = total_count(&response)?; + if response.quote.as_ref().map_or(0, Vec::len) < count { + // MSN ignores pageIndex for these lists. Re-request the reported total + // instead of silently treating the initial batch as the whole market. + response = fetch(count)?; + } + if response.quote.as_ref().map_or(0, Vec::len) < total_count(&response)? { + return Err(IdxError::ParseError( + "MSN screener returned an incomplete candidate list".into(), + )); + } + Ok(response) +} + fn msn_chart_type(period: &Period, interval: &Interval) -> Result<&'static str, IdxError> { if !matches!(interval, Interval::Day) { return Err(IdxError::Unsupported( @@ -335,10 +365,55 @@ fn msn_chart_type(period: &Period, interval: &Interval) -> Result<&'static str, #[cfg(test)] mod tests { - use super::msn_chart_type; + use super::{fetch_complete_screener, msn_chart_type}; + use crate::api::msn::raw_types::RawScreenerResponse; use crate::api::types::{Interval, Period}; use crate::error::IdxError; + fn screener_response(count: Option, returned: usize) -> RawScreenerResponse { + serde_json::from_value(serde_json::json!({ + "count": count, + "quote": (0..returned).map(|index| serde_json::json!({ + "symbol": format!("STOCK{index}"), + "price": 100 + })).collect::>() + })) + .expect("valid screener response") + } + + #[test] + fn full_screener_includes_candidates_beyond_initial_batch() { + let response = + fetch_complete_screener(|limit| Ok(screener_response(Some(600), limit.min(600)))) + .expect("complete list"); + let symbols: Vec<_> = response + .quote + .unwrap() + .into_iter() + .map(|quote| quote.symbol.unwrap()) + .collect(); + assert_eq!( + symbols, + (0..600) + .map(|index| format!("STOCK{index}")) + .collect::>() + ); + } + + #[test] + fn full_screener_rejects_a_truncated_total_response() { + let err = fetch_complete_screener(|_| Ok(screener_response(Some(600), 500))) + .expect_err("truncated list must not be ranked as complete"); + assert!(matches!(err, IdxError::ParseError(_))); + } + + #[test] + fn full_screener_requires_a_total_count() { + let err = fetch_complete_screener(|_| Ok(screener_response(None, 500))) + .expect_err("unknown completeness must not be treated as a full list"); + assert!(matches!(err, IdxError::ParseError(_))); + } + #[test] fn maps_supported_msn_chart_types() { assert_eq!( diff --git a/src/api/msn/mod.rs b/src/api/msn/mod.rs index 6189572..2278e6a 100644 --- a/src/api/msn/mod.rs +++ b/src/api/msn/mod.rs @@ -107,7 +107,12 @@ impl NewsProvider for MsnProvider { } impl ScreenerProvider for MsnProvider { - fn screener(&self, filter: &str, region: &str, limit: usize) -> Result, IdxError> { + fn screener( + &self, + filter: &str, + region: &str, + limit: Option, + ) -> Result, IdxError> { let raw = self.client.fetch_screener(filter, region, limit)?; parse_screener_results(&raw) } diff --git a/src/cli/stocks.rs b/src/cli/stocks.rs index a6ef78b..1d15d5f 100644 --- a/src/cli/stocks.rs +++ b/src/cli/stocks.rs @@ -622,9 +622,12 @@ pub fn handle( filter.as_str(), "high-volume" | "large-cap" | "top-performers" | "worst-performers" ); - let fetch_limit = if needs_full_fetch { 500 } else { *limit }; + let fetch_limit = (!needs_full_fetch).then_some(*limit); let bucket = cache_bucket(provider.kind(), "screen"); - let key = format!("{filter}:{region}:{fetch_limit}"); + let key = match fetch_limit { + Some(limit) => format!("{filter}:{region}:{limit}"), + None => format!("{filter}:{region}:all"), + }; let screener_provider = provider.screener_provider("screen")?; let mut quotes: Vec = fetch_with_cache( &cache,