feat: default ownership sync to published snapshot

This commit is contained in:
Rasyidan Akbar F. 2026-04-12 14:27:47 +07:00
commit aad29eaf23
4 changed files with 55 additions and 23 deletions

View file

@ -163,6 +163,7 @@
- [ ] Additional providers (Alpha Vantage, Twelve Data, IDX official) - [ ] Additional providers (Alpha Vantage, Twelve Data, IDX official)
## 🔬 Latest Smoke Findings (2026-04-02) ## 🔬 Latest Smoke Findings (2026-04-02)
- [x] `ownership sync` now falls back to the built-in published manifest URL on `2026-04-12`, so plain `idx ownership sync` works on a clean machine once the public snapshot release is available
- [x] Ownership snapshot publishing now has a live-source maintainer path on `2026-04-11`: `scripts/build-latest-ownership-snapshot.sh` discovers the current supported `above1` IDX/KSEI PDF, imports it into an isolated temp DB, emits release-ready SQLite + manifest artifacts, and records the source IDX metadata in the manifest - [x] Ownership snapshot publishing now has a live-source maintainer path on `2026-04-11`: `scripts/build-latest-ownership-snapshot.sh` discovers the current supported `above1` IDX/KSEI PDF, imports it into an isolated temp DB, emits release-ready SQLite + manifest artifacts, and records the source IDX metadata in the manifest
- [x] Verification on `2026-04-11`: `nix develop --command cargo test`, `bash -n scripts/build-latest-ownership-snapshot.sh`, and a real `nix develop --command scripts/build-latest-ownership-snapshot.sh --idx-bin ./target/debug/idx --output-dir tmp/ownership-snapshot-live` run all passed; the current live source produced `7214` rows for `955` tickers with `as_of_date=2026-03-31` - [x] Verification on `2026-04-11`: `nix develop --command cargo test`, `bash -n scripts/build-latest-ownership-snapshot.sh`, and a real `nix develop --command scripts/build-latest-ownership-snapshot.sh --idx-bin ./target/debug/idx --output-dir tmp/ownership-snapshot-live` run all passed; the current live source produced `7214` rows for `955` tickers with `as_of_date=2026-03-31`
- [x] Final release-hygiene pass on `2026-04-06`: crate metadata now declares `rust-version = 1.85`, README install docs now spell out Cargo helper-runtime expectations plus persistent `nix profile install`, and CI install smoke now runs the mock smoke matrix against the installed binary instead of only checking `idx version` - [x] Final release-hygiene pass on `2026-04-06`: crate metadata now declares `rust-version = 1.85`, README install docs now spell out Cargo helper-runtime expectations plus persistent `nix profile install`, and CI install smoke now runs the mock smoke matrix against the installed binary instead of only checking `idx version`

View file

@ -14,6 +14,8 @@ The manifest location is resolved in this order:
1. `idx ownership sync --manifest <path-or-url>` 1. `idx ownership sync --manifest <path-or-url>`
2. `IDX_OWNERSHIP_SNAPSHOT_MANIFEST` 2. `IDX_OWNERSHIP_SNAPSHOT_MANIFEST`
3. `ownership.snapshot_manifest` in `config.toml` 3. `ownership.snapshot_manifest` in `config.toml`
4. built-in published manifest URL:
`https://github.com/0xrsydn/idx-cli/releases/download/ownership-snapshot-current/ownership-snapshot-manifest.json`
The value can be either: The value can be either:
- a local path to a manifest JSON file - a local path to a manifest JSON file

View file

@ -47,7 +47,7 @@ pub enum OwnershipCommand {
#[command( #[command(
about = "Install or refresh a maintained ownership SQLite snapshot", about = "Install or refresh a maintained ownership SQLite snapshot",
long_about = "Install or refresh a maintained ownership SQLite snapshot.\n\nThis is the normal bootstrap/update path for ownership data.\n\nManifest lookup order:\n 1. `--manifest`\n 2. `IDX_OWNERSHIP_SNAPSHOT_MANIFEST`\n 3. `ownership.snapshot_manifest` in config", long_about = "Install or refresh a maintained ownership SQLite snapshot.\n\nThis is the normal bootstrap/update path for ownership data.\n\nManifest lookup order:\n 1. `--manifest`\n 2. `IDX_OWNERSHIP_SNAPSHOT_MANIFEST`\n 3. `ownership.snapshot_manifest` in config",
after_help = "Examples:\n idx ownership sync\n idx ownership sync --manifest /path/to/ownership-snapshot-manifest.json\n IDX_OWNERSHIP_SNAPSHOT_MANIFEST=https://example.com/latest.json idx ownership sync" after_help = "Examples:\n idx ownership sync\n idx ownership sync --manifest /path/to/ownership-snapshot-manifest.json\n IDX_OWNERSHIP_SNAPSHOT_MANIFEST=https://example.com/latest.json idx ownership sync\n\nIf none of the above are set, `ownership sync` falls back to the built-in published manifest URL."
)] )]
Sync(SyncArgs), Sync(SyncArgs),
/// Show all holders for a ticker (KSEI + Bing combined). /// Show all holders for a ticker (KSEI + Bing combined).
@ -100,7 +100,7 @@ pub struct ImportArgs {
#[derive(Debug, Args)] #[derive(Debug, Args)]
pub struct SyncArgs { pub struct SyncArgs {
/// Snapshot manifest location (URL or local path). If omitted, config/env lookup is used. /// Snapshot manifest location (URL or local path). If omitted, idx falls back through env/config and then the published default URL.
#[arg(long)] #[arg(long)]
pub manifest: Option<String>, pub manifest: Option<String>,
/// Replace the local DB even when it is already current or newer than the snapshot. /// Replace the local DB even when it is already current or newer than the snapshot.

View file

@ -14,6 +14,7 @@ use crate::ownership::types::OwnershipRelease;
pub const SNAPSHOT_MANIFEST_CONFIG_KEY: &str = "ownership.snapshot_manifest"; pub const SNAPSHOT_MANIFEST_CONFIG_KEY: &str = "ownership.snapshot_manifest";
pub const SNAPSHOT_MANIFEST_ENV: &str = "IDX_OWNERSHIP_SNAPSHOT_MANIFEST"; pub const SNAPSHOT_MANIFEST_ENV: &str = "IDX_OWNERSHIP_SNAPSHOT_MANIFEST";
pub const SNAPSHOT_MANIFEST_SCHEMA_VERSION: u32 = 1; pub const SNAPSHOT_MANIFEST_SCHEMA_VERSION: u32 = 1;
pub const DEFAULT_SNAPSHOT_MANIFEST_URL: &str = "https://github.com/0xrsydn/idx-cli/releases/download/ownership-snapshot-current/ownership-snapshot-manifest.json";
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"; const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
@ -90,30 +91,27 @@ struct SyncDecision {
} }
pub fn resolve_manifest_source(explicit: Option<&str>) -> Result<String, IdxError> { pub fn resolve_manifest_source(explicit: Option<&str>) -> Result<String, IdxError> {
if let Some(value) = explicit { let env_value = std::env::var(SNAPSHOT_MANIFEST_ENV).ok();
let trimmed = value.trim(); let config_value = get_config_value(SNAPSHOT_MANIFEST_CONFIG_KEY)?;
if !trimmed.is_empty() {
return Ok(trimmed.to_string()); resolve_manifest_source_with(explicit, env_value.as_deref(), config_value.as_deref())
}
fn resolve_manifest_source_with(
explicit: Option<&str>,
env_value: Option<&str>,
config_value: Option<&str>,
) -> Result<String, IdxError> {
for value in [explicit, env_value, config_value] {
if let Some(value) = value {
let trimmed = value.trim();
if !trimmed.is_empty() {
return Ok(trimmed.to_string());
}
} }
} }
if let Ok(value) = std::env::var(SNAPSHOT_MANIFEST_ENV) { Ok(DEFAULT_SNAPSHOT_MANIFEST_URL.to_string())
let trimmed = value.trim();
if !trimmed.is_empty() {
return Ok(trimmed.to_string());
}
}
if let Some(value) = get_config_value(SNAPSHOT_MANIFEST_CONFIG_KEY)? {
let trimmed = value.trim();
if !trimmed.is_empty() {
return Ok(trimmed.to_string());
}
}
Err(IdxError::InvalidInput(format!(
"ownership sync needs a snapshot manifest; pass `--manifest` or set `{SNAPSHOT_MANIFEST_CONFIG_KEY}` / `{SNAPSHOT_MANIFEST_ENV}`"
)))
} }
pub fn fetch_manifest(source: &str) -> Result<OwnershipSnapshotManifest, IdxError> { pub fn fetch_manifest(source: &str) -> Result<OwnershipSnapshotManifest, IdxError> {
@ -687,8 +685,9 @@ mod tests {
use rusqlite::Connection; use rusqlite::Connection;
use super::{ use super::{
OwnershipSnapshotArtifact, OwnershipSnapshotManifest, OwnershipSnapshotSource, DEFAULT_SNAPSHOT_MANIFEST_URL, OwnershipSnapshotArtifact, OwnershipSnapshotManifest,
OwnershipSyncAction, SNAPSHOT_MANIFEST_SCHEMA_VERSION, build_sync_decision, parse_manifest, OwnershipSnapshotSource, OwnershipSyncAction, SNAPSHOT_MANIFEST_SCHEMA_VERSION,
build_sync_decision, parse_manifest, resolve_manifest_source_with,
}; };
use crate::ownership::db::{ensure_schema, insert_release}; use crate::ownership::db::{ensure_schema, insert_release};
use crate::ownership::types::OwnershipRelease; use crate::ownership::types::OwnershipRelease;
@ -815,6 +814,36 @@ mod tests {
assert!(parsed.source.is_none()); assert!(parsed.source.is_none());
} }
#[test]
fn resolve_manifest_source_uses_built_in_default_when_unset() {
let resolved = resolve_manifest_source_with(None, None, None).expect("default manifest");
assert_eq!(resolved, DEFAULT_SNAPSHOT_MANIFEST_URL);
}
#[test]
fn resolve_manifest_source_prefers_explicit_then_env_then_config() {
let explicit = resolve_manifest_source_with(
Some("https://example.com/explicit.json"),
Some("https://example.com/env.json"),
Some("https://example.com/config.json"),
)
.expect("explicit manifest");
assert_eq!(explicit, "https://example.com/explicit.json");
let env = resolve_manifest_source_with(
Some(" "),
Some("https://example.com/env.json"),
Some("https://example.com/config.json"),
)
.expect("env manifest");
assert_eq!(env, "https://example.com/env.json");
let config =
resolve_manifest_source_with(None, Some(" "), Some("https://example.com/config.json"))
.expect("config manifest");
assert_eq!(config, "https://example.com/config.json");
}
#[test] #[test]
fn ensure_schema_can_store_release_metadata_needed_for_snapshots() { fn ensure_schema_can_store_release_metadata_needed_for_snapshots() {
let conn = Connection::open_in_memory().unwrap(); let conn = Connection::open_in_memory().unwrap();