From 8404b276286cef82ae222596d733fec88814b1a8 Mon Sep 17 00:00:00 2001 From: 0xrsydn Date: Tue, 31 Mar 2026 19:28:13 +0700 Subject: [PATCH] Add ownership snapshot sync --- FEATURE_SPEC.md | 41 +- TODO.md | 17 +- docs/ARCHITECTURE.md | 5 +- docs/OWNERSHIP_SYNC.md | 73 ++ scripts/build-ownership-snapshot.sh | 130 +++ src/cli/ownership.rs | 61 +- src/config.rs | 23 + src/ownership/db.rs | 6 +- src/ownership/mod.rs | 1 + src/ownership/parser.rs | 7 +- src/ownership/snapshot.rs | 762 ++++++++++++++++++ tests/cli.rs | 302 +++++++ .../ksei_above1_stext_excerpt_prev.xml | 29 + 13 files changed, 1415 insertions(+), 42 deletions(-) create mode 100644 docs/OWNERSHIP_SYNC.md create mode 100755 scripts/build-ownership-snapshot.sh create mode 100644 src/ownership/snapshot.rs create mode 100644 tests/fixtures/ksei_above1_stext_excerpt_prev.xml diff --git a/FEATURE_SPEC.md b/FEATURE_SPEC.md index 74f6b0f..1059f1c 100644 --- a/FEATURE_SPEC.md +++ b/FEATURE_SPEC.md @@ -1,6 +1,6 @@ # Feature Spec: Current Coverage and Remaining Core Work -**Status:** Active - updated against the current repo state on 2026-03-28 +**Status:** Active - updated against the current repo state on 2026-03-31 **Current focus:** Close the remaining core gaps before expanding into new market and watchlist surfaces **Reference:** `origin/dev/rubick` (endpoint reference only, not the current implementation plan) @@ -26,7 +26,7 @@ Use `TODO.md` as the execution log and smoke-history record. ## Verified Current State -- Current automated coverage is `168` tests: `110` unit and `58` integration. +- Current automated coverage is `176` tests: `115` unit and `61` integration. - Reusable smoke coverage exists via `scripts/live-smoke.sh`; command groups are documented in `docs/SMOKE.md`. - The latest smoke notes in `TODO.md` report passing live table and JSON checks for all shipped `stocks` commands. - Cache/offline parity, JSON startup-error handling, screener input validation, and the recent MSN output cleanups have already been completed. @@ -36,6 +36,7 @@ Use `TODO.md` as the execution log and smoke-history record. - Live verification on `2026-03-29` confirmed that the parser-compatible discovered source is currently the `Pemegang Saham di atas 1% (KSEI)` `lamp1` attachment, and `idx ownership import --url` now works end to end for that discovered BEI PDF. - The currently discoverable `above 5%` and `investor-type` BEI families remain different schemas and are now classified and rejected explicitly during import instead of falling through to a generic zero-row parse failure. - Ownership smoke coverage now includes a dedicated `ownership-import` group that verifies live supported import plus expected unsupported-family failures. +- `idx ownership sync` now installs maintained SQLite snapshots via a manifest-driven contract with checksum validation, conservative local replacement rules, and fixture-backed regression coverage for install/no-op/force-refresh behavior. This means the main gap is no longer endpoint coverage. The remaining work is architecture cleanup, a few correctness edge cases, and selective UX expansion on top of already-shipped commands. @@ -63,6 +64,7 @@ The remaining work is architecture cleanup, a few correctness edge cases, and se | Screener | `idx stocks screen` | Implemented with gaps | Validation landed; expression/preset workflow is still future work | | MSN charts | `idx stocks history --history-provider msn` | Missing | Explicit MSN history still returns unsupported for IDX | | KSEI ownership import/query | `idx ownership import --file`, `idx ownership import --url`, `idx ownership releases`, `idx ownership ticker` | Implemented | Local PDF import and SQLite-backed query flow are verified against the March 2026 KSEI release; remote IDX import now works for the discovered `above 1%` `lamp1` BEI attachment, and legacy `above 5%` / `investor-type` BEI report families are rejected explicitly | +| Ownership snapshot sync | `idx ownership sync` | Implemented | Manifest-driven SQLite snapshot install with checksum validation, conservative replacement/no-op rules, and publisher helper script | | Bing ownership CLI | `idx ownership import --fetch-bing` | Not implemented | Client groundwork exists, CLI import path is still deferred | --- @@ -84,6 +86,7 @@ The following items should no longer be treated as active backlog in this spec: - KSEI ownership parser hardening for the March 2026 live PDF layout, including the merged `DATE + SHARE_CODE` segment and `D`/`A` locality markers - Real KSEI ownership CLI verification from local file import into SQLite (`7261` rows across `955` tickers on `2026-03-28`) - Ownership remote-import hardening for the `above1` contract, including direct-PDF-only `--url` input, discovery status output, explicit legacy-schema rejection, and live ownership-import smoke coverage +- Ownership snapshot publishing/sync contract, including `idx ownership sync`, manifest/checksum validation, conservative local replacement rules, and the `scripts/build-ownership-snapshot.sh` publisher helper If any of the above regress, capture that in `TODO.md` as a new finding rather than reopening the old section here wholesale. @@ -145,32 +148,25 @@ Done when: - The retry/fallback story for Yahoo failures is deliberate and documented. - SMA200 behavior is either improved or clearly documented as expected. -#### 5. Automate IDX ownership source discovery and fetch +#### 5. Ownership fallback ingest and cross-check path Current state: -- `idx ownership import --file ` is now working and verified against the March 2026 KSEI release. -- Direct IDX announcement PDFs exist, but the monthly file path is not safely hardcodable because the asset filename is hashed. -- The official BEI listing page for this feed is `https://www.idx.co.id/id/berita/pengumuman/`, and the page’s own Nuxt client fetches announcement data from `GET /primary/NewsAnnouncement/GetAllAnnouncement`. -- The BEI endpoint is now reverse-engineered enough for `idx ownership discover` to locate the current `Pemegang Saham di atas 1% (KSEI)`, `Pemegang Saham di atas 5% (KSEI)`, and `Kepemilikan Saham Perusahaan Tercatat Berdasarkan Tipe Investor` hashed PDFs. -- Live verification on `2026-03-29` shows that the discoverable `above 1%` `lamp1` attachment matches the raw KSEI holder-register shape that the current parser imports successfully. -- Live verification on `2026-03-29` also shows that the currently discoverable `above 5%` and `investor-type` BEI families do not match the holder-register schema the current parser imports. -- Product direction as of `2026-03-30` is to standardize on the `above 1%` holder-register structure for remote import; the other discovered families are legacy inputs that should be rejected clearly rather than parsed. -- Plain `curl`/`ureq` requests to IDX-hosted PDFs still get `403` from Cloudflare, while `curl-impersonate` inside the project `nix develop` environment has already been verified to return a real PDF for a March 2026 ownership source URL. -- The KSEI archive remains a secondary upstream and currently exposes monthly ZIP files that can be used later as fallback or cross-check input, but it is no longer the primary roadmap target. +- Remote IDX discovery/fetch is already in place for the supported `above1` holder-register family. +- `idx ownership sync` now covers the maintained-snapshot distribution path on top of that import flow. +- The KSEI ZIP/TXT archive remains the main unresolved ownership follow-up and should stay a fallback/cross-check input, not the primary ingest path. Why it matters: -- The ownership feature now parses and stores live ownership data correctly from both local files and the currently discoverable `above 1%` BEI `lamp1` attachment. -- Batch 2 hardening is now complete; the remaining ownership roadmap is snapshot publishing / `ownership sync` plus optional fallback ingest and cross-check work. +- The core ownership product path is now stable: local import, remote IDX discovery/import, and maintained snapshot sync all exist. +- The remaining ownership work is about fallback resilience and validation against a secondary upstream. Done when: -- The CLI can discover the latest parser-compatible IDX ownership PDF URL from an IDX listing/announcement surface without hardcoded monthly paths. -- Remote fetches use the same browser-impersonation strategy already established elsewhere in the repo instead of the current bare `ureq` path. -- `idx ownership import --url` can fetch and parse the current `above 1%` IDX-hosted PDF reliably. -- Unsupported BEI report families are classified and rejected explicitly instead of failing later with a generic zero-row parse error. +- The KSEI ZIP/TXT archive ingest shape is defined. +- At least one monthly archive release is cross-checked against the IDX-PDF-derived SQLite state. +- Fallback ingest produces query-compatible data for `ownership releases`, `ticker`, and `changes`. Roadmap: -1. Publish maintained SQLite snapshot artifacts and add `idx ownership sync` after remote IDX import is stable. -2. Optionally add KSEI ZIP/TXT ingest later as fallback or cross-check input. +1. Add KSEI ZIP/TXT ingest later as fallback or cross-check input. +2. Decide whether that fallback stays maintainer-only or becomes a user-facing alternative path. ### P1 - UX and output contract cleanup @@ -195,10 +191,9 @@ Priority order: - Reuse the existing client groundwork in `src/api/msn/bing.rs`. - Define the import shape and output contract for `idx ownership import --fetch-bing`. -3. Ownership sync and snapshot distribution - - Keep IDX PDF discovery/fetch as the first milestone before starting this work. - - Publish maintained SQLite snapshot artifacts and add `idx ownership sync`. +3. Ownership fallback ingest and cross-check work - Treat the KSEI archive (`https://web.ksei.co.id/archive_download/holding_composition`) as fallback/backstop input, not the primary product ingest path. + - Define the ZIP/TXT ingest shape and verify compatibility with the IDX-PDF-derived SQLite contract. 4. Richer financial statements - Decide whether to stay with the current single-period model or add multi-period fetch support. diff --git a/TODO.md b/TODO.md index d01cc9a..ab69b26 100644 --- a/TODO.md +++ b/TODO.md @@ -96,14 +96,14 @@ - [x] Batch 2 verification: ownership-focused smoke checks cover successful remote import plus expected failure UX ### Batch 3 — Snapshot publishing + sync -- [ ] Design maintained SQLite snapshot publishing after remote IDX import is stable -- [ ] Add `idx ownership sync` -- [ ] Define manifest/checksum/update semantics and local DB replacement rules -- [ ] Add regression coverage for manifest parsing, checksum validation, no-op sync, and forced refresh -- [ ] Batch 3 verification: `cargo build` -- [ ] Batch 3 verification: `cargo clippy -- -D warnings` -- [ ] Batch 3 verification: `cargo test` -- [ ] Batch 3 verification: sync installs into an empty temp data dir, preserves query behavior, and no-ops when already current +- [x] Design maintained SQLite snapshot publishing after remote IDX import is stable +- [x] Add `idx ownership sync` +- [x] Define manifest/checksum/update semantics and local DB replacement rules +- [x] Add regression coverage for manifest parsing, checksum validation, no-op sync, and forced refresh +- [x] Batch 3 verification: `cargo build` +- [x] Batch 3 verification: `cargo clippy -- -D warnings` +- [x] Batch 3 verification: `cargo test` +- [x] Batch 3 verification: sync installs into an empty temp data dir, preserves query behavior, and no-ops when already current ### Batch 4 — KSEI ZIP/TXT fallback and cross-check path - [ ] Keep KSEI ZIP/TXT ingest as fallback and validation/backstop work, not the first milestone @@ -167,6 +167,7 @@ - [x] Valid-but-unsupported BEI PDFs now fail before row parsing with explicit schema-aware errors (`announcement_wrapper`, legacy `above5`, legacy `investor-type`) instead of the old generic `no KSEI rows parsed from PDF` path - [x] Regression coverage now covers default `ownership discover` behavior, status visibility, listing-page rejection, duplicate SHA imports, and explicit unsupported-schema detection with compact `stext` fixtures plus fake-`mutool` CLI tests - [x] New `ownership-import` smoke coverage now discovers the current live `above1`/`above5`/`investor-type` URLs, imports the supported `above1` attachment successfully, and confirms the legacy families fail with explicit unsupported-schema UX (`tmp/live-smoke/20260330-160201`) +- [x] New ownership snapshot sync coverage now verifies manifest parsing, checksum validation, install into an empty temp data dir, preserved query behavior for `releases`/`ticker`/`changes`, no-op sync when current, and `--force` refresh via fixture-backed local manifests on `2026-03-31` ## 🐛 Known Issues - [ ] Yahoo Finance returns 429 from datacenter IPs occasionally diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 3f11106..c0c240c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -55,8 +55,9 @@ Factory functions: Unlike the `stocks` module (live-fetch), ownership is **import-then-query**: 1. `idx ownership import` — ETL pipeline: fetch PDF/API → parse → normalize → load SQLite -2. All query commands read from local `~/.local/share/idx/ownership.db` -3. Fully offline after import +2. `idx ownership sync` — install a maintained SQLite snapshot via manifest + checksum validation +3. All query commands read from local `~/.local/share/idx/ownership.db` +4. Fully offline after import/sync ### Data Sources - **KSEI** — official ≥1% shareholder registry (monthly PDF from IDX) diff --git a/docs/OWNERSHIP_SYNC.md b/docs/OWNERSHIP_SYNC.md new file mode 100644 index 0000000..1990a3f --- /dev/null +++ b/docs/OWNERSHIP_SYNC.md @@ -0,0 +1,73 @@ +# Ownership Snapshot Sync + +`idx ownership sync` installs a maintained SQLite snapshot into the local ownership DB path. + +The command is intentionally manifest-driven so the repo can publish snapshots in GitHub releases, object storage, or a local filesystem path without changing the CLI. + +## Consumer Inputs + +The manifest location is resolved in this order: + +1. `idx ownership sync --manifest ` +2. `IDX_OWNERSHIP_SNAPSHOT_MANIFEST` +3. `ownership.snapshot_manifest` in `config.toml` + +The value can be either: +- a local path to a manifest JSON file +- a remote `http://` or `https://` URL + +## Manifest Contract + +Current schema version: `1` + +```json +{ + "schema_version": 1, + "generated_at": "2026-03-31T12:00:00Z", + "snapshot": { + "kind": "sqlite", + "compression": "none", + "version": "2026-02-27", + "download_url": "https://example.com/ownership-snapshot-2026-02-27.sqlite", + "sqlite_sha256": "<64-char hex sha256>", + "size_bytes": 123456, + "release_count": 2, + "latest_as_of_date": "2026-02-27", + "latest_release_sha256": "", + "latest_row_count": 7261, + "ticker_count": 955 + } +} +``` + +Semantics: +- `download_url` points to the SQLite artifact itself. +- `sqlite_sha256` and `size_bytes` are validated before install. +- `latest_*` and `release_count` are validated against the downloaded SQLite contents before replacement. + +## Replacement Rules + +Without `--force`, sync behaves conservatively: + +- no local DB: install snapshot +- empty local DB: replace with snapshot +- local latest release older than snapshot: replace with snapshot +- local latest release/date/count matches snapshot: no-op +- local latest release matches but has fewer releases than snapshot: replace to fill missing history +- local DB newer than snapshot: no-op and keep local data +- local DB diverges at the same latest date: no-op and require `--force` + +Replacement is staged through a temp file and validated before the local DB is swapped in. + +## Publisher Workflow + +Use `scripts/build-ownership-snapshot.sh` to copy a vetted ownership DB into a publishable artifact plus manifest: + +```bash +scripts/build-ownership-snapshot.sh \ + --db /path/to/ownership.db \ + --output-dir dist/ownership-snapshot \ + --base-url https://example.com/idx/ownership +``` + +If `--base-url` is omitted, the generated manifest uses the local artifact path as `download_url`, which is useful for local testing. diff --git a/scripts/build-ownership-snapshot.sh b/scripts/build-ownership-snapshot.sh new file mode 100755 index 0000000..9c1fabb --- /dev/null +++ b/scripts/build-ownership-snapshot.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: scripts/build-ownership-snapshot.sh --db --output-dir [--base-url ] + +Create a publishable ownership SQLite snapshot plus a manifest JSON file. + +Options: + --db Source ownership SQLite database + --output-dir Directory to write the copied snapshot and manifest + --base-url Optional URL prefix to use for snapshot.download_url + --help Show this help +EOF +} + +sha256_file() { + local path="$1" + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$path" | awk '{print $1}' + return 0 + fi + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$path" | awk '{print $1}' + return 0 + fi + + echo "sha256sum/shasum not found in PATH" >&2 + exit 1 +} + +DB_PATH="" +OUTPUT_DIR="" +BASE_URL="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --db) + DB_PATH="${2:-}" + shift 2 + ;; + --output-dir) + OUTPUT_DIR="${2:-}" + shift 2 + ;; + --base-url) + BASE_URL="${2:-}" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "$DB_PATH" || -z "$OUTPUT_DIR" ]]; then + echo "--db and --output-dir are required" >&2 + usage >&2 + exit 2 +fi + +if [[ ! -f "$DB_PATH" ]]; then + echo "ownership database not found: $DB_PATH" >&2 + exit 1 +fi + +if ! command -v sqlite3 >/dev/null 2>&1; then + echo "sqlite3 is required to build ownership snapshots" >&2 + exit 1 +fi + +mkdir -p "$OUTPUT_DIR" + +LATEST_AS_OF="$(sqlite3 "$DB_PATH" "SELECT as_of_date FROM ownership_releases ORDER BY as_of_date DESC, imported_at DESC LIMIT 1;")" +LATEST_RELEASE_SHA="$(sqlite3 "$DB_PATH" "SELECT sha256 FROM ownership_releases ORDER BY as_of_date DESC, imported_at DESC LIMIT 1;")" +LATEST_ROW_COUNT="$(sqlite3 "$DB_PATH" "SELECT row_count FROM ownership_releases ORDER BY as_of_date DESC, imported_at DESC LIMIT 1;")" +RELEASE_COUNT="$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM ownership_releases;")" +TICKER_COUNT="$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM tickers;")" + +if [[ -z "$LATEST_AS_OF" || -z "$LATEST_RELEASE_SHA" ]]; then + echo "ownership database has no imported releases: $DB_PATH" >&2 + exit 1 +fi + +ARTIFACT_NAME="ownership-snapshot-${LATEST_AS_OF}.sqlite" +ARTIFACT_PATH="$OUTPUT_DIR/$ARTIFACT_NAME" +MANIFEST_PATH="$OUTPUT_DIR/ownership-snapshot-manifest.json" + +cp "$DB_PATH" "$ARTIFACT_PATH" + +SQLITE_SHA256="$(sha256_file "$ARTIFACT_PATH")" +SIZE_BYTES="$(wc -c < "$ARTIFACT_PATH" | tr -d ' ')" +GENERATED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + +if [[ -n "$BASE_URL" ]]; then + DOWNLOAD_URL="${BASE_URL%/}/$ARTIFACT_NAME" +else + DOWNLOAD_URL="$ARTIFACT_PATH" +fi + +cat > "$MANIFEST_PATH" <, + /// Replace the local DB even when already current or newer. + #[arg(long)] + pub force: bool, +} + #[derive(Debug, Args)] pub struct TickerArgs { pub symbol: String, @@ -173,6 +185,7 @@ pub fn handle(cmd: &OwnershipCommand, config: &IdxConfig) -> Result<(), IdxError match cmd { OwnershipCommand::Discover(args) => handle_discover(args, config), OwnershipCommand::Import(args) => handle_import(args, config), + OwnershipCommand::Sync(args) => handle_sync(args, config), OwnershipCommand::Ticker(args) => handle_ticker(args, config), OwnershipCommand::Entity(args) => handle_entity(args, config), OwnershipCommand::Search(args) => handle_search(args, config), @@ -186,6 +199,51 @@ pub fn handle(cmd: &OwnershipCommand, config: &IdxConfig) -> Result<(), IdxError } } +fn handle_sync(args: &SyncArgs, config: &IdxConfig) -> Result<(), IdxError> { + let manifest_source = snapshot::resolve_manifest_source(args.manifest.as_deref())?; + let db_path = db::db_path(config)?; + let result = snapshot::sync_snapshot(&manifest_source, &db_path, args.force)?; + + if matches!(config.output, OutputFormat::Json) { + return json::print_json(&result); + } + + match result.action { + snapshot::OwnershipSyncAction::Installed => { + println!( + "Installed ownership snapshot {} ({} release(s), {} tickers) into {}.", + result.latest_as_of_date.format("%Y-%m-%d"), + result.release_count, + result.ticker_count, + result.db_path + ); + } + snapshot::OwnershipSyncAction::Updated => { + println!( + "Updated ownership snapshot to {} ({} release(s), {} tickers) in {}.", + result.latest_as_of_date.format("%Y-%m-%d"), + result.release_count, + result.ticker_count, + result.db_path + ); + } + snapshot::OwnershipSyncAction::Refreshed => { + println!( + "Refreshed ownership snapshot {} in {}.", + result.latest_as_of_date.format("%Y-%m-%d"), + result.db_path + ); + } + snapshot::OwnershipSyncAction::NoChange + | snapshot::OwnershipSyncAction::SkippedNewer + | snapshot::OwnershipSyncAction::SkippedDiverged => { + println!("{}", result.reason); + } + } + + Ok(()) +} + fn handle_discover(args: &DiscoverArgs, config: &IdxConfig) -> Result<(), IdxError> { if args.limit == 0 { return Err(IdxError::ParseError( @@ -855,6 +913,7 @@ fn resolve_pdf_input(args: &ImportArgs) -> Result, IdxE path.display() ))); } + return Ok(Some(ResolvedPdfInput { pdf_path: path.clone(), source_url: None, diff --git a/src/config.rs b/src/config.rs index b022255..2274ddd 100644 --- a/src/config.rs +++ b/src/config.rs @@ -258,6 +258,7 @@ const KNOWN_CONFIG_KEYS: &[&str] = &[ "cache.quote_ttl", "cache.fundamental_ttl", "ownership.db_path", + "ownership.snapshot_manifest", ]; /// Validates a config key and value before writing @@ -302,6 +303,10 @@ fn normalize_config_value(key: &str, value: &str) -> Result { + // Snapshot manifest can be an absolute/relative path or a URL. + Ok(toml::Value::String(value.to_string())) + } "general.color" => { if !value.eq_ignore_ascii_case("true") && !value.eq_ignore_ascii_case("false") { return Err(IdxError::InvalidInput(format!( @@ -556,6 +561,24 @@ mod tests { assert!(super::validate_config_key_value("ownership.db_path", "data/ownership.db").is_ok()); } + #[test] + fn validate_accepts_ownership_snapshot_manifest() { + assert!( + super::validate_config_key_value( + "ownership.snapshot_manifest", + "https://example.com/latest.json" + ) + .is_ok() + ); + assert!( + super::validate_config_key_value( + "ownership.snapshot_manifest", + "snapshots/latest.json" + ) + .is_ok() + ); + } + #[test] fn validate_accepts_valid_color() { assert!(super::validate_config_key_value("general.color", "true").is_ok()); diff --git a/src/ownership/db.rs b/src/ownership/db.rs index ca899aa..19a7035 100644 --- a/src/ownership/db.rs +++ b/src/ownership/db.rs @@ -128,8 +128,8 @@ pub fn ensure_schema(conn: &Connection) -> Result<(), IdxError> { } /// Open ownership database connection and run idempotent schema migration. -pub fn open_db(_config: &IdxConfig) -> Result { - let db_path = resolve_db_path()?; +pub fn open_db(config: &IdxConfig) -> Result { + let db_path = db_path(config)?; if let Some(parent) = db_path.parent() { fs::create_dir_all(parent).map_err(|e| IdxError::DatabaseError(e.to_string()))?; @@ -1088,7 +1088,7 @@ pub fn compute_concentration(percentages_bps: &[i64]) -> ConcentrationMetrics { } } -fn resolve_db_path() -> Result { +pub fn db_path(_config: &IdxConfig) -> Result { if let Some(custom_path) = get_config_value("ownership.db_path")? { let trimmed = custom_path.trim(); if !trimmed.is_empty() { diff --git a/src/ownership/mod.rs b/src/ownership/mod.rs index 07b2902..feaf860 100644 --- a/src/ownership/mod.rs +++ b/src/ownership/mod.rs @@ -4,4 +4,5 @@ pub mod graph; pub mod parser; pub mod remote; pub mod search; +pub mod snapshot; pub mod types; diff --git a/src/ownership/parser.rs b/src/ownership/parser.rs index aa819ed..223aa44 100644 --- a/src/ownership/parser.rs +++ b/src/ownership/parser.rs @@ -37,11 +37,8 @@ const HOLDER_REGISTER_SCHEMA_MARKERS: &[&str] = &[ ]; const ANNOUNCEMENT_WRAPPER_SCHEMA_MARKERS: &[&str] = &["TEXT=\"PENGUMUMAN\"", "PT BURSA EFEK INDONESIA (BEI)"]; -const ABOVE_FIVE_SCHEMA_MARKERS: &[&str] = &[ - "TEXT=\"INVS\"", - "REKENING TAMPUNGAN KSEI", - "CLOSED MEMBER-", -]; +const ABOVE_FIVE_SCHEMA_MARKERS: &[&str] = + &["TEXT=\"INVS\"", "REKENING TAMPUNGAN KSEI", "CLOSED MEMBER-"]; const INVESTOR_TYPE_SCHEMA_MARKERS: &[&str] = &[ "TEXT=\"STOCK_CODE\"", "TEXT=\"NUMBER_OF_SHARES\"", diff --git a/src/ownership/snapshot.rs b/src/ownership/snapshot.rs new file mode 100644 index 0000000..daac151 --- /dev/null +++ b/src/ownership/snapshot.rs @@ -0,0 +1,762 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use chrono::NaiveDate; +use rusqlite::{Connection, OpenFlags}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::config::get_config_value; +use crate::error::IdxError; +use crate::ownership::db; +use crate::ownership::types::OwnershipRelease; + +pub const SNAPSHOT_MANIFEST_CONFIG_KEY: &str = "ownership.snapshot_manifest"; +pub const SNAPSHOT_MANIFEST_ENV: &str = "IDX_OWNERSHIP_SNAPSHOT_MANIFEST"; +pub const SNAPSHOT_MANIFEST_SCHEMA_VERSION: u32 = 1; + +const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct OwnershipSnapshotManifest { + pub schema_version: u32, + pub generated_at: String, + pub snapshot: OwnershipSnapshotArtifact, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct OwnershipSnapshotArtifact { + pub kind: String, + pub compression: String, + pub version: String, + pub download_url: String, + pub sqlite_sha256: String, + pub size_bytes: u64, + pub release_count: usize, + pub latest_as_of_date: NaiveDate, + pub latest_release_sha256: String, + pub latest_row_count: usize, + pub ticker_count: usize, +} + +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum OwnershipSyncAction { + Installed, + Updated, + Refreshed, + NoChange, + SkippedNewer, + SkippedDiverged, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct OwnershipSyncResult { + pub action: OwnershipSyncAction, + pub manifest: String, + pub db_path: String, + pub snapshot_version: String, + pub latest_as_of_date: NaiveDate, + pub release_count: usize, + pub ticker_count: usize, + pub reason: String, +} + +#[derive(Debug, Clone)] +struct LocalSnapshotState { + latest_release: Option, + release_count: usize, +} + +#[derive(Debug, Clone)] +struct SyncDecision { + action: OwnershipSyncAction, + should_download: bool, + reason: String, +} + +pub fn resolve_manifest_source(explicit: Option<&str>) -> Result { + if let Some(value) = explicit { + let trimmed = value.trim(); + if !trimmed.is_empty() { + return Ok(trimmed.to_string()); + } + } + + if let Ok(value) = std::env::var(SNAPSHOT_MANIFEST_ENV) { + let trimmed = value.trim(); + if !trimmed.is_empty() { + return Ok(trimmed.to_string()); + } + } + + if let Some(value) = get_config_value(SNAPSHOT_MANIFEST_CONFIG_KEY)? { + let trimmed = value.trim(); + if !trimmed.is_empty() { + return Ok(trimmed.to_string()); + } + } + + Err(IdxError::InvalidInput(format!( + "ownership sync needs a snapshot manifest; pass `--manifest` or set `{SNAPSHOT_MANIFEST_CONFIG_KEY}` / `{SNAPSHOT_MANIFEST_ENV}`" + ))) +} + +pub fn fetch_manifest(source: &str) -> Result { + let raw = read_text(source, "ownership snapshot manifest")?; + parse_manifest(&raw) +} + +pub fn parse_manifest(raw: &str) -> Result { + let manifest: OwnershipSnapshotManifest = serde_json::from_str(raw).map_err(|e| { + IdxError::ParseError(format!( + "failed to parse ownership snapshot manifest JSON: {e}" + )) + })?; + validate_manifest(&manifest)?; + Ok(manifest) +} + +pub fn sync_snapshot( + manifest_source: &str, + db_path: &Path, + force: bool, +) -> Result { + let manifest = fetch_manifest(manifest_source)?; + let local_state = inspect_local_db(db_path)?; + let decision = build_sync_decision(local_state.as_ref(), &manifest.snapshot, force); + + if !decision.should_download { + return Ok(OwnershipSyncResult { + action: decision.action, + manifest: manifest_source.to_string(), + db_path: db_path.display().to_string(), + snapshot_version: manifest.snapshot.version.clone(), + latest_as_of_date: manifest.snapshot.latest_as_of_date, + release_count: manifest.snapshot.release_count, + ticker_count: manifest.snapshot.ticker_count, + reason: decision.reason, + }); + } + + let bytes = read_bytes(&manifest.snapshot.download_url, "ownership snapshot SQLite")?; + validate_downloaded_bytes(&bytes, &manifest.snapshot)?; + + let temp_path = build_temp_path(db_path); + if let Some(parent) = temp_path.parent() + && !parent.as_os_str().is_empty() + { + fs::create_dir_all(parent).map_err(|e| { + IdxError::Io(format!( + "failed to create snapshot temp directory {}: {e}", + parent.display() + )) + })?; + } + fs::write(&temp_path, &bytes).map_err(|e| { + IdxError::Io(format!( + "failed to write downloaded snapshot {}: {e}", + temp_path.display() + )) + })?; + + let install_result = (|| -> Result<(), IdxError> { + validate_snapshot_db(&temp_path, &manifest.snapshot)?; + install_snapshot_file(&temp_path, db_path) + })(); + + if install_result.is_err() { + let _ = fs::remove_file(&temp_path); + } + install_result?; + + Ok(OwnershipSyncResult { + action: decision.action, + manifest: manifest_source.to_string(), + db_path: db_path.display().to_string(), + snapshot_version: manifest.snapshot.version.clone(), + latest_as_of_date: manifest.snapshot.latest_as_of_date, + release_count: manifest.snapshot.release_count, + ticker_count: manifest.snapshot.ticker_count, + reason: decision.reason, + }) +} + +fn validate_manifest(manifest: &OwnershipSnapshotManifest) -> Result<(), IdxError> { + if manifest.schema_version != SNAPSHOT_MANIFEST_SCHEMA_VERSION { + return Err(IdxError::ParseError(format!( + "unsupported ownership snapshot manifest schema_version {}; expected {}", + manifest.schema_version, SNAPSHOT_MANIFEST_SCHEMA_VERSION + ))); + } + + if manifest.generated_at.trim().is_empty() { + return Err(IdxError::ParseError( + "ownership snapshot manifest is missing generated_at".to_string(), + )); + } + + let snapshot = &manifest.snapshot; + if snapshot.kind.trim() != "sqlite" { + return Err(IdxError::Unsupported(format!( + "ownership snapshot kind `{}` is not supported; expected `sqlite`", + snapshot.kind + ))); + } + if snapshot.compression.trim() != "none" { + return Err(IdxError::Unsupported(format!( + "ownership snapshot compression `{}` is not supported; expected `none`", + snapshot.compression + ))); + } + if snapshot.version.trim().is_empty() { + return Err(IdxError::ParseError( + "ownership snapshot manifest is missing snapshot.version".to_string(), + )); + } + if snapshot.download_url.trim().is_empty() { + return Err(IdxError::ParseError( + "ownership snapshot manifest is missing snapshot.download_url".to_string(), + )); + } + if snapshot.size_bytes == 0 { + return Err(IdxError::ParseError( + "ownership snapshot manifest reported size_bytes=0".to_string(), + )); + } + if snapshot.release_count == 0 { + return Err(IdxError::ParseError( + "ownership snapshot manifest reported release_count=0".to_string(), + )); + } + if snapshot.latest_row_count == 0 { + return Err(IdxError::ParseError( + "ownership snapshot manifest reported latest_row_count=0".to_string(), + )); + } + if snapshot.ticker_count == 0 { + return Err(IdxError::ParseError( + "ownership snapshot manifest reported ticker_count=0".to_string(), + )); + } + + validate_sha256_hex(&snapshot.sqlite_sha256, "snapshot.sqlite_sha256")?; + validate_sha256_hex( + &snapshot.latest_release_sha256, + "snapshot.latest_release_sha256", + )?; + + Ok(()) +} + +fn validate_sha256_hex(value: &str, field_name: &str) -> Result<(), IdxError> { + let trimmed = value.trim(); + if trimmed.len() != 64 || !trimmed.chars().all(|ch| ch.is_ascii_hexdigit()) { + return Err(IdxError::ParseError(format!( + "ownership snapshot manifest field `{field_name}` must be a 64-character hex sha256" + ))); + } + Ok(()) +} + +fn inspect_local_db(db_path: &Path) -> Result, IdxError> { + if !db_path.exists() { + return Ok(None); + } + + let conn = Connection::open(db_path).map_err(|e| { + IdxError::DatabaseError(format!("failed to open {}: {e}", db_path.display())) + })?; + db::ensure_schema(&conn)?; + let releases = db::query_releases(&conn)?; + + Ok(Some(LocalSnapshotState { + latest_release: releases.first().cloned(), + release_count: releases.len(), + })) +} + +fn build_sync_decision( + local: Option<&LocalSnapshotState>, + snapshot: &OwnershipSnapshotArtifact, + force: bool, +) -> SyncDecision { + if force { + return SyncDecision { + action: if local.is_some() { + OwnershipSyncAction::Refreshed + } else { + OwnershipSyncAction::Installed + }, + should_download: true, + reason: "force refresh requested".to_string(), + }; + } + + let Some(local) = local else { + return SyncDecision { + action: OwnershipSyncAction::Installed, + should_download: true, + reason: "local ownership database does not exist yet".to_string(), + }; + }; + + let Some(latest) = &local.latest_release else { + return SyncDecision { + action: OwnershipSyncAction::Updated, + should_download: true, + reason: "local ownership database exists but has no imported releases".to_string(), + }; + }; + + if latest.as_of_date > snapshot.latest_as_of_date { + return SyncDecision { + action: OwnershipSyncAction::SkippedNewer, + should_download: false, + reason: format!( + "local ownership database is newer than the published snapshot (local {} > snapshot {})", + latest.as_of_date.format("%Y-%m-%d"), + snapshot.latest_as_of_date.format("%Y-%m-%d") + ), + }; + } + + if latest.as_of_date < snapshot.latest_as_of_date { + return SyncDecision { + action: OwnershipSyncAction::Updated, + should_download: true, + reason: format!( + "published snapshot advances local ownership data from {} to {}", + latest.as_of_date.format("%Y-%m-%d"), + snapshot.latest_as_of_date.format("%Y-%m-%d") + ), + }; + } + + if latest.sha256 == snapshot.latest_release_sha256 { + if latest.row_count != snapshot.latest_row_count { + return SyncDecision { + action: OwnershipSyncAction::Updated, + should_download: true, + reason: "local ownership database has the same latest release date but different row_count metadata".to_string(), + }; + } + + if local.release_count < snapshot.release_count { + return SyncDecision { + action: OwnershipSyncAction::Updated, + should_download: true, + reason: format!( + "local ownership database is missing historical releases (local {} < snapshot {})", + local.release_count, snapshot.release_count + ), + }; + } + + if local.release_count == snapshot.release_count { + return SyncDecision { + action: OwnershipSyncAction::NoChange, + should_download: false, + reason: format!( + "ownership snapshot already current at {} with {} release(s)", + snapshot.latest_as_of_date.format("%Y-%m-%d"), + snapshot.release_count + ), + }; + } + } + + SyncDecision { + action: OwnershipSyncAction::SkippedDiverged, + should_download: false, + reason: format!( + "local ownership database differs from the published snapshot for {}; use --force to replace it", + snapshot.latest_as_of_date.format("%Y-%m-%d") + ), + } +} + +fn validate_downloaded_bytes( + bytes: &[u8], + snapshot: &OwnershipSnapshotArtifact, +) -> Result<(), IdxError> { + let size_bytes = u64::try_from(bytes.len()).map_err(|e| { + IdxError::ParseError(format!("snapshot download too large to validate: {e}")) + })?; + if size_bytes != snapshot.size_bytes { + return Err(IdxError::ParseError(format!( + "ownership snapshot size mismatch: manifest expected {} bytes, downloaded {} bytes", + snapshot.size_bytes, size_bytes + ))); + } + + let actual_sha256 = sha256_hex(bytes); + if actual_sha256 != snapshot.sqlite_sha256 { + return Err(IdxError::ParseError(format!( + "ownership snapshot checksum mismatch: manifest expected {}, downloaded {}", + snapshot.sqlite_sha256, actual_sha256 + ))); + } + + Ok(()) +} + +fn validate_snapshot_db(path: &Path, snapshot: &OwnershipSnapshotArtifact) -> Result<(), IdxError> { + let conn = + Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY).map_err(|e| { + IdxError::DatabaseError(format!( + "failed to open downloaded snapshot {}: {e}", + path.display() + )) + })?; + + let quick_check: String = conn + .query_row("PRAGMA quick_check(1)", [], |row| row.get(0)) + .map_err(|e| IdxError::DatabaseError(format!("snapshot quick_check failed: {e}")))?; + if quick_check.trim() != "ok" { + return Err(IdxError::DatabaseError(format!( + "snapshot quick_check failed: {quick_check}" + ))); + } + + let releases = db::query_releases(&conn)?; + if releases.len() != snapshot.release_count { + return Err(IdxError::ParseError(format!( + "downloaded snapshot release_count mismatch: manifest expected {}, sqlite has {}", + snapshot.release_count, + releases.len() + ))); + } + + let latest = releases.first().ok_or_else(|| { + IdxError::ParseError("downloaded snapshot sqlite had no ownership releases".to_string()) + })?; + + if latest.as_of_date != snapshot.latest_as_of_date { + return Err(IdxError::ParseError(format!( + "downloaded snapshot latest_as_of_date mismatch: manifest expected {}, sqlite has {}", + snapshot.latest_as_of_date.format("%Y-%m-%d"), + latest.as_of_date.format("%Y-%m-%d") + ))); + } + if latest.sha256 != snapshot.latest_release_sha256 { + return Err(IdxError::ParseError(format!( + "downloaded snapshot latest_release_sha256 mismatch: manifest expected {}, sqlite has {}", + snapshot.latest_release_sha256, latest.sha256 + ))); + } + if latest.row_count != snapshot.latest_row_count { + return Err(IdxError::ParseError(format!( + "downloaded snapshot latest_row_count mismatch: manifest expected {}, sqlite has {}", + snapshot.latest_row_count, latest.row_count + ))); + } + + let ticker_count: i64 = conn + .query_row("SELECT COUNT(*) FROM tickers", [], |row| row.get(0)) + .map_err(|e| IdxError::DatabaseError(format!("failed counting snapshot tickers: {e}")))?; + let ticker_count = usize::try_from(ticker_count) + .map_err(|e| IdxError::DatabaseError(format!("invalid snapshot ticker_count: {e}")))?; + if ticker_count != snapshot.ticker_count { + return Err(IdxError::ParseError(format!( + "downloaded snapshot ticker_count mismatch: manifest expected {}, sqlite has {}", + snapshot.ticker_count, ticker_count + ))); + } + + Ok(()) +} + +fn build_temp_path(db_path: &Path) -> PathBuf { + let file_name = db_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("ownership.db"); + let temp_name = format!(".{file_name}.sync-{}.tmp", fastrand::u64(..)); + match db_path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent.join(temp_name), + _ => PathBuf::from(temp_name), + } +} + +fn install_snapshot_file(temp_path: &Path, db_path: &Path) -> Result<(), IdxError> { + if let Some(parent) = db_path.parent() + && !parent.as_os_str().is_empty() + { + fs::create_dir_all(parent).map_err(|e| { + IdxError::Io(format!( + "failed to create ownership database directory {}: {e}", + parent.display() + )) + })?; + } + + let backup_path = build_backup_path(db_path); + if backup_path.exists() { + fs::remove_file(&backup_path).map_err(|e| { + IdxError::Io(format!( + "failed to remove stale backup {}: {e}", + backup_path.display() + )) + })?; + } + + let had_existing = db_path.exists(); + remove_sqlite_sidecars(db_path)?; + + if had_existing { + fs::rename(db_path, &backup_path).map_err(|e| { + IdxError::Io(format!( + "failed to move existing ownership database {} to backup {}: {e}", + db_path.display(), + backup_path.display() + )) + })?; + } + + if let Err(err) = fs::rename(temp_path, db_path) { + if had_existing { + let _ = fs::rename(&backup_path, db_path); + } + return Err(IdxError::Io(format!( + "failed to install ownership snapshot into {}: {err}", + db_path.display() + ))); + } + + if had_existing && backup_path.exists() { + fs::remove_file(&backup_path).map_err(|e| { + IdxError::Io(format!( + "failed to remove snapshot backup {}: {e}", + backup_path.display() + )) + })?; + } + + remove_sqlite_sidecars(db_path)?; + Ok(()) +} + +fn build_backup_path(db_path: &Path) -> PathBuf { + let file_name = db_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("ownership.db"); + let backup_name = format!(".{file_name}.sync-backup"); + match db_path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent.join(backup_name), + _ => PathBuf::from(backup_name), + } +} + +fn remove_sqlite_sidecars(db_path: &Path) -> Result<(), IdxError> { + for suffix in ["-wal", "-shm"] { + let file_name = format!( + "{}{suffix}", + db_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("ownership.db") + ); + let sidecar = match db_path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent.join(&file_name), + _ => PathBuf::from(&file_name), + }; + + if sidecar.exists() { + fs::remove_file(&sidecar).map_err(|e| { + IdxError::Io(format!( + "failed to remove SQLite sidecar {}: {e}", + sidecar.display() + )) + })?; + } + } + + Ok(()) +} + +fn sha256_hex(bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + let digest = hasher.finalize(); + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn read_text(source: &str, context: &str) -> Result { + if is_http_source(source) { + let response = ureq::get(source) + .header("User-Agent", USER_AGENT) + .header("Accept", "application/json,text/plain;q=0.9,*/*;q=0.8") + .call() + .map_err(|e| IdxError::Http(format!("failed to fetch {context}: {e}")))?; + let mut body = response.into_body(); + return body + .read_to_string() + .map_err(|e| IdxError::Http(format!("failed reading {context} body: {e}"))); + } + + let path = local_source_path(source); + fs::read_to_string(&path) + .map_err(|e| IdxError::Io(format!("failed to read {context} {}: {e}", path.display()))) +} + +fn read_bytes(source: &str, context: &str) -> Result, IdxError> { + if is_http_source(source) { + let response = ureq::get(source) + .header("User-Agent", USER_AGENT) + .header( + "Accept", + "application/octet-stream,application/x-sqlite3,*/*;q=0.8", + ) + .call() + .map_err(|e| IdxError::Http(format!("failed to fetch {context}: {e}")))?; + let mut body = response.into_body(); + return body + .read_to_vec() + .map_err(|e| IdxError::Http(format!("failed reading {context} body: {e}"))); + } + + let path = local_source_path(source); + fs::read(&path) + .map_err(|e| IdxError::Io(format!("failed to read {context} {}: {e}", path.display()))) +} + +fn is_http_source(source: &str) -> bool { + let normalized = source.trim().to_ascii_lowercase(); + normalized.starts_with("http://") || normalized.starts_with("https://") +} + +fn local_source_path(source: &str) -> PathBuf { + if let Some(path) = source.trim().strip_prefix("file://") { + return PathBuf::from(path); + } + PathBuf::from(source.trim()) +} + +#[cfg(test)] +mod tests { + use chrono::NaiveDate; + use rusqlite::Connection; + + use super::{ + OwnershipSnapshotArtifact, OwnershipSnapshotManifest, OwnershipSyncAction, + SNAPSHOT_MANIFEST_SCHEMA_VERSION, build_sync_decision, parse_manifest, + }; + use crate::ownership::db::{ensure_schema, insert_release}; + use crate::ownership::types::OwnershipRelease; + + #[test] + fn parse_manifest_rejects_unknown_schema_version() { + let raw = r#"{ + "schema_version": 99, + "generated_at": "2026-03-31T12:00:00Z", + "snapshot": { + "kind": "sqlite", + "compression": "none", + "version": "2026-02-27", + "download_url": "/tmp/ownership.sqlite", + "sqlite_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "size_bytes": 123, + "release_count": 2, + "latest_as_of_date": "2026-02-27", + "latest_release_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "latest_row_count": 100, + "ticker_count": 5 + } + }"#; + + let err = parse_manifest(raw).expect_err("schema_version should fail"); + assert!( + err.to_string() + .contains("unsupported ownership snapshot manifest schema_version") + ); + } + + #[test] + fn build_sync_decision_updates_same_latest_release_when_history_is_incomplete() { + let snapshot = OwnershipSnapshotArtifact { + kind: "sqlite".to_string(), + compression: "none".to_string(), + version: "2026-02-27".to_string(), + download_url: "/tmp/ownership.sqlite".to_string(), + sqlite_sha256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + .to_string(), + size_bytes: 123, + release_count: 2, + latest_as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(), + latest_release_sha256: + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(), + latest_row_count: 7261, + ticker_count: 955, + }; + let local = super::LocalSnapshotState { + latest_release: Some(OwnershipRelease { + id: 1, + source_url: None, + sha256: snapshot.latest_release_sha256.clone(), + as_of_date: snapshot.latest_as_of_date, + row_count: snapshot.latest_row_count, + imported_at: 0, + }), + release_count: 1, + }; + + let decision = build_sync_decision(Some(&local), &snapshot, false); + assert_eq!(decision.action, OwnershipSyncAction::Updated); + assert!(decision.should_download); + assert!(decision.reason.contains("missing historical releases")); + } + + #[test] + fn manifest_round_trip_parses_valid_payload() { + let manifest = OwnershipSnapshotManifest { + schema_version: SNAPSHOT_MANIFEST_SCHEMA_VERSION, + generated_at: "2026-03-31T12:00:00Z".to_string(), + snapshot: OwnershipSnapshotArtifact { + kind: "sqlite".to_string(), + compression: "none".to_string(), + version: "2026-02-27".to_string(), + download_url: "/tmp/ownership.sqlite".to_string(), + sqlite_sha256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + .to_string(), + size_bytes: 123, + release_count: 1, + latest_as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(), + latest_release_sha256: + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(), + latest_row_count: 7261, + ticker_count: 955, + }, + }; + + let raw = serde_json::to_string(&manifest).unwrap(); + let parsed = parse_manifest(&raw).expect("valid manifest"); + assert_eq!(parsed, manifest); + } + + #[test] + fn ensure_schema_can_store_release_metadata_needed_for_snapshots() { + let conn = Connection::open_in_memory().unwrap(); + ensure_schema(&conn).unwrap(); + insert_release( + &conn, + &OwnershipRelease { + id: 0, + source_url: Some("https://example.com/ownership.sqlite".to_string()), + sha256: "abc".to_string(), + as_of_date: NaiveDate::from_ymd_opt(2026, 2, 27).unwrap(), + row_count: 1, + imported_at: 1, + }, + ) + .unwrap(); + + let release_count: i64 = conn + .query_row("SELECT COUNT(*) FROM ownership_releases", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(release_count, 1); + } +} diff --git a/tests/cli.rs b/tests/cli.rs index f6ac8f6..d61cad0 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -8,6 +8,8 @@ use std::thread; use assert_cmd::Command; use predicates::prelude::*; +use rusqlite::Connection; +use sha2::{Digest, Sha256}; fn bin() -> Command { let current = std::env::current_exe().expect("current test executable path"); @@ -101,6 +103,59 @@ exit 1\n" bin_dir } +fn install_fake_mutool_routes(root: &Path, routes: &[(&str, &str)]) -> PathBuf { + let bin_dir = root.join("fake-bin-routes"); + fs::create_dir_all(&bin_dir).expect("create fake route bin dir"); + let mutool_path = bin_dir.join("mutool"); + + let mut script = String::from( + "#!/bin/sh\n\ +if [ \"$1\" = \"--help\" ]; then\n\ + exit 0\n\ +fi\n\ +if [ \"$1\" != \"convert\" ]; then\n\ + echo \"unexpected mutool args: $@\" >&2\n\ + exit 1\n\ +fi\n\ +last=\"\"\n\ +for arg in \"$@\"; do\n\ + last=\"$arg\"\n\ +done\n\ +case \"$last\" in\n", + ); + + for (index, (suffix, xml)) in routes.iter().enumerate() { + script.push_str(&format!( + " *{suffix})\n\ + cat <<'__IDX_XML_{index}__'\n\ +{xml}\n\ +__IDX_XML_{index}__\n\ + exit 0\n\ + ;;\n" + )); + } + + script.push_str( + " *)\n\ + echo \"unexpected mutool target: $last\" >&2\n\ + exit 1\n\ + ;;\n\ +esac\n", + ); + + fs::write(&mutool_path, script).expect("write fake routed mutool"); + #[cfg(unix)] + { + let mut perms = fs::metadata(&mutool_path) + .expect("fake routed mutool metadata") + .permissions(); + perms.set_mode(0o755); + fs::set_permissions(&mutool_path, perms).expect("set fake routed mutool perms"); + } + + bin_dir +} + fn prepend_path(dir: &Path) -> String { match std::env::var("PATH") { Ok(current) if !current.is_empty() => format!("{}:{current}", dir.display()), @@ -112,6 +167,116 @@ fn pdf_url(base: &str, name: &str) -> String { format!("{base}/{name}.pdf") } +fn sha256_hex(bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + let digest = hasher.finalize(); + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn snapshot_metadata(db_path: &Path) -> (usize, String, String, usize, usize) { + let conn = Connection::open(db_path).expect("open snapshot sqlite"); + let release_count: i64 = conn + .query_row("SELECT COUNT(*) FROM ownership_releases", [], |row| { + row.get(0) + }) + .expect("snapshot release count"); + let (latest_as_of_date, latest_release_sha256, latest_row_count): (String, String, i64) = conn + .query_row( + "SELECT as_of_date, sha256, row_count + FROM ownership_releases + ORDER BY as_of_date DESC, imported_at DESC + LIMIT 1", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .expect("snapshot latest release metadata"); + let ticker_count: i64 = conn + .query_row("SELECT COUNT(*) FROM tickers", [], |row| row.get(0)) + .expect("snapshot ticker count"); + + ( + usize::try_from(release_count).expect("release_count fits usize"), + latest_as_of_date, + latest_release_sha256, + usize::try_from(latest_row_count).expect("row_count fits usize"), + usize::try_from(ticker_count).expect("ticker_count fits usize"), + ) +} + +fn write_snapshot_manifest(path: &Path, db_path: &Path, checksum_override: Option<&str>) { + let bytes = fs::read(db_path).expect("read snapshot sqlite"); + let (release_count, latest_as_of_date, latest_release_sha256, latest_row_count, ticker_count) = + snapshot_metadata(db_path); + let manifest = serde_json::json!({ + "schema_version": 1, + "generated_at": "2026-03-31T12:00:00Z", + "snapshot": { + "kind": "sqlite", + "compression": "none", + "version": latest_as_of_date, + "download_url": db_path.to_str().expect("snapshot db path"), + "sqlite_sha256": checksum_override.unwrap_or(&sha256_hex(&bytes)), + "size_bytes": bytes.len(), + "release_count": release_count, + "latest_as_of_date": latest_as_of_date, + "latest_release_sha256": latest_release_sha256, + "latest_row_count": latest_row_count, + "ticker_count": ticker_count + } + }); + + fs::write( + path, + serde_json::to_string_pretty(&manifest).expect("serialize snapshot manifest"), + ) + .expect("write snapshot manifest"); +} + +fn prepare_snapshot_fixture(root: &Path) -> (PathBuf, PathBuf) { + let source_db = root.join("published-ownership.db"); + let manifest_path = root.join("ownership-snapshot-manifest.json"); + let first_pdf = root.join("release-2026-01-31.pdf"); + let second_pdf = root.join("release-2026-02-27.pdf"); + fs::write(&first_pdf, b"%PDF-1.7\n% snapshot fixture jan\n").expect("write jan pdf"); + fs::write(&second_pdf, b"%PDF-1.7\n% snapshot fixture feb\n").expect("write feb pdf"); + + let fake_mutool_dir = install_fake_mutool_routes( + root, + &[ + ( + "release-2026-01-31.pdf", + include_str!("fixtures/ksei_above1_stext_excerpt_prev.xml"), + ), + ( + "release-2026-02-27.pdf", + include_str!("fixtures/ksei_above1_stext_excerpt.xml"), + ), + ], + ); + + bin_with_root(root) + .args([ + "config", + "set", + "ownership.db_path", + source_db.to_str().unwrap(), + ]) + .assert() + .success(); + + for file in [&first_pdf, &second_pdf] { + bin_with_root(root) + .env("PATH", prepend_path(&fake_mutool_dir)) + .args(["ownership", "import", "--file", file.to_str().unwrap()]) + .assert() + .success(); + } + + write_snapshot_manifest(&manifest_path, &source_db, None); + (source_db, manifest_path) +} + #[test] fn help_works() { test_bin("help").arg("--help").assert().success(); @@ -622,6 +787,27 @@ fn config_set_and_get_ownership_db_path_round_trip() { .stdout(predicate::str::contains("/tmp/ownership.db")); } +#[test] +fn config_set_and_get_ownership_snapshot_manifest_round_trip() { + let root = test_env_dir("config-ownership-snapshot-manifest"); + + bin_with_root(&root) + .args([ + "config", + "set", + "ownership.snapshot_manifest", + "/tmp/ownership-latest.json", + ]) + .assert() + .success(); + + bin_with_root(&root) + .args(["config", "get", "ownership.snapshot_manifest"]) + .assert() + .success() + .stdout(predicate::str::contains("/tmp/ownership-latest.json")); +} + #[test] fn config_set_mixed_case_provider_does_not_break_future_loads() { let root = test_env_dir("config-mixed-case-provider"); @@ -989,6 +1175,122 @@ fn ownership_import_url_rejects_legacy_investor_type_pdf_schema() { )); } +#[test] +fn ownership_sync_installs_snapshot_and_preserves_query_behavior() { + let publisher_root = test_env_dir("ownership-sync-publisher"); + let (_source_db, manifest_path) = prepare_snapshot_fixture(&publisher_root); + + let sync_root = test_env_dir("ownership-sync-consumer"); + let target_db = sync_root.join("ownership.db"); + + bin_with_root(&sync_root) + .args([ + "config", + "set", + "ownership.db_path", + target_db.to_str().unwrap(), + ]) + .assert() + .success(); + bin_with_root(&sync_root) + .args([ + "config", + "set", + "ownership.snapshot_manifest", + manifest_path.to_str().unwrap(), + ]) + .assert() + .success(); + + bin_with_root(&sync_root) + .args(["ownership", "sync"]) + .assert() + .success() + .stdout(predicate::str::contains( + "Installed ownership snapshot 2026-02-27", + )); + + assert!(target_db.exists()); + + bin_with_root(&sync_root) + .args(["ownership", "releases"]) + .assert() + .success() + .stdout(predicate::str::contains("2026-02-27")) + .stdout(predicate::str::contains("2026-01-31")); + + bin_with_root(&sync_root) + .args(["ownership", "ticker", "AADI", "--source", "ksei"]) + .assert() + .success() + .stdout(predicate::str::contains("ADARO STRATEGIC INVESTMENTS")) + .stdout(predicate::str::contains("41.10%")); + + bin_with_root(&sync_root) + .args([ + "ownership", + "changes", + "--from", + "2026-01-31", + "--to", + "2026-02-27", + ]) + .assert() + .success() + .stdout(predicate::str::contains("AADI")) + .stdout(predicate::str::contains("INCREASED")) + .stdout(predicate::str::contains("+1.28%")); + + bin_with_root(&sync_root) + .args(["ownership", "sync"]) + .assert() + .success() + .stdout(predicate::str::contains("already current")); + + bin_with_root(&sync_root) + .args(["ownership", "sync", "--force"]) + .assert() + .success() + .stdout(predicate::str::contains( + "Refreshed ownership snapshot 2026-02-27", + )); +} + +#[test] +fn ownership_sync_rejects_checksum_mismatch() { + let publisher_root = test_env_dir("ownership-sync-bad-checksum"); + let (source_db, manifest_path) = prepare_snapshot_fixture(&publisher_root); + write_snapshot_manifest( + &manifest_path, + &source_db, + Some("cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"), + ); + + let sync_root = test_env_dir("ownership-sync-bad-checksum-consumer"); + let target_db = sync_root.join("ownership.db"); + + bin_with_root(&sync_root) + .args([ + "config", + "set", + "ownership.db_path", + target_db.to_str().unwrap(), + ]) + .assert() + .success(); + + bin_with_root(&sync_root) + .args([ + "ownership", + "sync", + "--manifest", + manifest_path.to_str().unwrap(), + ]) + .assert() + .failure() + .stderr(predicate::str::contains("checksum mismatch")); +} + #[test] fn technical_serves_stale_cache_on_provider_failure_with_warning() { let root = test_env_dir("technical-stale"); diff --git a/tests/fixtures/ksei_above1_stext_excerpt_prev.xml b/tests/fixtures/ksei_above1_stext_excerpt_prev.xml new file mode 100644 index 0000000..462af2f --- /dev/null +++ b/tests/fixtures/ksei_above1_stext_excerpt_prev.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +