diff --git a/crates/idx-core/src/ownership/snapshot.rs b/crates/idx-core/src/ownership/snapshot.rs index ad229ed..d23af4d 100644 --- a/crates/idx-core/src/ownership/snapshot.rs +++ b/crates/idx-core/src/ownership/snapshot.rs @@ -58,6 +58,12 @@ pub struct OwnershipSnapshotArtifact { pub latest_release_sha256: String, pub latest_row_count: usize, pub ticker_count: usize, + /// `sqlite_sha256` of earlier builds published for the same + /// `latest_as_of_date` that this snapshot replaces (e.g. after a parser + /// fix). A local database still byte-identical to one of them is updated + /// without `--force`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub replaces_sqlite_sha256: Vec, } #[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] @@ -85,6 +91,9 @@ pub struct OwnershipSyncResult { #[derive(Debug, Clone)] struct LocalSnapshotState { + /// SHA-256 of the database file. Queries leave a synced file + /// byte-identical, so this identifies an unmodified published snapshot. + file_sha256: String, latest_release: Option, release_count: usize, } @@ -307,6 +316,9 @@ fn validate_manifest(manifest: &OwnershipSnapshotManifest) -> Result<(), IdxErro &snapshot.latest_release_sha256, "snapshot.latest_release_sha256", )?; + for sha256 in &snapshot.replaces_sqlite_sha256 { + validate_sha256_hex(sha256, "snapshot.replaces_sqlite_sha256")?; + } Ok(()) } @@ -326,6 +338,12 @@ fn inspect_local_db(db_path: &Path) -> Result, IdxErr return Ok(None); } + // Hash before opening: schema setup may write to an empty database. + let bytes = fs::read(db_path) + .map_err(|e| IdxError::Io(format!("failed to read {}: {e}", db_path.display())))?; + let file_sha256 = sha256_hex(&bytes); + drop(bytes); + let conn = Connection::open(db_path).map_err(|e| { IdxError::DatabaseError(format!("failed to open {}: {e}", db_path.display())) })?; @@ -333,6 +351,7 @@ fn inspect_local_db(db_path: &Path) -> Result, IdxErr let releases = db::query_releases(&conn)?; Ok(Some(LocalSnapshotState { + file_sha256, latest_release: releases.first().cloned(), release_count: releases.len(), })) @@ -395,6 +414,33 @@ fn build_sync_decision( }; } + if local.file_sha256 == snapshot.sqlite_sha256 { + return SyncDecision { + action: OwnershipSyncAction::NoChange, + should_download: false, + reason: format!( + "ownership snapshot already current at {} with {} release(s)", + snapshot.latest_as_of_date.format("%Y-%m-%d"), + snapshot.release_count + ), + }; + } + + if snapshot + .replaces_sqlite_sha256 + .iter() + .any(|sha256| sha256.eq_ignore_ascii_case(&local.file_sha256)) + { + return SyncDecision { + action: OwnershipSyncAction::Updated, + should_download: true, + reason: format!( + "published snapshot for {} was rebuilt; replacing the earlier build", + snapshot.latest_as_of_date.format("%Y-%m-%d") + ), + }; + } + if latest.sha256 == snapshot.latest_release_sha256 { if latest.row_count != snapshot.latest_row_count { return SyncDecision { @@ -758,8 +804,11 @@ mod tests { "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(), latest_row_count: 7261, ticker_count: 955, + replaces_sqlite_sha256: Vec::new(), }; let local = super::LocalSnapshotState { + file_sha256: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + .to_string(), latest_release: Some(OwnershipRelease { id: 1, source_url: None, @@ -777,6 +826,99 @@ mod tests { assert!(decision.reason.contains("missing historical releases")); } + fn same_date_snapshot(replaces: Vec) -> OwnershipSnapshotArtifact { + OwnershipSnapshotArtifact { + kind: "sqlite".to_string(), + compression: "none".to_string(), + version: "2026-09-30".to_string(), + download_url: "/tmp/ownership.sqlite".to_string(), + sqlite_sha256: "a".repeat(64), + size_bytes: 123, + release_count: 3, + latest_as_of_date: NaiveDate::from_ymd_opt(2026, 9, 30).unwrap(), + latest_release_sha256: "b".repeat(64), + latest_row_count: 7154, + ticker_count: 961, + replaces_sqlite_sha256: replaces, + } + } + + /// A local database at the snapshot's date, release, and history. + fn same_release_local( + snapshot: &OwnershipSnapshotArtifact, + file_sha256: &str, + ) -> super::LocalSnapshotState { + super::LocalSnapshotState { + file_sha256: file_sha256.to_string(), + latest_release: Some(OwnershipRelease { + id: 3, + source_url: None, + sha256: snapshot.latest_release_sha256.clone(), + as_of_date: snapshot.latest_as_of_date, + row_count: snapshot.latest_row_count, + imported_at: 0, + }), + release_count: snapshot.release_count, + } + } + + #[test] + fn build_sync_decision_replaces_an_unmodified_earlier_build() { + let earlier = "e".repeat(64); + let snapshot = same_date_snapshot(vec![earlier.clone()]); + + let decision = build_sync_decision( + Some(&same_release_local(&snapshot, &earlier)), + &snapshot, + false, + ); + assert_eq!(decision.action, OwnershipSyncAction::Updated); + assert!(decision.should_download); + assert!( + decision.reason.contains("was rebuilt"), + "{}", + decision.reason + ); + + let current = same_release_local(&snapshot, &snapshot.sqlite_sha256); + let decision = build_sync_decision(Some(¤t), &snapshot, false); + assert_eq!(decision.action, OwnershipSyncAction::NoChange); + assert!(!decision.should_download); + } + + #[test] + fn build_sync_decision_keeps_a_locally_modified_database() { + // Same release metadata but a file matching no published build (local + // imports or entity edits): unchanged behavior, never replaced silently. + let snapshot = same_date_snapshot(vec!["e".repeat(64)]); + let local = same_release_local(&snapshot, &"f".repeat(64)); + + let decision = build_sync_decision(Some(&local), &snapshot, false); + assert_eq!(decision.action, OwnershipSyncAction::NoChange); + assert!(!decision.should_download); + } + + #[test] + fn manifest_rejects_malformed_replaced_sha256() { + let mut manifest = OwnershipSnapshotManifest { + schema_version: SNAPSHOT_MANIFEST_SCHEMA_VERSION, + generated_at: "2026-10-14T00:00:00Z".to_string(), + source: None, + snapshot: same_date_snapshot(vec!["not-a-sha".to_string()]), + }; + let err = parse_manifest(&serde_json::to_string(&manifest).unwrap()) + .expect_err("malformed replaced sha256"); + assert!(err.to_string().contains("replaces_sqlite_sha256"), "{err}"); + + manifest.snapshot.replaces_sqlite_sha256.clear(); + let raw = serde_json::to_string(&manifest).unwrap(); + assert!( + !raw.contains("replaces_sqlite_sha256"), + "omitted when empty: {raw}" + ); + parse_manifest(&raw).expect("manifest without replaced builds"); + } + #[test] fn manifest_round_trip_parses_valid_payload() { let manifest = OwnershipSnapshotManifest { @@ -805,6 +947,9 @@ mod tests { "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(), latest_row_count: 7261, ticker_count: 955, + replaces_sqlite_sha256: vec![ + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc".to_string(), + ], }, }; diff --git a/docs/OWNERSHIP_PUBLISH.md b/docs/OWNERSHIP_PUBLISH.md index 0b82831..81a8929 100644 --- a/docs/OWNERSHIP_PUBLISH.md +++ b/docs/OWNERSHIP_PUBLISH.md @@ -79,7 +79,9 @@ The publisher treats the published snapshot as current only when the manifest's `download_url` names this repo and tag and the referenced asset matches its SHA-256 and size. A missing or corrupt asset, a mismatched URL, an unreadable manifest, or any GitHub auth/network error means "publish", never "up to date". -`--force` uploads regardless. +`--force` uploads regardless. A forced rebuild for the already-published as-of +date adds the replaced build's `sqlite_sha256` to `replaces_sqlite_sha256` +(keeping the last 20), so clients still holding it update on their next sync. ## Run Result diff --git a/docs/OWNERSHIP_SYNC.md b/docs/OWNERSHIP_SYNC.md index bfdd045..e589ce4 100644 --- a/docs/OWNERSHIP_SYNC.md +++ b/docs/OWNERSHIP_SYNC.md @@ -48,7 +48,8 @@ Current schema version: `1` "latest_as_of_date": "2026-02-27", "latest_release_sha256": "", "latest_row_count": 7261, - "ticker_count": 955 + "ticker_count": 955, + "replaces_sqlite_sha256": [""] } } ``` @@ -62,6 +63,9 @@ Semantics: follow `download_url` and validate it with `sqlite_sha256` and `size_bytes`. - `sqlite_sha256` and `size_bytes` are validated before install. - `latest_*` and `release_count` are validated against the downloaded SQLite contents before replacement. +- `replaces_sqlite_sha256` (optional, omitted when empty) lists earlier builds + published for the same `latest_as_of_date` that this snapshot supersedes, for + example after a parser fix. Clients up to v0.2.x ignore it. ## Replacement Rules @@ -70,6 +74,9 @@ Without `--force`, sync behaves conservatively: - no local DB: install snapshot - empty local DB: replace with snapshot - local latest release older than snapshot: replace with snapshot +- local DB file is byte-identical to `sqlite_sha256`: no-op +- local DB file is byte-identical to a build in `replaces_sqlite_sha256`: replace + (queries never write to a synced file, so this means it was not modified) - local latest release/date/count matches snapshot: no-op - local latest release matches but has fewer releases than snapshot: replace to fill missing history - local DB newer than snapshot: no-op and keep local data diff --git a/scripts/publish-ownership-snapshot-test.sh b/scripts/publish-ownership-snapshot-test.sh index 3226c7e..796b564 100755 --- a/scripts/publish-ownership-snapshot-test.sh +++ b/scripts/publish-ownership-snapshot-test.sh @@ -396,6 +396,22 @@ if [[ "$old_asset" != "$new_asset" ]] && remote_asset_exists "$old_asset" && rem else fail "forced refresh keeps the previous content-addressed artifact (old=$old_asset new=$new_asset count=$(asset_count))" fi +old_sha="${old_asset#ownership-snapshot-2026-08-31-}" +old_sha="${old_sha%.sqlite}" +if [[ "$(remote_manifest_field '.snapshot.replaces_sqlite_sha256 | join(",")')" == "$old_sha" ]]; then + pass "same-date rebuild lists the build it replaces" +else + fail "same-date rebuild lists the build it replaces (got $(remote_manifest_field '.snapshot.replaces_sqlite_sha256'))" +fi +mid_sha="$(remote_manifest_field '.snapshot.sqlite_sha256')" +export FAKE_BUILD_VARIANT="second-replacement" +run_publisher --force +unset FAKE_BUILD_VARIANT +if [[ "$(remote_manifest_field '.snapshot.replaces_sqlite_sha256 | join(",")')" == "$old_sha,$mid_sha" ]]; then + pass "a second rebuild carries earlier replaced builds forward" +else + fail "a second rebuild carries earlier replaced builds forward (got $(remote_manifest_field '.snapshot.replaces_sqlite_sha256'))" +fi # =========================================================================== # 5. a trustworthy-looking manifest with a missing asset forces a republish diff --git a/scripts/publish-ownership-snapshot.sh b/scripts/publish-ownership-snapshot.sh index ebd2b7c..dd2c9c3 100755 --- a/scripts/publish-ownership-snapshot.sh +++ b/scripts/publish-ownership-snapshot.sh @@ -461,6 +461,25 @@ cp "$STAGED_MANIFEST_PATH" "$OUTPUT_DIR/ownership-snapshot-manifest.json" cp "$STAGED_SQLITE_PATH" "$OUTPUT_DIR/" MANIFEST_PATH="$OUTPUT_DIR/ownership-snapshot-manifest.json" + +# A rebuild for the published as-of date (e.g. after a parser fix) lists the +# builds it replaces, so clients still holding one update without --force. +if [[ "$BUILT_AS_OF" == "$PUBLISHED_AS_OF" && "$PUBLISHED_SHA256" =~ ^[0-9a-fA-F]{64}$ ]]; then + replaces_tmp="$PUBLISH_WORKDIR/manifest-replaces.json" + jq --arg published "$PUBLISHED_SHA256" \ + --argjson previous "$(jq -c '.snapshot.replaces_sqlite_sha256 // []' <<< "$PUBLISHED_MANIFEST" 2>/dev/null || echo '[]')" ' + .snapshot.sqlite_sha256 as $own + | .snapshot.replaces_sqlite_sha256 = + ([$previous[], $published] + | map(ascii_downcase) | map(select(. != ($own | ascii_downcase))) + | reduce .[] as $sha ([]; if index([$sha]) then . else . + [$sha] end) + | .[-20:]) + | if (.snapshot.replaces_sqlite_sha256 | length) == 0 + then del(.snapshot.replaces_sqlite_sha256) else . end + ' "$MANIFEST_PATH" > "$replaces_tmp" + mv "$replaces_tmp" "$MANIFEST_PATH" +fi + SQLITE_PATH="$OUTPUT_DIR/$(basename "$STAGED_SQLITE_PATH")" SQLITE_ASSET_NAME="$(basename "$STAGED_SQLITE_PATH")" LOCAL_SQLITE_BYTES="$(wc -c < "$SQLITE_PATH" | tr -d ' ')"