0xrsydn/idx-cli
1
mirror of https://github.com/0xrsydn/idx-cli.git synced 2026-11-07 01:35:36 +00:00
Code Issues Projects Releases 1 Packages Wiki Activity

fix(ownership): update syncs when a same-date snapshot is rebuilt

Browse source 
A snapshot republished for the same as-of date (the TRUE ticker fix)
kept the date, latest release, and release count, so sync saw nothing
to do and existing installs kept the bad data without --force.

The manifest gains an optional snapshot.replaces_sqlite_sha256 listing
earlier builds for that date. Queries never write to a synced file, so
a local database byte-identical to a listed build is replaced; one that
matches nothing (local imports or edits) still needs --force. The
publisher fills the list on a same-date rebuild. Older clients ignore
the field.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
hermes 2026-10-14 18:52:13 +00:00
commit ec17a1bd05
5 changed files with 191 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

145
crates/idx-core/src/ownership/snapshot.rs
View file

@ -58,6 +58,12 @@ pub struct OwnershipSnapshotArtifact {
pub latest_release_sha256: String,
pub latest_row_count: usize,
pub ticker_count: usize,
/// `sqlite_sha256` of earlier builds published for the same
/// `latest_as_of_date` that this snapshot replaces (e.g. after a parser
/// fix). A local database still byte-identical to one of them is updated
/// without `--force`.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub replaces_sqlite_sha256: Vec<String>,
}
#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
@ -85,6 +91,9 @@ pub struct OwnershipSyncResult {
#[derive(Debug, Clone)]
struct LocalSnapshotState {
/// SHA-256 of the database file. Queries leave a synced file
/// byte-identical, so this identifies an unmodified published snapshot.
file_sha256: String,
latest_release: Option<OwnershipRelease>,
release_count: usize,
}
@ -307,6 +316,9 @@ fn validate_manifest(manifest: &OwnershipSnapshotManifest) -> Result<(), IdxErro
&snapshot.latest_release_sha256,
"snapshot.latest_release_sha256",
)?;
for sha256 in &snapshot.replaces_sqlite_sha256 {
validate_sha256_hex(sha256, "snapshot.replaces_sqlite_sha256")?;
}
Ok(())
}
@ -326,6 +338,12 @@ fn inspect_local_db(db_path: &Path) -> Result<Option<LocalSnapshotState>, IdxErr
return Ok(None);
}
// Hash before opening: schema setup may write to an empty database.
let bytes = fs::read(db_path)
.map_err(|e| IdxError::Io(format!("failed to read {}: {e}", db_path.display())))?;
let file_sha256 = sha256_hex(&bytes);
drop(bytes);
let conn = Connection::open(db_path).map_err(|e| {
IdxError::DatabaseError(format!("failed to open {}: {e}", db_path.display()))
})?;
@ -333,6 +351,7 @@ fn inspect_local_db(db_path: &Path) -> Result<Option<LocalSnapshotState>, IdxErr
let releases = db::query_releases(&conn)?;
Ok(Some(LocalSnapshotState {
file_sha256,
latest_release: releases.first().cloned(),
release_count: releases.len(),
}))
@ -395,6 +414,33 @@ fn build_sync_decision(
};
}
if local.file_sha256 == snapshot.sqlite_sha256 {
return SyncDecision {
action: OwnershipSyncAction::NoChange,
should_download: false,
reason: format!(
"ownership snapshot already current at {} with {} release(s)",
snapshot.latest_as_of_date.format("%Y-%m-%d"),
snapshot.release_count
),
};
}
if snapshot
.replaces_sqlite_sha256
.iter()
.any(|sha256| sha256.eq_ignore_ascii_case(&local.file_sha256))
{
return SyncDecision {
action: OwnershipSyncAction::Updated,
should_download: true,
reason: format!(
"published snapshot for {} was rebuilt; replacing the earlier build",
snapshot.latest_as_of_date.format("%Y-%m-%d")
),
};
}
if latest.sha256 == snapshot.latest_release_sha256 {
if latest.row_count != snapshot.latest_row_count {
return SyncDecision {
@ -758,8 +804,11 @@ mod tests {
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(),
latest_row_count: 7261,
ticker_count: 955,
replaces_sqlite_sha256: Vec::new(),
};
let local = super::LocalSnapshotState {
file_sha256: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
.to_string(),
latest_release: Some(OwnershipRelease {
id: 1,
source_url: None,
@ -777,6 +826,99 @@ mod tests {
assert!(decision.reason.contains("missing historical releases"));
}
fn same_date_snapshot(replaces: Vec<String>) -> OwnershipSnapshotArtifact {
OwnershipSnapshotArtifact {
kind: "sqlite".to_string(),
compression: "none".to_string(),
version: "2026-09-30".to_string(),
download_url: "/tmp/ownership.sqlite".to_string(),
sqlite_sha256: "a".repeat(64),
size_bytes: 123,
release_count: 3,
latest_as_of_date: NaiveDate::from_ymd_opt(2026, 9, 30).unwrap(),
latest_release_sha256: "b".repeat(64),
latest_row_count: 7154,
ticker_count: 961,
replaces_sqlite_sha256: replaces,
}
}
/// A local database at the snapshot's date, release, and history.
fn same_release_local(
snapshot: &OwnershipSnapshotArtifact,
file_sha256: &str,
) -> super::LocalSnapshotState {
super::LocalSnapshotState {
file_sha256: file_sha256.to_string(),
latest_release: Some(OwnershipRelease {
id: 3,
source_url: None,
sha256: snapshot.latest_release_sha256.clone(),
as_of_date: snapshot.latest_as_of_date,
row_count: snapshot.latest_row_count,
imported_at: 0,
}),
release_count: snapshot.release_count,
}
}
#[test]
fn build_sync_decision_replaces_an_unmodified_earlier_build() {
let earlier = "e".repeat(64);
let snapshot = same_date_snapshot(vec![earlier.clone()]);
let decision = build_sync_decision(
Some(&same_release_local(&snapshot, &earlier)),
&snapshot,
false,
);
assert_eq!(decision.action, OwnershipSyncAction::Updated);
assert!(decision.should_download);
assert!(
decision.reason.contains("was rebuilt"),
"{}",
decision.reason
);
let current = same_release_local(&snapshot, &snapshot.sqlite_sha256);
let decision = build_sync_decision(Some(&current), &snapshot, false);
assert_eq!(decision.action, OwnershipSyncAction::NoChange);
assert!(!decision.should_download);
}
#[test]
fn build_sync_decision_keeps_a_locally_modified_database() {
// Same release metadata but a file matching no published build (local
// imports or entity edits): unchanged behavior, never replaced silently.
let snapshot = same_date_snapshot(vec!["e".repeat(64)]);
let local = same_release_local(&snapshot, &"f".repeat(64));
let decision = build_sync_decision(Some(&local), &snapshot, false);
assert_eq!(decision.action, OwnershipSyncAction::NoChange);
assert!(!decision.should_download);
}
#[test]
fn manifest_rejects_malformed_replaced_sha256() {
let mut manifest = OwnershipSnapshotManifest {
schema_version: SNAPSHOT_MANIFEST_SCHEMA_VERSION,
generated_at: "2026-10-14T00:00:00Z".to_string(),
source: None,
snapshot: same_date_snapshot(vec!["not-a-sha".to_string()]),
};
let err = parse_manifest(&serde_json::to_string(&manifest).unwrap())
.expect_err("malformed replaced sha256");
assert!(err.to_string().contains("replaces_sqlite_sha256"), "{err}");
manifest.snapshot.replaces_sqlite_sha256.clear();
let raw = serde_json::to_string(&manifest).unwrap();
assert!(
!raw.contains("replaces_sqlite_sha256"),
"omitted when empty: {raw}"
);
parse_manifest(&raw).expect("manifest without replaced builds");
}
#[test]
fn manifest_round_trip_parses_valid_payload() {
let manifest = OwnershipSnapshotManifest {
@ -805,6 +947,9 @@ mod tests {
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(),
latest_row_count: 7261,
ticker_count: 955,
replaces_sqlite_sha256: vec![
"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc".to_string(),
],
},
};

4
docs/OWNERSHIP_PUBLISH.md
View file

@ -79,7 +79,9 @@ The publisher treats the published snapshot as current only when the manifest's
`download_url` names this repo and tag and the referenced asset matches its
SHA-256 and size. A missing or corrupt asset, a mismatched URL, an unreadable
manifest, or any GitHub auth/network error means "publish", never "up to date".
`--force` uploads regardless.
`--force` uploads regardless. A forced rebuild for the already-published as-of
date adds the replaced build's `sqlite_sha256` to `replaces_sqlite_sha256`
(keeping the last 20), so clients still holding it update on their next sync.
## Run Result

9
docs/OWNERSHIP_SYNC.md
View file

@ -48,7 +48,8 @@ Current schema version: `1`
"latest_as_of_date": "2026-02-27",
"latest_release_sha256": "<latest ownership_releases.sha256>",
"latest_row_count": 7261,
"ticker_count": 955
"ticker_count": 955,
"replaces_sqlite_sha256": ["<sqlite_sha256 of an earlier build for this date>"]
}
}
```
@ -62,6 +63,9 @@ Semantics:
follow `download_url` and validate it with `sqlite_sha256` and `size_bytes`.
- `sqlite_sha256` and `size_bytes` are validated before install.
- `latest_*` and `release_count` are validated against the downloaded SQLite contents before replacement.
- `replaces_sqlite_sha256` (optional, omitted when empty) lists earlier builds
published for the same `latest_as_of_date` that this snapshot supersedes, for
example after a parser fix. Clients up to v0.2.x ignore it.
## Replacement Rules
@ -70,6 +74,9 @@ Without `--force`, sync behaves conservatively:
- no local DB: install snapshot
- empty local DB: replace with snapshot
- local latest release older than snapshot: replace with snapshot
- local DB file is byte-identical to `sqlite_sha256`: no-op
- local DB file is byte-identical to a build in `replaces_sqlite_sha256`: replace
(queries never write to a synced file, so this means it was not modified)
- local latest release/date/count matches snapshot: no-op
- local latest release matches but has fewer releases than snapshot: replace to fill missing history
- local DB newer than snapshot: no-op and keep local data

16
scripts/publish-ownership-snapshot-test.sh
View file

@ -396,6 +396,22 @@ if [[ "$old_asset" != "$new_asset" ]] && remote_asset_exists "$old_asset" && rem
else
fail "forced refresh keeps the previous content-addressed artifact (old=$old_asset new=$new_asset count=$(asset_count))"
fi
old_sha="${old_asset#ownership-snapshot-2026-08-31-}"
old_sha="${old_sha%.sqlite}"
if [[ "$(remote_manifest_field '.snapshot.replaces_sqlite_sha256 | join(",")')" == "$old_sha" ]]; then
pass "same-date rebuild lists the build it replaces"
else
fail "same-date rebuild lists the build it replaces (got $(remote_manifest_field '.snapshot.replaces_sqlite_sha256'))"
fi
mid_sha="$(remote_manifest_field '.snapshot.sqlite_sha256')"
export FAKE_BUILD_VARIANT="second-replacement"
run_publisher --force
unset FAKE_BUILD_VARIANT
if [[ "$(remote_manifest_field '.snapshot.replaces_sqlite_sha256 | join(",")')" == "$old_sha,$mid_sha" ]]; then
pass "a second rebuild carries earlier replaced builds forward"
else
fail "a second rebuild carries earlier replaced builds forward (got $(remote_manifest_field '.snapshot.replaces_sqlite_sha256'))"
fi
# ===========================================================================
# 5. a trustworthy-looking manifest with a missing asset forces a republish

19
scripts/publish-ownership-snapshot.sh
View file

@ -461,6 +461,25 @@ cp "$STAGED_MANIFEST_PATH" "$OUTPUT_DIR/ownership-snapshot-manifest.json"
cp "$STAGED_SQLITE_PATH" "$OUTPUT_DIR/"
MANIFEST_PATH="$OUTPUT_DIR/ownership-snapshot-manifest.json"
# A rebuild for the published as-of date (e.g. after a parser fix) lists the
# builds it replaces, so clients still holding one update without --force.
if [[ "$BUILT_AS_OF" == "$PUBLISHED_AS_OF" && "$PUBLISHED_SHA256" =~ ^[0-9a-fA-F]{64}$ ]]; then
replaces_tmp="$PUBLISH_WORKDIR/manifest-replaces.json"
jq --arg published "$PUBLISHED_SHA256" \
--argjson previous "$(jq -c '.snapshot.replaces_sqlite_sha256 // []' <<< "$PUBLISHED_MANIFEST" 2>/dev/null || echo '[]')" '
.snapshot.sqlite_sha256 as $own
| .snapshot.replaces_sqlite_sha256 =
([$previous[], $published]
| map(ascii_downcase) | map(select(. != ($own | ascii_downcase)))
| reduce .[] as $sha ([]; if index([$sha]) then . else . + [$sha] end)
| .[-20:])
| if (.snapshot.replaces_sqlite_sha256 | length) == 0
then del(.snapshot.replaces_sqlite_sha256) else . end
' "$MANIFEST_PATH" > "$replaces_tmp"
mv "$replaces_tmp" "$MANIFEST_PATH"
fi
SQLITE_PATH="$OUTPUT_DIR/$(basename "$STAGED_SQLITE_PATH")"
SQLITE_ASSET_NAME="$(basename "$STAGED_SQLITE_PATH")"
LOCAL_SQLITE_BYTES="$(wc -c < "$SQLITE_PATH" | tr -d ' ')"