mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-11-07 09:45:36 +00:00
fix(ownership): update syncs when a same-date snapshot is rebuilt
A snapshot republished for the same as-of date (the TRUE ticker fix) kept the date, latest release, and release count, so sync saw nothing to do and existing installs kept the bad data without --force. The manifest gains an optional snapshot.replaces_sqlite_sha256 listing earlier builds for that date. Queries never write to a synced file, so a local database byte-identical to a listed build is replaced; one that matches nothing (local imports or edits) still needs --force. The publisher fills the list on a same-date rebuild. Older clients ignore the field. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
cda12fe2db
commit
ec17a1bd05
5 changed files with 191 additions and 2 deletions
|
|
@ -58,6 +58,12 @@ pub struct OwnershipSnapshotArtifact {
|
||||||
pub latest_release_sha256: String,
|
pub latest_release_sha256: String,
|
||||||
pub latest_row_count: usize,
|
pub latest_row_count: usize,
|
||||||
pub ticker_count: usize,
|
pub ticker_count: usize,
|
||||||
|
/// `sqlite_sha256` of earlier builds published for the same
|
||||||
|
/// `latest_as_of_date` that this snapshot replaces (e.g. after a parser
|
||||||
|
/// fix). A local database still byte-identical to one of them is updated
|
||||||
|
/// without `--force`.
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub replaces_sqlite_sha256: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
|
||||||
|
|
@ -85,6 +91,9 @@ pub struct OwnershipSyncResult {
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
struct LocalSnapshotState {
|
struct LocalSnapshotState {
|
||||||
|
/// SHA-256 of the database file. Queries leave a synced file
|
||||||
|
/// byte-identical, so this identifies an unmodified published snapshot.
|
||||||
|
file_sha256: String,
|
||||||
latest_release: Option<OwnershipRelease>,
|
latest_release: Option<OwnershipRelease>,
|
||||||
release_count: usize,
|
release_count: usize,
|
||||||
}
|
}
|
||||||
|
|
@ -307,6 +316,9 @@ fn validate_manifest(manifest: &OwnershipSnapshotManifest) -> Result<(), IdxErro
|
||||||
&snapshot.latest_release_sha256,
|
&snapshot.latest_release_sha256,
|
||||||
"snapshot.latest_release_sha256",
|
"snapshot.latest_release_sha256",
|
||||||
)?;
|
)?;
|
||||||
|
for sha256 in &snapshot.replaces_sqlite_sha256 {
|
||||||
|
validate_sha256_hex(sha256, "snapshot.replaces_sqlite_sha256")?;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
@ -326,6 +338,12 @@ fn inspect_local_db(db_path: &Path) -> Result<Option<LocalSnapshotState>, IdxErr
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Hash before opening: schema setup may write to an empty database.
|
||||||
|
let bytes = fs::read(db_path)
|
||||||
|
.map_err(|e| IdxError::Io(format!("failed to read {}: {e}", db_path.display())))?;
|
||||||
|
let file_sha256 = sha256_hex(&bytes);
|
||||||
|
drop(bytes);
|
||||||
|
|
||||||
let conn = Connection::open(db_path).map_err(|e| {
|
let conn = Connection::open(db_path).map_err(|e| {
|
||||||
IdxError::DatabaseError(format!("failed to open {}: {e}", db_path.display()))
|
IdxError::DatabaseError(format!("failed to open {}: {e}", db_path.display()))
|
||||||
})?;
|
})?;
|
||||||
|
|
@ -333,6 +351,7 @@ fn inspect_local_db(db_path: &Path) -> Result<Option<LocalSnapshotState>, IdxErr
|
||||||
let releases = db::query_releases(&conn)?;
|
let releases = db::query_releases(&conn)?;
|
||||||
|
|
||||||
Ok(Some(LocalSnapshotState {
|
Ok(Some(LocalSnapshotState {
|
||||||
|
file_sha256,
|
||||||
latest_release: releases.first().cloned(),
|
latest_release: releases.first().cloned(),
|
||||||
release_count: releases.len(),
|
release_count: releases.len(),
|
||||||
}))
|
}))
|
||||||
|
|
@ -395,6 +414,33 @@ fn build_sync_decision(
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if local.file_sha256 == snapshot.sqlite_sha256 {
|
||||||
|
return SyncDecision {
|
||||||
|
action: OwnershipSyncAction::NoChange,
|
||||||
|
should_download: false,
|
||||||
|
reason: format!(
|
||||||
|
"ownership snapshot already current at {} with {} release(s)",
|
||||||
|
snapshot.latest_as_of_date.format("%Y-%m-%d"),
|
||||||
|
snapshot.release_count
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if snapshot
|
||||||
|
.replaces_sqlite_sha256
|
||||||
|
.iter()
|
||||||
|
.any(|sha256| sha256.eq_ignore_ascii_case(&local.file_sha256))
|
||||||
|
{
|
||||||
|
return SyncDecision {
|
||||||
|
action: OwnershipSyncAction::Updated,
|
||||||
|
should_download: true,
|
||||||
|
reason: format!(
|
||||||
|
"published snapshot for {} was rebuilt; replacing the earlier build",
|
||||||
|
snapshot.latest_as_of_date.format("%Y-%m-%d")
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if latest.sha256 == snapshot.latest_release_sha256 {
|
if latest.sha256 == snapshot.latest_release_sha256 {
|
||||||
if latest.row_count != snapshot.latest_row_count {
|
if latest.row_count != snapshot.latest_row_count {
|
||||||
return SyncDecision {
|
return SyncDecision {
|
||||||
|
|
@ -758,8 +804,11 @@ mod tests {
|
||||||
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(),
|
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(),
|
||||||
latest_row_count: 7261,
|
latest_row_count: 7261,
|
||||||
ticker_count: 955,
|
ticker_count: 955,
|
||||||
|
replaces_sqlite_sha256: Vec::new(),
|
||||||
};
|
};
|
||||||
let local = super::LocalSnapshotState {
|
let local = super::LocalSnapshotState {
|
||||||
|
file_sha256: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
|
||||||
|
.to_string(),
|
||||||
latest_release: Some(OwnershipRelease {
|
latest_release: Some(OwnershipRelease {
|
||||||
id: 1,
|
id: 1,
|
||||||
source_url: None,
|
source_url: None,
|
||||||
|
|
@ -777,6 +826,99 @@ mod tests {
|
||||||
assert!(decision.reason.contains("missing historical releases"));
|
assert!(decision.reason.contains("missing historical releases"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn same_date_snapshot(replaces: Vec<String>) -> OwnershipSnapshotArtifact {
|
||||||
|
OwnershipSnapshotArtifact {
|
||||||
|
kind: "sqlite".to_string(),
|
||||||
|
compression: "none".to_string(),
|
||||||
|
version: "2026-09-30".to_string(),
|
||||||
|
download_url: "/tmp/ownership.sqlite".to_string(),
|
||||||
|
sqlite_sha256: "a".repeat(64),
|
||||||
|
size_bytes: 123,
|
||||||
|
release_count: 3,
|
||||||
|
latest_as_of_date: NaiveDate::from_ymd_opt(2026, 9, 30).unwrap(),
|
||||||
|
latest_release_sha256: "b".repeat(64),
|
||||||
|
latest_row_count: 7154,
|
||||||
|
ticker_count: 961,
|
||||||
|
replaces_sqlite_sha256: replaces,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A local database at the snapshot's date, release, and history.
|
||||||
|
fn same_release_local(
|
||||||
|
snapshot: &OwnershipSnapshotArtifact,
|
||||||
|
file_sha256: &str,
|
||||||
|
) -> super::LocalSnapshotState {
|
||||||
|
super::LocalSnapshotState {
|
||||||
|
file_sha256: file_sha256.to_string(),
|
||||||
|
latest_release: Some(OwnershipRelease {
|
||||||
|
id: 3,
|
||||||
|
source_url: None,
|
||||||
|
sha256: snapshot.latest_release_sha256.clone(),
|
||||||
|
as_of_date: snapshot.latest_as_of_date,
|
||||||
|
row_count: snapshot.latest_row_count,
|
||||||
|
imported_at: 0,
|
||||||
|
}),
|
||||||
|
release_count: snapshot.release_count,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_sync_decision_replaces_an_unmodified_earlier_build() {
|
||||||
|
let earlier = "e".repeat(64);
|
||||||
|
let snapshot = same_date_snapshot(vec![earlier.clone()]);
|
||||||
|
|
||||||
|
let decision = build_sync_decision(
|
||||||
|
Some(&same_release_local(&snapshot, &earlier)),
|
||||||
|
&snapshot,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
assert_eq!(decision.action, OwnershipSyncAction::Updated);
|
||||||
|
assert!(decision.should_download);
|
||||||
|
assert!(
|
||||||
|
decision.reason.contains("was rebuilt"),
|
||||||
|
"{}",
|
||||||
|
decision.reason
|
||||||
|
);
|
||||||
|
|
||||||
|
let current = same_release_local(&snapshot, &snapshot.sqlite_sha256);
|
||||||
|
let decision = build_sync_decision(Some(¤t), &snapshot, false);
|
||||||
|
assert_eq!(decision.action, OwnershipSyncAction::NoChange);
|
||||||
|
assert!(!decision.should_download);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_sync_decision_keeps_a_locally_modified_database() {
|
||||||
|
// Same release metadata but a file matching no published build (local
|
||||||
|
// imports or entity edits): unchanged behavior, never replaced silently.
|
||||||
|
let snapshot = same_date_snapshot(vec!["e".repeat(64)]);
|
||||||
|
let local = same_release_local(&snapshot, &"f".repeat(64));
|
||||||
|
|
||||||
|
let decision = build_sync_decision(Some(&local), &snapshot, false);
|
||||||
|
assert_eq!(decision.action, OwnershipSyncAction::NoChange);
|
||||||
|
assert!(!decision.should_download);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn manifest_rejects_malformed_replaced_sha256() {
|
||||||
|
let mut manifest = OwnershipSnapshotManifest {
|
||||||
|
schema_version: SNAPSHOT_MANIFEST_SCHEMA_VERSION,
|
||||||
|
generated_at: "2026-10-14T00:00:00Z".to_string(),
|
||||||
|
source: None,
|
||||||
|
snapshot: same_date_snapshot(vec!["not-a-sha".to_string()]),
|
||||||
|
};
|
||||||
|
let err = parse_manifest(&serde_json::to_string(&manifest).unwrap())
|
||||||
|
.expect_err("malformed replaced sha256");
|
||||||
|
assert!(err.to_string().contains("replaces_sqlite_sha256"), "{err}");
|
||||||
|
|
||||||
|
manifest.snapshot.replaces_sqlite_sha256.clear();
|
||||||
|
let raw = serde_json::to_string(&manifest).unwrap();
|
||||||
|
assert!(
|
||||||
|
!raw.contains("replaces_sqlite_sha256"),
|
||||||
|
"omitted when empty: {raw}"
|
||||||
|
);
|
||||||
|
parse_manifest(&raw).expect("manifest without replaced builds");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn manifest_round_trip_parses_valid_payload() {
|
fn manifest_round_trip_parses_valid_payload() {
|
||||||
let manifest = OwnershipSnapshotManifest {
|
let manifest = OwnershipSnapshotManifest {
|
||||||
|
|
@ -805,6 +947,9 @@ mod tests {
|
||||||
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(),
|
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(),
|
||||||
latest_row_count: 7261,
|
latest_row_count: 7261,
|
||||||
ticker_count: 955,
|
ticker_count: 955,
|
||||||
|
replaces_sqlite_sha256: vec![
|
||||||
|
"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc".to_string(),
|
||||||
|
],
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -79,7 +79,9 @@ The publisher treats the published snapshot as current only when the manifest's
|
||||||
`download_url` names this repo and tag and the referenced asset matches its
|
`download_url` names this repo and tag and the referenced asset matches its
|
||||||
SHA-256 and size. A missing or corrupt asset, a mismatched URL, an unreadable
|
SHA-256 and size. A missing or corrupt asset, a mismatched URL, an unreadable
|
||||||
manifest, or any GitHub auth/network error means "publish", never "up to date".
|
manifest, or any GitHub auth/network error means "publish", never "up to date".
|
||||||
`--force` uploads regardless.
|
`--force` uploads regardless. A forced rebuild for the already-published as-of
|
||||||
|
date adds the replaced build's `sqlite_sha256` to `replaces_sqlite_sha256`
|
||||||
|
(keeping the last 20), so clients still holding it update on their next sync.
|
||||||
|
|
||||||
## Run Result
|
## Run Result
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -48,7 +48,8 @@ Current schema version: `1`
|
||||||
"latest_as_of_date": "2026-02-27",
|
"latest_as_of_date": "2026-02-27",
|
||||||
"latest_release_sha256": "<latest ownership_releases.sha256>",
|
"latest_release_sha256": "<latest ownership_releases.sha256>",
|
||||||
"latest_row_count": 7261,
|
"latest_row_count": 7261,
|
||||||
"ticker_count": 955
|
"ticker_count": 955,
|
||||||
|
"replaces_sqlite_sha256": ["<sqlite_sha256 of an earlier build for this date>"]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
@ -62,6 +63,9 @@ Semantics:
|
||||||
follow `download_url` and validate it with `sqlite_sha256` and `size_bytes`.
|
follow `download_url` and validate it with `sqlite_sha256` and `size_bytes`.
|
||||||
- `sqlite_sha256` and `size_bytes` are validated before install.
|
- `sqlite_sha256` and `size_bytes` are validated before install.
|
||||||
- `latest_*` and `release_count` are validated against the downloaded SQLite contents before replacement.
|
- `latest_*` and `release_count` are validated against the downloaded SQLite contents before replacement.
|
||||||
|
- `replaces_sqlite_sha256` (optional, omitted when empty) lists earlier builds
|
||||||
|
published for the same `latest_as_of_date` that this snapshot supersedes, for
|
||||||
|
example after a parser fix. Clients up to v0.2.x ignore it.
|
||||||
|
|
||||||
## Replacement Rules
|
## Replacement Rules
|
||||||
|
|
||||||
|
|
@ -70,6 +74,9 @@ Without `--force`, sync behaves conservatively:
|
||||||
- no local DB: install snapshot
|
- no local DB: install snapshot
|
||||||
- empty local DB: replace with snapshot
|
- empty local DB: replace with snapshot
|
||||||
- local latest release older than snapshot: replace with snapshot
|
- local latest release older than snapshot: replace with snapshot
|
||||||
|
- local DB file is byte-identical to `sqlite_sha256`: no-op
|
||||||
|
- local DB file is byte-identical to a build in `replaces_sqlite_sha256`: replace
|
||||||
|
(queries never write to a synced file, so this means it was not modified)
|
||||||
- local latest release/date/count matches snapshot: no-op
|
- local latest release/date/count matches snapshot: no-op
|
||||||
- local latest release matches but has fewer releases than snapshot: replace to fill missing history
|
- local latest release matches but has fewer releases than snapshot: replace to fill missing history
|
||||||
- local DB newer than snapshot: no-op and keep local data
|
- local DB newer than snapshot: no-op and keep local data
|
||||||
|
|
|
||||||
|
|
@ -396,6 +396,22 @@ if [[ "$old_asset" != "$new_asset" ]] && remote_asset_exists "$old_asset" && rem
|
||||||
else
|
else
|
||||||
fail "forced refresh keeps the previous content-addressed artifact (old=$old_asset new=$new_asset count=$(asset_count))"
|
fail "forced refresh keeps the previous content-addressed artifact (old=$old_asset new=$new_asset count=$(asset_count))"
|
||||||
fi
|
fi
|
||||||
|
old_sha="${old_asset#ownership-snapshot-2026-08-31-}"
|
||||||
|
old_sha="${old_sha%.sqlite}"
|
||||||
|
if [[ "$(remote_manifest_field '.snapshot.replaces_sqlite_sha256 | join(",")')" == "$old_sha" ]]; then
|
||||||
|
pass "same-date rebuild lists the build it replaces"
|
||||||
|
else
|
||||||
|
fail "same-date rebuild lists the build it replaces (got $(remote_manifest_field '.snapshot.replaces_sqlite_sha256'))"
|
||||||
|
fi
|
||||||
|
mid_sha="$(remote_manifest_field '.snapshot.sqlite_sha256')"
|
||||||
|
export FAKE_BUILD_VARIANT="second-replacement"
|
||||||
|
run_publisher --force
|
||||||
|
unset FAKE_BUILD_VARIANT
|
||||||
|
if [[ "$(remote_manifest_field '.snapshot.replaces_sqlite_sha256 | join(",")')" == "$old_sha,$mid_sha" ]]; then
|
||||||
|
pass "a second rebuild carries earlier replaced builds forward"
|
||||||
|
else
|
||||||
|
fail "a second rebuild carries earlier replaced builds forward (got $(remote_manifest_field '.snapshot.replaces_sqlite_sha256'))"
|
||||||
|
fi
|
||||||
|
|
||||||
# ===========================================================================
|
# ===========================================================================
|
||||||
# 5. a trustworthy-looking manifest with a missing asset forces a republish
|
# 5. a trustworthy-looking manifest with a missing asset forces a republish
|
||||||
|
|
|
||||||
|
|
@ -461,6 +461,25 @@ cp "$STAGED_MANIFEST_PATH" "$OUTPUT_DIR/ownership-snapshot-manifest.json"
|
||||||
cp "$STAGED_SQLITE_PATH" "$OUTPUT_DIR/"
|
cp "$STAGED_SQLITE_PATH" "$OUTPUT_DIR/"
|
||||||
|
|
||||||
MANIFEST_PATH="$OUTPUT_DIR/ownership-snapshot-manifest.json"
|
MANIFEST_PATH="$OUTPUT_DIR/ownership-snapshot-manifest.json"
|
||||||
|
|
||||||
|
# A rebuild for the published as-of date (e.g. after a parser fix) lists the
|
||||||
|
# builds it replaces, so clients still holding one update without --force.
|
||||||
|
if [[ "$BUILT_AS_OF" == "$PUBLISHED_AS_OF" && "$PUBLISHED_SHA256" =~ ^[0-9a-fA-F]{64}$ ]]; then
|
||||||
|
replaces_tmp="$PUBLISH_WORKDIR/manifest-replaces.json"
|
||||||
|
jq --arg published "$PUBLISHED_SHA256" \
|
||||||
|
--argjson previous "$(jq -c '.snapshot.replaces_sqlite_sha256 // []' <<< "$PUBLISHED_MANIFEST" 2>/dev/null || echo '[]')" '
|
||||||
|
.snapshot.sqlite_sha256 as $own
|
||||||
|
| .snapshot.replaces_sqlite_sha256 =
|
||||||
|
([$previous[], $published]
|
||||||
|
| map(ascii_downcase) | map(select(. != ($own | ascii_downcase)))
|
||||||
|
| reduce .[] as $sha ([]; if index([$sha]) then . else . + [$sha] end)
|
||||||
|
| .[-20:])
|
||||||
|
| if (.snapshot.replaces_sqlite_sha256 | length) == 0
|
||||||
|
then del(.snapshot.replaces_sqlite_sha256) else . end
|
||||||
|
' "$MANIFEST_PATH" > "$replaces_tmp"
|
||||||
|
mv "$replaces_tmp" "$MANIFEST_PATH"
|
||||||
|
fi
|
||||||
|
|
||||||
SQLITE_PATH="$OUTPUT_DIR/$(basename "$STAGED_SQLITE_PATH")"
|
SQLITE_PATH="$OUTPUT_DIR/$(basename "$STAGED_SQLITE_PATH")"
|
||||||
SQLITE_ASSET_NAME="$(basename "$STAGED_SQLITE_PATH")"
|
SQLITE_ASSET_NAME="$(basename "$STAGED_SQLITE_PATH")"
|
||||||
LOCAL_SQLITE_BYTES="$(wc -c < "$SQLITE_PATH" | tr -d ' ')"
|
LOCAL_SQLITE_BYTES="$(wc -c < "$SQLITE_PATH" | tr -d ' ')"
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue