Merge pull request #31 from 0xrsydn/fix/snapshot-size-limit

fix(ownership): sync snapshots over 10 MiB; refuse oversized publishes

docs/OWNERSHIP_SELF_HOSTED.md

@@ -103,6 +103,26 @@ idempotent run the exact day no longer matters.
 Pass `--history <n>` to include earlier months so `idx ownership changes`
 works straight from the synced snapshot (about 2.4 MB per extra month).
 
+### Snapshot size limit
+
+idx clients up to v0.2.3 cap the snapshot download at **10 MiB** (10,485,760
+bytes) and only accept uncompressed SQLite. A larger snapshot makes
+`idx ownership sync` fail for every existing install. This happened on
+2026-09-25 with `--history 5` (15.3 MB).
+
+| `--history` | months | size (2026-08-31) |
+| --- | --- | --- |
+| 2 | 3 | 8,171,520 |
+| 3 | 4 | 10,510,336 (too large) |
+| 5 | 6 | 15,290,368 (too large) |
+
+The builder therefore refuses snapshots above `--max-snapshot-bytes`
+(default 10485760). The run ends `RESULT: FAILED stage=build-snapshot` and the
+published snapshot stays unchanged. Newer clients accept snapshots up to the
+manifest's `size_bytes` (at most 1 GiB, buffered in memory);
+raise `--max-snapshot-bytes` and `--history` only after clients without the
+10 MiB cap are widespread.
+
 ## Clan Integration Later
 
 The clean split for `clan-private` is:

scripts/build-latest-ownership-snapshot.sh

@@ -21,6 +21,10 @@ Options:
   --history <n>          Also import the <n> previous monthly above-1% XLSX
                          reports (oldest first) so `ownership changes` works
                          from the snapshot (default: 0)
+  --max-snapshot-bytes <n>
+                         Refuse to emit a snapshot larger than this
+                         (default: 10485760, the download limit of idx
+                         clients <= v0.2.3; exceeding it breaks their sync)
   --keep-workdir         Keep the temp workdir instead of deleting it
   --help                 Show this help
 EOF
@@ -33,6 +37,7 @@ REPO_FULL_NAME="0xrsydn/idx-cli"
 RELEASE_TAG="ownership-snapshot-current"
 KEEP_WORKDIR="0"
 HISTORY="0"
+MAX_SNAPSHOT_BYTES="10485760"
 
 while [[ $# -gt 0 ]]; do
     case "$1" in
@@ -60,6 +65,10 @@ while [[ $# -gt 0 ]]; do
             HISTORY="${2:-}"
             shift 2
             ;;
+        --max-snapshot-bytes)
+            MAX_SNAPSHOT_BYTES="${2:-}"
+            shift 2
+            ;;
         --keep-workdir)
             KEEP_WORKDIR="1"
             shift
@@ -82,6 +91,11 @@ if [[ -z "$OUTPUT_DIR" ]]; then
     exit 2
 fi
 
+if ! [[ "$MAX_SNAPSHOT_BYTES" =~ ^[1-9][0-9]{0,17}$ ]]; then
+    echo "--max-snapshot-bytes must be a positive integer" >&2
+    exit 2
+fi
+
 if ! [[ "$HISTORY" =~ ^[0-9]+$ ]]; then
     echo "--history must be a non-negative integer" >&2
     exit 2
@@ -254,6 +268,26 @@ IMPORTED_ROW_COUNT="${release_fields[2]:-}"
 printf 'Building snapshot artifact and manifest...\n'
 "$BASIC_BUILDER" --db "$DB_PATH" --output-dir "$OUTPUT_DIR" --base-url "$BASE_URL"
 
+# Released clients cannot download a snapshot above their HTTP body limit, so
+# publishing one would break `idx ownership sync` for every existing install.
+SNAPSHOT_BYTES="$(jq -r '.snapshot.size_bytes // empty' "$MANIFEST_PATH")"
+if ! [[ "$SNAPSHOT_BYTES" =~ ^[0-9]+$ ]]; then
+    echo "built manifest has no valid snapshot.size_bytes" >&2
+    exit 1
+fi
+if (( SNAPSHOT_BYTES > MAX_SNAPSHOT_BYTES )); then
+    printf 'snapshot is %s bytes (%s monthly releases), above --max-snapshot-bytes %s; lower --history\n' \
+        "$SNAPSHOT_BYTES" "$(jq -r '.snapshot.release_count' "$MANIFEST_PATH")" "$MAX_SNAPSHOT_BYTES" >&2
+    # Remove the refused pair so a standalone run leaves nothing publishable.
+    refused_sqlite="$(jq -r '.snapshot.download_url // empty' "$MANIFEST_PATH")"
+    refused_sqlite="${refused_sqlite##*/}"
+    if [[ -n "$refused_sqlite" && "$refused_sqlite" != */* ]]; then
+        rm -f "$OUTPUT_DIR/$refused_sqlite"
+    fi
+    rm -f "$MANIFEST_PATH"
+    exit 1
+fi
+
 TMP_MANIFEST_PATH="$WORKDIR/ownership-snapshot-manifest.json"
 jq \
     --arg family "$DISCOVERED_FAMILY" \

scripts/publish-ownership-snapshot-test.sh

@@ -165,12 +165,14 @@ cat >"$scripts_dir/build-latest-ownership-snapshot.sh" <<'STUB_BUILDER'
 set -euo pipefail
 out=""
 history="0"
+max_bytes="default"
 repo="example/repo"
 tag="ownership-snapshot-current"
 while [[ $# -gt 0 ]]; do
     case "$1" in
         --output-dir) out="$2"; shift 2 ;;
         --history) history="$2"; shift 2 ;;
+        --max-snapshot-bytes) max_bytes="$2"; shift 2 ;;
         --repo) repo="$2"; shift 2 ;;
         --release-tag) tag="$2"; shift 2 ;;
         --idx-bin) shift 2 ;;
@@ -179,7 +181,11 @@ while [[ $# -gt 0 ]]; do
     esac
 done
 [[ -n "$out" ]] || { echo "stub builder: missing --output-dir" >&2; exit 2; }
-printf 'history=%s\n' "$history" >>"${FAKE_BUILDER_CALLS:?}"
+printf 'history=%s max_bytes=%s\n' "$history" "$max_bytes" >>"${FAKE_BUILDER_CALLS:?}"
+if [[ "${FAKE_BUILDER_TOO_LARGE:-0}" == "1" ]]; then
+    echo "snapshot is 15290368 bytes (6 monthly releases), above --max-snapshot-bytes $max_bytes; lower --history" >&2
+    exit 1
+fi
 
 as_of="${FAKE_BUILD_AS_OF:-2026-08-31}"
 if [[ -n "${FAKE_BUILD_RELEASE_COUNT:-}" ]]; then
@@ -666,6 +672,28 @@ cp "$state/nested-manifest.json" "$remote_manifest"
 run_publisher
 publish_ok "nested download URL cannot authorize a no-op" "RESULT: published 2026-08-31"
 
+# ===========================================================================
+# 8. an oversized snapshot fails the build and leaves the release untouched
+# ===========================================================================
+reset_remote
+run_publisher
+publish_ok "baseline publish before the size check" "RESULT: published 2026-08-31"
+manifest_before="$(sha256_of "$state/remote/assets/ownership-snapshot-manifest.json")"
+export FAKE_BUILDER_TOO_LARGE="1"
+run_publisher --force --max-snapshot-bytes 10485760
+unset FAKE_BUILDER_TOO_LARGE
+if [[ "$PUBLISH_STATUS" != "0" && "$PUBLISH_LAST" == "RESULT: FAILED stage=build-snapshot"* ]]; then
+    pass "oversized snapshot fails at build-snapshot"
+else
+    fail "oversized snapshot fails at build-snapshot (status=$PUBLISH_STATUS last='$PUBLISH_LAST')"
+fi
+expect_contains "size limit is passed to the builder" "$(tail -n 1 "$state/builder-calls")" "max_bytes=10485760"
+if [[ "$(sha256_of "$state/remote/assets/ownership-snapshot-manifest.json")" == "$manifest_before" ]]; then
+    pass "oversized snapshot leaves the published manifest untouched"
+else
+    fail "oversized snapshot leaves the published manifest untouched"
+fi
+
 if ((failures > 0)); then
     echo "publish-ownership-snapshot tests: ${failures} failure(s)" >&2
     exit 1

scripts/publish-ownership-snapshot.sh

@@ -35,6 +35,9 @@ Options:
   --history <n>          Request at least <n> previous monthly above-1% reports
                          in addition to the latest (default: 0). Fewer reports
                          are accepted when the source only has fewer months.
+  --max-snapshot-bytes <n>
+                         Passed to the builder: refuse snapshots above this
+                         size (default: 10485760, the limit of idx <= v0.2.3)
   --force                Upload even if the published snapshot is up to date
   --build                Run `cargo build` before publishing
   --keep-workdir         Keep the temp workdir created by the builder helper
@@ -49,6 +52,7 @@ RELEASE_TAG="ownership-snapshot-current"
 BUILD_FIRST="0"
 KEEP_WORKDIR="0"
 HISTORY="0"
+MAX_SNAPSHOT_BYTES=""
 FORCE="0"
 PUBLISH_WORKDIR=""
 STAGE="arguments"
@@ -109,6 +113,10 @@ while [[ $# -gt 0 ]]; do
             HISTORY="${2:-}"
             shift 2
             ;;
+        --max-snapshot-bytes)
+            MAX_SNAPSHOT_BYTES="${2:-}"
+            shift 2
+            ;;
         --force)
             FORCE="1"
             shift
@@ -393,6 +401,10 @@ build_args=(
     --history "$HISTORY"
 )
 
+if [[ -n "$MAX_SNAPSHOT_BYTES" ]]; then
+    build_args+=(--max-snapshot-bytes "$MAX_SNAPSHOT_BYTES")
+fi
+
 if [[ "$KEEP_WORKDIR" == "1" ]]; then
     build_args+=(--keep-workdir)
 fi

src/cli/ownership.rs

@@ -1102,8 +1102,10 @@ fn download_pdf(url: &str, target: &Path) -> Result<(), IdxError> {
         .call()
         .map_err(|e| IdxError::Http(format!("failed to download PDF: {e}")))?;
 
-    let mut body = response.into_body();
+    let bytes = response
-    let bytes = body
+        .into_body()
+        .with_config()
+        .limit(REMOTE_REPORT_LIMIT_BYTES)
         .read_to_vec()
         .map_err(|e| IdxError::Http(format!("failed reading PDF body: {e}")))?;
     remote::validate_pdf_payload(&bytes)?;
@@ -1118,6 +1120,9 @@ fn download_pdf(url: &str, target: &Path) -> Result<(), IdxError> {
     Ok(())
 }
 
+/// Above ureq's 10 MiB default: the monthly holder-register PDF can grow past it.
+const REMOTE_REPORT_LIMIT_BYTES: u64 = 256 * 1024 * 1024;
+
 fn download_xlsx(url: &str, target: &Path) -> Result<(), IdxError> {
     if is_idx_url(url) {
         return remote::download_idx_xlsx(url, target);
@@ -1132,6 +1137,8 @@ fn download_xlsx(url: &str, target: &Path) -> Result<(), IdxError> {
         .map_err(|e| IdxError::Http(format!("failed to download XLSX: {e}")))?;
     let bytes = response
         .into_body()
+        .with_config()
+        .limit(REMOTE_REPORT_LIMIT_BYTES)
         .read_to_vec()
         .map_err(|e| IdxError::Http(format!("failed reading XLSX body: {e}")))?;
     remote::validate_xlsx_payload(&bytes)?;

src/ownership/snapshot.rs

@@ -16,6 +16,12 @@ pub const SNAPSHOT_MANIFEST_ENV: &str = "IDX_OWNERSHIP_SNAPSHOT_MANIFEST";
 pub const SNAPSHOT_MANIFEST_SCHEMA_VERSION: u32 = 1;
 pub const DEFAULT_SNAPSHOT_MANIFEST_URL: &str = "https://github.com/0xrsydn/idx-cli/releases/download/ownership-snapshot-current/ownership-snapshot-manifest.json";
 
+/// ureq caps response bodies at 10 MiB by default. Snapshots with several
+/// months of history exceed that (history 3 is ~10.5 MB), so the download is
+/// bounded by the manifest's `size_bytes` instead, never above 1 GiB. The body
+/// is still verified against that size and the SHA-256 afterwards.
+pub(crate) const SNAPSHOT_DOWNLOAD_LIMIT_BYTES: u64 = 1024 * 1024 * 1024;
+
 const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
@@ -149,7 +155,20 @@ pub fn sync_snapshot(
         });
     }
 
-    let bytes = read_bytes(&manifest.snapshot.download_url, "ownership snapshot SQLite")?;
+    if manifest.snapshot.size_bytes > SNAPSHOT_DOWNLOAD_LIMIT_BYTES {
+        return Err(IdxError::InvalidInput(format!(
+            "ownership snapshot manifest reports {} bytes, above the {} byte limit",
+            manifest.snapshot.size_bytes, SNAPSHOT_DOWNLOAD_LIMIT_BYTES
+        )));
+    }
+    // A body larger than the manifest says is rejected while downloading
+    // (ureq fails once a body reaches its limit, hence the +1); the exact size
+    // and SHA-256 are verified afterwards.
+    let bytes = read_bytes(
+        &manifest.snapshot.download_url,
+        "ownership snapshot SQLite",
+        manifest.snapshot.size_bytes + 1,
+    )?;
     validate_downloaded_bytes(&bytes, &manifest.snapshot)?;
 
     let temp_path = build_temp_path(db_path);
@@ -644,7 +663,7 @@ fn read_text(source: &str, context: &str) -> Result<String, IdxError> {
         .map_err(|e| IdxError::Io(format!("failed to read {context} {}: {e}", path.display())))
 }
 
-fn read_bytes(source: &str, context: &str) -> Result<Vec<u8>, IdxError> {
+fn read_bytes(source: &str, context: &str, limit_bytes: u64) -> Result<Vec<u8>, IdxError> {
     if is_http_source(source) {
         let response = ureq::get(source)
             .header("User-Agent", USER_AGENT)
@@ -654,10 +673,16 @@ fn read_bytes(source: &str, context: &str) -> Result<Vec<u8>, IdxError> {
             )
             .call()
             .map_err(|e| IdxError::Http(format!("failed to fetch {context}: {e}")))?;
-        let mut body = response.into_body();
+        return response
-        return body
+            .into_body()
+            .with_config()
+            .limit(limit_bytes)
             .read_to_vec()
-            .map_err(|e| IdxError::Http(format!("failed reading {context} body: {e}")));
+            .map_err(|e| {
+                IdxError::Http(format!(
+                    "failed reading {context} body (larger than the manifest's size_bytes?): {e}"
+                ))
+            });
     }
 
     let path = local_source_path(source);

tests/cli.rs

@@ -1866,6 +1866,95 @@ fn ownership_import_file_zip_archive_supports_releases_ticker_and_changes() {
         .stdout(predicate::str::contains("DECREASED"));
 }
 
+#[test]
+fn ownership_sync_downloads_snapshot_larger_than_ten_mib_over_http() {
+    // ureq's default 10 MiB body limit broke `ownership sync` for every
+    // client when a 15.3 MB snapshot (5 months of history) was published.
+    let publisher_root = test_env_dir("ownership-sync-large-publisher");
+    let (source_db, manifest_path) = prepare_snapshot_fixture(&publisher_root);
+    {
+        let conn = Connection::open(&source_db).expect("open snapshot db");
+        conn.execute_batch("CREATE TABLE test_padding (data BLOB)")
+            .expect("create padding table");
+        conn.execute(
+            "INSERT INTO test_padding (data) VALUES (zeroblob(?1))",
+            [11 * 1024 * 1024],
+        )
+        .expect("insert padding");
+    }
+    let snapshot_bytes = fs::read(&source_db).expect("read padded snapshot");
+    assert!(snapshot_bytes.len() > 10 * 1024 * 1024);
+
+    write_snapshot_manifest(&manifest_path, &source_db, None);
+    let snapshot_url = spawn_single_response_server("application/octet-stream", snapshot_bytes);
+    let mut manifest: Value =
+        serde_json::from_str(&fs::read_to_string(&manifest_path).expect("read manifest"))
+            .expect("parse manifest");
+    manifest["snapshot"]["download_url"] = Value::String(format!("{snapshot_url}/snapshot.sqlite"));
+    fs::write(&manifest_path, manifest.to_string()).expect("write http manifest");
+
+    let sync_root = test_env_dir("ownership-sync-large-consumer");
+    let target_db = sync_root.join("ownership.db");
+    bin_with_root(&sync_root)
+        .args([
+            "config",
+            "set",
+            "ownership.db_path",
+            target_db.to_str().unwrap(),
+        ])
+        .assert()
+        .success();
+
+    bin_with_root(&sync_root)
+        .args([
+            "ownership",
+            "sync",
+            "--manifest",
+            manifest_path.to_str().unwrap(),
+        ])
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("Installed ownership snapshot"));
+}
+
+#[test]
+fn ownership_sync_rejects_body_larger_than_manifest_size() {
+    let publisher_root = test_env_dir("ownership-sync-oversized-body");
+    let (source_db, manifest_path) = prepare_snapshot_fixture(&publisher_root);
+    write_snapshot_manifest(&manifest_path, &source_db, None);
+    let mut body = fs::read(&source_db).expect("read snapshot");
+    body.extend_from_slice(&[0u8; 4096]);
+    let snapshot_url = spawn_single_response_server("application/octet-stream", body);
+    let mut manifest: Value =
+        serde_json::from_str(&fs::read_to_string(&manifest_path).expect("read manifest"))
+            .expect("parse manifest");
+    manifest["snapshot"]["download_url"] = Value::String(format!("{snapshot_url}/snapshot.sqlite"));
+    fs::write(&manifest_path, manifest.to_string()).expect("write http manifest");
+
+    let sync_root = test_env_dir("ownership-sync-oversized-body-consumer");
+    bin_with_root(&sync_root)
+        .args([
+            "config",
+            "set",
+            "ownership.db_path",
+            sync_root.join("ownership.db").to_str().unwrap(),
+        ])
+        .assert()
+        .success();
+    bin_with_root(&sync_root)
+        .args([
+            "ownership",
+            "sync",
+            "--manifest",
+            manifest_path.to_str().unwrap(),
+        ])
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains(
+            "larger than the manifest's size_bytes",
+        ));
+}
+
 #[test]
 fn ownership_sync_installs_snapshot_and_preserves_query_behavior() {
     let publisher_root = test_env_dir("ownership-sync-publisher");