mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-11-07 01:35:36 +00:00
fix(ownership): sync snapshots over 10 MiB; refuse oversized publishes
On 2026-09-25 a snapshot built with --history 5 (15.3 MB) was published, and `idx ownership sync` failed for every client: "the response body is larger than request limit: 10485760" (ureq's default body cap). It was republished by hand with --history 2 (8.2 MB). - snapshot download: raise the ureq body limit to 1 GiB; the file is still verified against the manifest's size and SHA-256. The PDF/XLSX report downloads get a 256 MiB limit for the same reason. - CLI test: sync a >10 MiB snapshot over HTTP (fails without the fix with the exact production error). - build-latest-ownership-snapshot.sh: --max-snapshot-bytes (default 10485760, the limit of released clients <= v0.2.3); an oversized snapshot fails the build, so the published one stays untouched. - publish-ownership-snapshot.sh passes --max-snapshot-bytes through; publisher tests cover the failure and that the release is unchanged. - docs: size table and guidance. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E
This commit is contained in:
hermes
parent
7225f210ac
commit
de58d04da9
7 changed files with 156 additions and 5 deletions
|
|
@ -103,6 +103,25 @@ idempotent run the exact day no longer matters.
|
|||
Pass `--history <n>` to include earlier months so `idx ownership changes`
|
||||
works straight from the synced snapshot (about 2.4 MB per extra month).
|
||||
|
||||
### Snapshot size limit
|
||||
|
||||
idx clients up to v0.2.3 cap the snapshot download at **10 MiB** (10,485,760
|
||||
bytes) and only accept uncompressed SQLite. A larger snapshot makes
|
||||
`idx ownership sync` fail for every existing install. This happened on
|
||||
2026-09-25 with `--history 5` (15.3 MB).
|
||||
|
||||
| `--history` | months | size (2026-08-31) |
|
||||
| --- | --- | --- |
|
||||
| 2 | 3 | 8,171,520 |
|
||||
| 3 | 4 | 10,510,336 (too large) |
|
||||
| 5 | 6 | 15,290,368 (too large) |
|
||||
|
||||
The builder therefore refuses snapshots above `--max-snapshot-bytes`
|
||||
(default 10485760). The run ends `RESULT: FAILED stage=build-snapshot` and the
|
||||
published snapshot stays unchanged. Newer clients stream downloads up to 1 GiB;
|
||||
raise `--max-snapshot-bytes` and `--history` only after clients without the
|
||||
10 MiB cap are widespread.
|
||||
|
||||
## Clan Integration Later
|
||||
|
||||
The clean split for `clan-private` is:
|
||||
|
|
|
|||
|
|
@ -21,6 +21,10 @@ Options:
|
|||
--history <n> Also import the <n> previous monthly above-1% XLSX
|
||||
reports (oldest first) so `ownership changes` works
|
||||
from the snapshot (default: 0)
|
||||
--max-snapshot-bytes <n>
|
||||
Refuse to emit a snapshot larger than this
|
||||
(default: 10485760, the download limit of idx
|
||||
clients <= v0.2.3; exceeding it breaks their sync)
|
||||
--keep-workdir Keep the temp workdir instead of deleting it
|
||||
--help Show this help
|
||||
EOF
|
||||
|
|
@ -33,6 +37,7 @@ REPO_FULL_NAME="0xrsydn/idx-cli"
|
|||
RELEASE_TAG="ownership-snapshot-current"
|
||||
KEEP_WORKDIR="0"
|
||||
HISTORY="0"
|
||||
MAX_SNAPSHOT_BYTES="10485760"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
|
|
@ -60,6 +65,10 @@ while [[ $# -gt 0 ]]; do
|
|||
HISTORY="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--max-snapshot-bytes)
|
||||
MAX_SNAPSHOT_BYTES="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--keep-workdir)
|
||||
KEEP_WORKDIR="1"
|
||||
shift
|
||||
|
|
@ -82,6 +91,11 @@ if [[ -z "$OUTPUT_DIR" ]]; then
|
|||
exit 2
|
||||
fi
|
||||
|
||||
if ! [[ "$MAX_SNAPSHOT_BYTES" =~ ^[1-9][0-9]{0,17}$ ]]; then
|
||||
echo "--max-snapshot-bytes must be a positive integer" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if ! [[ "$HISTORY" =~ ^[0-9]+$ ]]; then
|
||||
echo "--history must be a non-negative integer" >&2
|
||||
exit 2
|
||||
|
|
@ -254,6 +268,19 @@ IMPORTED_ROW_COUNT="${release_fields[2]:-}"
|
|||
printf 'Building snapshot artifact and manifest...\n'
|
||||
"$BASIC_BUILDER" --db "$DB_PATH" --output-dir "$OUTPUT_DIR" --base-url "$BASE_URL"
|
||||
|
||||
# Released clients cannot download a snapshot above their HTTP body limit, so
|
||||
# publishing one would break `idx ownership sync` for every existing install.
|
||||
SNAPSHOT_BYTES="$(jq -r '.snapshot.size_bytes // empty' "$MANIFEST_PATH")"
|
||||
if ! [[ "$SNAPSHOT_BYTES" =~ ^[0-9]+$ ]]; then
|
||||
echo "built manifest has no valid snapshot.size_bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
if (( SNAPSHOT_BYTES > MAX_SNAPSHOT_BYTES )); then
|
||||
printf 'snapshot is %s bytes (%s monthly releases), above --max-snapshot-bytes %s; lower --history\n' \
|
||||
"$SNAPSHOT_BYTES" "$(jq -r '.snapshot.release_count' "$MANIFEST_PATH")" "$MAX_SNAPSHOT_BYTES" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TMP_MANIFEST_PATH="$WORKDIR/ownership-snapshot-manifest.json"
|
||||
jq \
|
||||
--arg family "$DISCOVERED_FAMILY" \
|
||||
|
|
|
|||
|
|
@ -165,12 +165,14 @@ cat >"$scripts_dir/build-latest-ownership-snapshot.sh" <<'STUB_BUILDER'
|
|||
set -euo pipefail
|
||||
out=""
|
||||
history="0"
|
||||
max_bytes="default"
|
||||
repo="example/repo"
|
||||
tag="ownership-snapshot-current"
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--output-dir) out="$2"; shift 2 ;;
|
||||
--history) history="$2"; shift 2 ;;
|
||||
--max-snapshot-bytes) max_bytes="$2"; shift 2 ;;
|
||||
--repo) repo="$2"; shift 2 ;;
|
||||
--release-tag) tag="$2"; shift 2 ;;
|
||||
--idx-bin) shift 2 ;;
|
||||
|
|
@ -179,7 +181,11 @@ while [[ $# -gt 0 ]]; do
|
|||
esac
|
||||
done
|
||||
[[ -n "$out" ]] || { echo "stub builder: missing --output-dir" >&2; exit 2; }
|
||||
printf 'history=%s\n' "$history" >>"${FAKE_BUILDER_CALLS:?}"
|
||||
printf 'history=%s max_bytes=%s\n' "$history" "$max_bytes" >>"${FAKE_BUILDER_CALLS:?}"
|
||||
if [[ "${FAKE_BUILDER_TOO_LARGE:-0}" == "1" ]]; then
|
||||
echo "snapshot is 15290368 bytes (6 monthly releases), above --max-snapshot-bytes $max_bytes; lower --history" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
as_of="${FAKE_BUILD_AS_OF:-2026-08-31}"
|
||||
if [[ -n "${FAKE_BUILD_RELEASE_COUNT:-}" ]]; then
|
||||
|
|
@ -666,6 +672,28 @@ cp "$state/nested-manifest.json" "$remote_manifest"
|
|||
run_publisher
|
||||
publish_ok "nested download URL cannot authorize a no-op" "RESULT: published 2026-08-31"
|
||||
|
||||
# ===========================================================================
|
||||
# 8. an oversized snapshot fails the build and leaves the release untouched
|
||||
# ===========================================================================
|
||||
reset_remote
|
||||
run_publisher
|
||||
publish_ok "baseline publish before the size check" "RESULT: published 2026-08-31"
|
||||
manifest_before="$(sha256_of "$state/remote/assets/ownership-snapshot-manifest.json")"
|
||||
export FAKE_BUILDER_TOO_LARGE="1"
|
||||
run_publisher --force --max-snapshot-bytes 10485760
|
||||
unset FAKE_BUILDER_TOO_LARGE
|
||||
if [[ "$PUBLISH_STATUS" != "0" && "$PUBLISH_LAST" == "RESULT: FAILED stage=build-snapshot"* ]]; then
|
||||
pass "oversized snapshot fails at build-snapshot"
|
||||
else
|
||||
fail "oversized snapshot fails at build-snapshot (status=$PUBLISH_STATUS last='$PUBLISH_LAST')"
|
||||
fi
|
||||
expect_contains "size limit is passed to the builder" "$(tail -n 1 "$state/builder-calls")" "max_bytes=10485760"
|
||||
if [[ "$(sha256_of "$state/remote/assets/ownership-snapshot-manifest.json")" == "$manifest_before" ]]; then
|
||||
pass "oversized snapshot leaves the published manifest untouched"
|
||||
else
|
||||
fail "oversized snapshot leaves the published manifest untouched"
|
||||
fi
|
||||
|
||||
if ((failures > 0)); then
|
||||
echo "publish-ownership-snapshot tests: ${failures} failure(s)" >&2
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -35,6 +35,9 @@ Options:
|
|||
--history <n> Request at least <n> previous monthly above-1% reports
|
||||
in addition to the latest (default: 0). Fewer reports
|
||||
are accepted when the source only has fewer months.
|
||||
--max-snapshot-bytes <n>
|
||||
Passed to the builder: refuse snapshots above this
|
||||
size (default: 10485760, the limit of idx <= v0.2.3)
|
||||
--force Upload even if the published snapshot is up to date
|
||||
--build Run `cargo build` before publishing
|
||||
--keep-workdir Keep the temp workdir created by the builder helper
|
||||
|
|
@ -49,6 +52,7 @@ RELEASE_TAG="ownership-snapshot-current"
|
|||
BUILD_FIRST="0"
|
||||
KEEP_WORKDIR="0"
|
||||
HISTORY="0"
|
||||
MAX_SNAPSHOT_BYTES=""
|
||||
FORCE="0"
|
||||
PUBLISH_WORKDIR=""
|
||||
STAGE="arguments"
|
||||
|
|
@ -109,6 +113,10 @@ while [[ $# -gt 0 ]]; do
|
|||
HISTORY="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--max-snapshot-bytes)
|
||||
MAX_SNAPSHOT_BYTES="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--force)
|
||||
FORCE="1"
|
||||
shift
|
||||
|
|
@ -393,6 +401,10 @@ build_args=(
|
|||
--history "$HISTORY"
|
||||
)
|
||||
|
||||
if [[ -n "$MAX_SNAPSHOT_BYTES" ]]; then
|
||||
build_args+=(--max-snapshot-bytes "$MAX_SNAPSHOT_BYTES")
|
||||
fi
|
||||
|
||||
if [[ "$KEEP_WORKDIR" == "1" ]]; then
|
||||
build_args+=(--keep-workdir)
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -1102,8 +1102,10 @@ fn download_pdf(url: &str, target: &Path) -> Result<(), IdxError> {
|
|||
.call()
|
||||
.map_err(|e| IdxError::Http(format!("failed to download PDF: {e}")))?;
|
||||
|
||||
let mut body = response.into_body();
|
||||
let bytes = body
|
||||
let bytes = response
|
||||
.into_body()
|
||||
.with_config()
|
||||
.limit(REMOTE_REPORT_LIMIT_BYTES)
|
||||
.read_to_vec()
|
||||
.map_err(|e| IdxError::Http(format!("failed reading PDF body: {e}")))?;
|
||||
remote::validate_pdf_payload(&bytes)?;
|
||||
|
|
@ -1118,6 +1120,9 @@ fn download_pdf(url: &str, target: &Path) -> Result<(), IdxError> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Above ureq's 10 MiB default: the monthly holder-register PDF can grow past it.
|
||||
const REMOTE_REPORT_LIMIT_BYTES: u64 = 256 * 1024 * 1024;
|
||||
|
||||
fn download_xlsx(url: &str, target: &Path) -> Result<(), IdxError> {
|
||||
if is_idx_url(url) {
|
||||
return remote::download_idx_xlsx(url, target);
|
||||
|
|
@ -1132,6 +1137,8 @@ fn download_xlsx(url: &str, target: &Path) -> Result<(), IdxError> {
|
|||
.map_err(|e| IdxError::Http(format!("failed to download XLSX: {e}")))?;
|
||||
let bytes = response
|
||||
.into_body()
|
||||
.with_config()
|
||||
.limit(REMOTE_REPORT_LIMIT_BYTES)
|
||||
.read_to_vec()
|
||||
.map_err(|e| IdxError::Http(format!("failed reading XLSX body: {e}")))?;
|
||||
remote::validate_xlsx_payload(&bytes)?;
|
||||
|
|
|
|||
|
|
@ -16,6 +16,11 @@ pub const SNAPSHOT_MANIFEST_ENV: &str = "IDX_OWNERSHIP_SNAPSHOT_MANIFEST";
|
|||
pub const SNAPSHOT_MANIFEST_SCHEMA_VERSION: u32 = 1;
|
||||
pub const DEFAULT_SNAPSHOT_MANIFEST_URL: &str = "https://github.com/0xrsydn/idx-cli/releases/download/ownership-snapshot-current/ownership-snapshot-manifest.json";
|
||||
|
||||
/// ureq caps response bodies at 10 MiB by default. Snapshots with several
|
||||
/// months of history exceed that (history 3 is ~10.5 MB), and the download is
|
||||
/// verified against the manifest's size and SHA-256 anyway, so allow up to 1 GiB.
|
||||
pub(crate) const SNAPSHOT_DOWNLOAD_LIMIT_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
|
||||
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
|
|
@ -654,8 +659,10 @@ fn read_bytes(source: &str, context: &str) -> Result<Vec<u8>, IdxError> {
|
|||
)
|
||||
.call()
|
||||
.map_err(|e| IdxError::Http(format!("failed to fetch {context}: {e}")))?;
|
||||
let mut body = response.into_body();
|
||||
return body
|
||||
return response
|
||||
.into_body()
|
||||
.with_config()
|
||||
.limit(SNAPSHOT_DOWNLOAD_LIMIT_BYTES)
|
||||
.read_to_vec()
|
||||
.map_err(|e| IdxError::Http(format!("failed reading {context} body: {e}")));
|
||||
}
|
||||
|
|
|
|||
51
tests/cli.rs
51
tests/cli.rs
|
|
@ -1866,6 +1866,57 @@ fn ownership_import_file_zip_archive_supports_releases_ticker_and_changes() {
|
|||
.stdout(predicate::str::contains("DECREASED"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_sync_downloads_snapshot_larger_than_ten_mib_over_http() {
|
||||
// ureq's default 10 MiB body limit broke `ownership sync` for every
|
||||
// client when a 15.3 MB snapshot (5 months of history) was published.
|
||||
let publisher_root = test_env_dir("ownership-sync-large-publisher");
|
||||
let (source_db, manifest_path) = prepare_snapshot_fixture(&publisher_root);
|
||||
{
|
||||
let conn = Connection::open(&source_db).expect("open snapshot db");
|
||||
conn.execute_batch("CREATE TABLE test_padding (data BLOB)")
|
||||
.expect("create padding table");
|
||||
conn.execute(
|
||||
"INSERT INTO test_padding (data) VALUES (zeroblob(?1))",
|
||||
[11 * 1024 * 1024],
|
||||
)
|
||||
.expect("insert padding");
|
||||
}
|
||||
let snapshot_bytes = fs::read(&source_db).expect("read padded snapshot");
|
||||
assert!(snapshot_bytes.len() > 10 * 1024 * 1024);
|
||||
|
||||
write_snapshot_manifest(&manifest_path, &source_db, None);
|
||||
let snapshot_url = spawn_single_response_server("application/octet-stream", snapshot_bytes);
|
||||
let mut manifest: Value =
|
||||
serde_json::from_str(&fs::read_to_string(&manifest_path).expect("read manifest"))
|
||||
.expect("parse manifest");
|
||||
manifest["snapshot"]["download_url"] = Value::String(format!("{snapshot_url}/snapshot.sqlite"));
|
||||
fs::write(&manifest_path, manifest.to_string()).expect("write http manifest");
|
||||
|
||||
let sync_root = test_env_dir("ownership-sync-large-consumer");
|
||||
let target_db = sync_root.join("ownership.db");
|
||||
bin_with_root(&sync_root)
|
||||
.args([
|
||||
"config",
|
||||
"set",
|
||||
"ownership.db_path",
|
||||
target_db.to_str().unwrap(),
|
||||
])
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
bin_with_root(&sync_root)
|
||||
.args([
|
||||
"ownership",
|
||||
"sync",
|
||||
"--manifest",
|
||||
manifest_path.to_str().unwrap(),
|
||||
])
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Installed ownership snapshot"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_sync_installs_snapshot_and_preserves_query_behavior() {
|
||||
let publisher_root = test_env_dir("ownership-sync-publisher");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue