mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-11-07 01:35:36 +00:00
fix(ownership): sync snapshots over 10 MiB; refuse oversized publishes
On 2026-09-25 a snapshot built with --history 5 (15.3 MB) was published, and `idx ownership sync` failed for every client: "the response body is larger than request limit: 10485760" (ureq's default body cap). It was republished by hand with --history 2 (8.2 MB). - snapshot download: raise the ureq body limit to 1 GiB; the file is still verified against the manifest's size and SHA-256. The PDF/XLSX report downloads get a 256 MiB limit for the same reason. - CLI test: sync a >10 MiB snapshot over HTTP (fails without the fix with the exact production error). - build-latest-ownership-snapshot.sh: --max-snapshot-bytes (default 10485760, the limit of released clients <= v0.2.3); an oversized snapshot fails the build, so the published one stays untouched. - publish-ownership-snapshot.sh passes --max-snapshot-bytes through; publisher tests cover the failure and that the release is unchanged. - docs: size table and guidance. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E
This commit is contained in:
hermes
parent
7225f210ac
commit
de58d04da9
7 changed files with 156 additions and 5 deletions
|
|
@ -21,6 +21,10 @@ Options:
|
|||
--history <n> Also import the <n> previous monthly above-1% XLSX
|
||||
reports (oldest first) so `ownership changes` works
|
||||
from the snapshot (default: 0)
|
||||
--max-snapshot-bytes <n>
|
||||
Refuse to emit a snapshot larger than this
|
||||
(default: 10485760, the download limit of idx
|
||||
clients <= v0.2.3; exceeding it breaks their sync)
|
||||
--keep-workdir Keep the temp workdir instead of deleting it
|
||||
--help Show this help
|
||||
EOF
|
||||
|
|
@ -33,6 +37,7 @@ REPO_FULL_NAME="0xrsydn/idx-cli"
|
|||
RELEASE_TAG="ownership-snapshot-current"
|
||||
KEEP_WORKDIR="0"
|
||||
HISTORY="0"
|
||||
MAX_SNAPSHOT_BYTES="10485760"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
|
|
@ -60,6 +65,10 @@ while [[ $# -gt 0 ]]; do
|
|||
HISTORY="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--max-snapshot-bytes)
|
||||
MAX_SNAPSHOT_BYTES="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--keep-workdir)
|
||||
KEEP_WORKDIR="1"
|
||||
shift
|
||||
|
|
@ -82,6 +91,11 @@ if [[ -z "$OUTPUT_DIR" ]]; then
|
|||
exit 2
|
||||
fi
|
||||
|
||||
if ! [[ "$MAX_SNAPSHOT_BYTES" =~ ^[1-9][0-9]{0,17}$ ]]; then
|
||||
echo "--max-snapshot-bytes must be a positive integer" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if ! [[ "$HISTORY" =~ ^[0-9]+$ ]]; then
|
||||
echo "--history must be a non-negative integer" >&2
|
||||
exit 2
|
||||
|
|
@ -254,6 +268,19 @@ IMPORTED_ROW_COUNT="${release_fields[2]:-}"
|
|||
printf 'Building snapshot artifact and manifest...\n'
|
||||
"$BASIC_BUILDER" --db "$DB_PATH" --output-dir "$OUTPUT_DIR" --base-url "$BASE_URL"
|
||||
|
||||
# Released clients cannot download a snapshot above their HTTP body limit, so
|
||||
# publishing one would break `idx ownership sync` for every existing install.
|
||||
SNAPSHOT_BYTES="$(jq -r '.snapshot.size_bytes // empty' "$MANIFEST_PATH")"
|
||||
if ! [[ "$SNAPSHOT_BYTES" =~ ^[0-9]+$ ]]; then
|
||||
echo "built manifest has no valid snapshot.size_bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
if (( SNAPSHOT_BYTES > MAX_SNAPSHOT_BYTES )); then
|
||||
printf 'snapshot is %s bytes (%s monthly releases), above --max-snapshot-bytes %s; lower --history\n' \
|
||||
"$SNAPSHOT_BYTES" "$(jq -r '.snapshot.release_count' "$MANIFEST_PATH")" "$MAX_SNAPSHOT_BYTES" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TMP_MANIFEST_PATH="$WORKDIR/ownership-snapshot-manifest.json"
|
||||
jq \
|
||||
--arg family "$DISCOVERED_FAMILY" \
|
||||
|
|
|
|||
|
|
@ -165,12 +165,14 @@ cat >"$scripts_dir/build-latest-ownership-snapshot.sh" <<'STUB_BUILDER'
|
|||
set -euo pipefail
|
||||
out=""
|
||||
history="0"
|
||||
max_bytes="default"
|
||||
repo="example/repo"
|
||||
tag="ownership-snapshot-current"
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--output-dir) out="$2"; shift 2 ;;
|
||||
--history) history="$2"; shift 2 ;;
|
||||
--max-snapshot-bytes) max_bytes="$2"; shift 2 ;;
|
||||
--repo) repo="$2"; shift 2 ;;
|
||||
--release-tag) tag="$2"; shift 2 ;;
|
||||
--idx-bin) shift 2 ;;
|
||||
|
|
@ -179,7 +181,11 @@ while [[ $# -gt 0 ]]; do
|
|||
esac
|
||||
done
|
||||
[[ -n "$out" ]] || { echo "stub builder: missing --output-dir" >&2; exit 2; }
|
||||
printf 'history=%s\n' "$history" >>"${FAKE_BUILDER_CALLS:?}"
|
||||
printf 'history=%s max_bytes=%s\n' "$history" "$max_bytes" >>"${FAKE_BUILDER_CALLS:?}"
|
||||
if [[ "${FAKE_BUILDER_TOO_LARGE:-0}" == "1" ]]; then
|
||||
echo "snapshot is 15290368 bytes (6 monthly releases), above --max-snapshot-bytes $max_bytes; lower --history" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
as_of="${FAKE_BUILD_AS_OF:-2026-08-31}"
|
||||
if [[ -n "${FAKE_BUILD_RELEASE_COUNT:-}" ]]; then
|
||||
|
|
@ -666,6 +672,28 @@ cp "$state/nested-manifest.json" "$remote_manifest"
|
|||
run_publisher
|
||||
publish_ok "nested download URL cannot authorize a no-op" "RESULT: published 2026-08-31"
|
||||
|
||||
# ===========================================================================
|
||||
# 8. an oversized snapshot fails the build and leaves the release untouched
|
||||
# ===========================================================================
|
||||
reset_remote
|
||||
run_publisher
|
||||
publish_ok "baseline publish before the size check" "RESULT: published 2026-08-31"
|
||||
manifest_before="$(sha256_of "$state/remote/assets/ownership-snapshot-manifest.json")"
|
||||
export FAKE_BUILDER_TOO_LARGE="1"
|
||||
run_publisher --force --max-snapshot-bytes 10485760
|
||||
unset FAKE_BUILDER_TOO_LARGE
|
||||
if [[ "$PUBLISH_STATUS" != "0" && "$PUBLISH_LAST" == "RESULT: FAILED stage=build-snapshot"* ]]; then
|
||||
pass "oversized snapshot fails at build-snapshot"
|
||||
else
|
||||
fail "oversized snapshot fails at build-snapshot (status=$PUBLISH_STATUS last='$PUBLISH_LAST')"
|
||||
fi
|
||||
expect_contains "size limit is passed to the builder" "$(tail -n 1 "$state/builder-calls")" "max_bytes=10485760"
|
||||
if [[ "$(sha256_of "$state/remote/assets/ownership-snapshot-manifest.json")" == "$manifest_before" ]]; then
|
||||
pass "oversized snapshot leaves the published manifest untouched"
|
||||
else
|
||||
fail "oversized snapshot leaves the published manifest untouched"
|
||||
fi
|
||||
|
||||
if ((failures > 0)); then
|
||||
echo "publish-ownership-snapshot tests: ${failures} failure(s)" >&2
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -35,6 +35,9 @@ Options:
|
|||
--history <n> Request at least <n> previous monthly above-1% reports
|
||||
in addition to the latest (default: 0). Fewer reports
|
||||
are accepted when the source only has fewer months.
|
||||
--max-snapshot-bytes <n>
|
||||
Passed to the builder: refuse snapshots above this
|
||||
size (default: 10485760, the limit of idx <= v0.2.3)
|
||||
--force Upload even if the published snapshot is up to date
|
||||
--build Run `cargo build` before publishing
|
||||
--keep-workdir Keep the temp workdir created by the builder helper
|
||||
|
|
@ -49,6 +52,7 @@ RELEASE_TAG="ownership-snapshot-current"
|
|||
BUILD_FIRST="0"
|
||||
KEEP_WORKDIR="0"
|
||||
HISTORY="0"
|
||||
MAX_SNAPSHOT_BYTES=""
|
||||
FORCE="0"
|
||||
PUBLISH_WORKDIR=""
|
||||
STAGE="arguments"
|
||||
|
|
@ -109,6 +113,10 @@ while [[ $# -gt 0 ]]; do
|
|||
HISTORY="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--max-snapshot-bytes)
|
||||
MAX_SNAPSHOT_BYTES="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--force)
|
||||
FORCE="1"
|
||||
shift
|
||||
|
|
@ -393,6 +401,10 @@ build_args=(
|
|||
--history "$HISTORY"
|
||||
)
|
||||
|
||||
if [[ -n "$MAX_SNAPSHOT_BYTES" ]]; then
|
||||
build_args+=(--max-snapshot-bytes "$MAX_SNAPSHOT_BYTES")
|
||||
fi
|
||||
|
||||
if [[ "$KEEP_WORKDIR" == "1" ]]; then
|
||||
build_args+=(--keep-workdir)
|
||||
fi
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue