mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-11-07 01:35:36 +00:00
Merge pull request #31 from 0xrsydn/fix/snapshot-size-limit
fix(ownership): sync snapshots over 10 MiB; refuse oversized publishes
This commit is contained in:
brutaljokerz
GitHub
commit
568159b186
7 changed files with 223 additions and 8 deletions
|
|
@ -103,6 +103,26 @@ idempotent run the exact day no longer matters.
|
|||
Pass `--history <n>` to include earlier months so `idx ownership changes`
|
||||
works straight from the synced snapshot (about 2.4 MB per extra month).
|
||||
|
||||
### Snapshot size limit
|
||||
|
||||
idx clients up to v0.2.3 cap the snapshot download at **10 MiB** (10,485,760
|
||||
bytes) and only accept uncompressed SQLite. A larger snapshot makes
|
||||
`idx ownership sync` fail for every existing install. This happened on
|
||||
2026-09-25 with `--history 5` (15.3 MB).
|
||||
|
||||
| `--history` | months | size (2026-08-31) |
|
||||
| --- | --- | --- |
|
||||
| 2 | 3 | 8,171,520 |
|
||||
| 3 | 4 | 10,510,336 (too large) |
|
||||
| 5 | 6 | 15,290,368 (too large) |
|
||||
|
||||
The builder therefore refuses snapshots above `--max-snapshot-bytes`
|
||||
(default 10485760). The run ends `RESULT: FAILED stage=build-snapshot` and the
|
||||
published snapshot stays unchanged. Newer clients accept snapshots up to the
|
||||
manifest's `size_bytes` (at most 1 GiB, buffered in memory);
|
||||
raise `--max-snapshot-bytes` and `--history` only after clients without the
|
||||
10 MiB cap are widespread.
|
||||
|
||||
## Clan Integration Later
|
||||
|
||||
The clean split for `clan-private` is:
|
||||
|
|
|
|||
|
|
@ -21,6 +21,10 @@ Options:
|
|||
--history <n> Also import the <n> previous monthly above-1% XLSX
|
||||
reports (oldest first) so `ownership changes` works
|
||||
from the snapshot (default: 0)
|
||||
--max-snapshot-bytes <n>
|
||||
Refuse to emit a snapshot larger than this
|
||||
(default: 10485760, the download limit of idx
|
||||
clients <= v0.2.3; exceeding it breaks their sync)
|
||||
--keep-workdir Keep the temp workdir instead of deleting it
|
||||
--help Show this help
|
||||
EOF
|
||||
|
|
@ -33,6 +37,7 @@ REPO_FULL_NAME="0xrsydn/idx-cli"
|
|||
RELEASE_TAG="ownership-snapshot-current"
|
||||
KEEP_WORKDIR="0"
|
||||
HISTORY="0"
|
||||
MAX_SNAPSHOT_BYTES="10485760"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
|
|
@ -60,6 +65,10 @@ while [[ $# -gt 0 ]]; do
|
|||
HISTORY="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--max-snapshot-bytes)
|
||||
MAX_SNAPSHOT_BYTES="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--keep-workdir)
|
||||
KEEP_WORKDIR="1"
|
||||
shift
|
||||
|
|
@ -82,6 +91,11 @@ if [[ -z "$OUTPUT_DIR" ]]; then
|
|||
exit 2
|
||||
fi
|
||||
|
||||
if ! [[ "$MAX_SNAPSHOT_BYTES" =~ ^[1-9][0-9]{0,17}$ ]]; then
|
||||
echo "--max-snapshot-bytes must be a positive integer" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if ! [[ "$HISTORY" =~ ^[0-9]+$ ]]; then
|
||||
echo "--history must be a non-negative integer" >&2
|
||||
exit 2
|
||||
|
|
@ -254,6 +268,26 @@ IMPORTED_ROW_COUNT="${release_fields[2]:-}"
|
|||
printf 'Building snapshot artifact and manifest...\n'
|
||||
"$BASIC_BUILDER" --db "$DB_PATH" --output-dir "$OUTPUT_DIR" --base-url "$BASE_URL"
|
||||
|
||||
# Released clients cannot download a snapshot above their HTTP body limit, so
|
||||
# publishing one would break `idx ownership sync` for every existing install.
|
||||
SNAPSHOT_BYTES="$(jq -r '.snapshot.size_bytes // empty' "$MANIFEST_PATH")"
|
||||
if ! [[ "$SNAPSHOT_BYTES" =~ ^[0-9]+$ ]]; then
|
||||
echo "built manifest has no valid snapshot.size_bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
if (( SNAPSHOT_BYTES > MAX_SNAPSHOT_BYTES )); then
|
||||
printf 'snapshot is %s bytes (%s monthly releases), above --max-snapshot-bytes %s; lower --history\n' \
|
||||
"$SNAPSHOT_BYTES" "$(jq -r '.snapshot.release_count' "$MANIFEST_PATH")" "$MAX_SNAPSHOT_BYTES" >&2
|
||||
# Remove the refused pair so a standalone run leaves nothing publishable.
|
||||
refused_sqlite="$(jq -r '.snapshot.download_url // empty' "$MANIFEST_PATH")"
|
||||
refused_sqlite="${refused_sqlite##*/}"
|
||||
if [[ -n "$refused_sqlite" && "$refused_sqlite" != */* ]]; then
|
||||
rm -f "$OUTPUT_DIR/$refused_sqlite"
|
||||
fi
|
||||
rm -f "$MANIFEST_PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TMP_MANIFEST_PATH="$WORKDIR/ownership-snapshot-manifest.json"
|
||||
jq \
|
||||
--arg family "$DISCOVERED_FAMILY" \
|
||||
|
|
|
|||
|
|
@ -165,12 +165,14 @@ cat >"$scripts_dir/build-latest-ownership-snapshot.sh" <<'STUB_BUILDER'
|
|||
set -euo pipefail
|
||||
out=""
|
||||
history="0"
|
||||
max_bytes="default"
|
||||
repo="example/repo"
|
||||
tag="ownership-snapshot-current"
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--output-dir) out="$2"; shift 2 ;;
|
||||
--history) history="$2"; shift 2 ;;
|
||||
--max-snapshot-bytes) max_bytes="$2"; shift 2 ;;
|
||||
--repo) repo="$2"; shift 2 ;;
|
||||
--release-tag) tag="$2"; shift 2 ;;
|
||||
--idx-bin) shift 2 ;;
|
||||
|
|
@ -179,7 +181,11 @@ while [[ $# -gt 0 ]]; do
|
|||
esac
|
||||
done
|
||||
[[ -n "$out" ]] || { echo "stub builder: missing --output-dir" >&2; exit 2; }
|
||||
printf 'history=%s\n' "$history" >>"${FAKE_BUILDER_CALLS:?}"
|
||||
printf 'history=%s max_bytes=%s\n' "$history" "$max_bytes" >>"${FAKE_BUILDER_CALLS:?}"
|
||||
if [[ "${FAKE_BUILDER_TOO_LARGE:-0}" == "1" ]]; then
|
||||
echo "snapshot is 15290368 bytes (6 monthly releases), above --max-snapshot-bytes $max_bytes; lower --history" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
as_of="${FAKE_BUILD_AS_OF:-2026-08-31}"
|
||||
if [[ -n "${FAKE_BUILD_RELEASE_COUNT:-}" ]]; then
|
||||
|
|
@ -666,6 +672,28 @@ cp "$state/nested-manifest.json" "$remote_manifest"
|
|||
run_publisher
|
||||
publish_ok "nested download URL cannot authorize a no-op" "RESULT: published 2026-08-31"
|
||||
|
||||
# ===========================================================================
|
||||
# 8. an oversized snapshot fails the build and leaves the release untouched
|
||||
# ===========================================================================
|
||||
reset_remote
|
||||
run_publisher
|
||||
publish_ok "baseline publish before the size check" "RESULT: published 2026-08-31"
|
||||
manifest_before="$(sha256_of "$state/remote/assets/ownership-snapshot-manifest.json")"
|
||||
export FAKE_BUILDER_TOO_LARGE="1"
|
||||
run_publisher --force --max-snapshot-bytes 10485760
|
||||
unset FAKE_BUILDER_TOO_LARGE
|
||||
if [[ "$PUBLISH_STATUS" != "0" && "$PUBLISH_LAST" == "RESULT: FAILED stage=build-snapshot"* ]]; then
|
||||
pass "oversized snapshot fails at build-snapshot"
|
||||
else
|
||||
fail "oversized snapshot fails at build-snapshot (status=$PUBLISH_STATUS last='$PUBLISH_LAST')"
|
||||
fi
|
||||
expect_contains "size limit is passed to the builder" "$(tail -n 1 "$state/builder-calls")" "max_bytes=10485760"
|
||||
if [[ "$(sha256_of "$state/remote/assets/ownership-snapshot-manifest.json")" == "$manifest_before" ]]; then
|
||||
pass "oversized snapshot leaves the published manifest untouched"
|
||||
else
|
||||
fail "oversized snapshot leaves the published manifest untouched"
|
||||
fi
|
||||
|
||||
if ((failures > 0)); then
|
||||
echo "publish-ownership-snapshot tests: ${failures} failure(s)" >&2
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -35,6 +35,9 @@ Options:
|
|||
--history <n> Request at least <n> previous monthly above-1% reports
|
||||
in addition to the latest (default: 0). Fewer reports
|
||||
are accepted when the source only has fewer months.
|
||||
--max-snapshot-bytes <n>
|
||||
Passed to the builder: refuse snapshots above this
|
||||
size (default: 10485760, the limit of idx <= v0.2.3)
|
||||
--force Upload even if the published snapshot is up to date
|
||||
--build Run `cargo build` before publishing
|
||||
--keep-workdir Keep the temp workdir created by the builder helper
|
||||
|
|
@ -49,6 +52,7 @@ RELEASE_TAG="ownership-snapshot-current"
|
|||
BUILD_FIRST="0"
|
||||
KEEP_WORKDIR="0"
|
||||
HISTORY="0"
|
||||
MAX_SNAPSHOT_BYTES=""
|
||||
FORCE="0"
|
||||
PUBLISH_WORKDIR=""
|
||||
STAGE="arguments"
|
||||
|
|
@ -109,6 +113,10 @@ while [[ $# -gt 0 ]]; do
|
|||
HISTORY="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--max-snapshot-bytes)
|
||||
MAX_SNAPSHOT_BYTES="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--force)
|
||||
FORCE="1"
|
||||
shift
|
||||
|
|
@ -393,6 +401,10 @@ build_args=(
|
|||
--history "$HISTORY"
|
||||
)
|
||||
|
||||
if [[ -n "$MAX_SNAPSHOT_BYTES" ]]; then
|
||||
build_args+=(--max-snapshot-bytes "$MAX_SNAPSHOT_BYTES")
|
||||
fi
|
||||
|
||||
if [[ "$KEEP_WORKDIR" == "1" ]]; then
|
||||
build_args+=(--keep-workdir)
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -1102,8 +1102,10 @@ fn download_pdf(url: &str, target: &Path) -> Result<(), IdxError> {
|
|||
.call()
|
||||
.map_err(|e| IdxError::Http(format!("failed to download PDF: {e}")))?;
|
||||
|
||||
let mut body = response.into_body();
|
||||
let bytes = body
|
||||
let bytes = response
|
||||
.into_body()
|
||||
.with_config()
|
||||
.limit(REMOTE_REPORT_LIMIT_BYTES)
|
||||
.read_to_vec()
|
||||
.map_err(|e| IdxError::Http(format!("failed reading PDF body: {e}")))?;
|
||||
remote::validate_pdf_payload(&bytes)?;
|
||||
|
|
@ -1118,6 +1120,9 @@ fn download_pdf(url: &str, target: &Path) -> Result<(), IdxError> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Above ureq's 10 MiB default: the monthly holder-register PDF can grow past it.
|
||||
const REMOTE_REPORT_LIMIT_BYTES: u64 = 256 * 1024 * 1024;
|
||||
|
||||
fn download_xlsx(url: &str, target: &Path) -> Result<(), IdxError> {
|
||||
if is_idx_url(url) {
|
||||
return remote::download_idx_xlsx(url, target);
|
||||
|
|
@ -1132,6 +1137,8 @@ fn download_xlsx(url: &str, target: &Path) -> Result<(), IdxError> {
|
|||
.map_err(|e| IdxError::Http(format!("failed to download XLSX: {e}")))?;
|
||||
let bytes = response
|
||||
.into_body()
|
||||
.with_config()
|
||||
.limit(REMOTE_REPORT_LIMIT_BYTES)
|
||||
.read_to_vec()
|
||||
.map_err(|e| IdxError::Http(format!("failed reading XLSX body: {e}")))?;
|
||||
remote::validate_xlsx_payload(&bytes)?;
|
||||
|
|
|
|||
|
|
@ -16,6 +16,12 @@ pub const SNAPSHOT_MANIFEST_ENV: &str = "IDX_OWNERSHIP_SNAPSHOT_MANIFEST";
|
|||
pub const SNAPSHOT_MANIFEST_SCHEMA_VERSION: u32 = 1;
|
||||
pub const DEFAULT_SNAPSHOT_MANIFEST_URL: &str = "https://github.com/0xrsydn/idx-cli/releases/download/ownership-snapshot-current/ownership-snapshot-manifest.json";
|
||||
|
||||
/// ureq caps response bodies at 10 MiB by default. Snapshots with several
|
||||
/// months of history exceed that (history 3 is ~10.5 MB), so the download is
|
||||
/// bounded by the manifest's `size_bytes` instead, never above 1 GiB. The body
|
||||
/// is still verified against that size and the SHA-256 afterwards.
|
||||
pub(crate) const SNAPSHOT_DOWNLOAD_LIMIT_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
|
||||
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
|
|
@ -149,7 +155,20 @@ pub fn sync_snapshot(
|
|||
});
|
||||
}
|
||||
|
||||
let bytes = read_bytes(&manifest.snapshot.download_url, "ownership snapshot SQLite")?;
|
||||
if manifest.snapshot.size_bytes > SNAPSHOT_DOWNLOAD_LIMIT_BYTES {
|
||||
return Err(IdxError::InvalidInput(format!(
|
||||
"ownership snapshot manifest reports {} bytes, above the {} byte limit",
|
||||
manifest.snapshot.size_bytes, SNAPSHOT_DOWNLOAD_LIMIT_BYTES
|
||||
)));
|
||||
}
|
||||
// A body larger than the manifest says is rejected while downloading
|
||||
// (ureq fails once a body reaches its limit, hence the +1); the exact size
|
||||
// and SHA-256 are verified afterwards.
|
||||
let bytes = read_bytes(
|
||||
&manifest.snapshot.download_url,
|
||||
"ownership snapshot SQLite",
|
||||
manifest.snapshot.size_bytes + 1,
|
||||
)?;
|
||||
validate_downloaded_bytes(&bytes, &manifest.snapshot)?;
|
||||
|
||||
let temp_path = build_temp_path(db_path);
|
||||
|
|
@ -644,7 +663,7 @@ fn read_text(source: &str, context: &str) -> Result<String, IdxError> {
|
|||
.map_err(|e| IdxError::Io(format!("failed to read {context} {}: {e}", path.display())))
|
||||
}
|
||||
|
||||
fn read_bytes(source: &str, context: &str) -> Result<Vec<u8>, IdxError> {
|
||||
fn read_bytes(source: &str, context: &str, limit_bytes: u64) -> Result<Vec<u8>, IdxError> {
|
||||
if is_http_source(source) {
|
||||
let response = ureq::get(source)
|
||||
.header("User-Agent", USER_AGENT)
|
||||
|
|
@ -654,10 +673,16 @@ fn read_bytes(source: &str, context: &str) -> Result<Vec<u8>, IdxError> {
|
|||
)
|
||||
.call()
|
||||
.map_err(|e| IdxError::Http(format!("failed to fetch {context}: {e}")))?;
|
||||
let mut body = response.into_body();
|
||||
return body
|
||||
return response
|
||||
.into_body()
|
||||
.with_config()
|
||||
.limit(limit_bytes)
|
||||
.read_to_vec()
|
||||
.map_err(|e| IdxError::Http(format!("failed reading {context} body: {e}")));
|
||||
.map_err(|e| {
|
||||
IdxError::Http(format!(
|
||||
"failed reading {context} body (larger than the manifest's size_bytes?): {e}"
|
||||
))
|
||||
});
|
||||
}
|
||||
|
||||
let path = local_source_path(source);
|
||||
|
|
|
|||
89
tests/cli.rs
89
tests/cli.rs
|
|
@ -1866,6 +1866,95 @@ fn ownership_import_file_zip_archive_supports_releases_ticker_and_changes() {
|
|||
.stdout(predicate::str::contains("DECREASED"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_sync_downloads_snapshot_larger_than_ten_mib_over_http() {
|
||||
// ureq's default 10 MiB body limit broke `ownership sync` for every
|
||||
// client when a 15.3 MB snapshot (5 months of history) was published.
|
||||
let publisher_root = test_env_dir("ownership-sync-large-publisher");
|
||||
let (source_db, manifest_path) = prepare_snapshot_fixture(&publisher_root);
|
||||
{
|
||||
let conn = Connection::open(&source_db).expect("open snapshot db");
|
||||
conn.execute_batch("CREATE TABLE test_padding (data BLOB)")
|
||||
.expect("create padding table");
|
||||
conn.execute(
|
||||
"INSERT INTO test_padding (data) VALUES (zeroblob(?1))",
|
||||
[11 * 1024 * 1024],
|
||||
)
|
||||
.expect("insert padding");
|
||||
}
|
||||
let snapshot_bytes = fs::read(&source_db).expect("read padded snapshot");
|
||||
assert!(snapshot_bytes.len() > 10 * 1024 * 1024);
|
||||
|
||||
write_snapshot_manifest(&manifest_path, &source_db, None);
|
||||
let snapshot_url = spawn_single_response_server("application/octet-stream", snapshot_bytes);
|
||||
let mut manifest: Value =
|
||||
serde_json::from_str(&fs::read_to_string(&manifest_path).expect("read manifest"))
|
||||
.expect("parse manifest");
|
||||
manifest["snapshot"]["download_url"] = Value::String(format!("{snapshot_url}/snapshot.sqlite"));
|
||||
fs::write(&manifest_path, manifest.to_string()).expect("write http manifest");
|
||||
|
||||
let sync_root = test_env_dir("ownership-sync-large-consumer");
|
||||
let target_db = sync_root.join("ownership.db");
|
||||
bin_with_root(&sync_root)
|
||||
.args([
|
||||
"config",
|
||||
"set",
|
||||
"ownership.db_path",
|
||||
target_db.to_str().unwrap(),
|
||||
])
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
bin_with_root(&sync_root)
|
||||
.args([
|
||||
"ownership",
|
||||
"sync",
|
||||
"--manifest",
|
||||
manifest_path.to_str().unwrap(),
|
||||
])
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Installed ownership snapshot"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_sync_rejects_body_larger_than_manifest_size() {
|
||||
let publisher_root = test_env_dir("ownership-sync-oversized-body");
|
||||
let (source_db, manifest_path) = prepare_snapshot_fixture(&publisher_root);
|
||||
write_snapshot_manifest(&manifest_path, &source_db, None);
|
||||
let mut body = fs::read(&source_db).expect("read snapshot");
|
||||
body.extend_from_slice(&[0u8; 4096]);
|
||||
let snapshot_url = spawn_single_response_server("application/octet-stream", body);
|
||||
let mut manifest: Value =
|
||||
serde_json::from_str(&fs::read_to_string(&manifest_path).expect("read manifest"))
|
||||
.expect("parse manifest");
|
||||
manifest["snapshot"]["download_url"] = Value::String(format!("{snapshot_url}/snapshot.sqlite"));
|
||||
fs::write(&manifest_path, manifest.to_string()).expect("write http manifest");
|
||||
|
||||
let sync_root = test_env_dir("ownership-sync-oversized-body-consumer");
|
||||
bin_with_root(&sync_root)
|
||||
.args([
|
||||
"config",
|
||||
"set",
|
||||
"ownership.db_path",
|
||||
sync_root.join("ownership.db").to_str().unwrap(),
|
||||
])
|
||||
.assert()
|
||||
.success();
|
||||
bin_with_root(&sync_root)
|
||||
.args([
|
||||
"ownership",
|
||||
"sync",
|
||||
"--manifest",
|
||||
manifest_path.to_str().unwrap(),
|
||||
])
|
||||
.assert()
|
||||
.failure()
|
||||
.stderr(predicate::str::contains(
|
||||
"larger than the manifest's size_bytes",
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ownership_sync_installs_snapshot_and_preserves_query_behavior() {
|
||||
let publisher_root = test_env_dir("ownership-sync-publisher");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue