From 9f4ed5b4a5d2132e63899bd2f74d70fafffb80f2 Mon Sep 17 00:00:00 2001 From: hermes Date: Fri, 25 Sep 2026 10:05:07 +0000 Subject: [PATCH] fix(ownership): verify snapshot assets before publishing manifests --- .github/workflows/ci.yml | 3 + .../workflows/publish-ownership-snapshot.yml | 61 +- FEATURE_SPEC.md | 2 +- TODO.md | 8 + docs/OWNERSHIP_PUBLISH.md | 86 ++- docs/OWNERSHIP_SELF_HOSTED.md | 26 +- docs/OWNERSHIP_SYNC.md | 7 +- scripts/build-latest-ownership-snapshot.sh | 21 +- scripts/build-ownership-snapshot.sh | 14 +- scripts/publish-ownership-snapshot-test.sh | 673 ++++++++++++++++++ scripts/publish-ownership-snapshot.sh | 340 +++++++-- 11 files changed, 1127 insertions(+), 114 deletions(-) create mode 100755 scripts/publish-ownership-snapshot-test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ecb6a09..38423a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,9 @@ jobs: - name: Test run: cargo test + - name: Publish ownership snapshot tests + run: scripts/publish-ownership-snapshot-test.sh + - name: Package run: cargo package --locked diff --git a/.github/workflows/publish-ownership-snapshot.yml b/.github/workflows/publish-ownership-snapshot.yml index e392355..3204670 100644 --- a/.github/workflows/publish-ownership-snapshot.yml +++ b/.github/workflows/publish-ownership-snapshot.yml @@ -31,54 +31,31 @@ jobs: - name: Install Nix uses: cachix/install-nix-action@v30 - - name: Build idx - run: nix develop --command cargo build + - name: Build ownership publisher + run: nix build .#ownership-publisher - - name: Build snapshot artifacts from latest supported IDX/KSEI source + - name: Publish ownership snapshot + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + RELEASE_TAG: ${{ inputs.release_tag }} + HISTORY: ${{ inputs.history }} run: | - nix develop --command scripts/build-latest-ownership-snapshot.sh \ - --idx-bin ./target/debug/idx \ + set -euo pipefail + + # The packaged publisher owns the full protocol: discovery, import, + # content-addressed SQLite upload + SHA-256 verification, and the + # manifest as the final commit point. It is a no-op when current. + ./result/bin/idx-ownership-publish \ --output-dir dist/ownership-snapshot \ - --repo "${{ github.repository }}" \ - --release-tag "${{ inputs.release_tag }}" \ - --history "${{ inputs.history }}" + --repo "$REPO" \ + --release-tag "$RELEASE_TAG" \ + --history "$HISTORY" - name: Upload workflow artifact copy + # A legitimate up-to-date run produces no new local artifacts. uses: actions/upload-artifact@v4 with: name: ownership-snapshot-${{ github.run_id }} path: dist/ownership-snapshot/* - if-no-files-found: error - - - name: Ensure stable snapshot release exists - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - - tag="${{ inputs.release_tag }}" - repo="${{ github.repository }}" - - if gh release view "$tag" --repo "$repo" >/dev/null 2>&1; then - echo "Release $tag already exists." - exit 0 - fi - - gh release create "$tag" \ - --repo "$repo" \ - --target "${{ github.sha }}" \ - --title "Ownership Snapshot Current" \ - --notes "Stable release for idx ownership snapshot artifacts consumed by \`idx ownership sync\`." \ - --latest=false - - - name: Upload snapshot assets to stable release - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - - gh release upload "${{ inputs.release_tag }}" \ - dist/ownership-snapshot/ownership-snapshot-manifest.json \ - dist/ownership-snapshot/ownership-snapshot-*.sqlite \ - --repo "${{ github.repository }}" \ - --clobber + if-no-files-found: ignore diff --git a/FEATURE_SPEC.md b/FEATURE_SPEC.md index 966f0ec..055fa1e 100644 --- a/FEATURE_SPEC.md +++ b/FEATURE_SPEC.md @@ -87,7 +87,7 @@ The following items should no longer be treated as active backlog in this spec: - KSEI ownership parser hardening for the March 2026 live PDF layout, including the merged `DATE + SHARE_CODE` segment and `D`/`A` locality markers - Real KSEI ownership CLI verification from local file import into SQLite (`7261` rows across `955` tickers on `2026-03-28`) - Ownership remote-import hardening for the `above1` contract, including direct-PDF-only `--url` input, discovery status output, explicit legacy-schema rejection, and live ownership-import smoke coverage -- Ownership snapshot publishing/sync contract, including `idx ownership sync`, manifest/checksum validation, conservative local replacement rules, and the `scripts/build-ownership-snapshot.sh` publisher helper +- Ownership snapshot publishing/sync contract, including `idx ownership sync`, manifest/checksum validation, conservative local replacement rules, and the `scripts/build-ownership-snapshot.sh` publisher helper; the publisher now uses an immutable content-addressed SQLite asset, uploads and verifies it before the manifest commit point, and makes history-aware no-op decisions - KSEI archive ZIP/TXT fallback ingest for local `.zip`/`.txt` files, including cross-check coverage against the `above1` PDF holder-register fixture and compatible `releases`/`ticker`/`changes` verification If any of the above regress, capture that in `TODO.md` as a new finding rather than reopening the old section here wholesale. diff --git a/TODO.md b/TODO.md index 8b44ce1..27c2ce1 100644 --- a/TODO.md +++ b/TODO.md @@ -162,6 +162,14 @@ - [ ] Additional providers (Alpha Vantage, Twelve Data, IDX official) ## 🔬 Latest Smoke Findings (2026-04-02) +- [x] Publish hardening batch on `2026-09-25`: `scripts/publish-ownership-snapshot.sh` now uploads an immutable content-addressed SQLite asset (date + sha256) and verifies it before publishing the manifest as the final commit point, so a partial upload no longer leaves a manifest that points at a missing asset; the manifest is trusted for a no-op only when the referenced asset matches the manifest SHA-256 and size at the configured repo/tag, and GitHub auth/network errors never become "up-to-date" +- [x] Publish integrity follow-up on `2026-09-25`: no-op and upload verification now check the full SHA-256, not just the filename and size; the publisher uses GitHub's `sha256:` asset digest when present and falls back to `gh release download` + local hashing for legacy assets, rejects a `download_url` that names a different repo/tag, and refuses to publish the manifest when the uploaded asset's SHA-256 does not match +- [x] `--history` is bounded (maximum `1000`, leading zeros stripped before conversion) so an enormous value cannot wrap negative and look already satisfied +- [x] The GitHub publish workflow now builds the packaged `idx-ownership-publisher` and routes publication through it with inputs passed as environment variables; the workflow artifact upload uses `if-no-files-found: ignore` so a legitimate up-to-date run does not fail +- [x] `--keep-workdir` no longer prints after the `RESULT:` line: the retained path is announced when the workdir is created and the EXIT cleanup is silent +- [x] `--history` changes are now honored for same-date data: the publisher compares `1 + min(--history, available XLSX months)` against the published `release_count` before and after building, validates `--history` before early exits, and does not rebuild on every run when the source has fewer historical months than requested +- [x] New offline regression harness `scripts/publish-ownership-snapshot-test.sh` (wired into `.github/workflows/ci.yml`) covers successful publish/no-op, partial SQLite upload and retry, manifest upload failure and retry, forced same-date refresh preserving the previous artifact, a missing referenced asset forcing republish, same-size remote corruption with and without digest metadata, upload corruption rejected before the manifest commit (digest and digest-less), a manifest URL for a different repo/tag, legacy PDF post-build no-op, history upgrade, limited available history, `--history` bounds, `--keep-workdir` output ordering, and auth/network failure not becoming up-to-date +- [x] Verification on `2026-09-25`: `bash -n` on the changed shell scripts and `scripts/publish-ownership-snapshot-test.sh` all passed - [x] `ownership sync` now falls back to the built-in published manifest URL on `2026-04-12`, so plain `idx ownership sync` works on a clean machine once the public snapshot release is available - [x] Ownership snapshot publishing now has a live-source maintainer path on `2026-04-11`: `scripts/build-latest-ownership-snapshot.sh` discovers the current supported `above1` IDX/KSEI PDF, imports it into an isolated temp DB, emits release-ready SQLite + manifest artifacts, and records the source IDX metadata in the manifest - [x] Verification on `2026-04-11`: `nix develop --command cargo test`, `bash -n scripts/build-latest-ownership-snapshot.sh`, and a real `nix develop --command scripts/build-latest-ownership-snapshot.sh --idx-bin ./target/debug/idx --output-dir tmp/ownership-snapshot-live` run all passed; the current live source produced `7214` rows for `955` tickers with `as_of_date=2026-03-31` diff --git a/docs/OWNERSHIP_PUBLISH.md b/docs/OWNERSHIP_PUBLISH.md index 05821fe..8112004 100644 --- a/docs/OWNERSHIP_PUBLISH.md +++ b/docs/OWNERSHIP_PUBLISH.md @@ -17,7 +17,7 @@ Use a dedicated stable release tag for snapshot assets: - tag: `ownership-snapshot-current` - manifest asset: `ownership-snapshot-manifest.json` -- SQLite asset: `ownership-snapshot-YYYY-MM-DD.sqlite` +- SQLite asset: `ownership-snapshot-YYYY-MM-DD-.sqlite` (immutable, content-addressed) Recommended public manifest URL: @@ -36,14 +36,14 @@ Run the publisher helper inside `nix develop` so `mutool` and the ```bash nix develop --command cargo build -nix develop --command scripts/build-latest-ownership-snapshot.sh \ +nix develop --command scripts/publish-ownership-snapshot.sh \ --idx-bin ./target/debug/idx \ --output-dir dist/ownership-snapshot \ --repo 0xrsydn/idx-cli \ --release-tag ownership-snapshot-current ``` -The script performs these steps: +The helper performs these steps: 1. `idx -o json ownership discover --family above1 --limit 50` (newest supported report first; XLSX from the Data Kepemilikan Saham page since June 2026) @@ -68,16 +68,69 @@ The resulting manifest records: That metadata is additive. Existing sync clients can still parse the manifest. +## Publication Protocol + +The manifest is the commit point that `idx ownership sync` follows. The +publisher therefore never uploads the manifest and the SQLite asset in one +non-atomic call. It uses this order: + +1. Build the snapshot and stage the manifest plus one content-addressed SQLite + file. +2. Upload the SQLite asset first. The name includes the artifact's SHA-256, so + the same content always has the same name and a retry is idempotent. +3. Read the release assets back and confirm the SQLite asset exists with the + expected SHA-256 and size. Use GitHub's `sha256:` digest metadata when it is + present; for legacy assets without a digest, download the asset through `gh` + and hash it. Fail before touching the manifest if verification fails. +4. Upload the manifest last and verify that its `snapshot.sqlite_sha256` + matches the staged manifest. + +Consequences: + +- A partial upload (manifest uploaded, SQLite missing) can no longer happen. +- If either upload fails, the job exits non-zero and the next run retries. + Previously downloaded manifests remain valid because the old SQLite asset is retained. +- GitHub does not replace the manifest transactionally. A failed replacement + can leave the manifest unavailable until a retry succeeds. +- The publisher treats a published manifest as current only when `download_url` + names this repo and release tag and the referenced asset matches the manifest + SHA-256 and size. A missing or corrupt asset, a mismatched URL, or any GitHub + authentication/network error means "publish", never "up to date". + +## History Coverage + +`--history ` requests the latest report plus up to `` earlier monthly +XLSX reports. The publisher compares the request with the months the source +actually exposes: + +- desired releases = `1 + min(--history, available earlier XLSX months)` +- a published snapshot with the same as-of date and at least the desired number + of releases is a no-op +- a published snapshot with the same as-of date but fewer releases is + republished to add history +- when the source has fewer historical months than requested, the publisher + accepts the smaller coverage and does not rebuild on every run + ## Upload Step -After the local build succeeds, upload these two files to the -`ownership-snapshot-current` GitHub release: +Prefer the publisher helper, which implements the protocol above: -- `dist/ownership-snapshot/ownership-snapshot-manifest.json` -- `dist/ownership-snapshot/ownership-snapshot-YYYY-MM-DD.sqlite` +```bash +scripts/publish-ownership-snapshot.sh \ + --idx-bin ./target/debug/idx \ + --output-dir dist/ownership-snapshot \ + --repo 0xrsydn/idx-cli \ + --release-tag ownership-snapshot-current +``` -Only after this manual flow is reliable should the repo automate it in -GitHub Actions. +If you upload by hand, upload these files to the `ownership-snapshot-current` +GitHub release in this order: + +1. `dist/ownership-snapshot/ownership-snapshot-YYYY-MM-DD-.sqlite` +2. `dist/ownership-snapshot/ownership-snapshot-manifest.json` + +Never upload the manifest before its SQLite asset is present. Only after this +manual flow is reliable should the repo automate it in GitHub Actions. ## GitHub Actions Workflow @@ -87,15 +140,20 @@ The repo now includes a manual workflow at Current behavior: - trigger: `workflow_dispatch` only -- builds `idx` inside `nix develop` -- runs `scripts/build-latest-ownership-snapshot.sh` -- uploads the generated files as workflow artifacts -- creates the stable release tag if needed -- uploads the manifest and SQLite asset to that release with `--clobber` +- builds the packaged publisher with `nix build .#ownership-publisher` +- runs `idx-ownership-publish` from that package, which performs discovery, + import, snapshot build, and the full upload protocol in one pass +- uploads any locally produced artifacts as a workflow artifact copy with + `if-no-files-found: ignore`, so a legitimate up-to-date run still succeeds +- does not run a separate release-create or asset-upload step This is intentionally manual-first. Add a schedule only after a few successful publish runs confirm the live source remains stable enough. +The workflow passes its inputs to the shell through environment variables and +routes publication through the same fixed publisher used by self-hosted hosts, +so it cannot drift from the CLI protocol. + ## Self-Hosted Automation GitHub-hosted Actions are still not sufficient for this job on their own. Live diff --git a/docs/OWNERSHIP_SELF_HOSTED.md b/docs/OWNERSHIP_SELF_HOSTED.md index d41cf9b..74c9443 100644 --- a/docs/OWNERSHIP_SELF_HOSTED.md +++ b/docs/OWNERSHIP_SELF_HOSTED.md @@ -50,14 +50,30 @@ The publisher is idempotent and prints exactly one final line: | Last line | Exit | Meaning | | --- | --- | --- | -| `RESULT: published ` | 0 | a newer snapshot was uploaded | -| `RESULT: up-to-date ` | 0 | the published snapshot already has the latest report; nothing uploaded | -| `RESULT: FAILED stage= (exit N)` | N | failed in `arguments`, `build`, `preflight`, `discover`, `build-snapshot`, `stage-output`, or `upload` | +| `RESULT: published ` | 0 | a newer snapshot, or a larger history, was uploaded | +| `RESULT: up-to-date ` | 0 | the published snapshot already has the latest report and requested history; nothing uploaded | +| `RESULT: FAILED stage= (exit N)` | N | failed in `arguments`, `build`, `preflight`, `discover`, `build-snapshot`, `stage-output`, `upload`, or `publish` | -It checks the published manifest first and, for XLSX sources, compares as-of -dates before downloading anything, so it is cheap to run **daily**. `--force` +It checks the published manifest and verifies that the manifest's `download_url` +names this repo and release tag, and that the referenced SQLite asset matches +the manifest SHA-256 and size, before it skips anything. It uses GitHub's +`sha256:` digest metadata when available and downloads legacy assets without a +digest to hash them. A missing or corrupt asset, a mismatched URL, an unreadable +manifest, or a GitHub authentication/network error is treated as "publish", +never as up to date. The upload path verifies the full SHA-256 of the uploaded +asset before it publishes the manifest. +For XLSX sources it also compares the requested `--history` coverage with the +months the source actually exposes, so a history increase is published once and +a request larger than the available history does not rebuild forever. `--history` +is bounded (maximum 1000) so an enormous value cannot wrap and look already +satisfied. The checks use discovery only, so a daily run stays cheap. `--force` uploads regardless. +The SQLite asset is immutable and content-addressed. The publisher uploads and +verifies it before it uploads the manifest, which is the commit point consumers +follow. A partial upload therefore exits non-zero and is repaired by the next +run. + `idx-ownership-freshness` is the independent alarm: it exits 1 when the published `latest_as_of_date` is older than `--max-age-days` (default 40). The source is monthly and lands 2-3 days after month end, so 40 days only trips diff --git a/docs/OWNERSHIP_SYNC.md b/docs/OWNERSHIP_SYNC.md index cb6d720..725b98a 100644 --- a/docs/OWNERSHIP_SYNC.md +++ b/docs/OWNERSHIP_SYNC.md @@ -42,7 +42,7 @@ Current schema version: `1` "kind": "sqlite", "compression": "none", "version": "2026-02-27", - "download_url": "https://example.com/ownership-snapshot-2026-02-27.sqlite", + "download_url": "https://example.com/ownership-snapshot-2026-02-27-.sqlite", "sqlite_sha256": "<64-char hex sha256>", "size_bytes": 123456, "release_count": 2, @@ -56,7 +56,10 @@ Current schema version: `1` Semantics: - `source` is optional provenance metadata describing the IDX/KSEI PDF used to build the snapshot. -- `download_url` points to the SQLite artifact itself. +- `download_url` points to the SQLite artifact itself. The filename is an + implementation detail: published assets are immutable and content-addressed + (`ownership-snapshot-YYYY-MM-DD-.sqlite`), so consumers must always + follow `download_url` and validate it with `sqlite_sha256` and `size_bytes`. - `sqlite_sha256` and `size_bytes` are validated before install. - `latest_*` and `release_count` are validated against the downloaded SQLite contents before replacement. diff --git a/scripts/build-latest-ownership-snapshot.sh b/scripts/build-latest-ownership-snapshot.sh index 9b787b4..9fcff7e 100755 --- a/scripts/build-latest-ownership-snapshot.sh +++ b/scripts/build-latest-ownership-snapshot.sh @@ -86,8 +86,23 @@ if ! [[ "$HISTORY" =~ ^[0-9]+$ ]]; then echo "--history must be a non-negative integer" >&2 exit 2 fi -# Canonicalize so "08" is 8, not an invalid octal literal in (( )). -HISTORY=$((10#$HISTORY)) +# Strip leading zeros, then bound the value before arithmetic. A raw value +# with thousands of digits would wrap negative in $((10#$HISTORY)). +HISTORY_DIGITS="${HISTORY#"${HISTORY%%[!0]*}"}" +MAX_HISTORY=1000 +if [[ -z "$HISTORY_DIGITS" ]]; then + HISTORY=0 +else + if (( ${#HISTORY_DIGITS} > 6 )); then + echo "--history must be at most $MAX_HISTORY" >&2 + exit 2 + fi + HISTORY=$((10#$HISTORY_DIGITS)) + if (( HISTORY > MAX_HISTORY )); then + echo "--history must be at most $MAX_HISTORY" >&2 + exit 2 + fi +fi if [[ -z "$BASE_URL" ]]; then BASE_URL="https://github.com/${REPO_FULL_NAME}/releases/download/${RELEASE_TAG}" @@ -264,4 +279,4 @@ mv "$TMP_MANIFEST_PATH" "$MANIFEST_PATH" printf 'Prepared release-ready snapshot from %s (%s)\n' \ "$DISCOVERED_PDF_URL" "$IMPORTED_AS_OF_DATE" printf 'Manifest URL target: %s/ownership-snapshot-manifest.json\n' "$BASE_URL" -printf 'SQLite URL target: %s/ownership-snapshot-%s.sqlite\n' "$BASE_URL" "$IMPORTED_AS_OF_DATE" +printf 'SQLite URL target: %s\n' "$(jq -r '.snapshot.download_url // empty' "$MANIFEST_PATH")" diff --git a/scripts/build-ownership-snapshot.sh b/scripts/build-ownership-snapshot.sh index 9c1fabb..997b70b 100755 --- a/scripts/build-ownership-snapshot.sh +++ b/scripts/build-ownership-snapshot.sh @@ -90,13 +90,19 @@ if [[ -z "$LATEST_AS_OF" || -z "$LATEST_RELEASE_SHA" ]]; then exit 1 fi -ARTIFACT_NAME="ownership-snapshot-${LATEST_AS_OF}.sqlite" -ARTIFACT_PATH="$OUTPUT_DIR/$ARTIFACT_NAME" MANIFEST_PATH="$OUTPUT_DIR/ownership-snapshot-manifest.json" +TMP_ARTIFACT_PATH="$OUTPUT_DIR/.ownership-snapshot-${LATEST_AS_OF}.sqlite.tmp.$$" +# Remove a partial copy if the script fails before the atomic rename below. +trap 'rm -f "$TMP_ARTIFACT_PATH"' EXIT +cp "$DB_PATH" "$TMP_ARTIFACT_PATH" -cp "$DB_PATH" "$ARTIFACT_PATH" +# Immutable, content-addressed name. The publisher relies on this to upload and +# verify the SQLite asset before it publishes the manifest that references it. +SQLITE_SHA256="$(sha256_file "$TMP_ARTIFACT_PATH")" +ARTIFACT_NAME="ownership-snapshot-${LATEST_AS_OF}-${SQLITE_SHA256}.sqlite" +ARTIFACT_PATH="$OUTPUT_DIR/$ARTIFACT_NAME" +mv "$TMP_ARTIFACT_PATH" "$ARTIFACT_PATH" -SQLITE_SHA256="$(sha256_file "$ARTIFACT_PATH")" SIZE_BYTES="$(wc -c < "$ARTIFACT_PATH" | tr -d ' ')" GENERATED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" diff --git a/scripts/publish-ownership-snapshot-test.sh b/scripts/publish-ownership-snapshot-test.sh new file mode 100755 index 0000000..a8ca808 --- /dev/null +++ b/scripts/publish-ownership-snapshot-test.sh @@ -0,0 +1,673 @@ +#!/usr/bin/env bash +# Offline regression tests for scripts/publish-ownership-snapshot.sh. +# +# Uses a fake `gh`, a fake `idx`, and a stub builder. No live network is touched. +# The test exercises the publication protocol (content-addressed SQLite uploaded +# and verified before the manifest commit point), the history-aware no-op logic, +# and the RESULT contract. +set -Eeuo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work="$(mktemp -d "${TMPDIR:-/tmp}/idx-publish-test.XXXXXX")" +state="$work/state" +bin_dir="$work/bin" +scripts_dir="$work/scripts" +out_dir="$work/out" +publisher="$scripts_dir/publish-ownership-snapshot.sh" + +cleanup() { + rm -rf "$work" +} +trap cleanup EXIT + +failures=0 +pass() { echo "ok - $1"; } +fail() { + echo "FAIL - $1" >&2 + failures=$((failures + 1)) +} + +mkdir -p "$bin_dir" "$scripts_dir" "$out_dir" +cp "$repo_root/scripts/publish-ownership-snapshot.sh" "$publisher" +chmod +x "$publisher" + +# --- fake gh --------------------------------------------------------------- +cat >"$bin_dir/gh" <<'FAKE_GH' +#!/usr/bin/env bash +set -euo pipefail +state="${FAKE_GH_STATE:?}" +remote="$state/remote" +assets="$remote/assets" +mkdir -p "$assets" + +if [[ -n "${FAKE_GH_FAIL_ALL:-}" ]]; then + echo "fake gh: simulated network/auth failure" >&2 + exit 1 +fi + +file_sha() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +cmd="${1:-} ${2:-}" +case "$cmd" in + "release download") + shift 2 + pattern="" + output="" + while [[ $# -gt 0 ]]; do + case "$1" in + --pattern) pattern="$2"; shift 2 ;; + --output) output="$2"; shift 2 ;; + --repo) shift 2 ;; + --clobber) shift ;; + *) shift ;; + esac + done + [[ -n "$pattern" ]] || pattern="ownership-snapshot-manifest.json" + if [[ ! -f "$assets/$pattern" ]]; then + exit 1 + fi + if [[ "$output" == "-" || -z "$output" ]]; then + cat "$assets/$pattern" + else + cp "$assets/$pattern" "$output" + fi + ;; + "release view") + if [[ ! -f "$remote/.release-exists" ]]; then + exit 1 + fi + if [[ "$*" == *"--json assets"* ]]; then + printf '{"assets":[' + first=1 + for f in "$assets"/*; do + [[ -e "$f" ]] || continue + name="$(basename "$f")" + size="$(wc -c <"$f" | tr -d ' ')" + if [[ "$first" == "1" ]]; then first=0; else printf ','; fi + if [[ -n "${FAKE_GH_EMIT_DIGEST:-}" ]]; then + printf '{"name":"%s","size":%s,"digest":"sha256:%s"}' "$name" "$size" "$(file_sha "$f")" + else + printf '{"name":"%s","size":%s}' "$name" "$size" + fi + done + printf ']}' + else + exit 0 + fi + ;; + "release create") + touch "$remote/.release-exists" + ;; + "release upload") + shift 3 + files=() + while [[ $# -gt 0 ]]; do + case "$1" in + --repo) shift 2 ;; + --clobber) shift ;; + *) files+=("$1"); shift ;; + esac + done + for f in "${files[@]}"; do + name="$(basename "$f")" + if [[ -n "${FAKE_GH_FAIL_UPLOAD_ALWAYS:-}" && "$name" == *"$FAKE_GH_FAIL_UPLOAD_ALWAYS"* ]]; then + echo "fake gh: upload refused for $name" >&2 + exit 1 + fi + if [[ -n "${FAKE_GH_FAIL_UPLOAD_ONCE:-}" && "$name" == *"$FAKE_GH_FAIL_UPLOAD_ONCE"* && ! -f "$state/fail-once-used" ]]; then + touch "$state/fail-once-used" + echo "fake gh: simulated one-time upload failure for $name" >&2 + exit 1 + fi + mkdir -p "$assets" + cp "$f" "$assets/$name" + if [[ -n "${FAKE_GH_CORRUPT_UPLOAD_MATCH:-}" && "$name" == *"$FAKE_GH_CORRUPT_UPLOAD_MATCH"* ]]; then + # Flip the first byte but keep the length, to model silent upload corruption. + first_byte="$(head -c 1 "$assets/$name")" + if [[ "$first_byte" == "Z" ]]; then repl="Y"; else repl="Z"; fi + printf '%s' "$repl" | dd of="$assets/$name" bs=1 count=1 conv=notrunc status=none + fi + done + ;; + *) + echo "fake gh: unexpected command: $*" >&2 + exit 99 + ;; +esac +FAKE_GH +chmod +x "$bin_dir/gh" + +# --- fake idx -------------------------------------------------------------- +cat >"$bin_dir/idx" <<'FAKE_IDX' +#!/usr/bin/env bash +set -euo pipefail +if [[ "${1:-}" == "version" ]]; then + exit 0 +fi +if [[ "$*" == *"ownership discover"* ]]; then + printf '%s\n' "${FAKE_DISCOVER_JSON:?FAKE_DISCOVER_JSON not set}" + exit 0 +fi +echo "fake idx: unexpected args: $*" >&2 +exit 99 +FAKE_IDX +chmod +x "$bin_dir/idx" + +# --- stub builder ---------------------------------------------------------- +cat >"$scripts_dir/build-latest-ownership-snapshot.sh" <<'STUB_BUILDER' +#!/usr/bin/env bash +set -euo pipefail +out="" +history="0" +repo="example/repo" +tag="ownership-snapshot-current" +while [[ $# -gt 0 ]]; do + case "$1" in + --output-dir) out="$2"; shift 2 ;; + --history) history="$2"; shift 2 ;; + --repo) repo="$2"; shift 2 ;; + --release-tag) tag="$2"; shift 2 ;; + --idx-bin) shift 2 ;; + --keep-workdir) shift ;; + *) shift ;; + esac +done +[[ -n "$out" ]] || { echo "stub builder: missing --output-dir" >&2; exit 2; } +printf 'history=%s\n' "$history" >>"${FAKE_BUILDER_CALLS:?}" + +as_of="${FAKE_BUILD_AS_OF:-2026-08-31}" +if [[ -n "${FAKE_BUILD_RELEASE_COUNT:-}" ]]; then + count="$FAKE_BUILD_RELEASE_COUNT" +else + avail="${FAKE_AVAILABLE_HISTORY:-$history}" + if (( history < avail )); then + count=$(( 1 + history )) + else + count=$(( 1 + avail )) + fi +fi +variant="${FAKE_BUILD_VARIANT:-}" + +tmp="$out/.stub-sqlite.tmp" +printf 'stub-sqlite as_of=%s count=%s variant=%s' "$as_of" "$count" "$variant" >"$tmp" +if command -v sha256sum >/dev/null 2>&1; then + sha="$(sha256sum "$tmp" | awk '{print $1}')" +else + sha="$(shasum -a 256 "$tmp" | awk '{print $1}')" +fi +size="$(wc -c <"$tmp" | tr -d ' ')" +name="ownership-snapshot-${as_of}-${sha}.sqlite" +mv "$tmp" "$out/$name" +url="https://github.com/${repo}/releases/download/${tag}/${name}" +cat >"$out/ownership-snapshot-manifest.json" </dev/null || true + +PUBLISH_OUT="" +PUBLISH_STATUS="" +PUBLISH_LAST="" + +run_publisher() { + local out status + set +e + out="$(PATH="$bin_dir:$PATH" "$publisher" \ + --idx-bin "$bin_dir/idx" \ + --output-dir "$out_dir" \ + --repo example/repo \ + --release-tag ownership-snapshot-current "$@" 2>&1)" + status=$? + set -e + PUBLISH_OUT="$out" + PUBLISH_STATUS="$status" + PUBLISH_LAST="$(printf '%s\n' "$out" | tail -n 1)" +} + +reset_remote() { + rm -rf "$state/remote" "$state/builder-calls" "$state/fail-once-used" + mkdir -p "$state/remote/assets" + rm -rf "$out_dir" + mkdir -p "$out_dir" +} + +builder_calls() { + if [[ -f "$state/builder-calls" ]]; then + wc -l <"$state/builder-calls" | tr -d ' ' + else + echo 0 + fi +} + +asset_count() { + find "$state/remote/assets" -type f -name '*.sqlite' 2>/dev/null | wc -l | tr -d ' ' +} + +remote_manifest_field() { + jq -r "$1" "$state/remote/assets/ownership-snapshot-manifest.json" 2>/dev/null || true +} + +remote_asset_exists() { + [[ -f "$state/remote/assets/$1" ]] +} + +# Flip the first byte of a remote asset but keep its length. +corrupt_remote_asset_same_length() { + local name="$1" + local path="$state/remote/assets/$name" + local first_byte repl + first_byte="$(head -c 1 "$path")" + if [[ "$first_byte" == "Z" ]]; then repl="Y"; else repl="Z"; fi + printf '%s' "$repl" | dd of="$path" bs=1 count=1 conv=notrunc status=none +} + +sha256_of() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +publish_ok() { + local label="$1" expected="$2" + if [[ "$PUBLISH_STATUS" == "0" && "$PUBLISH_LAST" == "$expected" ]]; then + pass "$label" + else + fail "$label (status=$PUBLISH_STATUS last='$PUBLISH_LAST')" + printf '%s\n' "$PUBLISH_OUT" >&2 + fi +} + +expect_contains() { + local label="$1" haystack="$2" needle="$3" + if [[ "$haystack" == *"$needle"* ]]; then + pass "$label" + else + fail "$label (missing '$needle')" + printf '%s\n' "$haystack" >&2 + fi +} + +# =========================================================================== +# 1. successful publish, then a cheap no-op on the next run +# =========================================================================== +reset_remote +run_publisher +publish_ok "publishes the first snapshot" "RESULT: published 2026-08-31" +if [[ "$(builder_calls)" == "1" ]]; then + pass "first publish builds once" +else + fail "first publish builds once (calls=$(builder_calls))" +fi +if remote_asset_exists "$(remote_manifest_field '.snapshot.download_url' | sed 's#.*/##')"; then + pass "published manifest points at a present asset" +else + fail "published manifest points at a present asset" +fi + +run_publisher +publish_ok "second run is up-to-date without rebuilding" "RESULT: up-to-date 2026-08-31" +if [[ "$(builder_calls)" == "1" ]]; then + pass "no-op run skips the builder" +else + fail "no-op run skips the builder (calls=$(builder_calls))" +fi + +# =========================================================================== +# 2. partial SQLite upload fails, then a retry repairs the publication +# =========================================================================== +reset_remote +export FAKE_GH_FAIL_UPLOAD_ONCE=".sqlite" +run_publisher +if [[ "$PUBLISH_STATUS" != "0" ]] && [[ "$PUBLISH_LAST" == "RESULT: FAILED stage=upload"* ]]; then + pass "partial SQLite upload failure is reported and exits non-zero" +else + fail "partial SQLite upload failure is reported (status=$PUBLISH_STATUS last='$PUBLISH_LAST')" +fi +if ! remote_asset_exists "ownership-snapshot-manifest.json"; then + pass "manifest is not published while the SQLite asset is missing" +else + fail "manifest is not published while the SQLite asset is missing" +fi + +unset FAKE_GH_FAIL_UPLOAD_ONCE +run_publisher +publish_ok "retry publishes after a partial upload" "RESULT: published 2026-08-31" + +# =========================================================================== +# 3. manifest upload fails after SQLite succeeds, then a retry repairs it +# =========================================================================== +reset_remote +export FAKE_GH_FAIL_UPLOAD_ONCE="ownership-snapshot-manifest.json" +run_publisher +if [[ "$PUBLISH_STATUS" != "0" ]] && [[ "$PUBLISH_LAST" == "RESULT: FAILED stage=publish"* ]]; then + pass "manifest upload failure is reported and exits non-zero" +else + fail "manifest upload failure is reported (status=$PUBLISH_STATUS last='$PUBLISH_LAST')" +fi +if [[ "$(asset_count)" == "1" ]]; then + pass "SQLite artifact from the failed manifest publish is retained" +else + fail "SQLite artifact from the failed manifest publish is retained (count=$(asset_count))" +fi + +unset FAKE_GH_FAIL_UPLOAD_ONCE +run_publisher +publish_ok "retry publishes the manifest after a manifest failure" "RESULT: published 2026-08-31" + +# =========================================================================== +# 4. forced same-date refresh preserves the previous artifact +# =========================================================================== +reset_remote +run_publisher +old_asset="$(remote_manifest_field '.snapshot.download_url' | sed 's#.*/##')" +export FAKE_BUILD_VARIANT="replacement" +run_publisher --force +publish_ok "forced same-date refresh publishes" "RESULT: published 2026-08-31" +unset FAKE_BUILD_VARIANT +new_asset="$(remote_manifest_field '.snapshot.download_url' | sed 's#.*/##')" +if [[ "$old_asset" != "$new_asset" ]] && remote_asset_exists "$old_asset" && remote_asset_exists "$new_asset"; then + pass "forced refresh keeps the previous content-addressed artifact" +else + fail "forced refresh keeps the previous content-addressed artifact (old=$old_asset new=$new_asset count=$(asset_count))" +fi + +# =========================================================================== +# 5. a trustworthy-looking manifest with a missing asset forces a republish +# =========================================================================== +reset_remote +mkdir -p "$state/remote/assets" +touch "$state/remote/.release-exists" +cat >"$state/remote/assets/ownership-snapshot-manifest.json" <<'JSON' +{"schema_version":1,"generated_at":"2026-08-31T00:00:00Z","snapshot":{"kind":"sqlite","compression":"none","version":"2026-08-31","download_url":"https://github.com/example/repo/releases/download/ownership-snapshot-current/ownership-snapshot-2026-08-31-missing.sqlite","sqlite_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","size_bytes":999,"release_count":1,"latest_as_of_date":"2026-08-31","latest_release_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","latest_row_count":10,"ticker_count":5}} +JSON +run_publisher +publish_ok "missing referenced asset forces a republish" "RESULT: published 2026-08-31" +if remote_asset_exists "$(remote_manifest_field '.snapshot.download_url' | sed 's#.*/##')"; then + pass "republished manifest points at a present asset" +else + fail "republished manifest points at a present asset" +fi + +# =========================================================================== +# 5b. same-size corruption of a legacy (digest-less) asset forces republish +# =========================================================================== +reset_remote +run_publisher +published_asset="$(remote_manifest_field '.snapshot.download_url' | sed 's#.*/##')" +corrupt_remote_asset_same_length "$published_asset" +run_publisher +publish_ok "same-size corruption of a legacy asset forces republish" "RESULT: published 2026-08-31" +republished_asset="$(remote_manifest_field '.snapshot.download_url' | sed 's#.*/##')" +if [[ "$(sha256_of "$state/remote/assets/$republished_asset")" == "$(remote_manifest_field '.snapshot.sqlite_sha256')" ]]; then + pass "republished legacy asset matches the manifest sha256" +else + fail "republished legacy asset matches the manifest sha256" +fi + +# =========================================================================== +# 5c. sha256 digest metadata detects same-size corruption +# =========================================================================== +reset_remote +export FAKE_GH_EMIT_DIGEST="1" +run_publisher +run_publisher +publish_ok "intact digest-verified asset no-ops" "RESULT: up-to-date 2026-08-31" +published_asset="$(remote_manifest_field '.snapshot.download_url' | sed 's#.*/##')" +corrupt_remote_asset_same_length "$published_asset" +run_publisher +publish_ok "same-size corruption with digest metadata forces republish" "RESULT: published 2026-08-31" +unset FAKE_GH_EMIT_DIGEST + +# =========================================================================== +# 5d. upload corruption cannot commit the manifest (digest and digest-less) +# =========================================================================== +reset_remote +export FAKE_GH_CORRUPT_UPLOAD_MATCH=".sqlite" +export FAKE_GH_EMIT_DIGEST="1" +run_publisher +if [[ "$PUBLISH_STATUS" != "0" ]] && [[ "$PUBLISH_LAST" == "RESULT: FAILED stage=upload"* ]]; then + pass "digest-verified upload corruption is rejected" +else + fail "digest-verified upload corruption is rejected (status=$PUBLISH_STATUS last='$PUBLISH_LAST')" +fi +if ! remote_asset_exists "ownership-snapshot-manifest.json"; then + pass "corrupt upload does not commit a manifest (digest mode)" +else + fail "corrupt upload does not commit a manifest (digest mode)" +fi +unset FAKE_GH_EMIT_DIGEST +reset_remote +export FAKE_GH_CORRUPT_UPLOAD_MATCH=".sqlite" +run_publisher +if [[ "$PUBLISH_STATUS" != "0" ]] && [[ "$PUBLISH_LAST" == "RESULT: FAILED stage=upload"* ]]; then + pass "digest-less upload corruption is rejected after fetch and hash" +else + fail "digest-less upload corruption is rejected (status=$PUBLISH_STATUS last='$PUBLISH_LAST')" +fi +if ! remote_asset_exists "ownership-snapshot-manifest.json"; then + pass "corrupt upload does not commit a manifest (digest-less mode)" +else + fail "corrupt upload does not commit a manifest (digest-less mode)" +fi +unset FAKE_GH_CORRUPT_UPLOAD_MATCH + +# =========================================================================== +# 5e. a manifest whose URL names a different repo/tag is not trusted +# =========================================================================== +reset_remote +run_publisher +asset_name="$(remote_manifest_field '.snapshot.download_url' | sed 's#.*/##')" +manifest_path="$state/remote/assets/ownership-snapshot-manifest.json" +jq --arg url "https://github.com/other/repo/releases/download/other-tag/$asset_name" \ + '.snapshot.download_url = $url' "$manifest_path" >"$manifest_path.tmp" +mv "$manifest_path.tmp" "$manifest_path" +run_publisher +publish_ok "manifest URL for a different repo/tag is not trusted" "RESULT: published 2026-08-31" + +# =========================================================================== +# 6. legacy PDF discovery ends in a post-build no-op +# =========================================================================== +reset_remote +run_publisher +calls_before="$(builder_calls)" +export FAKE_DISCOVER_JSON='[{"status":"supported","format":"pdf","as_of_date":null,"pdf_url":"https://example.invalid/legacy.pdf"}]' +export FAKE_BUILD_RELEASE_COUNT="1" +run_publisher +publish_ok "legacy PDF source still no-ops after build" "RESULT: up-to-date 2026-08-31" +if [[ "$(builder_calls)" == "$((calls_before + 1))" ]]; then + pass "legacy PDF source runs the builder once" +else + fail "legacy PDF source runs the builder once (before=$calls_before after=$(builder_calls))" +fi +unset FAKE_BUILD_RELEASE_COUNT +export FAKE_DISCOVER_JSON='[{"status":"supported","format":"xlsx","as_of_date":"2026-08-31"}]' + +# =========================================================================== +# 7. a larger --history upgrades an existing same-date snapshot +# =========================================================================== +reset_remote +run_publisher +export FAKE_AVAILABLE_HISTORY="6" +export FAKE_DISCOVER_JSON='[ + {"status":"supported","format":"xlsx","as_of_date":"2026-08-31"}, + {"status":"supported","format":"xlsx","as_of_date":"2026-07-31"}, + {"status":"supported","format":"xlsx","as_of_date":"2026-06-30"}, + {"status":"supported","format":"xlsx","as_of_date":"2026-05-31"}, + {"status":"supported","format":"xlsx","as_of_date":"2026-04-30"}, + {"status":"supported","format":"xlsx","as_of_date":"2026-03-31"}, + {"status":"supported","format":"xlsx","as_of_date":"2026-02-28"} +]' +run_publisher --history 5 +publish_ok "history increase republishes the same date" "RESULT: published 2026-08-31" +if [[ "$(remote_manifest_field '.snapshot.release_count')" == "6" ]]; then + pass "history increase publishes the requested coverage" +else + fail "history increase publishes the requested coverage (rc=$(remote_manifest_field '.snapshot.release_count'))" +fi +export FAKE_AVAILABLE_HISTORY="0" +export FAKE_DISCOVER_JSON='[{"status":"supported","format":"xlsx","as_of_date":"2026-08-31"}]' + +# =========================================================================== +# 8. limited available history does not rebuild forever +# =========================================================================== +reset_remote +export FAKE_AVAILABLE_HISTORY="2" +export FAKE_DISCOVER_JSON='[ + {"status":"supported","format":"xlsx","as_of_date":"2026-08-31"}, + {"status":"supported","format":"xlsx","as_of_date":"2026-07-31"}, + {"status":"supported","format":"xlsx","as_of_date":"2026-06-30"} +]' +run_publisher --history 2 +publish_ok "limited-history baseline publishes" "RESULT: published 2026-08-31" +calls_before="$(builder_calls)" +run_publisher --history 5 +publish_ok "requesting more than available history no-ops" "RESULT: up-to-date 2026-08-31" +if [[ "$(builder_calls)" == "$calls_before" ]]; then + pass "limited available history does not rebuild repeatedly" +else + fail "limited available history does not rebuild repeatedly (before=$calls_before after=$(builder_calls))" +fi +export FAKE_AVAILABLE_HISTORY="0" +export FAKE_DISCOVER_JSON='[{"status":"supported","format":"xlsx","as_of_date":"2026-08-31"}]' + +# =========================================================================== +# 9. invalid arguments fail before an early exit +# =========================================================================== +reset_remote +run_publisher +calls_before="$(builder_calls)" +run_publisher --history abc +if [[ "$PUBLISH_STATUS" == "2" ]] && [[ "$PUBLISH_LAST" == "RESULT: FAILED stage=arguments (exit 2)" ]]; then + pass "invalid --history fails even when the snapshot is current" +else + fail "invalid --history fails on early exit (status=$PUBLISH_STATUS last='$PUBLISH_LAST')" +fi +if [[ "$(builder_calls)" == "$calls_before" ]]; then + pass "invalid arguments do not run the builder" +else + fail "invalid arguments do not run the builder" +fi + +run_publisher --history 1001 +if [[ "$PUBLISH_STATUS" == "2" ]] && [[ "$PUBLISH_LAST" == "RESULT: FAILED stage=arguments (exit 2)" ]]; then + pass "over-max --history is rejected" +else + fail "over-max --history is rejected (status=$PUBLISH_STATUS last='$PUBLISH_LAST')" +fi + +run_publisher --history 999999999999999999999999999999 +if [[ "$PUBLISH_STATUS" == "2" ]] && [[ "$PUBLISH_LAST" == "RESULT: FAILED stage=arguments (exit 2)" ]]; then + pass "enormous --history is rejected without wrapping negative" +else + fail "enormous --history is rejected (status=$PUBLISH_STATUS last='$PUBLISH_LAST')" +fi + +run_publisher --history 1000 +publish_ok "boundary --history 1000 is accepted" "RESULT: up-to-date 2026-08-31" + +# =========================================================================== +# 9b. --keep-workdir keeps RESULT as the final output line +# =========================================================================== +reset_remote +run_publisher --keep-workdir +if [[ "$PUBLISH_LAST" == "RESULT: published 2026-08-31" ]]; then + pass "--keep-workdir keeps RESULT as the final line" +else + fail "--keep-workdir keeps RESULT as the final line (last='$PUBLISH_LAST')" +fi +expect_contains "--keep-workdir announces the retained path" "$PUBLISH_OUT" "Keeping publish workdir:" +after_result="$(printf '%s\n' "$PUBLISH_OUT" | awk '/^RESULT:/{seen=1; next} seen{print}')" +if [[ -z "$after_result" ]]; then + pass "--keep-workdir prints nothing after RESULT" +else + fail "--keep-workdir prints nothing after RESULT (trailing='$after_result')" +fi + +# =========================================================================== +# 10. auth/network errors never become a trustworthy up-to-date +# =========================================================================== +reset_remote +export FAKE_GH_FAIL_ALL="1" +run_publisher +unset FAKE_GH_FAIL_ALL +if [[ "$PUBLISH_LAST" != *"up-to-date"* ]] && [[ "$PUBLISH_STATUS" != "0" ]]; then + pass "gh failure is reported, not converted into up-to-date" +else + fail "gh failure is reported, not converted into up-to-date (status=$PUBLISH_STATUS last='$PUBLISH_LAST')" +fi + +# =========================================================================== +# 11. build-ownership-snapshot.sh emits a content-addressed, self-consistent pair +# =========================================================================== +if command -v sqlite3 >/dev/null 2>&1; then + content_dir="$work/content-addressed" + mkdir -p "$content_dir" + db="$content_dir/source.db" + sqlite3 "$db" <<'SQL' +CREATE TABLE ownership_releases (as_of_date TEXT, sha256 TEXT, row_count INTEGER, imported_at INTEGER); +CREATE TABLE tickers (code TEXT); +INSERT INTO ownership_releases VALUES ('2026-08-31', 'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', 10, 1); +INSERT INTO tickers VALUES ('BBCA'); +SQL + set +e + "$repo_root/scripts/build-ownership-snapshot.sh" \ + --db "$db" \ + --output-dir "$content_dir/out" \ + --base-url "https://example.invalid/base" >"$content_dir/build.log" 2>&1 + build_status=$? + set -e + artifacts=("$content_dir/out"/ownership-snapshot-*.sqlite) + if [[ "$build_status" == "0" && "${#artifacts[@]}" == "1" ]]; then + artifact_name="$(basename "${artifacts[0]}")" + artifact_sha="$(jq -r '.snapshot.sqlite_sha256' "$content_dir/out/ownership-snapshot-manifest.json")" + artifact_url="$(jq -r '.snapshot.download_url' "$content_dir/out/ownership-snapshot-manifest.json")" + if [[ "$artifact_name" =~ ^ownership-snapshot-2026-08-31-[0-9a-f]{64}\.sqlite$ ]] \ + && [[ "$artifact_name" == *"$artifact_sha"* ]] \ + && [[ "$artifact_url" == "https://example.invalid/base/$artifact_name" ]]; then + pass "builder emits a content-addressed, self-consistent artifact pair" + else + fail "builder emits a content-addressed pair (name=$artifact_name sha=$artifact_sha url=$artifact_url)" + fi + else + fail "builder emits one artifact (status=$build_status count=${#artifacts[@]})" + cat "$content_dir/build.log" >&2 + fi +else + echo "skip - sqlite3 not found; content-addressed builder check skipped" >&2 +fi + +# A matching asset basename must not authorize a broken nested download URL. +reset_remote +run_publisher +remote_manifest="$state/remote/assets/ownership-snapshot-manifest.json" +jq '.snapshot.download_url |= sub("/ownership-snapshot-current/"; "/ownership-snapshot-current/not-an-asset/")' \ + "$remote_manifest" > "$state/nested-manifest.json" +cp "$state/nested-manifest.json" "$remote_manifest" +run_publisher +publish_ok "nested download URL cannot authorize a no-op" "RESULT: published 2026-08-31" + +if ((failures > 0)); then + echo "publish-ownership-snapshot tests: ${failures} failure(s)" >&2 + exit 1 +fi +echo "publish-ownership-snapshot tests: all passed" diff --git a/scripts/publish-ownership-snapshot.sh b/scripts/publish-ownership-snapshot.sh index 68b0cbf..0f13f7c 100755 --- a/scripts/publish-ownership-snapshot.sh +++ b/scripts/publish-ownership-snapshot.sh @@ -10,8 +10,17 @@ Build and publish the latest supported IDX/KSEI ownership snapshot to the stable GitHub release used by `idx ownership sync`. Safe to run daily: unless --force is given, it skips the upload when the -published snapshot already has the latest as-of date. The last line is always -one of: +published snapshot already has the latest as-of date and at least the requested +historical coverage. The published manifest is trusted only when the SQLite +artifact it references is at the configured repo/tag and matches the manifest +SHA-256 and size. + +The immutable, content-addressed SQLite artifact is uploaded and verified +before the manifest is published, and the manifest is the final commit point. +A partial upload therefore never leaves consumers following a manifest whose +artifact is missing, and a retry repairs the publication. + +The last line is always one of: RESULT: published RESULT: up-to-date RESULT: FAILED stage= (exit ) @@ -23,8 +32,9 @@ Options: (default: 0xrsydn/idx-cli) --release-tag Stable GitHub release tag used for snapshot assets (default: ownership-snapshot-current) - --history Also include the previous monthly above-1% reports - in the snapshot (passed to the builder; default: 0) + --history Request at least previous monthly above-1% reports + in addition to the latest (default: 0). Fewer reports + are accepted when the source only has fewer months. --force Upload even if the published snapshot is up to date --build Run `cargo build` before publishing --keep-workdir Keep the temp workdir created by the builder helper @@ -59,6 +69,22 @@ on_error() { printf 'RESULT: FAILED stage=%s (exit %s)\n' "$STAGE" "$status" >&2 exit "$status" } + +cleanup() { + if [[ -z "$PUBLISH_WORKDIR" ]]; then + return + fi + + # Keep the RESULT line as the final line: never print here. The retained + # path is announced when the workdir is created. + if [[ "$KEEP_WORKDIR" == "1" ]]; then + return + fi + + rm -rf "$PUBLISH_WORKDIR" +} + +trap cleanup EXIT trap on_error ERR while [[ $# -gt 0 ]]; do @@ -111,10 +137,51 @@ if [[ -z "$OUTPUT_DIR" ]]; then fail 2 "--output-dir is required" fi +# Validate arguments before any early exit so bad input always fails. +if ! [[ "$HISTORY" =~ ^[0-9]+$ ]]; then + usage >&2 + fail 2 "--history must be a non-negative integer" +fi +# Strip leading zeros, then bound the value before arithmetic. A raw value +# with thousands of digits would wrap negative in $((10#$HISTORY)) and could +# make a history request look already satisfied. +HISTORY_DIGITS="${HISTORY#"${HISTORY%%[!0]*}"}" +MAX_HISTORY=1000 +if [[ -z "$HISTORY_DIGITS" ]]; then + HISTORY=0 +else + if (( ${#HISTORY_DIGITS} > 6 )); then + usage >&2 + fail 2 "--history must be at most $MAX_HISTORY" + fi + HISTORY=$((10#$HISTORY_DIGITS)) + if (( HISTORY > MAX_HISTORY )); then + usage >&2 + fail 2 "--history must be at most $MAX_HISTORY" + fi +fi + +if [[ -z "$IDX_BIN" ]]; then + usage >&2 + fail 2 "--idx-bin must not be empty" +fi +if [[ -z "$REPO_FULL_NAME" ]]; then + usage >&2 + fail 2 "--repo must not be empty" +fi +if [[ -z "$RELEASE_TAG" ]]; then + usage >&2 + fail 2 "--release-tag must not be empty" +fi + if ! command -v gh >/dev/null 2>&1; then fail 1 "gh is required for GitHub release upload" fi +if ! command -v jq >/dev/null 2>&1; then + fail 1 "jq is required for manifest and release metadata processing" +fi + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BUILDER="$SCRIPT_DIR/build-latest-ownership-snapshot.sh" @@ -125,21 +192,11 @@ fi mkdir -p "$OUTPUT_DIR" PUBLISH_WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/idx-ownership-publish.XXXXXX")" -cleanup() { - if [[ -z "$PUBLISH_WORKDIR" ]]; then - return - fi +if [[ "$KEEP_WORKDIR" == "1" ]]; then + printf 'Keeping publish workdir: %s\n' "$PUBLISH_WORKDIR" +fi - if [[ "$KEEP_WORKDIR" == "1" ]]; then - printf 'Kept publish workdir: %s\n' "$PUBLISH_WORKDIR" - return - fi - - rm -rf "$PUBLISH_WORKDIR" -} -trap cleanup EXIT - -# Published manifest, or empty when none exists yet. +# Published manifest, or empty when none exists or it cannot be read. published_manifest() { gh release download "$RELEASE_TAG" \ --repo "$REPO_FULL_NAME" \ @@ -147,6 +204,122 @@ published_manifest() { --output - 2>/dev/null || true } +# Release assets as JSON. Fails when the release or the GitHub API is not +# readable; callers must treat that as "not up to date", never as current. +remote_assets_json() { + gh release view "$RELEASE_TAG" \ + --repo "$REPO_FULL_NAME" \ + --json assets 2>/dev/null +} + +# Size of one release asset in bytes, or empty when it is not present. +remote_asset_size() { + local assets_json="$1" + local asset_name="$2" + jq -r --arg name "$asset_name" \ + '.assets[]? | select(.name == $name) | .size' \ + <<< "$assets_json" 2>/dev/null || true +} + +sha256_file() { + local path="$1" + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$path" | awk '{print $1}' + return 0 + fi + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$path" | awk '{print $1}' + return 0 + fi + fail 1 "sha256sum/shasum not found in PATH" +} + +# Require the exact asset URL. A matching basename under an extra path is +# not a valid GitHub release download URL. +published_url_matches() { + local url="$1" + local asset_name="${url##*/}" + [[ "$asset_name" =~ ^[A-Za-z0-9._-]+\.sqlite$ ]] || return 1 + [[ "$url" == "https://github.com/${REPO_FULL_NAME}/releases/download/${RELEASE_TAG}/${asset_name}" ]] +} + +# Full SHA-256 of a release asset. Prefer GitHub's sha256 digest metadata and +# fall back to downloading the asset for legacy releases that predate it. +# Prints the lowercase hex digest, or returns non-zero when unavailable. +remote_asset_sha256() { + local assets_json="$1" + local asset_name="$2" + local digest hex tmp result + + digest="$(jq -r --arg name "$asset_name" \ + '.assets[]? | select(.name == $name) | (.digest // empty)' \ + <<< "$assets_json" 2>/dev/null || true)" + if [[ "$digest" == sha256:* ]]; then + hex="${digest#sha256:}" + hex="$(printf '%s' "$hex" | tr 'A-F' 'a-f')" + if [[ "$hex" =~ ^[0-9a-f]{64}$ ]]; then + printf '%s' "$hex" + return 0 + fi + fi + + tmp="$PUBLISH_WORKDIR/.verify-${asset_name}.$$" + if gh release download "$RELEASE_TAG" \ + --repo "$REPO_FULL_NAME" \ + --pattern "$asset_name" \ + --output "$tmp" \ + --clobber >/dev/null 2>&1; then + result="$(sha256_file "$tmp")" + rm -f "$tmp" + printf '%s' "$result" + return 0 + fi + rm -f "$tmp" + return 1 +} + +# Populate LATEST_SOURCE_AS_OF and DESIRED_RELEASE_COUNT from discovery. +# DESIRED_RELEASE_COUNT is 1 + min(--history, available XLSX months), so a job +# asked for more history than the source has does not rebuild forever. +# Returns non-zero when discovery is unusable; callers then build. +compute_requested_coverage() { + local json latest available wanted + if ! json="$("$IDX_BIN" -o json ownership discover --family above1 --limit 50 2>/dev/null)"; then + return 1 + fi + + latest="$( + jq -r ' + if type == "array" and length > 0 and .[0].status == "supported" + then (.[0].as_of_date // empty) else empty end + ' <<< "$json" 2>/dev/null || true + )" + if ! [[ "$latest" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then + return 1 + fi + + available="$( + jq -r --arg latest "$latest" ' + [ .[] + | select(.status == "supported" and .format == "xlsx" + and .as_of_date != null and .as_of_date < $latest) + ] + | map(.as_of_date) | unique | length + ' <<< "$json" 2>/dev/null || true + )" + if ! [[ "$available" =~ ^[0-9]+$ ]]; then + return 1 + fi + + wanted="$HISTORY" + if (( wanted > available )); then + wanted="$available" + fi + LATEST_SOURCE_AS_OF="$latest" + DESIRED_RELEASE_COUNT=$(( 1 + wanted )) + return 0 +} + STAGE="build" if [[ "$BUILD_FIRST" == "1" ]]; then printf 'Building idx...\n' @@ -158,23 +331,56 @@ if ! "$IDX_BIN" version >/dev/null 2>&1; then fail 1 "failed to run idx binary: $IDX_BIN; build the CLI first or pass --idx-bin " fi -# Best effort: a missing or unreadable published manifest means "publish". +# Best effort read. A missing or unreadable manifest means "publish"; it is +# trusted for a no-op only after the referenced asset is verified. PUBLISHED_MANIFEST="$(published_manifest)" PUBLISHED_AS_OF="$(jq -r '.snapshot.latest_as_of_date // empty' <<< "$PUBLISHED_MANIFEST" 2>/dev/null || true)" +PUBLISHED_RELEASE_COUNT="$(jq -r '.snapshot.release_count // empty' <<< "$PUBLISHED_MANIFEST" 2>/dev/null || true)" +PUBLISHED_DOWNLOAD_URL="$(jq -r '.snapshot.download_url // empty' <<< "$PUBLISHED_MANIFEST" 2>/dev/null || true)" +PUBLISHED_SIZE_BYTES="$(jq -r '.snapshot.size_bytes // empty' <<< "$PUBLISHED_MANIFEST" 2>/dev/null || true)" +PUBLISHED_SHA256="$(jq -r '.snapshot.sqlite_sha256 // empty' <<< "$PUBLISHED_MANIFEST" 2>/dev/null || true)" +PUBLISHED_ASSET_NAME="${PUBLISHED_DOWNLOAD_URL##*/}" +PUBLISHED_TRUSTWORTHY="0" -if [[ "$FORCE" != "1" && -n "$PUBLISHED_AS_OF" ]]; then +if [[ "$PUBLISHED_AS_OF" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] \ + && [[ "$PUBLISHED_RELEASE_COUNT" =~ ^[0-9]+$ ]] \ + && [[ "$PUBLISHED_SIZE_BYTES" =~ ^[0-9]+$ ]] \ + && [[ "$PUBLISHED_SHA256" =~ ^[0-9a-fA-F]{64}$ ]] \ + && [[ -n "$PUBLISHED_ASSET_NAME" ]] \ + && published_url_matches "$PUBLISHED_DOWNLOAD_URL"; then + if PUBLISHED_ASSETS_JSON="$(remote_assets_json)"; then + published_asset_size="$(remote_asset_size "$PUBLISHED_ASSETS_JSON" "$PUBLISHED_ASSET_NAME")" + if [[ "$published_asset_size" == "$PUBLISHED_SIZE_BYTES" ]]; then + published_asset_sha="$(remote_asset_sha256 "$PUBLISHED_ASSETS_JSON" "$PUBLISHED_ASSET_NAME" 2>/dev/null)" || published_asset_sha="" + if [[ "$published_asset_sha" == "$PUBLISHED_SHA256" ]]; then + PUBLISHED_TRUSTWORTHY="1" + fi + fi + fi +fi + +LATEST_SOURCE_AS_OF="" +DESIRED_RELEASE_COUNT="" +if [[ "$FORCE" != "1" && "$PUBLISHED_TRUSTWORTHY" == "1" ]]; then STAGE="discover" - # Cheap pre-check: XLSX reports carry their as-of date, so an unchanged - # month is detected without downloading or building anything. - LATEST_AS_OF="$( - "$IDX_BIN" -o json ownership discover --family above1 --limit 1 | - jq -r '.[0] | select(.status == "supported") | .as_of_date // empty' - )" - if [[ -n "$LATEST_AS_OF" && ! "$LATEST_AS_OF" > "$PUBLISHED_AS_OF" ]]; then - printf 'Published snapshot %s is current (latest source report: %s).\n' \ - "$PUBLISHED_AS_OF" "$LATEST_AS_OF" - printf 'RESULT: up-to-date %s\n' "$PUBLISHED_AS_OF" - exit 0 + if compute_requested_coverage; then + if [[ "$LATEST_SOURCE_AS_OF" < "$PUBLISHED_AS_OF" ]]; then + printf 'Published snapshot %s is newer than the latest source report %s; nothing to do.\n' \ + "$PUBLISHED_AS_OF" "$LATEST_SOURCE_AS_OF" + printf 'RESULT: up-to-date %s\n' "$PUBLISHED_AS_OF" + exit 0 + fi + if [[ "$LATEST_SOURCE_AS_OF" == "$PUBLISHED_AS_OF" ]] \ + && (( DESIRED_RELEASE_COUNT <= PUBLISHED_RELEASE_COUNT )); then + printf 'Published snapshot %s is current (latest source report: %s, releases: %s).\n' \ + "$PUBLISHED_AS_OF" "$LATEST_SOURCE_AS_OF" "$PUBLISHED_RELEASE_COUNT" + printf 'RESULT: up-to-date %s\n' "$PUBLISHED_AS_OF" + exit 0 + fi + if [[ "$LATEST_SOURCE_AS_OF" == "$PUBLISHED_AS_OF" ]]; then + printf 'Published snapshot %s has %s release(s) but history=%s needs %s; rebuilding.\n' \ + "$PUBLISHED_AS_OF" "$PUBLISHED_RELEASE_COUNT" "$HISTORY" "$DESIRED_RELEASE_COUNT" + fi fi fi @@ -201,7 +407,6 @@ fi shopt -s nullglob sqlite_matches=("$PUBLISH_WORKDIR"/ownership-snapshot-*.sqlite) -existing_snapshot_paths=("$OUTPUT_DIR"/ownership-snapshot-*.sqlite) shopt -u nullglob if [[ "${#sqlite_matches[@]}" -ne 1 ]]; then @@ -209,23 +414,33 @@ if [[ "${#sqlite_matches[@]}" -ne 1 ]]; then fi STAGED_SQLITE_PATH="${sqlite_matches[0]}" -BUILT_AS_OF="$(jq -r '.snapshot.latest_as_of_date // empty' "$STAGED_MANIFEST_PATH")" +BUILT_AS_OF="$(jq -r '.snapshot.latest_as_of_date // empty' "$STAGED_MANIFEST_PATH" 2>/dev/null || true)" if ! [[ "$BUILT_AS_OF" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then fail 1 "built manifest has no valid snapshot.latest_as_of_date: '$BUILT_AS_OF'" fi +BUILT_RELEASE_COUNT="$(jq -r '.snapshot.release_count // empty' "$STAGED_MANIFEST_PATH" 2>/dev/null || true)" +if ! [[ "$BUILT_RELEASE_COUNT" =~ ^[0-9]+$ ]] || (( BUILT_RELEASE_COUNT < 1 )); then + fail 1 "built manifest has no valid snapshot.release_count: '$BUILT_RELEASE_COUNT'" +fi # Post-build check covers legacy PDF sources, whose as-of date is only known -# after import. -if [[ "$FORCE" != "1" && -n "$PUBLISHED_AS_OF" && ! "$BUILT_AS_OF" > "$PUBLISHED_AS_OF" ]]; then - printf 'Built snapshot %s is not newer than published %s; skipping upload.\n' \ - "$BUILT_AS_OF" "$PUBLISHED_AS_OF" - printf 'RESULT: up-to-date %s\n' "$PUBLISHED_AS_OF" - exit 0 +# after import, and confirms that a history-only change is worth publishing. +if [[ "$FORCE" != "1" && "$PUBLISHED_TRUSTWORTHY" == "1" ]]; then + if [[ "$BUILT_AS_OF" < "$PUBLISHED_AS_OF" ]] \ + || { [[ "$BUILT_AS_OF" == "$PUBLISHED_AS_OF" ]] && (( BUILT_RELEASE_COUNT <= PUBLISHED_RELEASE_COUNT )); }; then + printf 'Built snapshot %s (releases: %s) is not newer than published %s (releases: %s); skipping upload.\n' \ + "$BUILT_AS_OF" "$BUILT_RELEASE_COUNT" "$PUBLISHED_AS_OF" "$PUBLISHED_RELEASE_COUNT" + printf 'RESULT: up-to-date %s\n' "$PUBLISHED_AS_OF" + exit 0 + fi fi STAGE="stage-output" rm -f "$OUTPUT_DIR/ownership-snapshot-manifest.json" +shopt -s nullglob +existing_snapshot_paths=("$OUTPUT_DIR"/ownership-snapshot-*.sqlite) +shopt -u nullglob if [[ "${#existing_snapshot_paths[@]}" -gt 0 ]]; then rm -f "${existing_snapshot_paths[@]}" fi @@ -235,6 +450,16 @@ cp "$STAGED_SQLITE_PATH" "$OUTPUT_DIR/" MANIFEST_PATH="$OUTPUT_DIR/ownership-snapshot-manifest.json" SQLITE_PATH="$OUTPUT_DIR/$(basename "$STAGED_SQLITE_PATH")" +SQLITE_ASSET_NAME="$(basename "$STAGED_SQLITE_PATH")" +LOCAL_SQLITE_BYTES="$(wc -c < "$SQLITE_PATH" | tr -d ' ')" +LOCAL_SQLITE_SHA="$(jq -r '.snapshot.sqlite_sha256 // empty' "$MANIFEST_PATH" 2>/dev/null || true)" +if ! [[ "$LOCAL_SQLITE_SHA" =~ ^[0-9a-fA-F]{64}$ ]]; then + fail 1 "staged manifest has no valid snapshot.sqlite_sha256: '$LOCAL_SQLITE_SHA'" +fi +LOCAL_FILE_SHA="$(sha256_file "$SQLITE_PATH")" +if [[ "$LOCAL_FILE_SHA" != "$LOCAL_SQLITE_SHA" ]]; then + fail 1 "staged SQLite file hash $LOCAL_FILE_SHA does not match manifest sqlite_sha256 $LOCAL_SQLITE_SHA" +fi STAGE="upload" if gh release view "$RELEASE_TAG" --repo "$REPO_FULL_NAME" >/dev/null 2>&1; then @@ -254,13 +479,42 @@ else --latest=false fi -printf 'Uploading snapshot assets to %s...\n' "$RELEASE_TAG" -gh release upload "$RELEASE_TAG" \ - "$MANIFEST_PATH" \ - "$SQLITE_PATH" \ +# Upload the immutable SQLite artifact first. The content-addressed name makes a +# retry idempotent and keeps the previous valid artifact on --force replacements. +REMOTE_ASSETS_JSON="$(remote_assets_json)" || fail 1 "failed to read release assets for $RELEASE_TAG" +existing_size="$(remote_asset_size "$REMOTE_ASSETS_JSON" "$SQLITE_ASSET_NAME")" +existing_sha="$(remote_asset_sha256 "$REMOTE_ASSETS_JSON" "$SQLITE_ASSET_NAME" 2>/dev/null)" || existing_sha="" +if [[ "$existing_size" == "$LOCAL_SQLITE_BYTES" && "$existing_sha" == "$LOCAL_SQLITE_SHA" ]]; then + printf 'SQLite asset already present and verified: %s\n' "$SQLITE_ASSET_NAME" +else + printf 'Uploading SQLite asset %s...\n' "$SQLITE_ASSET_NAME" + gh release upload "$RELEASE_TAG" "$SQLITE_PATH" \ + --repo "$REPO_FULL_NAME" \ + --clobber +fi + +# Do not publish the manifest until the exact artifact is visible in the +# release and its full SHA-256 matches the staged manifest. +REMOTE_ASSETS_JSON="$(remote_assets_json)" || fail 1 "failed to re-read release assets for $RELEASE_TAG" +verified_size="$(remote_asset_size "$REMOTE_ASSETS_JSON" "$SQLITE_ASSET_NAME")" +verified_sha="$(remote_asset_sha256 "$REMOTE_ASSETS_JSON" "$SQLITE_ASSET_NAME" 2>/dev/null)" || verified_sha="" +if [[ "$verified_size" != "$LOCAL_SQLITE_BYTES" || "$verified_sha" != "$LOCAL_SQLITE_SHA" ]]; then + fail 1 "SQLite asset integrity verification failed: expected $SQLITE_ASSET_NAME sha256=$LOCAL_SQLITE_SHA size=$LOCAL_SQLITE_BYTES in $RELEASE_TAG, found sha256='${verified_sha:-missing}' size='${verified_size:-missing}'" +fi + +# The manifest is the consumer commit point, so it is published last. +STAGE="publish" +printf 'Publishing manifest ownership-snapshot-manifest.json...\n' +gh release upload "$RELEASE_TAG" "$MANIFEST_PATH" \ --repo "$REPO_FULL_NAME" \ --clobber +REMOTE_MANIFEST="$(published_manifest)" +REMOTE_MANIFEST_SHA="$(jq -r '.snapshot.sqlite_sha256 // empty' <<< "$REMOTE_MANIFEST" 2>/dev/null || true)" +if [[ -z "$LOCAL_SQLITE_SHA" || "$REMOTE_MANIFEST_SHA" != "$LOCAL_SQLITE_SHA" ]]; then + fail 1 "published manifest verification failed: remote sqlite_sha256='${REMOTE_MANIFEST_SHA}' expected '$LOCAL_SQLITE_SHA'" +fi + printf 'Published manifest: https://github.com/%s/releases/download/%s/ownership-snapshot-manifest.json\n' \ "$REPO_FULL_NAME" "$RELEASE_TAG" printf 'Published SQLite: %s\n' "$SQLITE_PATH"