From 1f6f0ee02e490f97bccfd20f4aeef9c826fa6b61 Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 24 Sep 2026 21:40:02 +0000 Subject: [PATCH 1/2] feat(ownership): idempotent, packaged snapshot publisher The self-hosted publish timer has not produced an upload since it was enabled (manifest last updated 2026-07-14, a manual run), and failures were invisible. Make the job safe to run daily and easy to deploy: - publish-ownership-snapshot.sh: skip the upload when the published manifest already has the latest as-of date (pre-check via discover for XLSX sources, post-build check for legacy PDF); --force to override. Always ends with one line: RESULT: published|up-to-date or RESULT: FAILED stage= (exit N). gh release create no longer requires a git checkout. - check-ownership-snapshot-freshness.sh: exit 1 when the published latest_as_of_date is older than --max-age-days (default 40). - flake: packages.ownership-publisher bundles the scripts with a pinned idx (curl-impersonate + mupdf) and jq/gh/sqlite/curl as bin/idx-ownership-publish and bin/idx-ownership-freshness, so hosts need no checkout, nix develop or cargo build at run time. Built in CI. - contrib/systemd + docs: daily 06:00 UTC timer, run semantics table, freshness alarm, packaged usage. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E --- .github/workflows/ci.yml | 5 ++ .../idx-ownership-snapshot-publish.service | 5 +- .../idx-ownership-snapshot-publish.timer | 6 +- docs/OWNERSHIP_SELF_HOSTED.md | 85 ++++++++++--------- flake.nix | 48 +++++++++++ scripts/check-ownership-snapshot-freshness.sh | 62 ++++++++++++++ scripts/publish-ownership-snapshot.sh | 74 +++++++++++++++- 7 files changed, 239 insertions(+), 46 deletions(-) create mode 100755 scripts/check-ownership-snapshot-freshness.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a47a2ab..ecb6a09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,11 @@ jobs: - name: Run default app run: nix run .#default -- version + - name: Build ownership publisher + run: | + nix build .#ownership-publisher + ./result/bin/idx-ownership-publish --help + install-script: runs-on: ${{ matrix.os }} strategy: diff --git a/contrib/systemd/idx-ownership-snapshot-publish.service b/contrib/systemd/idx-ownership-snapshot-publish.service index 0e7351f..8064e24 100644 --- a/contrib/systemd/idx-ownership-snapshot-publish.service +++ b/contrib/systemd/idx-ownership-snapshot-publish.service @@ -5,7 +5,8 @@ After=network-online.target [Service] Type=oneshot -WorkingDirectory=/srv/idx-cli Environment=HOME=/var/lib/idx-ownership-snapshot EnvironmentFile=/etc/idx-ownership-snapshot.env -ExecStart=/bin/sh -lc 'nix develop --command scripts/publish-ownership-snapshot.sh --build --idx-bin ./target/debug/idx --output-dir /var/lib/idx-ownership-snapshot/current --repo 0xrsydn/idx-cli --release-tag ownership-snapshot-current' +# Build once with `nix build github:0xrsydn/idx-cli#ownership-publisher -o /opt/idx-ownership-publisher`. +# Idempotent: exits 0 with "RESULT: up-to-date" when nothing newer exists. +ExecStart=/opt/idx-ownership-publisher/bin/idx-ownership-publish --output-dir /var/lib/idx-ownership-snapshot/current --repo 0xrsydn/idx-cli --release-tag ownership-snapshot-current --history 5 diff --git a/contrib/systemd/idx-ownership-snapshot-publish.timer b/contrib/systemd/idx-ownership-snapshot-publish.timer index 17b127e..7d7b13a 100644 --- a/contrib/systemd/idx-ownership-snapshot-publish.timer +++ b/contrib/systemd/idx-ownership-snapshot-publish.timer @@ -1,8 +1,10 @@ [Unit] -Description=Run monthly idx ownership snapshot publish job +Description=Daily idx ownership snapshot publish check [Timer] -OnCalendar=*-*-02 09:00:00 +# The publisher is idempotent, so a daily run retries automatically when IDX +# posts late or a run is blocked; most days end with "RESULT: up-to-date". +OnCalendar=*-*-* 06:00:00 UTC Persistent=true RandomizedDelaySec=30m Unit=idx-ownership-snapshot-publish.service diff --git a/docs/OWNERSHIP_SELF_HOSTED.md b/docs/OWNERSHIP_SELF_HOSTED.md index b51962a..9530c6b 100644 --- a/docs/OWNERSHIP_SELF_HOSTED.md +++ b/docs/OWNERSHIP_SELF_HOSTED.md @@ -22,9 +22,44 @@ simple: No Nix setup is required for end users. Nix is only an implementation detail for the maintainer host if that host already uses NixOS. -## Manual Command +## Packaged Publisher (recommended) -From a checked-out repo with GitHub auth already configured: +The flake exposes a self-contained publisher, so the host needs no checkout, +`nix develop`, or cargo build at run time: + +```bash +nix build github:0xrsydn/idx-cli#ownership-publisher +./result/bin/idx-ownership-publish --output-dir /var/lib/idx/ownership-snapshot/current --history 5 +./result/bin/idx-ownership-freshness --max-age-days 40 +``` + +`idx-ownership-publish` wraps `scripts/publish-ownership-snapshot.sh` with a +pinned `idx` (itself wrapped with curl-impersonate and mupdf) plus jq, gh, +sqlite and curl. On NixOS, add this repo as a flake input and run +`inputs.idx-cli.packages.${system}.ownership-publisher`; bump the input +(`nix flake update idx-cli`) to pick up fixes. + +## Run Semantics + +The publisher is idempotent and prints exactly one final line: + +| Last line | Exit | Meaning | +| --- | --- | --- | +| `RESULT: published ` | 0 | a newer snapshot was uploaded | +| `RESULT: up-to-date ` | 0 | the published snapshot already has the latest report; nothing uploaded | +| `RESULT: FAILED stage= (exit N)` | N | failed in `check-published`, `discover`, `build-snapshot`, `stage-output`, or `upload` | + +It checks the published manifest first and, for XLSX sources, compares as-of +dates before downloading anything, so it is cheap to run **daily**. `--force` +uploads regardless. + +`idx-ownership-freshness` is the independent alarm: it exits 1 when the +published `latest_as_of_date` is older than `--max-age-days` (default 40). The +source is monthly and lands 2-3 days after month end, so 40 days only trips +when publishing has actually stopped working. Wire both to an `OnFailure=` +notification. + +## Manual Command (from a checkout) ```bash nix develop --command scripts/publish-ownership-snapshot.sh \ @@ -35,47 +70,17 @@ nix develop --command scripts/publish-ownership-snapshot.sh \ --release-tag ownership-snapshot-current ``` -That helper: +## systemd Units -1. optionally builds `idx` -2. discovers the latest supported IDX/KSEI above-1% report (XLSX on the IDX - Data Kepemilikan Saham page since the 2026-05-29 report; PDF announcements - before that) -3. imports it into an isolated temp DB -4. emits the SQLite snapshot and manifest -5. ensures the stable GitHub release exists -6. uploads the manifest and SQLite asset with `--clobber` +Examples: `contrib/systemd/idx-ownership-snapshot-publish.service` and +`contrib/systemd/idx-ownership-snapshot-publish.timer` (daily at 06:00 UTC). -## systemd Service +Observed XLSX upload times (HTTP Last-Modified): 2026-06-03, +2026-07-02 03:55Z, 2026-08-02 01:50Z, 2026-09-02 03:51Z. With a daily, +idempotent run the exact day no longer matters. -Example service: [contrib/systemd/idx-ownership-snapshot-publish.service](/Users/rasyidanakbar/Development/myApp/idx-cli/contrib/systemd/idx-ownership-snapshot-publish.service) - -Important assumptions: - -- the repo checkout lives at `/srv/idx-cli` -- a writable publish directory exists at `/var/lib/idx-ownership-snapshot/current` -- `GH_TOKEN` is provided via an env file such as `/etc/idx-ownership-snapshot.env` -- the host can run `nix develop` - -## systemd Timer - -Example timer: [contrib/systemd/idx-ownership-snapshot-publish.timer](/Users/rasyidanakbar/Development/myApp/idx-cli/contrib/systemd/idx-ownership-snapshot-publish.timer) - -The sample timer uses: - -- `OnCalendar=*-*-02 09:00:00` -- `Persistent=true` -- `RandomizedDelaySec=30m` - -That is intentionally conservative. The ownership source is monthly, but the -exact publish day can drift. Observed XLSX upload times (HTTP Last-Modified): -2026-06-03, 2026-07-02 03:55Z, 2026-08-02 01:50Z, 2026-09-02 03:51Z, so a run -on the 2nd can occasionally land before the file exists; the 3rd is safer. -Start with an early-month schedule and adjust after observing a few real runs. - -Pass `--history ` (for example via `services.idxOwnershipPublish.extraArgs = -[ "--history" "5" ];` in `clan-private`) to include earlier months so -`idx ownership changes` works straight from the synced snapshot. +Pass `--history ` to include earlier months so `idx ownership changes` +works straight from the synced snapshot (about 2.4 MB per extra month). ## Clan Integration Later diff --git a/flake.nix b/flake.nix index ee09631..1fc7352 100644 --- a/flake.nix +++ b/flake.nix @@ -48,14 +48,62 @@ --prefix PATH : "${lib.makeBinPath runtimeDeps}" ''; }; + # Self-contained ownership snapshot publisher for scheduled hosts + # (e.g. a NixOS systemd timer): no checkout, `nix develop` or cargo + # build at run time. Wraps the repo scripts with a pinned `idx`. + publisherTools = with pkgs; [ + bash + coreutils + curl + gawk + gh + gnugrep + gnused + jq + sqlite + ]; + ownershipPublisher = pkgs.stdenvNoCC.mkDerivation { + pname = "idx-ownership-publisher"; + version = cargoManifest.package.version; + src = lib.fileset.toSource { + root = ./.; + fileset = lib.fileset.unions [ + ./scripts/build-ownership-snapshot.sh + ./scripts/build-latest-ownership-snapshot.sh + ./scripts/publish-ownership-snapshot.sh + ./scripts/check-ownership-snapshot-freshness.sh + ]; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + dontBuild = true; + installPhase = '' + runHook preInstall + mkdir -p "$out/libexec/idx-ownership" "$out/bin" + cp scripts/*.sh "$out/libexec/idx-ownership/" + chmod +x "$out"/libexec/idx-ownership/*.sh + patchShebangs "$out/libexec/idx-ownership" + + makeWrapper "$out/libexec/idx-ownership/publish-ownership-snapshot.sh" \ + "$out/bin/idx-ownership-publish" \ + --prefix PATH : "${lib.makeBinPath ([ idxPackage ] ++ publisherTools)}" \ + --add-flags "--idx-bin ${idxPackage}/bin/idx" + makeWrapper "$out/libexec/idx-ownership/check-ownership-snapshot-freshness.sh" \ + "$out/bin/idx-ownership-freshness" \ + --prefix PATH : "${lib.makeBinPath publisherTools}" + runHook postInstall + ''; + meta.mainProgram = "idx-ownership-publish"; + }; in { packages.default = idxPackage; + packages.ownership-publisher = ownershipPublisher; apps.default = { type = "app"; program = "${idxPackage}/bin/idx"; }; checks.default = idxPackage; + checks.ownership-publisher = ownershipPublisher; devShells.default = pkgs.mkShell { inputsFrom = [ idxPackage ]; diff --git a/scripts/check-ownership-snapshot-freshness.sh b/scripts/check-ownership-snapshot-freshness.sh new file mode 100755 index 0000000..16752e7 --- /dev/null +++ b/scripts/check-ownership-snapshot-freshness.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Fail when the published ownership snapshot is older than expected, so a +# publish job that keeps failing (or keeps finding nothing new) gets noticed. + +set -euo pipefail + +usage() { + cat <<'USAGE' +Usage: scripts/check-ownership-snapshot-freshness.sh [options] + +Options: + --manifest-url Published manifest (default: the ownership-snapshot-current + release of 0xrsydn/idx-cli) + --max-age-days Allowed age of latest_as_of_date in days (default: 40; + the source is monthly and lands 2-3 days after month end) + --now Reference date instead of today (for testing) + --help Show this help + +Exit status: 0 fresh, 1 stale, 2 usage error, 3 manifest unreadable. +USAGE +} + +MANIFEST_URL="https://github.com/0xrsydn/idx-cli/releases/download/ownership-snapshot-current/ownership-snapshot-manifest.json" +MAX_AGE_DAYS="40" +NOW="$(date -u +%F)" + +while [[ $# -gt 0 ]]; do + case "$1" in + --manifest-url) MANIFEST_URL="${2:-}"; shift 2 ;; + --max-age-days) MAX_AGE_DAYS="${2:-}"; shift 2 ;; + --now) NOW="${2:-}"; shift 2 ;; + --help|-h) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac +done + +if ! [[ "$MAX_AGE_DAYS" =~ ^[0-9]+$ ]]; then + echo "--max-age-days must be a non-negative integer" >&2 + exit 2 +fi + +if [[ "$MANIFEST_URL" =~ ^https?:// ]]; then + manifest="$(curl --fail --silent --show-error --location --retry 3 "$MANIFEST_URL")" || { + echo "FRESHNESS: unreadable manifest $MANIFEST_URL" >&2 + exit 3 + } +else + manifest="$(cat "$MANIFEST_URL")" || exit 3 +fi + +as_of="$(jq -r '.snapshot.latest_as_of_date // empty' <<< "$manifest" 2>/dev/null || true)" +if [[ -z "$as_of" ]]; then + echo "FRESHNESS: manifest has no snapshot.latest_as_of_date" >&2 + exit 3 +fi + +age_days=$(( ($(date -u -d "$NOW" +%s) - $(date -u -d "$as_of" +%s)) / 86400 )) +if (( age_days > 10#$MAX_AGE_DAYS )); then + printf 'FRESHNESS: stale latest_as_of_date=%s age_days=%d max=%s\n' "$as_of" "$age_days" "$MAX_AGE_DAYS" >&2 + exit 1 +fi +printf 'FRESHNESS: ok latest_as_of_date=%s age_days=%d max=%s\n' "$as_of" "$age_days" "$MAX_AGE_DAYS" diff --git a/scripts/publish-ownership-snapshot.sh b/scripts/publish-ownership-snapshot.sh index 6bbc60d..4761c5b 100755 --- a/scripts/publish-ownership-snapshot.sh +++ b/scripts/publish-ownership-snapshot.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -euo pipefail +set -Eeuo pipefail usage() { cat <<'EOF' @@ -9,6 +9,13 @@ Usage: scripts/publish-ownership-snapshot.sh --output-dir [options] Build and publish the latest supported IDX/KSEI ownership snapshot to the stable GitHub release used by `idx ownership sync`. +Safe to run daily: unless --force is given, it skips the upload when the +published snapshot already has the latest as-of date. The last line is always +one of: + RESULT: published + RESULT: up-to-date + RESULT: FAILED stage= (exit ) + Options: --idx-bin idx binary to use (default: ./target/debug/idx) --output-dir Directory to write the copied snapshot and manifest @@ -18,6 +25,7 @@ Options: (default: ownership-snapshot-current) --history Also include the previous monthly above-1% reports in the snapshot (passed to the builder; default: 0) + --force Upload even if the published snapshot is up to date --build Run `cargo build` before publishing --keep-workdir Keep the temp workdir created by the builder helper --help Show this help @@ -31,7 +39,16 @@ RELEASE_TAG="ownership-snapshot-current" BUILD_FIRST="0" KEEP_WORKDIR="0" HISTORY="0" +FORCE="0" PUBLISH_WORKDIR="" +STAGE="arguments" + +on_error() { + local status=$? + printf 'RESULT: FAILED stage=%s (exit %s)\n' "$STAGE" "$status" >&2 + exit "$status" +} +trap on_error ERR while [[ $# -gt 0 ]]; do case "$1" in @@ -55,6 +72,10 @@ while [[ $# -gt 0 ]]; do HISTORY="${2:-}" shift 2 ;; + --force) + FORCE="1" + shift + ;; --build) BUILD_FIRST="1" shift @@ -111,6 +132,15 @@ cleanup() { } trap cleanup EXIT +# Published manifest, or empty when none exists yet. +published_manifest() { + gh release download "$RELEASE_TAG" \ + --repo "$REPO_FULL_NAME" \ + --pattern ownership-snapshot-manifest.json \ + --output - 2>/dev/null || true +} + +STAGE="build" if [[ "$BUILD_FIRST" == "1" ]]; then printf 'Building idx...\n' cargo build @@ -122,6 +152,27 @@ if ! "$IDX_BIN" version >/dev/null 2>&1; then exit 1 fi +STAGE="check-published" +PUBLISHED_MANIFEST="$(published_manifest)" +PUBLISHED_AS_OF="$(jq -r '.snapshot.latest_as_of_date // empty' <<< "$PUBLISHED_MANIFEST" 2>/dev/null || true)" + +if [[ "$FORCE" != "1" && -n "$PUBLISHED_AS_OF" ]]; then + STAGE="discover" + # Cheap pre-check: XLSX reports carry their as-of date, so an unchanged + # month is detected without downloading or building anything. + LATEST_AS_OF="$( + "$IDX_BIN" -o json ownership discover --family above1 --limit 1 | + jq -r '.[0] | select(.status == "supported") | .as_of_date // empty' + )" + if [[ -n "$LATEST_AS_OF" && ! "$LATEST_AS_OF" > "$PUBLISHED_AS_OF" ]]; then + printf 'Published snapshot %s is current (latest source report: %s).\n' \ + "$PUBLISHED_AS_OF" "$LATEST_AS_OF" + printf 'RESULT: up-to-date %s\n' "$PUBLISHED_AS_OF" + exit 0 + fi +fi + +STAGE="build-snapshot" build_args=( --idx-bin "$IDX_BIN" --output-dir "$PUBLISH_WORKDIR" @@ -154,6 +205,18 @@ if [[ "${#sqlite_matches[@]}" -ne 1 ]]; then fi STAGED_SQLITE_PATH="${sqlite_matches[0]}" +BUILT_AS_OF="$(jq -r '.snapshot.latest_as_of_date' "$STAGED_MANIFEST_PATH")" + +# Post-build check covers legacy PDF sources, whose as-of date is only known +# after import. +if [[ "$FORCE" != "1" && -n "$PUBLISHED_AS_OF" && ! "$BUILT_AS_OF" > "$PUBLISHED_AS_OF" ]]; then + printf 'Built snapshot %s is not newer than published %s; skipping upload.\n' \ + "$BUILT_AS_OF" "$PUBLISHED_AS_OF" + printf 'RESULT: up-to-date %s\n' "$PUBLISHED_AS_OF" + exit 0 +fi + +STAGE="stage-output" rm -f "$OUTPUT_DIR/ownership-snapshot-manifest.json" if [[ "${#existing_snapshot_paths[@]}" -gt 0 ]]; then @@ -166,13 +229,19 @@ cp "$STAGED_SQLITE_PATH" "$OUTPUT_DIR/" MANIFEST_PATH="$OUTPUT_DIR/ownership-snapshot-manifest.json" SQLITE_PATH="$OUTPUT_DIR/$(basename "$STAGED_SQLITE_PATH")" +STAGE="upload" if gh release view "$RELEASE_TAG" --repo "$REPO_FULL_NAME" >/dev/null 2>&1; then printf 'Release %s already exists.\n' "$RELEASE_TAG" else printf 'Creating stable snapshot release %s...\n' "$RELEASE_TAG" + target_args=() + # Packaged runs (Nix store) have no git checkout; let GitHub pick the default branch. + if git_head="$(git rev-parse HEAD 2>/dev/null)"; then + target_args=(--target "$git_head") + fi gh release create "$RELEASE_TAG" \ --repo "$REPO_FULL_NAME" \ - --target "$(git rev-parse HEAD)" \ + "${target_args[@]}" \ --title "Ownership Snapshot Current" \ --notes "Stable release for idx ownership snapshot artifacts consumed by \`idx ownership sync\`." \ --latest=false @@ -188,3 +257,4 @@ gh release upload "$RELEASE_TAG" \ printf 'Published manifest: https://github.com/%s/releases/download/%s/ownership-snapshot-manifest.json\n' \ "$REPO_FULL_NAME" "$RELEASE_TAG" printf 'Published SQLite: %s\n' "$SQLITE_PATH" +printf 'RESULT: published %s\n' "$BUILT_AS_OF" From 6628d3b2e9d65b8219e5479c0af6a365ace44cbf Mon Sep 17 00:00:00 2001 From: hermes Date: Thu, 24 Sep 2026 21:50:51 +0000 Subject: [PATCH 2/2] fix(ownership): make publisher RESULT contract hold on every path Addresses review on #29: - drop errtrace: a failure inside $(...) printed two RESULT lines - explicit failures go through fail(), which always prints RESULT: FAILED stage=...; idx check gets its own `preflight` stage; the never-failing published-manifest read is no longer a stage - validate the built latest_as_of_date (`// empty` + date regex) so a corrupt manifest cannot compare as "newer" - freshness: validate --now (exit 2) and latest_as_of_date (exit 3) before calling date - flake: package an explicit script list instead of scripts/*.sh - docs: GH_TOKEN prerequisite for the packaged publisher Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E --- docs/OWNERSHIP_SELF_HOSTED.md | 7 ++- flake.nix | 16 ++++--- scripts/check-ownership-snapshot-freshness.sh | 8 +++- scripts/publish-ownership-snapshot.sh | 43 +++++++++++-------- 4 files changed, 46 insertions(+), 28 deletions(-) diff --git a/docs/OWNERSHIP_SELF_HOSTED.md b/docs/OWNERSHIP_SELF_HOSTED.md index 9530c6b..d41cf9b 100644 --- a/docs/OWNERSHIP_SELF_HOSTED.md +++ b/docs/OWNERSHIP_SELF_HOSTED.md @@ -33,6 +33,11 @@ nix build github:0xrsydn/idx-cli#ownership-publisher ./result/bin/idx-ownership-freshness --max-age-days 40 ``` +The publisher needs `GH_TOKEN` in its environment: a token that can create +and upload assets on the target repo's releases (the examples read it from +`/etc/idx-ownership-snapshot.env`; clan-private uses a Clan var). The +freshness check needs no token. + `idx-ownership-publish` wraps `scripts/publish-ownership-snapshot.sh` with a pinned `idx` (itself wrapped with curl-impersonate and mupdf) plus jq, gh, sqlite and curl. On NixOS, add this repo as a flake input and run @@ -47,7 +52,7 @@ The publisher is idempotent and prints exactly one final line: | --- | --- | --- | | `RESULT: published ` | 0 | a newer snapshot was uploaded | | `RESULT: up-to-date ` | 0 | the published snapshot already has the latest report; nothing uploaded | -| `RESULT: FAILED stage= (exit N)` | N | failed in `check-published`, `discover`, `build-snapshot`, `stage-output`, or `upload` | +| `RESULT: FAILED stage= (exit N)` | N | failed in `arguments`, `build`, `preflight`, `discover`, `build-snapshot`, `stage-output`, or `upload` | It checks the published manifest first and, for XLSX sources, compares as-of dates before downloading anything, so it is cheap to run **daily**. `--force` diff --git a/flake.nix b/flake.nix index 1fc7352..7b5d1b1 100644 --- a/flake.nix +++ b/flake.nix @@ -62,24 +62,26 @@ jq sqlite ]; + publisherScripts = [ + ./scripts/build-ownership-snapshot.sh + ./scripts/build-latest-ownership-snapshot.sh + ./scripts/publish-ownership-snapshot.sh + ./scripts/check-ownership-snapshot-freshness.sh + ]; ownershipPublisher = pkgs.stdenvNoCC.mkDerivation { pname = "idx-ownership-publisher"; version = cargoManifest.package.version; src = lib.fileset.toSource { root = ./.; - fileset = lib.fileset.unions [ - ./scripts/build-ownership-snapshot.sh - ./scripts/build-latest-ownership-snapshot.sh - ./scripts/publish-ownership-snapshot.sh - ./scripts/check-ownership-snapshot-freshness.sh - ]; + fileset = lib.fileset.unions publisherScripts; }; nativeBuildInputs = [ pkgs.makeWrapper ]; dontBuild = true; installPhase = '' runHook preInstall mkdir -p "$out/libexec/idx-ownership" "$out/bin" - cp scripts/*.sh "$out/libexec/idx-ownership/" + cp ${lib.concatMapStringsSep " " (script: "scripts/${baseNameOf script}") publisherScripts} \ + "$out/libexec/idx-ownership/" chmod +x "$out"/libexec/idx-ownership/*.sh patchShebangs "$out/libexec/idx-ownership" diff --git a/scripts/check-ownership-snapshot-freshness.sh b/scripts/check-ownership-snapshot-freshness.sh index 16752e7..ab81946 100755 --- a/scripts/check-ownership-snapshot-freshness.sh +++ b/scripts/check-ownership-snapshot-freshness.sh @@ -38,6 +38,10 @@ if ! [[ "$MAX_AGE_DAYS" =~ ^[0-9]+$ ]]; then echo "--max-age-days must be a non-negative integer" >&2 exit 2 fi +if ! [[ "$NOW" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] || ! date -u -d "$NOW" +%s >/dev/null 2>&1; then + echo "--now must be a YYYY-MM-DD date" >&2 + exit 2 +fi if [[ "$MANIFEST_URL" =~ ^https?:// ]]; then manifest="$(curl --fail --silent --show-error --location --retry 3 "$MANIFEST_URL")" || { @@ -49,8 +53,8 @@ else fi as_of="$(jq -r '.snapshot.latest_as_of_date // empty' <<< "$manifest" 2>/dev/null || true)" -if [[ -z "$as_of" ]]; then - echo "FRESHNESS: manifest has no snapshot.latest_as_of_date" >&2 +if ! [[ "$as_of" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] || ! date -u -d "$as_of" +%s >/dev/null 2>&1; then + echo "FRESHNESS: manifest has no valid snapshot.latest_as_of_date ('$as_of')" >&2 exit 3 fi diff --git a/scripts/publish-ownership-snapshot.sh b/scripts/publish-ownership-snapshot.sh index 4761c5b..68b0cbf 100755 --- a/scripts/publish-ownership-snapshot.sh +++ b/scripts/publish-ownership-snapshot.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -Eeuo pipefail +set -euo pipefail usage() { cat <<'EOF' @@ -43,6 +43,17 @@ FORCE="0" PUBLISH_WORKDIR="" STAGE="arguments" +# Explicit failures: print the reason, then the RESULT line, then exit. +fail() { + local status="$1" + shift + printf '%s\n' "$*" >&2 + printf 'RESULT: FAILED stage=%s (exit %s)\n' "$STAGE" "$status" >&2 + exit "$status" +} + +# Unexpected command failures (set -e). errtrace is deliberately off: with +# it, a failure inside $(...) would run this in the subshell and the parent. on_error() { local status=$? printf 'RESULT: FAILED stage=%s (exit %s)\n' "$STAGE" "$status" >&2 @@ -89,30 +100,26 @@ while [[ $# -gt 0 ]]; do exit 0 ;; *) - echo "unknown argument: $1" >&2 usage >&2 - exit 2 + fail 2 "unknown argument: $1" ;; esac done if [[ -z "$OUTPUT_DIR" ]]; then - echo "--output-dir is required" >&2 usage >&2 - exit 2 + fail 2 "--output-dir is required" fi if ! command -v gh >/dev/null 2>&1; then - echo "gh is required for GitHub release upload" >&2 - exit 1 + fail 1 "gh is required for GitHub release upload" fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BUILDER="$SCRIPT_DIR/build-latest-ownership-snapshot.sh" if [[ ! -x "$BUILDER" ]]; then - echo "required helper script is missing or not executable: $BUILDER" >&2 - exit 1 + fail 1 "required helper script is missing or not executable: $BUILDER" fi mkdir -p "$OUTPUT_DIR" @@ -146,13 +153,12 @@ if [[ "$BUILD_FIRST" == "1" ]]; then cargo build fi +STAGE="preflight" if ! "$IDX_BIN" version >/dev/null 2>&1; then - echo "failed to run idx binary: $IDX_BIN" >&2 - echo "build the CLI first or pass --idx-bin " >&2 - exit 1 + fail 1 "failed to run idx binary: $IDX_BIN; build the CLI first or pass --idx-bin " fi -STAGE="check-published" +# Best effort: a missing or unreadable published manifest means "publish". PUBLISHED_MANIFEST="$(published_manifest)" PUBLISHED_AS_OF="$(jq -r '.snapshot.latest_as_of_date // empty' <<< "$PUBLISHED_MANIFEST" 2>/dev/null || true)" @@ -190,8 +196,7 @@ printf 'Preparing latest ownership snapshot artifacts...\n' STAGED_MANIFEST_PATH="$PUBLISH_WORKDIR/ownership-snapshot-manifest.json" if [[ ! -f "$STAGED_MANIFEST_PATH" ]]; then - echo "manifest was not generated: $STAGED_MANIFEST_PATH" >&2 - exit 1 + fail 1 "manifest was not generated: $STAGED_MANIFEST_PATH" fi shopt -s nullglob @@ -200,12 +205,14 @@ existing_snapshot_paths=("$OUTPUT_DIR"/ownership-snapshot-*.sqlite) shopt -u nullglob if [[ "${#sqlite_matches[@]}" -ne 1 ]]; then - echo "expected exactly one SQLite artifact in $PUBLISH_WORKDIR" >&2 - exit 1 + fail 1 "expected exactly one SQLite artifact in $PUBLISH_WORKDIR" fi STAGED_SQLITE_PATH="${sqlite_matches[0]}" -BUILT_AS_OF="$(jq -r '.snapshot.latest_as_of_date' "$STAGED_MANIFEST_PATH")" +BUILT_AS_OF="$(jq -r '.snapshot.latest_as_of_date // empty' "$STAGED_MANIFEST_PATH")" +if ! [[ "$BUILT_AS_OF" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then + fail 1 "built manifest has no valid snapshot.latest_as_of_date: '$BUILT_AS_OF'" +fi # Post-build check covers legacy PDF sources, whose as-of date is only known # after import.