diff --git a/.github/workflows/publish-ownership-snapshot.yml b/.github/workflows/publish-ownership-snapshot.yml index 3204670..179c76e 100644 --- a/.github/workflows/publish-ownership-snapshot.yml +++ b/.github/workflows/publish-ownership-snapshot.yml @@ -26,10 +26,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: Install Nix - uses: cachix/install-nix-action@v30 + uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 - name: Build ownership publisher run: nix build .#ownership-publisher @@ -54,7 +54,7 @@ jobs: - name: Upload workflow artifact copy # A legitimate up-to-date run produces no new local artifacts. - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: ownership-snapshot-${{ github.run_id }} path: dist/ownership-snapshot/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 135a50f..7dc0bdd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,8 +20,9 @@ on: - "scripts/install.js" - "install.sh" +# Read-only by default; only the publish job writes the release. permissions: - contents: write + contents: read env: RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }} @@ -49,12 +50,12 @@ jobs: steps: - name: Check out source - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref }} - name: Install Rust toolchain and target - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable with: targets: ${{ matrix.target }} @@ -83,7 +84,7 @@ jobs: ./idx-mcp-${{ matrix.asset }} --help >/dev/null - name: Upload target artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: idx-${{ matrix.asset }} path: | @@ -96,9 +97,11 @@ jobs: needs: build if: github.event_name != 'pull_request' runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Check out install script - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref }} sparse-checkout: install.sh @@ -106,7 +109,7 @@ jobs: path: src - name: Download target artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: pattern: idx-* merge-multiple: true @@ -119,7 +122,7 @@ jobs: > SHA256SUMS - name: Attach binaries to the version tag - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 with: tag_name: ${{ env.RELEASE_TAG }} files: | diff --git a/install.sh b/install.sh index af4f8b6..91b90de 100755 --- a/install.sh +++ b/install.sh @@ -97,8 +97,11 @@ resolve_latest_version() { fetch "$api_url/repos/$REPO/releases?per_page=30" "$releases" || err "could not list releases; pass --version to skip the lookup" + # One field per line first: on a single-line (compact) response the greedy + # match would otherwise pick the last tag, not the newest. tag="$( - sed -n 's/.*"tag_name"[[:space:]]*:[[:space:]]*"\(v[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\)".*/\1/p' "$releases" | + tr ',' '\n' < "$releases" | + sed -n 's/.*"tag_name"[[:space:]]*:[[:space:]]*"\(v[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\)".*/\1/p' | head -n 1 )" [ -n "$tag" ] || err "no vX.Y.Z release found for $REPO" diff --git a/scripts/audit-msn-fundamentals.sh b/scripts/audit-msn-fundamentals.sh index 593c8f1..b36825e 100755 --- a/scripts/audit-msn-fundamentals.sh +++ b/scripts/audit-msn-fundamentals.sh @@ -130,7 +130,9 @@ resolve_tickers() { done IFS="$old_ifs" else - mapfile -t TICKER_LIST < <(cut -f1 "$ROOT_DIR/src/api/msn/symbol_ids.tsv") + symbol_map="$ROOT_DIR/crates/idx-core/src/api/msn/symbol_ids.tsv" + [[ -f "$symbol_map" ]] || { echo "symbol map not found: $symbol_map" >&2; exit 1; } + mapfile -t TICKER_LIST < <(cut -f1 "$symbol_map") fi if [[ ${#TICKER_LIST[@]} -eq 0 ]]; then diff --git a/scripts/build-latest-ownership-snapshot.sh b/scripts/build-latest-ownership-snapshot.sh index 0c5b434..0641f25 100755 --- a/scripts/build-latest-ownership-snapshot.sh +++ b/scripts/build-latest-ownership-snapshot.sh @@ -163,8 +163,11 @@ DB_PATH="$XDG_DATA_HOME/idx/ownership.db" printf 'Discovering latest supported IDX/KSEI above-1%% report...\n' "$IDX_BIN" -o json ownership discover --family above1 --limit 50 > "$DISCOVERY_JSON" -discovery_payload="$( - jq -r ' +# NUL-separated, so a field containing newlines (a report title) cannot +# shift the fields after it. Written to a file: command substitution drops +# NULs, and a jq failure must still stop the script under set -e. +DISCOVERY_FIELDS="$WORKDIR/discovery-fields" +jq --raw-output0 ' if type != "array" or length == 0 then error("ownership discover returned no reports") else @@ -184,9 +187,8 @@ discovery_payload="$( ] | .[] end - ' "$DISCOVERY_JSON" -)" -mapfile -t discovery_fields <<< "$discovery_payload" + ' "$DISCOVERY_JSON" > "$DISCOVERY_FIELDS" +mapfile -d '' -t discovery_fields < "$DISCOVERY_FIELDS" DISCOVERED_FAMILY="${discovery_fields[0]:-}" DISCOVERED_LISTING_PAGE_URL="${discovery_fields[1]:-}" @@ -308,6 +310,14 @@ jq \ original_filename: ($original_filename | if . == "" then null else . end) } ' "$MANIFEST_PATH" > "$TMP_MANIFEST_PATH" +# Clients reject a manifest missing any of these, so a bad one would block +# every `idx ownership sync`; refuse to produce it. +if ! jq -e '[.source.pdf_url, .source.title, .source.publish_date, .snapshot.version] + | all(type == "string" and (gsub("^\\s+|\\s+$"; "") | length > 0))' \ + "$TMP_MANIFEST_PATH" > /dev/null; then + echo "built manifest is missing source.pdf_url, source.title, source.publish_date, or snapshot.version" >&2 + exit 1 +fi mv "$TMP_MANIFEST_PATH" "$MANIFEST_PATH" printf 'Prepared release-ready snapshot from %s (%s)\n' \ diff --git a/scripts/build-ownership-snapshot.sh b/scripts/build-ownership-snapshot.sh index b485c62..d626209 100755 --- a/scripts/build-ownership-snapshot.sh +++ b/scripts/build-ownership-snapshot.sh @@ -94,6 +94,15 @@ MANIFEST_PATH="$OUTPUT_DIR/ownership-snapshot-manifest.json" TMP_ARTIFACT_PATH="$OUTPUT_DIR/.ownership-snapshot-${LATEST_AS_OF}.sqlite.tmp.$$" # Remove a partial copy if the script fails before the atomic rename below. trap 'rm -f "$TMP_ARTIFACT_PATH"' EXIT +# Committed data can still sit in the -wal file while another connection has +# the database open; checkpoint it into the main file first (a no-op for a +# closed database, so its bytes and hash are unchanged), and refuse to copy +# when that cannot complete. +checkpoint="$(sqlite3 "$DB_PATH" "PRAGMA wal_checkpoint(TRUNCATE);")" +if [[ "${checkpoint%%|*}" != "0" ]]; then + echo "ownership database is busy; could not checkpoint its WAL before copying: $DB_PATH" >&2 + exit 1 +fi cp "$DB_PATH" "$TMP_ARTIFACT_PATH" # Immutable, content-addressed name. The publisher relies on this to upload and diff --git a/scripts/install-sh-test.sh b/scripts/install-sh-test.sh index 4bcff0e..769e91c 100755 --- a/scripts/install-sh-test.sh +++ b/scripts/install-sh-test.sh @@ -191,6 +191,17 @@ else fail "expands quoted ~ and tolerates trailing slashes"; cat "${work}/out9" >&2 fi +# 10. a compact single-line releases response still resolves the newest tag. +printf '%s' '[{"tag_name":"ownership-snapshot-current"},{"tag_name":"v0.2.4","name":"v0.2.4"},{"tag_name":"v0.2.3"}]' \ + >"${api_dir}/releases" +dir="${work}/compact" +if run_installer "$test_shell" "$installer" --dir "$dir" >"${work}/out10" 2>&1 && + [[ "$("${dir}/idx")" == "idx fake v0.2.4" ]]; then + pass "resolves the newest tag from a compact releases response" +else + fail "resolves the newest tag from a compact releases response"; cat "${work}/out10" >&2 +fi + if ((failures > 0)); then echo "install.sh tests: ${failures} failure(s) with ${test_shell}" >&2 exit 1