diff --git a/.github/workflows/publish-ownership-snapshot.yml b/.github/workflows/publish-ownership-snapshot.yml index cb1baa4..e392355 100644 --- a/.github/workflows/publish-ownership-snapshot.yml +++ b/.github/workflows/publish-ownership-snapshot.yml @@ -7,6 +7,10 @@ on: description: Stable GitHub release tag to publish the snapshot assets to required: true default: ownership-snapshot-current + history: + description: Number of earlier monthly above-1% reports to include for `ownership changes` + required: false + default: "5" permissions: contents: write @@ -36,7 +40,8 @@ jobs: --idx-bin ./target/debug/idx \ --output-dir dist/ownership-snapshot \ --repo "${{ github.repository }}" \ - --release-tag "${{ inputs.release_tag }}" + --release-tag "${{ inputs.release_tag }}" \ + --history "${{ inputs.history }}" - name: Upload workflow artifact copy uses: actions/upload-artifact@v4 diff --git a/docs/OWNERSHIP_PUBLISH.md b/docs/OWNERSHIP_PUBLISH.md index 278a16e..05821fe 100644 --- a/docs/OWNERSHIP_PUBLISH.md +++ b/docs/OWNERSHIP_PUBLISH.md @@ -45,9 +45,11 @@ nix develop --command scripts/build-latest-ownership-snapshot.sh \ The script performs these steps: -1. `idx -o json ownership discover --family above1 --limit 1` +1. `idx -o json ownership discover --family above1 --limit 50` (newest supported + report first; XLSX from the Data Kepemilikan Saham page since June 2026) 2. verifies the discovered report is the current supported import path -3. imports that PDF into an isolated temp ownership DB +3. imports that report (plus `--history ` earlier months, oldest first) into + an isolated temp ownership DB 4. checks that the imported release metadata is non-empty and tied to the same source URL 5. runs `scripts/build-ownership-snapshot.sh` to emit the SQLite artifact and diff --git a/docs/OWNERSHIP_SELF_HOSTED.md b/docs/OWNERSHIP_SELF_HOSTED.md index 7457fbb..57be012 100644 --- a/docs/OWNERSHIP_SELF_HOSTED.md +++ b/docs/OWNERSHIP_SELF_HOSTED.md @@ -38,7 +38,9 @@ nix develop --command scripts/publish-ownership-snapshot.sh \ That helper: 1. optionally builds `idx` -2. discovers the latest supported IDX/KSEI PDF +2. discovers the latest supported IDX/KSEI above-1% report (XLSX on the IDX + Data Kepemilikan Saham page since the 2026-05-29 report; PDF announcements + before that) 3. imports it into an isolated temp DB 4. emits the SQLite snapshot and manifest 5. ensures the stable GitHub release exists @@ -66,7 +68,13 @@ The sample timer uses: - `RandomizedDelaySec=30m` That is intentionally conservative. The ownership source is monthly, but the -exact publish day can drift. Start with an early-month schedule and adjust after +exact publish day can drift. Observed XLSX upload times (HTTP Last-Modified): +2026-06-03, 2026-07-02 03:55Z, 2026-08-02 01:50Z, 2026-09-02 03:51Z, so a run +on the 2nd can occasionally land before the file exists; the 3rd is safer. + +Pass `--history ` (for example via `services.idxOwnershipPublish.extraArgs = +[ "--history" "5" ];` in `clan-private`) to include earlier months so +`idx ownership changes` works straight from the synced snapshot. Start with an early-month schedule and adjust after observing a few real runs. ## Clan Integration Later diff --git a/scripts/build-latest-ownership-snapshot.sh b/scripts/build-latest-ownership-snapshot.sh index d69b54f..72da16d 100755 --- a/scripts/build-latest-ownership-snapshot.sh +++ b/scripts/build-latest-ownership-snapshot.sh @@ -6,7 +6,8 @@ usage() { cat <<'EOF' Usage: scripts/build-latest-ownership-snapshot.sh --output-dir [options] -Discover the latest supported IDX/KSEI ownership PDF, import it into an isolated +Discover the latest supported IDX/KSEI above-1% report (XLSX from the IDX Data +Kepemilikan Saham page, or a legacy PDF announcement), import it into an isolated ownership database, and emit GitHub-release-ready snapshot artifacts plus manifest. Options: @@ -17,6 +18,9 @@ Options: (default: 0xrsydn/idx-cli) --release-tag Stable GitHub release tag used for the default base URL (default: ownership-snapshot-current) + --history Also import the previous monthly above-1% XLSX + reports (oldest first) so `ownership changes` works + from the snapshot (default: 0) --keep-workdir Keep the temp workdir instead of deleting it --help Show this help EOF @@ -28,6 +32,7 @@ BASE_URL="" REPO_FULL_NAME="0xrsydn/idx-cli" RELEASE_TAG="ownership-snapshot-current" KEEP_WORKDIR="0" +HISTORY="0" while [[ $# -gt 0 ]]; do case "$1" in @@ -51,6 +56,10 @@ while [[ $# -gt 0 ]]; do RELEASE_TAG="${2:-}" shift 2 ;; + --history) + HISTORY="${2:-}" + shift 2 + ;; --keep-workdir) KEEP_WORKDIR="1" shift @@ -73,6 +82,11 @@ if [[ -z "$OUTPUT_DIR" ]]; then exit 2 fi +if ! [[ "$HISTORY" =~ ^[0-9]+$ ]]; then + echo "--history must be a non-negative integer" >&2 + exit 2 +fi + if [[ -z "$BASE_URL" ]]; then BASE_URL="https://github.com/${REPO_FULL_NAME}/releases/download/${RELEASE_TAG}" fi @@ -115,8 +129,8 @@ RELEASES_JSON="$WORKDIR/releases.json" MANIFEST_PATH="$OUTPUT_DIR/ownership-snapshot-manifest.json" DB_PATH="$XDG_DATA_HOME/idx/ownership.db" -printf 'Discovering latest supported IDX/KSEI ownership PDF...\n' -"$IDX_BIN" -o json ownership discover --family above1 --limit 1 > "$DISCOVERY_JSON" +printf 'Discovering latest supported IDX/KSEI above-1%% report...\n' +"$IDX_BIN" -o json ownership discover --family above1 --limit 50 > "$DISCOVERY_JSON" discovery_payload="$( jq -r ' @@ -151,7 +165,27 @@ DISCOVERED_TITLE="${discovery_fields[4]:-}" DISCOVERED_PUBLISH_DATE="${discovery_fields[5]:-}" DISCOVERED_ORIGINAL_FILENAME="${discovery_fields[6]:-}" -printf 'Importing discovered PDF into isolated DB...\n' +if (( HISTORY > 0 )); then + # Earlier supported XLSX months, one per as-of date, newest first; import oldest first. + mapfile -t history_urls < <( + jq -r --arg latest "$DISCOVERED_PDF_URL" --argjson n "$HISTORY" ' + (map(select(.pdf_url == $latest)) | .[0].as_of_date // "") as $latest_as_of + | [ .[] + | select(.status == "supported" and .format == "xlsx" and .as_of_date != null) + | select($latest_as_of == "" or .as_of_date < $latest_as_of) ] + | unique_by(.as_of_date) + | sort_by(.as_of_date) | reverse + | .[:$n] | reverse + | .[].pdf_url + ' "$DISCOVERY_JSON" + ) + printf 'Importing %d earlier monthly report(s) for history...\n' "${#history_urls[@]}" + for url in "${history_urls[@]}"; do + "$IDX_BIN" ownership import --url "$url" + done +fi + +printf 'Importing discovered report into isolated DB...\n' "$IDX_BIN" ownership import --url "$DISCOVERED_PDF_URL" printf 'Inspecting imported release metadata...\n' diff --git a/scripts/publish-ownership-snapshot.sh b/scripts/publish-ownership-snapshot.sh index 22a4d36..6bbc60d 100755 --- a/scripts/publish-ownership-snapshot.sh +++ b/scripts/publish-ownership-snapshot.sh @@ -16,6 +16,8 @@ Options: (default: 0xrsydn/idx-cli) --release-tag Stable GitHub release tag used for snapshot assets (default: ownership-snapshot-current) + --history Also include the previous monthly above-1% reports + in the snapshot (passed to the builder; default: 0) --build Run `cargo build` before publishing --keep-workdir Keep the temp workdir created by the builder helper --help Show this help @@ -28,6 +30,7 @@ REPO_FULL_NAME="0xrsydn/idx-cli" RELEASE_TAG="ownership-snapshot-current" BUILD_FIRST="0" KEEP_WORKDIR="0" +HISTORY="0" PUBLISH_WORKDIR="" while [[ $# -gt 0 ]]; do @@ -48,6 +51,10 @@ while [[ $# -gt 0 ]]; do RELEASE_TAG="${2:-}" shift 2 ;; + --history) + HISTORY="${2:-}" + shift 2 + ;; --build) BUILD_FIRST="1" shift @@ -120,6 +127,7 @@ build_args=( --output-dir "$PUBLISH_WORKDIR" --repo "$REPO_FULL_NAME" --release-tag "$RELEASE_TAG" + --history "$HISTORY" ) if [[ "$KEEP_WORKDIR" == "1" ]]; then