diff --git a/.github/workflows/publish-ownership-snapshot.yml b/.github/workflows/publish-ownership-snapshot.yml new file mode 100644 index 0000000..cb1baa4 --- /dev/null +++ b/.github/workflows/publish-ownership-snapshot.yml @@ -0,0 +1,79 @@ +name: Publish Ownership Snapshot + +on: + workflow_dispatch: + inputs: + release_tag: + description: Stable GitHub release tag to publish the snapshot assets to + required: true + default: ownership-snapshot-current + +permissions: + contents: write + +concurrency: + group: ownership-snapshot-publish + cancel-in-progress: false + +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Nix + uses: cachix/install-nix-action@v30 + + - name: Build idx + run: nix develop --command cargo build + + - name: Build snapshot artifacts from latest supported IDX/KSEI source + run: | + nix develop --command scripts/build-latest-ownership-snapshot.sh \ + --idx-bin ./target/debug/idx \ + --output-dir dist/ownership-snapshot \ + --repo "${{ github.repository }}" \ + --release-tag "${{ inputs.release_tag }}" + + - name: Upload workflow artifact copy + uses: actions/upload-artifact@v4 + with: + name: ownership-snapshot-${{ github.run_id }} + path: dist/ownership-snapshot/* + if-no-files-found: error + + - name: Ensure stable snapshot release exists + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + + tag="${{ inputs.release_tag }}" + repo="${{ github.repository }}" + + if gh release view "$tag" --repo "$repo" >/dev/null 2>&1; then + echo "Release $tag already exists." + exit 0 + fi + + gh release create "$tag" \ + --repo "$repo" \ + --target "${{ github.sha }}" \ + --title "Ownership Snapshot Current" \ + --notes "Stable release for idx ownership snapshot artifacts consumed by \`idx ownership sync\`." \ + --latest=false + + - name: Upload snapshot assets to stable release + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + + gh release upload "${{ inputs.release_tag }}" \ + dist/ownership-snapshot/ownership-snapshot-manifest.json \ + dist/ownership-snapshot/ownership-snapshot-*.sqlite \ + --repo "${{ github.repository }}" \ + --clobber diff --git a/TODO.md b/TODO.md index b37e35d..38eeb06 100644 --- a/TODO.md +++ b/TODO.md @@ -163,6 +163,8 @@ - [ ] Additional providers (Alpha Vantage, Twelve Data, IDX official) ## 🔬 Latest Smoke Findings (2026-04-02) +- [x] Ownership snapshot publishing now has a live-source maintainer path on `2026-04-11`: `scripts/build-latest-ownership-snapshot.sh` discovers the current supported `above1` IDX/KSEI PDF, imports it into an isolated temp DB, emits release-ready SQLite + manifest artifacts, and records the source IDX metadata in the manifest +- [x] Verification on `2026-04-11`: `nix develop --command cargo test`, `bash -n scripts/build-latest-ownership-snapshot.sh`, and a real `nix develop --command scripts/build-latest-ownership-snapshot.sh --idx-bin ./target/debug/idx --output-dir tmp/ownership-snapshot-live` run all passed; the current live source produced `7214` rows for `955` tickers with `as_of_date=2026-03-31` - [x] Final release-hygiene pass on `2026-04-06`: crate metadata now declares `rust-version = 1.85`, README install docs now spell out Cargo helper-runtime expectations plus persistent `nix profile install`, and CI install smoke now runs the mock smoke matrix against the installed binary instead of only checking `idx version` - [x] Verification on `2026-04-06`: `nix develop --command cargo build`, `nix develop --command cargo clippy -- -D warnings`, `nix develop --command cargo test`, `nix develop --command cargo package --allow-dirty --locked`, and `scripts/live-smoke.sh --bin ./tmp/release-install/bin/idx --no-build --mode mock` all passed - [x] Publish review blocker batch on `2026-04-05`: the default Nix app/package path builds again without relying on an untracked runtime module, `ownership import --force` now re-imports the same release SHA atomically, and current ownership views (`ticker`, `entity`, `cross-holders`, `concentration`, `graph`) now scope KSEI data to the latest imported release instead of blending historical snapshots diff --git a/docs/OWNERSHIP_PUBLISH.md b/docs/OWNERSHIP_PUBLISH.md new file mode 100644 index 0000000..d5b92ee --- /dev/null +++ b/docs/OWNERSHIP_PUBLISH.md @@ -0,0 +1,95 @@ +# Ownership Snapshot Publishing + +This document covers the maintainer workflow for producing and publishing the +ownership snapshot artifacts consumed by `idx ownership sync`. + +## Goals + +- Build the snapshot from the currently discoverable supported IDX/KSEI source, + not from an ad hoc local DB. +- Keep the published manifest URL stable for end users. +- Record source provenance in the manifest so the published SQLite artifact can + be traced back to the live IDX announcement and PDF URL used to build it. + +## Recommended GitHub Releases Layout + +Use a dedicated stable release tag for snapshot assets: + +- tag: `ownership-snapshot-current` +- manifest asset: `ownership-snapshot-manifest.json` +- SQLite asset: `ownership-snapshot-YYYY-MM-DD.sqlite` + +Recommended public manifest URL: + +```text +https://github.com/0xrsydn/idx-cli/releases/download/ownership-snapshot-current/ownership-snapshot-manifest.json +``` + +Avoid `.../releases/latest/download/...` if normal app releases and snapshot +publishes share the same repository. The repo's "latest" release can drift away +from the ownership snapshot release. + +## Manual Maintainer Flow + +Run the publisher helper inside `nix develop` so `mutool` and the +`curl-impersonate` helper are available: + +```bash +nix develop --command cargo build +nix develop --command scripts/build-latest-ownership-snapshot.sh \ + --idx-bin ./target/debug/idx \ + --output-dir dist/ownership-snapshot \ + --repo 0xrsydn/idx-cli \ + --release-tag ownership-snapshot-current +``` + +The script performs these steps: + +1. `idx -o json ownership discover --family above1 --limit 1` +2. verifies the discovered report is the current supported import path +3. imports that PDF into an isolated temp ownership DB +4. checks that the imported release metadata is non-empty and tied to the same + source URL +5. runs `scripts/build-ownership-snapshot.sh` to emit the SQLite artifact and + base manifest +6. enriches the manifest with `source` provenance metadata + +The resulting manifest records: + +- `source.family` +- `source.listing_page_url` +- `source.query_url` +- `source.pdf_url` +- `source.title` +- `source.publish_date` +- `source.original_filename` + +That metadata is additive. Existing sync clients can still parse the manifest. + +## Upload Step + +After the local build succeeds, upload these two files to the +`ownership-snapshot-current` GitHub release: + +- `dist/ownership-snapshot/ownership-snapshot-manifest.json` +- `dist/ownership-snapshot/ownership-snapshot-YYYY-MM-DD.sqlite` + +Only after this manual flow is reliable should the repo automate it in +GitHub Actions. + +## GitHub Actions Workflow + +The repo now includes a manual workflow at +`.github/workflows/publish-ownership-snapshot.yml`. + +Current behavior: + +- trigger: `workflow_dispatch` only +- builds `idx` inside `nix develop` +- runs `scripts/build-latest-ownership-snapshot.sh` +- uploads the generated files as workflow artifacts +- creates the stable release tag if needed +- uploads the manifest and SQLite asset to that release with `--clobber` + +This is intentionally manual-first. Add a schedule only after a few successful +publish runs confirm the live source remains stable enough. diff --git a/docs/OWNERSHIP_SYNC.md b/docs/OWNERSHIP_SYNC.md index 1990a3f..0fd0a71 100644 --- a/docs/OWNERSHIP_SYNC.md +++ b/docs/OWNERSHIP_SYNC.md @@ -4,6 +4,9 @@ The command is intentionally manifest-driven so the repo can publish snapshots in GitHub releases, object storage, or a local filesystem path without changing the CLI. +The recommended GitHub publish layout and maintainer workflow are documented in +`docs/OWNERSHIP_PUBLISH.md`. + ## Consumer Inputs The manifest location is resolved in this order: @@ -24,6 +27,15 @@ Current schema version: `1` { "schema_version": 1, "generated_at": "2026-03-31T12:00:00Z", + "source": { + "family": "above1", + "listing_page_url": "https://www.idx.co.id/id/berita/pengumuman/", + "query_url": "https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?...", + "pdf_url": "https://www.idx.co.id/StaticData/NewsAndAnnouncement/...pdf", + "title": "Pemegang Saham di atas 1% (KSEI)", + "publish_date": "2026-03-10T00:00:00", + "original_filename": "b9b638e5a8_8928aca255.pdf" + }, "snapshot": { "kind": "sqlite", "compression": "none", @@ -41,6 +53,7 @@ Current schema version: `1` ``` Semantics: +- `source` is optional provenance metadata describing the IDX/KSEI PDF used to build the snapshot. - `download_url` points to the SQLite artifact itself. - `sqlite_sha256` and `size_bytes` are validated before install. - `latest_*` and `release_count` are validated against the downloaded SQLite contents before replacement. @@ -71,3 +84,7 @@ scripts/build-ownership-snapshot.sh \ ``` If `--base-url` is omitted, the generated manifest uses the local artifact path as `download_url`, which is useful for local testing. + +For the maintainer flow that discovers the latest supported IDX/KSEI source +first and then builds GitHub-release-ready assets, use +`scripts/build-latest-ownership-snapshot.sh`. diff --git a/scripts/build-latest-ownership-snapshot.sh b/scripts/build-latest-ownership-snapshot.sh new file mode 100755 index 0000000..d69b54f --- /dev/null +++ b/scripts/build-latest-ownership-snapshot.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: scripts/build-latest-ownership-snapshot.sh --output-dir [options] + +Discover the latest supported IDX/KSEI ownership PDF, import it into an isolated +ownership database, and emit GitHub-release-ready snapshot artifacts plus manifest. + +Options: + --idx-bin idx binary to use (default: ./target/debug/idx) + --output-dir Directory to write the copied snapshot and manifest + --base-url Public URL prefix to use for snapshot.download_url + --repo GitHub repo used for the default base URL + (default: 0xrsydn/idx-cli) + --release-tag Stable GitHub release tag used for the default base URL + (default: ownership-snapshot-current) + --keep-workdir Keep the temp workdir instead of deleting it + --help Show this help +EOF +} + +IDX_BIN="./target/debug/idx" +OUTPUT_DIR="" +BASE_URL="" +REPO_FULL_NAME="0xrsydn/idx-cli" +RELEASE_TAG="ownership-snapshot-current" +KEEP_WORKDIR="0" + +while [[ $# -gt 0 ]]; do + case "$1" in + --idx-bin) + IDX_BIN="${2:-}" + shift 2 + ;; + --output-dir) + OUTPUT_DIR="${2:-}" + shift 2 + ;; + --base-url) + BASE_URL="${2:-}" + shift 2 + ;; + --repo) + REPO_FULL_NAME="${2:-}" + shift 2 + ;; + --release-tag) + RELEASE_TAG="${2:-}" + shift 2 + ;; + --keep-workdir) + KEEP_WORKDIR="1" + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "$OUTPUT_DIR" ]]; then + echo "--output-dir is required" >&2 + usage >&2 + exit 2 +fi + +if [[ -z "$BASE_URL" ]]; then + BASE_URL="https://github.com/${REPO_FULL_NAME}/releases/download/${RELEASE_TAG}" +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "jq is required for manifest/source metadata processing" >&2 + exit 1 +fi + +if ! "$IDX_BIN" version >/dev/null 2>&1; then + echo "failed to run idx binary: $IDX_BIN" >&2 + echo "build the CLI first or pass --idx-bin " >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BASIC_BUILDER="$SCRIPT_DIR/build-ownership-snapshot.sh" +if [[ ! -x "$BASIC_BUILDER" ]]; then + echo "required helper script is missing or not executable: $BASIC_BUILDER" >&2 + exit 1 +fi + +WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/idx-ownership-snapshot.XXXXXX")" +cleanup() { + if [[ "$KEEP_WORKDIR" == "1" ]]; then + printf 'Kept workdir: %s\n' "$WORKDIR" + return + fi + rm -rf "$WORKDIR" +} +trap cleanup EXIT + +export XDG_DATA_HOME="$WORKDIR/data" +export XDG_CACHE_HOME="$WORKDIR/cache" +export XDG_CONFIG_HOME="$WORKDIR/config" +mkdir -p "$XDG_DATA_HOME" "$XDG_CACHE_HOME" "$XDG_CONFIG_HOME" + +DISCOVERY_JSON="$WORKDIR/discovery.json" +RELEASES_JSON="$WORKDIR/releases.json" +MANIFEST_PATH="$OUTPUT_DIR/ownership-snapshot-manifest.json" +DB_PATH="$XDG_DATA_HOME/idx/ownership.db" + +printf 'Discovering latest supported IDX/KSEI ownership PDF...\n' +"$IDX_BIN" -o json ownership discover --family above1 --limit 1 > "$DISCOVERY_JSON" + +discovery_payload="$( + jq -r ' + if type != "array" or length == 0 then + error("ownership discover returned no reports") + else + .[0] + | if .status != "supported" then + error("latest discovered report is not supported: \(.status // "null")") + else . + end + | [ + (.family // error("discovered report is missing family")), + (.listing_page_url // error("discovered report is missing listing_page_url")), + (.query_url // error("discovered report is missing query_url")), + (.pdf_url // error("discovered report is missing pdf_url")), + (.title // error("discovered report is missing title")), + (.publish_date // error("discovered report is missing publish_date")), + (.original_filename // "") + ] + | .[] + end + ' "$DISCOVERY_JSON" +)" +mapfile -t discovery_fields <<< "$discovery_payload" + +DISCOVERED_FAMILY="${discovery_fields[0]:-}" +DISCOVERED_LISTING_PAGE_URL="${discovery_fields[1]:-}" +DISCOVERED_QUERY_URL="${discovery_fields[2]:-}" +DISCOVERED_PDF_URL="${discovery_fields[3]:-}" +DISCOVERED_TITLE="${discovery_fields[4]:-}" +DISCOVERED_PUBLISH_DATE="${discovery_fields[5]:-}" +DISCOVERED_ORIGINAL_FILENAME="${discovery_fields[6]:-}" + +printf 'Importing discovered PDF into isolated DB...\n' +"$IDX_BIN" ownership import --url "$DISCOVERED_PDF_URL" + +printf 'Inspecting imported release metadata...\n' +"$IDX_BIN" -o json ownership releases > "$RELEASES_JSON" + +release_payload="$( + jq -r --arg expected_source "$DISCOVERED_PDF_URL" ' + if type != "array" or length == 0 then + error("ownership releases returned no imported releases") + else + .[0] + | if (.source_url // "") != $expected_source then + error( + "latest imported release source_url mismatch: \(.source_url // "null") != \($expected_source)" + ) + else . + end + | [ + (.as_of_date // error("latest imported release is missing as_of_date")), + ( + (.sha256 // "") + | if test("^[0-9A-Fa-f]{64}$") then . + else error("latest imported release has invalid sha256") + end + ), + ( + (.row_count // 0) + | if type == "number" and . > 0 then tostring + else error("latest imported release has invalid row_count") + end + ) + ] + | .[] + end + ' "$RELEASES_JSON" +)" +mapfile -t release_fields <<< "$release_payload" + +IMPORTED_AS_OF_DATE="${release_fields[0]:-}" +IMPORTED_RELEASE_SHA256="${release_fields[1]:-}" +IMPORTED_ROW_COUNT="${release_fields[2]:-}" + +printf 'Building snapshot artifact and manifest...\n' +"$BASIC_BUILDER" --db "$DB_PATH" --output-dir "$OUTPUT_DIR" --base-url "$BASE_URL" + +TMP_MANIFEST_PATH="$WORKDIR/ownership-snapshot-manifest.json" +jq \ + --arg family "$DISCOVERED_FAMILY" \ + --arg listing_page_url "$DISCOVERED_LISTING_PAGE_URL" \ + --arg query_url "$DISCOVERED_QUERY_URL" \ + --arg pdf_url "$DISCOVERED_PDF_URL" \ + --arg title "$DISCOVERED_TITLE" \ + --arg publish_date "$DISCOVERED_PUBLISH_DATE" \ + --arg original_filename "$DISCOVERED_ORIGINAL_FILENAME" \ + ' + .source = { + family: $family, + listing_page_url: $listing_page_url, + query_url: $query_url, + pdf_url: $pdf_url, + title: $title, + publish_date: $publish_date, + original_filename: ($original_filename | if . == "" then null else . end) + } + ' "$MANIFEST_PATH" > "$TMP_MANIFEST_PATH" +mv "$TMP_MANIFEST_PATH" "$MANIFEST_PATH" + +printf 'Prepared release-ready snapshot from %s (%s)\n' \ + "$DISCOVERED_PDF_URL" "$IMPORTED_AS_OF_DATE" +printf 'Manifest URL target: %s/ownership-snapshot-manifest.json\n' "$BASE_URL" +printf 'SQLite URL target: %s/ownership-snapshot-%s.sqlite\n' "$BASE_URL" "$IMPORTED_AS_OF_DATE" diff --git a/src/ownership/snapshot.rs b/src/ownership/snapshot.rs index daac151..e6cff7b 100644 --- a/src/ownership/snapshot.rs +++ b/src/ownership/snapshot.rs @@ -21,9 +21,23 @@ const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/ pub struct OwnershipSnapshotManifest { pub schema_version: u32, pub generated_at: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, pub snapshot: OwnershipSnapshotArtifact, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct OwnershipSnapshotSource { + pub family: String, + pub listing_page_url: String, + pub query_url: String, + pub pdf_url: String, + pub title: String, + pub publish_date: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub original_filename: Option, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct OwnershipSnapshotArtifact { pub kind: String, @@ -196,6 +210,39 @@ fn validate_manifest(manifest: &OwnershipSnapshotManifest) -> Result<(), IdxErro )); } + if let Some(source) = &manifest.source { + if source.family.trim().is_empty() { + return Err(IdxError::ParseError( + "ownership snapshot manifest is missing source.family".to_string(), + )); + } + if source.listing_page_url.trim().is_empty() { + return Err(IdxError::ParseError( + "ownership snapshot manifest is missing source.listing_page_url".to_string(), + )); + } + if source.query_url.trim().is_empty() { + return Err(IdxError::ParseError( + "ownership snapshot manifest is missing source.query_url".to_string(), + )); + } + if source.pdf_url.trim().is_empty() { + return Err(IdxError::ParseError( + "ownership snapshot manifest is missing source.pdf_url".to_string(), + )); + } + if source.title.trim().is_empty() { + return Err(IdxError::ParseError( + "ownership snapshot manifest is missing source.title".to_string(), + )); + } + if source.publish_date.trim().is_empty() { + return Err(IdxError::ParseError( + "ownership snapshot manifest is missing source.publish_date".to_string(), + )); + } + } + let snapshot = &manifest.snapshot; if snapshot.kind.trim() != "sqlite" { return Err(IdxError::Unsupported(format!( @@ -640,8 +687,8 @@ mod tests { use rusqlite::Connection; use super::{ - OwnershipSnapshotArtifact, OwnershipSnapshotManifest, OwnershipSyncAction, - SNAPSHOT_MANIFEST_SCHEMA_VERSION, build_sync_decision, parse_manifest, + OwnershipSnapshotArtifact, OwnershipSnapshotManifest, OwnershipSnapshotSource, + OwnershipSyncAction, SNAPSHOT_MANIFEST_SCHEMA_VERSION, build_sync_decision, parse_manifest, }; use crate::ownership::db::{ensure_schema, insert_release}; use crate::ownership::types::OwnershipRelease; @@ -713,6 +760,15 @@ mod tests { let manifest = OwnershipSnapshotManifest { schema_version: SNAPSHOT_MANIFEST_SCHEMA_VERSION, generated_at: "2026-03-31T12:00:00Z".to_string(), + source: Some(OwnershipSnapshotSource { + family: "above1".to_string(), + listing_page_url: "https://www.idx.co.id/id/berita/pengumuman/".to_string(), + query_url: "https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%201&pageNumber=1&pageSize=10&lang=id".to_string(), + pdf_url: "https://www.idx.co.id/StaticData/sample.pdf".to_string(), + title: "Pemegang Saham di atas 1% (KSEI)".to_string(), + publish_date: "2026-03-10T00:00:00".to_string(), + original_filename: Some("sample.pdf".to_string()), + }), snapshot: OwnershipSnapshotArtifact { kind: "sqlite".to_string(), compression: "none".to_string(), @@ -735,6 +791,30 @@ mod tests { assert_eq!(parsed, manifest); } + #[test] + fn manifest_without_source_metadata_still_parses() { + let raw = r#"{ + "schema_version": 1, + "generated_at": "2026-03-31T12:00:00Z", + "snapshot": { + "kind": "sqlite", + "compression": "none", + "version": "2026-02-27", + "download_url": "/tmp/ownership.sqlite", + "sqlite_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "size_bytes": 123, + "release_count": 1, + "latest_as_of_date": "2026-02-27", + "latest_release_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "latest_row_count": 7261, + "ticker_count": 955 + } + }"#; + + let parsed = parse_manifest(raw).expect("old manifest shape should still parse"); + assert!(parsed.source.is_none()); + } + #[test] fn ensure_schema_can_store_release_metadata_needed_for_snapshots() { let conn = Connection::open_in_memory().unwrap();