0xrsydn/idx-cli
1
mirror of https://github.com/0xrsydn/idx-cli.git synced 2026-11-07 09:45:36 +00:00
Code Issues Projects Releases 1 Packages Wiki Activity
idx-cli/src/ownership/remote.rs
hermes bb39e98fab fix(ownership): harden data-page discovery per review 
Addresses review on #26:
- strip ?query/#fragment before the .xlsx/.pdf extension check
- skip page entries without a parseable as-of date instead of emitting
  an empty publish_date
- warn on stderr when the data page fails or parses to zero entries
  while announcement results still exist, instead of silently
  degrading to announcement-only results

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E
2026-09-24 21:11:56 +00:00

1214 lines
42 KiB
Rust
Raw Blame History

use std::fs;
use std::path::Path;
use serde::{Deserialize, Serialize};
use crate::curl_impersonate;
use crate::error::IdxError;
pub const IDX_ANNOUNCEMENT_LISTING_URL: &str = "https://www.idx.co.id/id/berita/pengumuman/";
const DEFAULT_IDX_ANNOUNCEMENT_API_URL: &str =
"https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement";
const IDX_ANNOUNCEMENT_API_ENV: &str = "IDX_OWNERSHIP_ANNOUNCEMENT_API_URL";
const IDX_ANNOUNCEMENT_LISTING_ENV: &str = "IDX_OWNERSHIP_ANNOUNCEMENT_PAGE_URL";
const IDX_ANNOUNCEMENT_PAGE_SIZE: usize = 10;
/// Since June 2026 the monthly above-1% list (XLSX) is published only here.
pub const IDX_SHARE_OWNERSHIP_PAGE_URL: &str =
"https://www.idx.co.id/id/perusahaan-tercatat/data-kepemilikan-saham/";
const IDX_SHARE_OWNERSHIP_PAGE_ENV: &str = "IDX_OWNERSHIP_DATA_PAGE_URL";
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
#[derive(Debug, Clone, Deserialize)]
pub struct AnnouncementPage {
#[serde(rename = "Items", default)]
pub items: Vec<AnnouncementItem>,
#[serde(rename = "ItemCount")]
pub item_count: Option<usize>,
#[serde(rename = "PageCount")]
pub page_count: Option<usize>,
}
#[derive(Debug, Clone, Deserialize)]
pub struct AnnouncementItem {
#[serde(rename = "PublishDate")]
pub publish_date: String,
#[serde(rename = "Title")]
pub title: String,
#[serde(rename = "AnnouncementType")]
pub announcement_type: Option<String>,
#[serde(rename = "Code")]
pub code: Option<String>,
#[serde(rename = "Attachments", default)]
pub attachments: Vec<AnnouncementAttachment>,
#[serde(rename = "PdfPath")]
pub pdf_path: Option<String>,
}
#[derive(Debug, Clone, Deserialize)]
pub struct AnnouncementAttachment {
#[serde(rename = "FullSavePath")]
pub full_save_path: String,
#[serde(rename = "OriginalFilename")]
pub original_filename: Option<String>,
#[serde(rename = "PDFFilename")]
pub pdf_filename: Option<String>,
#[serde(rename = "IsAttachment")]
pub is_attachment: Option<serde_json::Value>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum OwnershipReportFamily {
AboveOnePercent,
AboveFivePercent,
InvestorTypeBreakdown,
}
impl OwnershipReportFamily {
pub fn cli_name(self) -> &'static str {
match self {
Self::AboveOnePercent => "above1",
Self::AboveFivePercent => "above5",
Self::InvestorTypeBreakdown => "investor-type",
}
}
pub fn label(self) -> &'static str {
match self {
Self::AboveOnePercent => "Above 1%",
Self::AboveFivePercent => "Above 5%",
Self::InvestorTypeBreakdown => "Investor Type Breakdown",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum OwnershipReportStatus {
Supported,
AnnouncementOnly,
Unsupported,
}
impl OwnershipReportStatus {
pub fn label(self) -> &'static str {
match self {
Self::Supported => "supported",
Self::AnnouncementOnly => "announcement_only",
Self::Unsupported => "unsupported",
}
}
fn sort_rank(self) -> u8 {
match self {
Self::Supported => 0,
Self::AnnouncementOnly => 1,
Self::Unsupported => 2,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct DiscoveredOwnershipPdf {
pub family: OwnershipReportFamily,
pub status: OwnershipReportStatus,
pub listing_page_url: String,
pub query_url: String,
pub pdf_url: String,
pub title: String,
pub publish_date: String,
pub code: Option<String>,
pub original_filename: Option<String>,
pub is_attachment: bool,
/// File format of `pdf_url`: `pdf` (announcement attachment) or `xlsx`
/// (Data Kepemilikan Saham page).
pub format: String,
/// Report as-of date (`YYYY-MM-DD`) when the source states it. The data
/// page shows no publish date, so its entries use this as `publish_date`.
pub as_of_date: Option<String>,
}
struct DiscoveryQuery {
family: OwnershipReportFamily,
keywords: &'static str,
title_needles: &'static [&'static str],
}
const DISCOVERY_QUERIES: &[DiscoveryQuery] = &[
DiscoveryQuery {
family: OwnershipReportFamily::AboveOnePercent,
keywords: "pemegang saham di atas 1",
title_needles: &["PEMEGANG SAHAM DI ATAS 1", "SHAREHOLDERS ABOVE 1"],
},
DiscoveryQuery {
family: OwnershipReportFamily::AboveFivePercent,
keywords: "pemegang saham di atas 5",
title_needles: &["PEMEGANG SAHAM DI ATAS 5", "SHAREHOLDERS ABOVE 5"],
},
DiscoveryQuery {
family: OwnershipReportFamily::InvestorTypeBreakdown,
keywords: "kepemilikan saham perusahaan tercatat",
title_needles: &[
"KEPEMILIKAN SAHAM PERUSAHAAN TERCATAT BERDASARKAN TIPE INVESTOR",
"DATA KSEI TERKAIT KEPEMILIKAN SAHAM PERUSAHAAN TERCATAT",
],
},
];
pub fn announcement_listing_url() -> String {
std::env::var(IDX_ANNOUNCEMENT_LISTING_ENV)
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| IDX_ANNOUNCEMENT_LISTING_URL.to_string())
}
pub fn share_ownership_page_url() -> String {
std::env::var(IDX_SHARE_OWNERSHIP_PAGE_ENV)
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| IDX_SHARE_OWNERSHIP_PAGE_URL.to_string())
}
pub fn announcement_api_url() -> String {
std::env::var(IDX_ANNOUNCEMENT_API_ENV)
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| DEFAULT_IDX_ANNOUNCEMENT_API_URL.to_string())
}
pub fn build_announcement_query_url(
keywords: &str,
page_number: usize,
page_size: usize,
) -> String {
format!(
"{}?keywords={}&pageNumber={page_number}&pageSize={page_size}&lang=id",
announcement_api_url(),
percent_encode(keywords),
)
}
pub fn parse_announcement_page(raw: &str) -> Result<AnnouncementPage, IdxError> {
let trimmed = raw.trim();
if trimmed.is_empty() {
return Err(IdxError::Http(
"IDX ownership discovery returned an empty announcement payload; the IDX announcement API did not return any JSON items".to_string(),
));
}
let normalized = trimmed.to_ascii_lowercase();
if normalized.starts_with("<!doctype html") || normalized.starts_with("<html") {
return Err(IdxError::Http(
"IDX ownership discovery returned HTML instead of announcement JSON; verify the announcement endpoint or try again later".to_string(),
));
}
serde_json::from_str(trimmed)
.map_err(|e| IdxError::ParseError(format!("failed to parse IDX announcement JSON: {e}")))
}
pub fn select_latest_ownership_reports(
page: &AnnouncementPage,
query_url: &str,
family: OwnershipReportFamily,
) -> Result<Vec<DiscoveredOwnershipPdf>, IdxError> {
let Some(query) = DISCOVERY_QUERIES
.iter()
.find(|query| query.family == family)
else {
return Err(IdxError::Http(format!(
"failed to discover {} IDX ownership reports from {query_url}: unknown report family",
family.cli_name()
)));
};
let mut matches: Vec<&AnnouncementItem> = page
.items
.iter()
.filter(|item| item_matches_family(item, query))
.collect();
matches.sort_by(|left, right| right.publish_date.cmp(&left.publish_date));
let Some(item) = matches.into_iter().next() else {
return Err(IdxError::Http(format!(
"failed to discover {} IDX ownership reports from {query_url}: no matching announcement found",
family.cli_name()
)));
};
let mut attachments = item_pdf_attachments(item)
.into_iter()
.map(|attachment| {
let is_attachment = attachment_is_attachment(&attachment);
let original_filename = attachment
.original_filename
.clone()
.or(attachment.pdf_filename.clone())
.map(|value| value.trim().to_string());
let status = classify_report_status(family, is_attachment);
DiscoveredOwnershipPdf {
family,
status,
listing_page_url: announcement_listing_url(),
query_url: query_url.to_string(),
pdf_url: attachment.full_save_path,
title: item.title.clone(),
publish_date: item.publish_date.clone(),
code: clean_option(item.code.as_deref()),
original_filename,
is_attachment,
format: "pdf".to_string(),
as_of_date: None,
}
})
.collect::<Vec<_>>();
attachments.sort_by(|left, right| {
left.status
.sort_rank()
.cmp(&right.status.sort_rank())
.then_with(|| right.is_attachment.cmp(&left.is_attachment))
.then_with(|| left.original_filename.cmp(&right.original_filename))
});
if attachments.is_empty() {
return Err(IdxError::Http(format!(
"failed to discover {} IDX ownership reports from {query_url}: matching announcement had no PDF attachments",
family.cli_name()
)));
}
Ok(attachments)
}
pub fn discover_idx_ownership_reports(
family_filter: Option<OwnershipReportFamily>,
limit: usize,
) -> Result<Vec<DiscoveredOwnershipPdf>, IdxError> {
let mut discovered = Vec::new();
let mut errors = Vec::new();
for query in DISCOVERY_QUERIES
.iter()
.filter(|query| family_filter.is_none_or(|family| family == query.family))
{
let query_url = build_announcement_query_url(query.keywords, 1, IDX_ANNOUNCEMENT_PAGE_SIZE);
match fetch_text(
"IDX ownership announcement discovery",
&query_url,
&json_headers(),
&validate_json_payload,
)
.and_then(|raw| parse_announcement_page(&raw))
.and_then(|page| select_latest_ownership_reports(&page, &query_url, query.family))
{
Ok(mut reports) => discovered.append(&mut reports),
Err(err) => errors.push(err.to_string()),
}
}
let mut page_error = None;
match fetch_text(
"IDX share ownership data page discovery",
&share_ownership_page_url(),
&page_headers(),
&validate_share_ownership_page,
) {
Ok(html) => {
let mut page_reports = parse_share_ownership_page(&html, &share_ownership_page_url());
if page_reports.is_empty() {
// The page loaded (it has its file list) but nothing parsed:
// the payload shape changed. Say so instead of silently
// falling back to announcement-only results.
page_error = Some(format!(
"IDX share ownership data page {} loaded but no report entries could be parsed; the page layout may have changed",
share_ownership_page_url()
));
}
page_reports
.retain(|report| family_filter.is_none_or(|family| family == report.family));
// Keep the full above-1% history (snapshot backfill needs it), but
// only the newest entry of the unsupported daily/breakdown series.
page_reports.sort_by(|left, right| right.publish_date.cmp(&left.publish_date));
let mut seen_unsupported = Vec::new();
page_reports.retain(|report| {
if report.family == OwnershipReportFamily::AboveOnePercent {
return true;
}
if seen_unsupported.contains(&report.family) {
return false;
}
seen_unsupported.push(report.family);
true
});
discovered.extend(page_reports);
}
Err(err) => page_error = Some(err.to_string()),
}
discovered.sort_by(|left, right| {
right
.publish_date
.cmp(&left.publish_date)
.then_with(|| left.status.sort_rank().cmp(&right.status.sort_rank()))
.then_with(|| right.is_attachment.cmp(&left.is_attachment))
.then_with(|| left.original_filename.cmp(&right.original_filename))
});
if limit < discovered.len() {
discovered.truncate(limit);
}
if !discovered.is_empty()
&& let Some(err) = &page_error
{
// Announcement results still exist, but since June 2026 the newest
// above-1% reports are only on the data page: don't hide its failure.
crate::runtime::warn(err);
}
errors.extend(page_error);
if discovered.is_empty() {
let detail = if errors.is_empty() {
"no matching ownership reports found".to_string()
} else {
errors.join("; ")
};
return Err(IdxError::Http(format!(
"failed to discover IDX ownership reports from {} or {}: {detail}",
announcement_listing_url(),
share_ownership_page_url()
)));
}
Ok(discovered)
}
pub fn download_idx_pdf(url: &str, target: &Path) -> Result<(), IdxError> {
let bytes = fetch_bytes(
"IDX ownership PDF download",
url,
&pdf_headers(),
&validate_pdf_payload,
)?;
fs::write(target, &bytes).map_err(|e| {
IdxError::Io(format!(
"failed writing cached PDF {}: {e}",
target.display()
))
})?;
Ok(())
}
pub fn download_idx_xlsx(url: &str, target: &Path) -> Result<(), IdxError> {
let bytes = fetch_bytes(
"IDX ownership XLSX download",
url,
&xlsx_headers(),
&validate_xlsx_payload,
)?;
fs::write(target, &bytes).map_err(|e| {
IdxError::Io(format!(
"failed writing cached XLSX {}: {e}",
target.display()
))
})?;
Ok(())
}
/// XLSX is a ZIP container; Cloudflare challenges come back as HTML.
pub fn validate_xlsx_payload(bytes: &[u8]) -> Result<(), IdxError> {
if bytes.starts_with(b"PK\x03\x04") {
return Ok(());
}
if looks_like_html(bytes) {
return Err(IdxError::Http(
"IDX ownership download returned HTML (likely a Cloudflare challenge) instead of an XLSX workbook".to_string(),
));
}
Err(IdxError::Http(
"IDX ownership download did not look like an XLSX workbook".to_string(),
))
}
fn validate_json_payload(bytes: &[u8]) -> Result<(), IdxError> {
if looks_like_html(bytes) {
return Err(IdxError::Http(
"IDX ownership discovery returned HTML (likely a Cloudflare challenge) instead of announcement JSON".to_string(),
));
}
Ok(())
}
fn validate_share_ownership_page(bytes: &[u8]) -> Result<(), IdxError> {
if contains_bytes(bytes, b"Prospectus") {
return Ok(());
}
Err(IdxError::Http(
"IDX share ownership data page did not contain its file list (likely a Cloudflare challenge)".to_string(),
))
}
fn looks_like_html(bytes: &[u8]) -> bool {
let head: Vec<u8> = bytes
.iter()
.skip_while(|byte| byte.is_ascii_whitespace())
.take(256)
.map(u8::to_ascii_lowercase)
.collect();
head.starts_with(b"<!doctype html") || head.starts_with(b"<html")
}
fn contains_bytes(haystack: &[u8], needle: &[u8]) -> bool {
haystack
.windows(needle.len())
.any(|window| window == needle)
}
/// Parse the Data Kepemilikan Saham page. The page is server-rendered by
/// Nuxt and embeds its file list as JS object literals such as
/// `{Description:"Pemegang Saham di Atas 1% per 31 Agustus 2026",
/// Prospectus:"https:\u002F\u002Fwww.idx.co.id\u002FMedia\u002F...xlsx",...}`.
pub fn parse_share_ownership_page(html: &str, page_url: &str) -> Vec<DiscoveredOwnershipPdf> {
let mut reports = Vec::new();
let mut rest = html;
while let Some(start) = find_js_key(rest, "Description") {
let (description, after_description) = match read_js_string(&rest[start..]) {
Some(value) => value,
None => {
rest = &rest[start..];
continue;
}
};
let object_tail = &rest[start + after_description..];
// The URL belongs to the same object literal, so it must come before
// the next Description key.
let window_end = find_js_key(object_tail, "Description").unwrap_or(object_tail.len());
let window = &object_tail[..window_end];
rest = object_tail;
let Some(url_start) = find_js_key(window, "Prospectus") else {
continue;
};
let Some((url, _)) = read_js_string(&window[url_start..]) else {
continue;
};
if let Some(report) = classify_page_entry(description.trim(), url.trim(), page_url) {
reports.push(report);
}
}
reports
}
/// Offset just past `Key:` / `"Key":` (the opening quote of the value).
fn find_js_key(haystack: &str, key: &str) -> Option<usize> {
let bare = format!("{key}:\"");
let quoted = format!("\"{key}\":\"");
[bare, quoted]
.iter()
.filter_map(|needle| {
haystack
.find(needle.as_str())
.map(|index| index + needle.len() - 1)
})
.min()
}
/// Read a JS/JSON string literal starting at the opening quote. Returns the
/// decoded value and the byte offset just past the closing quote.
fn read_js_string(input: &str) -> Option<(String, usize)> {
let mut chars = input.char_indices();
if chars.next()?.1 != '"' {
return None;
}
let mut value = String::new();
while let Some((index, ch)) = chars.next() {
match ch {
'"' => return Some((value, index + 1)),
'\\' => match chars.next()?.1 {
'u' => {
let hex: String = (0..4)
.filter_map(|_| chars.next().map(|(_, c)| c))
.collect();
value.push(char::from_u32(u32::from_str_radix(&hex, 16).ok()?)?);
}
'n' => value.push('\n'),
't' => value.push('\t'),
other => value.push(other),
},
_ => value.push(ch),
}
}
None
}
fn classify_page_entry(
description: &str,
url: &str,
page_url: &str,
) -> Option<DiscoveredOwnershipPdf> {
let upper = description.to_uppercase();
let family = if upper.contains("DI ATAS 1%") || upper.contains("ABOVE 1%") {
OwnershipReportFamily::AboveOnePercent
} else if upper.contains("DI ATAS 5%") || upper.contains("ABOVE 5%") {
OwnershipReportFamily::AboveFivePercent
} else if upper.contains("TIPE INVESTOR")
|| upper.contains("KLASIFIKASI")
|| upper.contains("INVESTOR TYPE")
|| upper.contains("CLASSIFICATION")
{
OwnershipReportFamily::InvestorTypeBreakdown
} else {
return None;
};
let path = url.split(['?', '#']).next().unwrap_or_default();
let file_name = path.rsplit('/').next().unwrap_or_default().to_string();
let lower_name = file_name.to_ascii_lowercase();
let format = if lower_name.ends_with(".xlsx") {
"xlsx"
} else if lower_name.ends_with(".pdf") {
"pdf"
} else {
return None;
};
let status = if family == OwnershipReportFamily::AboveOnePercent && format == "xlsx" {
OwnershipReportStatus::Supported
} else {
OwnershipReportStatus::Unsupported
};
// Without an as-of date the entry cannot be ordered against other
// months, and the snapshot manifest needs a date, so skip it.
let as_of_date = parse_indonesian_as_of_date(description)?.to_string();
Some(DiscoveredOwnershipPdf {
family,
status,
listing_page_url: page_url.to_string(),
query_url: page_url.to_string(),
pdf_url: url.to_string(),
title: description.to_string(),
publish_date: as_of_date.clone(),
code: None,
original_filename: (!file_name.is_empty()).then_some(file_name),
is_attachment: true,
format: format.to_string(),
as_of_date: Some(as_of_date),
})
}
/// `"... per 31 Agustus 2026"` -> 2026-08-31 (Indonesian or English months).
fn parse_indonesian_as_of_date(description: &str) -> Option<chrono::NaiveDate> {
let tokens: Vec<String> = description
.split_whitespace()
.map(|token| {
token
.trim_matches(|c: char| !c.is_alphanumeric())
.to_lowercase()
})
.collect();
tokens.windows(3).rev().find_map(|window| {
let day = window[0].parse::<u32>().ok()?;
let month = match window[1].as_str() {
"januari" | "january" | "jan" => 1,
"februari" | "february" | "feb" => 2,
"maret" | "march" | "mar" => 3,
"april" | "apr" => 4,
"mei" | "may" => 5,
"juni" | "june" | "jun" => 6,
"juli" | "july" | "jul" => 7,
"agustus" | "august" | "agu" | "aug" => 8,
"september" | "sep" => 9,
"oktober" | "october" | "okt" | "oct" => 10,
"november" | "nov" => 11,
"desember" | "december" | "des" | "dec" => 12,
_ => return None,
};
let year = window[2].parse::<i32>().ok()?;
chrono::NaiveDate::from_ymd_opt(year, month, day)
})
}
pub fn validate_pdf_payload(bytes: &[u8]) -> Result<(), IdxError> {
if bytes.is_empty() {
return Err(IdxError::Http(
"IDX ownership download returned an empty response".to_string(),
));
}
let trimmed = bytes
.iter()
.skip_while(|byte| byte.is_ascii_whitespace())
.copied()
.collect::<Vec<_>>();
if trimmed.starts_with(b"%PDF-") {
return Ok(());
}
let preview = String::from_utf8_lossy(&trimmed[..trimmed.len().min(256)]).to_ascii_lowercase();
if preview.contains("<!doctype html") || preview.contains("<html") {
return Err(IdxError::Http(
"IDX ownership download returned HTML instead of a PDF/direct attachment".to_string(),
));
}
Err(IdxError::Http(
"IDX ownership download did not look like a PDF/direct attachment".to_string(),
))
}
fn fetch_text(
stage: &str,
url: &str,
headers: &[(String, String)],
accept: &dyn Fn(&[u8]) -> Result<(), IdxError>,
) -> Result<String, IdxError> {
let bytes = fetch_bytes(stage, url, headers, accept)?;
String::from_utf8(bytes)
.map_err(|e| IdxError::Http(format!("failed to decode {stage} response as utf-8: {e}")))
}
fn fetch_bytes(
stage: &str,
url: &str,
headers: &[(String, String)],
accept: &dyn Fn(&[u8]) -> Result<(), IdxError>,
) -> Result<Vec<u8>, IdxError> {
let mut args = vec![
"--silent".to_string(),
"--show-error".to_string(),
"--location".to_string(),
"--fail".to_string(),
"--compressed".to_string(),
];
for (name, value) in headers {
args.push("--header".to_string());
args.push(format!("{name}: {value}"));
}
args.push(url.to_string());
let output = curl_impersonate::run_owned_with_fallback(stage, &args, accept)?;
Ok(output.stdout)
}
fn json_headers() -> Vec<(String, String)> {
vec![
("User-Agent".to_string(), USER_AGENT.to_string()),
(
"Accept".to_string(),
"application/json,text/plain,*/*".to_string(),
),
(
"Accept-Language".to_string(),
"id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7".to_string(),
),
("Referer".to_string(), announcement_listing_url()),
]
}
fn page_headers() -> Vec<(String, String)> {
vec![
("User-Agent".to_string(), USER_AGENT.to_string()),
(
"Accept".to_string(),
"text/html,application/xhtml+xml,*/*;q=0.8".to_string(),
),
(
"Accept-Language".to_string(),
"id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7".to_string(),
),
]
}
fn xlsx_headers() -> Vec<(String, String)> {
vec![
("User-Agent".to_string(), USER_AGENT.to_string()),
(
"Accept".to_string(),
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/octet-stream,*/*;q=0.8".to_string(),
),
(
"Accept-Language".to_string(),
"id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7".to_string(),
),
("Referer".to_string(), share_ownership_page_url()),
]
}
fn pdf_headers() -> Vec<(String, String)> {
vec![
("User-Agent".to_string(), USER_AGENT.to_string()),
(
"Accept".to_string(),
"application/pdf,application/octet-stream,*/*;q=0.8".to_string(),
),
(
"Accept-Language".to_string(),
"id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7".to_string(),
),
("Referer".to_string(), announcement_listing_url()),
]
}
fn item_matches_family(item: &AnnouncementItem, query: &DiscoveryQuery) -> bool {
let title = item.title.to_ascii_uppercase();
query
.title_needles
.iter()
.any(|needle| title.contains(needle))
}
fn item_pdf_attachments(item: &AnnouncementItem) -> Vec<AnnouncementAttachment> {
let attachments = if item.attachments.is_empty() {
item.pdf_path
.as_deref()
.and_then(parse_pdf_path)
.unwrap_or_default()
} else {
item.attachments.clone()
};
attachments
.into_iter()
.filter(|attachment| {
attachment
.full_save_path
.to_ascii_lowercase()
.ends_with(".pdf")
})
.collect()
}
fn attachment_label(attachment: &AnnouncementAttachment) -> String {
attachment
.original_filename
.as_deref()
.or(attachment.pdf_filename.as_deref())
.unwrap_or_default()
.trim()
.to_ascii_lowercase()
}
fn classify_report_status(
family: OwnershipReportFamily,
is_attachment: bool,
) -> OwnershipReportStatus {
match (family, is_attachment) {
(OwnershipReportFamily::AboveOnePercent, true) => OwnershipReportStatus::Supported,
(OwnershipReportFamily::AboveOnePercent, false) => OwnershipReportStatus::AnnouncementOnly,
_ => OwnershipReportStatus::Unsupported,
}
}
fn parse_pdf_path(raw: &str) -> Option<Vec<AnnouncementAttachment>> {
serde_json::from_str::<Vec<AnnouncementAttachment>>(raw).ok()
}
fn attachment_is_attachment(attachment: &AnnouncementAttachment) -> bool {
match attachment.is_attachment.as_ref() {
Some(serde_json::Value::Bool(value)) => *value,
Some(serde_json::Value::Number(value)) => value.as_i64() == Some(1),
Some(serde_json::Value::String(value)) => {
matches!(value.trim(), "1" | "true" | "TRUE" | "True")
}
_ => attachment_label(attachment).contains("lamp"),
}
}
fn clean_option(value: Option<&str>) -> Option<String> {
value
.map(str::trim)
.filter(|value| !value.is_empty())
.map(str::to_string)
}
fn percent_encode(value: &str) -> String {
let mut encoded = String::with_capacity(value.len());
for byte in value.bytes() {
match byte {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
encoded.push(byte as char)
}
b' ' => encoded.push_str("%20"),
other => encoded.push_str(&format!("%{other:02X}")),
}
}
encoded
}
#[cfg(test)]
mod tests {
use super::{
AnnouncementPage, IDX_ANNOUNCEMENT_LISTING_URL, OwnershipReportFamily,
OwnershipReportStatus, build_announcement_query_url, parse_announcement_page,
parse_indonesian_as_of_date, parse_share_ownership_page, select_latest_ownership_reports,
validate_pdf_payload, validate_share_ownership_page, validate_xlsx_payload,
};
use crate::error::IdxError;
#[test]
fn builds_announcement_query_url() {
let url = build_announcement_query_url("pemegang saham di atas 5", 1, 10);
assert_eq!(
url,
"https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%205&pageNumber=1&pageSize=10&lang=id"
);
}
#[test]
fn parses_fixture_and_selects_latest_above_5_reports() {
let raw = include_str!("../../tests/fixtures/idx_announcement_kepemilikan.json");
let page = parse_announcement_page(raw).expect("announcement fixture should parse");
let discovered = select_latest_ownership_reports(
&page,
"https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%205&pageNumber=1&pageSize=10&lang=id",
OwnershipReportFamily::AboveFivePercent,
)
.expect("should select ownership reports");
assert_eq!(discovered.len(), 2);
assert_eq!(
discovered[0].family,
OwnershipReportFamily::AboveFivePercent
);
assert_eq!(discovered[0].listing_page_url, IDX_ANNOUNCEMENT_LISTING_URL);
assert_eq!(discovered[0].publish_date, "2026-03-27T16:34:20");
assert_eq!(
discovered[0].pdf_url,
"https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/4f5c4efc6f_bf70f249ac_lamp1.pdf"
);
assert_eq!(
discovered[0].original_filename.as_deref(),
Some("20260327_Semua Emiten Saham_Pengumuman Bursa_32055594_lamp1.pdf")
);
assert_eq!(discovered[0].status, OwnershipReportStatus::Unsupported);
assert!(discovered[0].is_attachment);
assert_eq!(
discovered[1].pdf_url,
"https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/5d31bb6f49_announcement.pdf"
);
assert_eq!(
discovered[1].original_filename.as_deref(),
Some("20260327_Semua Emiten Saham_Pengumuman Bursa_32055594.pdf")
);
assert!(!discovered[1].is_attachment);
}
#[test]
fn selects_above_1_reports() {
let raw = r#"{
"Items": [
{
"PublishDate": "2026-03-10T12:09:09",
"Title": "Pemegang Saham di atas 1% (KSEI)",
"AnnouncementType": "",
"Code": " Semua Emiten Saham ",
"Attachments": [
{
"PDFFilename": "d67ebf37e6_10d4080288.pdf",
"FullSavePath": "https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/d67ebf37e6_10d4080288.pdf",
"IsAttachment": 0,
"OriginalFilename": "20260310_Semua Emiten Saham_Pengumuman Bursa_32052554.pdf"
},
{
"PDFFilename": "b9b638e5a8_8928aca255.pdf",
"FullSavePath": "https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/b9b638e5a8_8928aca255.pdf",
"IsAttachment": 1,
"OriginalFilename": "20260310_Semua Emiten Saham_Pengumuman Bursa_32052554_lamp1.pdf"
}
],
"PdfPath": ""
}
],
"ItemCount": 1,
"PageCount": 1
}"#;
let page = parse_announcement_page(raw).expect("above-1 fixture should parse");
let discovered = select_latest_ownership_reports(
&page,
"https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%201&pageNumber=1&pageSize=10&lang=id",
OwnershipReportFamily::AboveOnePercent,
)
.expect("should select above-1 reports");
assert_eq!(discovered.len(), 2);
assert_eq!(discovered[0].family, OwnershipReportFamily::AboveOnePercent);
assert_eq!(discovered[0].status, OwnershipReportStatus::Supported);
assert_eq!(discovered[0].code.as_deref(), Some("Semua Emiten Saham"));
assert_eq!(
discovered[0].original_filename.as_deref(),
Some("20260310_Semua Emiten Saham_Pengumuman Bursa_32052554_lamp1.pdf")
);
assert!(discovered[0].is_attachment);
assert_eq!(
discovered[1].status,
OwnershipReportStatus::AnnouncementOnly
);
assert!(!discovered[1].is_attachment);
}
#[test]
fn selects_investor_type_breakdown_reports() {
let raw = r#"{
"Items": [
{
"PublishDate": "2026-03-02T16:14:37",
"Title": "Data KSEI terkait Kepemilikan Saham Perusahaan Tercatat Berdasarkan Tipe Investor per 27 Februari 2026",
"AnnouncementType": "",
"Code": " ",
"Attachments": [
{
"PDFFilename": "20260302_Pengumuman Bursa_32040089.pdf",
"FullSavePath": "https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/43025665c8_fda8953269.pdf",
"IsAttachment": 0,
"OriginalFilename": "20260302_Pengumuman Bursa_32040089.pdf"
},
{
"PDFFilename": "20260302_Pengumuman Bursa_32040089_lamp1.pdf",
"FullSavePath": "https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/db5b5a86e1_2e2b3d976a.pdf",
"IsAttachment": 1,
"OriginalFilename": "20260302_Pengumuman Bursa_32040089_lamp1.pdf"
}
],
"PdfPath": ""
}
],
"ItemCount": 1,
"PageCount": 1
}"#;
let page = parse_announcement_page(raw).expect("investor-type fixture should parse");
let discovered = select_latest_ownership_reports(
&page,
"https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=kepemilikan%20saham%20perusahaan%20tercatat&pageNumber=1&pageSize=10&lang=id",
OwnershipReportFamily::InvestorTypeBreakdown,
)
.expect("should select investor-type reports");
assert_eq!(discovered.len(), 2);
assert_eq!(
discovered[0].original_filename.as_deref(),
Some("20260302_Pengumuman Bursa_32040089_lamp1.pdf")
);
assert_eq!(discovered[0].status, OwnershipReportStatus::Unsupported);
assert_eq!(
discovered[1].original_filename.as_deref(),
Some("20260302_Pengumuman Bursa_32040089.pdf")
);
}
#[test]
fn falls_back_to_pdf_path_when_attachments_are_missing() {
let raw = r#"{
"Items": [
{
"PublishDate": "2026-03-04T08:54:00",
"Title": "Pemegang Saham di atas 5% (KSEI)",
"AnnouncementType": "",
"Code": "Semua Emiten Saham",
"Attachments": [],
"PdfPath": "[{\"PDFFilename\":\"20260305_LKS_KSEI_000043_lamp1.pdf\",\"FullSavePath\":\"https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/ec835d7451_c30c886a1a.pdf\",\"IsAttachment\":\"1\",\"OriginalFilename\":\"20260305_LKS_KSEI_000043_lamp1.pdf\"}]"
}
],
"ItemCount": 1,
"PageCount": 1
}"#;
let page: AnnouncementPage = parse_announcement_page(raw).expect("fallback page parses");
let discovered = select_latest_ownership_reports(
&page,
"https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%205&pageNumber=1&pageSize=10&lang=id",
OwnershipReportFamily::AboveFivePercent,
)
.expect("fallback attachment should be parsed");
assert_eq!(discovered.len(), 1);
assert_eq!(
discovered[0].pdf_url,
"https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/ec835d7451_c30c886a1a.pdf"
);
assert_eq!(discovered[0].code.as_deref(), Some("Semua Emiten Saham"));
assert!(discovered[0].is_attachment);
}
#[test]
fn rejects_html_instead_of_announcement_json() {
let err = parse_announcement_page("<!doctype html><html><body>blocked</body></html>")
.expect_err("html response must fail");
assert!(matches!(err, IdxError::Http(_)));
}
#[test]
fn rejects_empty_announcement_payload() {
let err = parse_announcement_page(" ").expect_err("empty payload must fail");
assert!(matches!(err, IdxError::Http(_)));
}
#[test]
fn select_latest_reports_errors_when_family_is_missing() {
let raw = r#"{"Items":[],"ItemCount":0,"PageCount":0}"#;
let page = parse_announcement_page(raw).expect("empty page parses");
let err = select_latest_ownership_reports(
&page,
"https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%201&pageNumber=1&pageSize=10&lang=id",
OwnershipReportFamily::AboveOnePercent,
)
.expect_err("missing announcement should fail");
assert!(matches!(err, IdxError::Http(_)));
assert!(err.to_string().contains("no matching announcement found"));
}
#[test]
fn select_latest_reports_errors_when_matching_item_has_no_pdf() {
let raw = r#"{
"Items": [
{
"PublishDate": "2026-03-10T12:09:09",
"Title": "Pemegang Saham di atas 1% (KSEI)",
"AnnouncementType": "",
"Code": "Semua Emiten Saham",
"Attachments": [],
"PdfPath": ""
}
],
"ItemCount": 1,
"PageCount": 1
}"#;
let page = parse_announcement_page(raw).expect("page parses");
let err = select_latest_ownership_reports(
&page,
"https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%201&pageNumber=1&pageSize=10&lang=id",
OwnershipReportFamily::AboveOnePercent,
)
.expect_err("missing pdf attachments should fail");
assert!(matches!(err, IdxError::Http(_)));
assert!(err.to_string().contains("no PDF attachments"));
}
#[test]
fn accepts_pdf_header() {
validate_pdf_payload(b"%PDF-1.7\n1 0 obj\n").expect("pdf header should pass");
}
#[test]
fn rejects_html_instead_of_pdf() {
let err = validate_pdf_payload(b"<!doctype html><html><body>blocked</body></html>")
.expect_err("html body must fail");
assert!(matches!(err, IdxError::Http(_)));
assert!(err.to_string().contains("PDF/direct attachment"));
}
#[test]
fn parses_share_ownership_page_entries() {
let page_url = "https://www.idx.co.id/id/perusahaan-tercatat/data-kepemilikan-saham/";
let reports = parse_share_ownership_page(
include_str!("../../tests/fixtures/idx_share_ownership_page_excerpt.html"),
page_url,
);
assert_eq!(reports.len(), 7);
let above1: Vec<_> = reports
.iter()
.filter(|r| r.family == OwnershipReportFamily::AboveOnePercent)
.collect();
assert_eq!(above1.len(), 3);
assert!(
above1
.iter()
.all(|r| r.status == OwnershipReportStatus::Supported)
);
assert!(above1.iter().all(|r| r.format == "xlsx"));
let august = above1
.iter()
.find(|r| r.as_of_date.as_deref() == Some("2026-08-31"))
.expect("August above-1% entry");
assert_eq!(
august.pdf_url,
"https://www.idx.co.id/Media/fahlw1o2/peng-2026-08-00017-satu-persen.xlsx"
);
assert_eq!(august.publish_date, "2026-08-31");
assert_eq!(august.listing_page_url, page_url);
assert_eq!(
august.original_filename.as_deref(),
Some("peng-2026-08-00017-satu-persen.xlsx")
);
let above5 = reports
.iter()
.filter(|r| r.family == OwnershipReportFamily::AboveFivePercent)
.count();
let breakdown = reports
.iter()
.filter(|r| r.family == OwnershipReportFamily::InvestorTypeBreakdown)
.count();
assert_eq!((above5, breakdown), (2, 2));
assert!(
reports
.iter()
.filter(|r| r.family != OwnershipReportFamily::AboveOnePercent)
.all(|r| r.status == OwnershipReportStatus::Unsupported)
);
}
#[test]
fn share_ownership_page_handles_query_strings_and_dateless_entries() {
let html = r#"{Description:"Pemegang Saham di Atas 1% per 31 Agustus 2026",Prospectus:"https://www.idx.co.id/Media/x/peng-satu-persen.xlsx?download=1",ListingDate:r},{Description:"Pemegang Saham di Atas 1%",Prospectus:"https://www.idx.co.id/Media/y/no-date.xlsx",ListingDate:r}"#;
let reports = parse_share_ownership_page(html, "p");
assert_eq!(
reports.len(),
1,
"dateless entry skipped, query-string entry kept"
);
assert_eq!(reports[0].format, "xlsx");
assert_eq!(reports[0].status, OwnershipReportStatus::Supported);
assert_eq!(
reports[0].original_filename.as_deref(),
Some("peng-satu-persen.xlsx")
);
assert_eq!(reports[0].publish_date, "2026-08-31");
}
#[test]
fn share_ownership_page_without_payload_yields_nothing() {
assert!(parse_share_ownership_page("<html>Just a moment...</html>", "p").is_empty());
assert!(validate_share_ownership_page(b"<html>Just a moment...</html>").is_err());
}
#[test]
fn parses_indonesian_and_english_as_of_dates() {
let date = |y, m, d| chrono::NaiveDate::from_ymd_opt(y, m, d);
assert_eq!(
parse_indonesian_as_of_date("Pemegang Saham di Atas 1% per 31 Agustus 2026"),
date(2026, 8, 31)
);
assert_eq!(
parse_indonesian_as_of_date("Shareholders above 1% as of 29 May 2026"),
date(2026, 5, 29)
);
assert_eq!(
parse_indonesian_as_of_date("Pemegang Saham di Atas 1%"),
None
);
}
#[test]
fn validates_xlsx_payloads() {
assert!(validate_xlsx_payload(b"PK\x03\x04rest-of-zip").is_ok());
let err = validate_xlsx_payload(b"<!DOCTYPE html><title>Just a moment...</title>")
.expect_err("html challenge rejected");
assert!(err.to_string().contains("Cloudflare"), "{err}");
assert!(validate_xlsx_payload(b"%PDF-1.7").is_err());
}
}
Reference in a new issue View git blame Copy permalink