0xrsydn/idx-cli
1
mirror of https://github.com/0xrsydn/idx-cli.git synced 2026-08-07 01:33:52 +00:00
Code Issues Projects Releases 1 Packages Wiki Activity

ownership: add IDX above1 discovery and import

Browse source 
Refactor KSEI PDF parsing for the live above-1 holder-register layout, add IDX announcement discovery plus browser-impersonated PDF fetches, and update the ownership docs to mark Batch 1 complete and scope Batch 2 around above1 hardening and unsupported legacy inputs.
This commit is contained in:
Rasyidan Akbar F. 2026-03-30 11:57:14 +07:00
commit a36648fd89
13 changed files with 1716 additions and 464 deletions
Download patch file Download diff file Expand all files Collapse all files

58
src/api/yahoo/client.rs
View file

@ -1,9 +1,10 @@
use std::path::{Path, PathBuf};
use std::process::{Command, Output};
use std::process::Command;
use std::sync::Mutex;
use std::time::Duration;
use crate::api::types::{Interval, Period};
use crate::curl_impersonate;
use crate::error::IdxError;
use super::raw_types::{ChartResponse, QuoteSummaryResponse};
@ -117,56 +118,8 @@ impl YahooClient {
Ok(cookies.join("; "))
}
fn chrome_curl_binary() -> Option<&'static str> {
// curl-impersonate-chrome ships per-version binaries (curl_chrome131 etc).
// Try latest versions first; no --impersonate flag needed; the binary is the impersonation.
const CANDIDATES: &[&str] = &[
"curl_chrome136",
"curl_chrome133a",
"curl_chrome131",
"curl_chrome124",
"curl_chrome120",
"curl_chrome116",
];
CANDIDATES
.iter()
.copied()
.find(|bin| Command::new(bin).arg("--version").output().is_ok())
}
fn run_curl(stage: &str, binary: &str, args: &[&str]) -> Result<Output, IdxError> {
let output = Command::new(binary).args(args).output().map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
return IdxError::Http(format!(
"curl-impersonate binary '{binary}' not found; install nixpkgs#curl-impersonate-chrome"
));
}
IdxError::Http(format!("failed to run {binary} for Yahoo {stage}: {e}"))
})?;
if output.status.success() {
return Ok(output);
}
let stderr = String::from_utf8_lossy(&output.stderr);
let detail = stderr.trim();
Err(IdxError::Http(format!(
"Yahoo {stage} {binary} failed (status {}): {}",
output.status,
if detail.is_empty() {
"no output"
} else {
detail
}
)))
}
fn fetch_crumb_via_curl(&self) -> Result<String, IdxError> {
let binary = Self::chrome_curl_binary().ok_or_else(|| {
IdxError::Http(
"no curl_chrome* binary found; install nixpkgs#curl-impersonate-chrome".to_string(),
)
})?;
let binary = curl_impersonate::chrome_curl_binary()?;
let cookie_jar = Self::cookie_jar_path();
let cookie_jar_str = cookie_jar.to_str().ok_or_else(|| {
@ -190,9 +143,8 @@ impl YahooClient {
.output();
// Step 2: fetch crumb with cookie jar (Chrome TLS fingerprint + A3 cookie).
let output = Self::run_curl(
"crumb fetch",
binary,
let output = curl_impersonate::run(
"Yahoo crumb fetch",
&["--silent", "--cookie", cookie_jar_str, CRUMB_FETCH_URL],
)?;

112
src/cli/ownership.rs
View file

@ -18,7 +18,7 @@ use crate::output::table::format_idr;
use crate::ownership::types::{
ChangeType, FlowSignal, HolderRow, KseiHolding, OwnershipRelease, OwnershipSource,
};
use crate::ownership::{db, entities, graph, parser, search};
use crate::ownership::{db, entities, graph, parser, remote, search};
#[derive(Debug, Args)]
pub struct OwnershipCmd {
@ -28,6 +28,8 @@ pub struct OwnershipCmd {
#[derive(Debug, Subcommand)]
pub enum OwnershipCommand {
/// Discover the latest IDX-hosted ownership report URLs.
Discover(DiscoverArgs),
/// Import ownership data from KSEI PDF or Bing API.
Import(ImportArgs),
/// Show all holders for a ticker (KSEI + Bing combined).
@ -52,9 +54,19 @@ pub enum OwnershipCommand {
Releases,
}
#[derive(Debug, Args)]
pub struct DiscoverArgs {
/// Report family to discover: all, above1, above5, or investor-type.
#[arg(long, default_value = "all")]
pub family: String,
/// Maximum number of discovered report URLs to print.
#[arg(long, default_value_t = 6)]
pub limit: usize,
}
#[derive(Debug, Args)]
pub struct ImportArgs {
/// URL to KSEI ownership PDF.
/// URL to a remote ownership PDF.
#[arg(long)]
pub url: Option<String>,
/// Path to local KSEI PDF file.
@ -159,6 +171,7 @@ pub enum ResolveCommand {
pub fn handle(cmd: &OwnershipCommand, config: &IdxConfig) -> Result<(), IdxError> {
match cmd {
OwnershipCommand::Discover(args) => handle_discover(args, config),
OwnershipCommand::Import(args) => handle_import(args, config),
OwnershipCommand::Ticker(args) => handle_ticker(args, config),
OwnershipCommand::Entity(args) => handle_entity(args, config),
@ -173,6 +186,45 @@ pub fn handle(cmd: &OwnershipCommand, config: &IdxConfig) -> Result<(), IdxError
}
}
fn handle_discover(args: &DiscoverArgs, config: &IdxConfig) -> Result<(), IdxError> {
if args.limit == 0 {
return Err(IdxError::ParseError(
"--limit must be greater than 0".to_string(),
));
}
let family = parse_discovery_family(&args.family)?;
let reports = remote::discover_idx_ownership_reports(family, args.limit)?;
if matches!(config.output, OutputFormat::Json) {
return json::print_json(&reports);
}
let mut table = Table::new();
table
.load_preset(UTF8_FULL)
.set_content_arrangement(ContentArrangement::Dynamic)
.set_header(vec!["DATE", "FAMILY", "KIND", "FILE", "TITLE", "URL"]);
for report in reports {
table.add_row(vec![
Cell::new(report.publish_date.split('T').next().unwrap_or("-")),
Cell::new(report.family.label()),
Cell::new(if report.is_attachment {
"attachment"
} else {
"main"
}),
Cell::new(report.original_filename.unwrap_or_else(|| "-".to_string())),
Cell::new(report.title),
Cell::new(report.pdf_url),
]);
}
println!("{table}");
Ok(())
}
fn handle_ticker(args: &TickerArgs, config: &IdxConfig) -> Result<(), IdxError> {
let conn = db::open_db(config)?;
let symbol = args.symbol.trim().to_uppercase();
@ -660,19 +712,19 @@ fn handle_import(args: &ImportArgs, config: &IdxConfig) -> Result<(), IdxError>
}
}
let Some(pdf_path) = resolve_pdf_input(args)? else {
let Some(pdf_input) = resolve_pdf_input(args)? else {
return Ok(());
};
let conn = db::open_db(config)?;
let sha256 = sha256_file(&pdf_path)?;
let sha256 = sha256_file(&pdf_input.pdf_path)?;
if !args.force && db::release_exists(&conn, &sha256)? {
println!("Release already imported (sha256: {sha256}). Use --force to re-import.");
return Ok(());
}
let raw_rows = parser::parse_ksei_pdf(&pdf_path)?;
let raw_rows = parser::parse_ksei_pdf(&pdf_input.pdf_path)?;
if raw_rows.is_empty() {
return Err(IdxError::ParseError(
"no KSEI rows parsed from PDF".to_string(),
@ -718,7 +770,7 @@ fn handle_import(args: &ImportArgs, config: &IdxConfig) -> Result<(), IdxError>
let release = OwnershipRelease {
id: 0,
source_url: args.url.clone(),
source_url: pdf_input.source_url,
sha256,
as_of_date,
row_count: inserted_rows,
@ -769,7 +821,30 @@ fn handle_releases(config: &IdxConfig) -> Result<(), IdxError> {
Ok(())
}
fn resolve_pdf_input(args: &ImportArgs) -> Result<Option<PathBuf>, IdxError> {
fn parse_discovery_family(raw: &str) -> Result<Option<remote::OwnershipReportFamily>, IdxError> {
match raw.trim().to_ascii_lowercase().as_str() {
"all" => Ok(None),
"above1" | "above-1" | "above_1" => {
Ok(Some(remote::OwnershipReportFamily::AboveOnePercent))
}
"above5" | "above-5" | "above_5" => {
Ok(Some(remote::OwnershipReportFamily::AboveFivePercent))
}
"investor-type" | "investor_type" | "investortype" => {
Ok(Some(remote::OwnershipReportFamily::InvestorTypeBreakdown))
}
_ => Err(IdxError::ParseError(
"invalid --family, expected: all|above1|above5|investor-type".to_string(),
)),
}
}
struct ResolvedPdfInput {
pdf_path: PathBuf,
source_url: Option<String>,
}
fn resolve_pdf_input(args: &ImportArgs) -> Result<Option<ResolvedPdfInput>, IdxError> {
if let Some(path) = &args.file {
if !path.exists() {
return Err(IdxError::Io(format!(
@ -777,13 +852,19 @@ fn resolve_pdf_input(args: &ImportArgs) -> Result<Option<PathBuf>, IdxError> {
path.display()
)));
}
return Ok(Some(path.clone()));
return Ok(Some(ResolvedPdfInput {
pdf_path: path.clone(),
source_url: None,
}));
}
if let Some(url) = &args.url {
let target = cache_pdf_path(url)?;
download_pdf(url, &target)?;
return Ok(Some(target));
return Ok(Some(ResolvedPdfInput {
pdf_path: target,
source_url: Some(url.clone()),
}));
}
Ok(None)
@ -815,6 +896,10 @@ fn cache_pdf_path(url: &str) -> Result<PathBuf, IdxError> {
}
fn download_pdf(url: &str, target: &Path) -> Result<(), IdxError> {
if is_idx_url(url) {
return remote::download_idx_pdf(url, target);
}
let response = ureq::get(url)
.header(
"User-Agent",
@ -830,6 +915,7 @@ fn download_pdf(url: &str, target: &Path) -> Result<(), IdxError> {
let bytes = body
.read_to_vec()
.map_err(|e| IdxError::Http(format!("failed reading PDF body: {e}")))?;
remote::validate_pdf_payload(&bytes)?;
fs::write(target, &bytes).map_err(|e| {
IdxError::Io(format!(
@ -841,6 +927,14 @@ fn download_pdf(url: &str, target: &Path) -> Result<(), IdxError> {
Ok(())
}
fn is_idx_url(url: &str) -> bool {
let normalized = url.trim().to_ascii_lowercase();
normalized.starts_with("https://www.idx.co.id/")
|| normalized.starts_with("http://www.idx.co.id/")
|| normalized.starts_with("https://idx.co.id/")
|| normalized.starts_with("http://idx.co.id/")
}
fn sha256_file(path: &Path) -> Result<String, IdxError> {
let bytes = fs::read(path).map_err(|e| {
IdxError::Io(format!(

67
src/curl_impersonate.rs Normal file
View file

@ -0,0 +1,67 @@
use std::process::{Command, Output};
use crate::error::IdxError;
const CANDIDATES: &[&str] = &[
"curl_chrome142",
"curl_chrome136",
"curl_chrome133a",
"curl_chrome131",
"curl_chrome124",
"curl_chrome120",
"curl_chrome116",
];
const OVERRIDE_ENV: &str = "IDX_CURL_IMPERSONATE_BIN";
pub fn chrome_curl_binary() -> Result<String, IdxError> {
if let Ok(value) = std::env::var(OVERRIDE_ENV) {
let trimmed = value.trim();
if !trimmed.is_empty() {
return Ok(trimmed.to_string());
}
}
CANDIDATES
.iter()
.copied()
.find(|bin| Command::new(bin).arg("--version").output().is_ok())
.map(str::to_string)
.ok_or_else(|| {
IdxError::Http(format!(
"no curl_chrome* binary found; set {OVERRIDE_ENV} or install nixpkgs#curl-impersonate-chrome"
))
})
}
pub fn run(stage: &str, args: &[&str]) -> Result<Output, IdxError> {
let owned_args: Vec<String> = args.iter().map(|arg| (*arg).to_string()).collect();
run_owned(stage, &owned_args)
}
pub fn run_owned(stage: &str, args: &[String]) -> Result<Output, IdxError> {
let binary = chrome_curl_binary()?;
let output = Command::new(&binary).args(args).output().map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
return IdxError::Http(format!(
"curl-impersonate binary '{binary}' not found; set {OVERRIDE_ENV} or install nixpkgs#curl-impersonate-chrome"
));
}
IdxError::Http(format!("failed to run {binary} for {stage}: {e}"))
})?;
if output.status.success() {
return Ok(output);
}
let stderr = String::from_utf8_lossy(&output.stderr);
let detail = stderr.trim();
Err(IdxError::Http(format!(
"{stage} {binary} failed (status {}): {}",
output.status,
if detail.is_empty() {
"no output"
} else {
detail
}
)))
}

1
src/main.rs
View file

@ -3,6 +3,7 @@ mod api;
mod cache;
mod cli;
mod config;
mod curl_impersonate;
mod error;
mod output;
#[cfg(feature = "ownership")]

2
src/ownership/entities.rs
View file

@ -215,7 +215,7 @@ fn normalize_investor_type(raw: &str) -> Option<InvestorTypeCode> {
fn normalize_locality(raw: &str) -> Option<Locality> {
match raw.trim().to_uppercase().as_str() {
"L" => Some(Locality::Local),
"L" | "D" => Some(Locality::Local),
"F" | "A" => Some(Locality::Foreign),
_ => None,
}

1
src/ownership/mod.rs
View file

@ -2,5 +2,6 @@ pub mod db;
pub mod entities;
pub mod graph;
pub mod parser;
pub mod remote;
pub mod search;
pub mod types;

520
src/ownership/parser.rs
View file

@ -3,32 +3,36 @@ use std::path::Path;
use std::process::Command;
use quick_xml::Reader;
use quick_xml::events::Event;
use quick_xml::events::{BytesStart, Event};
use crate::error::IdxError;
use crate::ownership::types::KseiRawRow;
/// Character grid for a single PDF page: y-coord → column-index → sorted (x, char) pairs.
type PageGrid = HashMap<i32, HashMap<usize, Vec<(i32, char)>>>;
const Y_TOLERANCE: f32 = 0.8;
const HEADER_MATCH_MIN: usize = 4;
/// Inclusive-left, exclusive-right X ranges for each KSEI data column.
const COLUMN_BOUNDS: [(f32, f32); 12] = [
(15.0, 52.0), // date
(52.0, 70.0), // share_code
(70.0, 167.0), // issuer_name
(167.0, 432.0), // investor_name
(432.0, 463.0), // investor_type
(463.0, 497.0), // local_foreign
(497.0, 558.0), // nationality
(558.0, 615.0), // domicile
(615.0, 653.0), // holdings_scripless
(653.0, 692.0), // holdings_scrip
(692.0, 745.0), // total_holding_shares
(745.0, 800.0), // percentage
const HEADER_LABELS: &[&str] = &[
"DATE",
"SHARECODE",
"ISSUERNAME",
"INVESTORNAME",
"INVESTORTYPE",
"LOCALFOREIGN",
"NATIONALITY",
"DOMICILE",
"HOLDINGSSCRIPLESS",
"HOLDINGSSCRIP",
"TOTALHOLDINGSHARES",
"PERCENTAGE",
];
#[derive(Debug, Clone)]
struct PageLine {
x: f32,
y: f32,
text: String,
}
/// Parse a KSEI ownership PDF into raw rows.
/// Shells out to `mutool` for XML extraction, then parses with quick-xml.
pub fn parse_ksei_pdf(path: &Path) -> Result<Vec<KseiRawRow>, IdxError> {
@ -65,76 +69,24 @@ pub fn parse_stext_xml(xml: &str) -> Result<Vec<KseiRawRow>, IdxError> {
reader.config_mut().trim_text(false);
let mut rows: Vec<KseiRawRow> = Vec::new();
let mut current_page: Option<PageGrid> = None;
let mut current_page: Option<Vec<PageLine>> = None;
let mut buf = Vec::new();
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(e)) if e.name().as_ref() == b"page" => {
current_page = Some(HashMap::new());
current_page = Some(Vec::new());
}
Ok(Event::Empty(e)) if e.name().as_ref() == b"char" => {
if let Some(page) = current_page.as_mut() {
let mut x: Option<f32> = None;
let mut y: Option<f32> = None;
let mut c: Option<char> = None;
for attr_result in e.attributes().with_checks(false) {
let attr = attr_result.map_err(|err| {
IdxError::PdfParseError(format!("invalid XML attribute: {err}"))
})?;
match attr.key.as_ref() {
b"x" => {
let s = attr.decode_and_unescape_value(reader.decoder()).map_err(
|err| {
IdxError::PdfParseError(format!(
"invalid XML x attribute: {err}"
))
},
)?;
x = s.parse::<f32>().ok();
}
b"y" => {
let s = attr.decode_and_unescape_value(reader.decoder()).map_err(
|err| {
IdxError::PdfParseError(format!(
"invalid XML y attribute: {err}"
))
},
)?;
y = s.parse::<f32>().ok();
}
b"c" => {
let s = attr.decode_and_unescape_value(reader.decoder()).map_err(
|err| {
IdxError::PdfParseError(format!(
"invalid XML char attribute: {err}"
))
},
)?;
c = s.chars().next();
}
_ => {}
}
}
if let (Some(x_val), Some(y_val), Some(ch)) = (x, y, c)
&& let Some(col_idx) = x_to_column(x_val)
{
let yb = y_bucket(y_val);
let xi = (x_val * 100.0).round() as i32;
page.entry(yb)
.or_default()
.entry(col_idx)
.or_default()
.push((xi, ch));
}
Ok(Event::Start(e)) if e.name().as_ref() == b"line" => {
if let Some(page) = current_page.as_mut()
&& let Some(line) = parse_line_attrs(&reader, &e)?
{
page.push(line);
}
}
Ok(Event::End(e)) if e.name().as_ref() == b"page" => {
if let Some(page) = current_page.take() {
rows.extend(extract_rows_from_page(page));
rows.extend(extract_rows_from_page(&page));
}
}
Ok(Event::Eof) => break,
@ -164,67 +116,263 @@ pub fn check_mutool() -> Result<(), IdxError> {
Ok(())
}
fn extract_rows_from_page(page: PageGrid) -> Vec<KseiRawRow> {
let mut page_rows = Vec::new();
fn parse_line_attrs(
reader: &Reader<&[u8]>,
event: &BytesStart<'_>,
) -> Result<Option<PageLine>, IdxError> {
let mut bbox: Option<String> = None;
let mut text: Option<String> = None;
let mut y_keys: Vec<i32> = page.keys().copied().collect();
for attr_result in event.attributes().with_checks(false) {
let attr = attr_result
.map_err(|err| IdxError::PdfParseError(format!("invalid XML attribute: {err}")))?;
match attr.key.as_ref() {
b"bbox" => {
bbox = Some(
attr.decode_and_unescape_value(reader.decoder())
.map_err(|err| {
IdxError::PdfParseError(format!("invalid XML bbox attribute: {err}"))
})?
.to_string(),
);
}
b"text" => {
text = Some(
attr.decode_and_unescape_value(reader.decoder())
.map_err(|err| {
IdxError::PdfParseError(format!("invalid XML text attribute: {err}"))
})?
.to_string(),
);
}
_ => {}
}
}
let Some(text) = text.map(|value| normalize_spaces(&value)) else {
return Ok(None);
};
if text.is_empty() {
return Ok(None);
}
let Some(bbox) = bbox else {
return Ok(None);
};
let mut parts = bbox.split_whitespace();
let x = parts.next().and_then(|value| value.parse::<f32>().ok());
let y = parts.next().and_then(|value| value.parse::<f32>().ok());
match (x, y) {
(Some(x), Some(y)) => Ok(Some(PageLine { x, y, text })),
_ => Ok(None),
}
}
fn extract_rows_from_page(page: &[PageLine]) -> Vec<KseiRawRow> {
let mut rows_by_y: HashMap<i32, Vec<PageLine>> = HashMap::new();
for line in page {
rows_by_y
.entry(y_bucket(line.y))
.or_default()
.push(line.clone());
}
let mut y_keys: Vec<i32> = rows_by_y.keys().copied().collect();
y_keys.sort_unstable();
let mut rows = Vec::new();
for y in y_keys {
let mut row = KseiRawRow {
date: String::new(),
share_code: String::new(),
issuer_name: String::new(),
investor_name: String::new(),
investor_type: String::new(),
local_foreign: String::new(),
nationality: String::new(),
domicile: String::new(),
holdings_scripless: String::new(),
holdings_scrip: String::new(),
total_holding_shares: String::new(),
percentage: String::new(),
let Some(lines) = rows_by_y.get(&y) else {
continue;
};
if let Some(col_map) = page.get(&y) {
for (col_idx, chars) in col_map {
let mut sorted = chars.clone();
sorted.sort_by_key(|(x, _)| *x);
let text = normalize_spaces(&sorted.iter().map(|(_, c)| c).collect::<String>());
assign_column(&mut row, *col_idx, text);
}
let mut sorted = lines.clone();
sorted.sort_by(|left, right| left.x.total_cmp(&right.x));
let texts: Vec<String> = sorted.into_iter().map(|line| line.text).collect();
if is_header_row(&texts) {
continue;
}
let row = parse_row_segments(&texts);
if is_data_row(&row) {
page_rows.push(row);
rows.push(row);
}
}
page_rows
rows
}
fn assign_column(row: &mut KseiRawRow, col_idx: usize, value: String) {
match col_idx {
0 => row.date = value,
1 => row.share_code = value,
2 => row.issuer_name = value,
3 => row.investor_name = value,
4 => row.investor_type = value,
5 => row.local_foreign = value,
6 => row.nationality = value,
7 => row.domicile = value,
8 => row.holdings_scripless = value,
9 => row.holdings_scrip = value,
10 => row.total_holding_shares = value,
11 => row.percentage = value,
_ => {}
}
}
fn x_to_column(x: f32) -> Option<usize> {
COLUMN_BOUNDS
fn is_header_row(texts: &[String]) -> bool {
texts
.iter()
.position(|(left, right)| x >= *left && x < *right)
.filter(|text| HEADER_LABELS.contains(&normalize_header_label(text).as_str()))
.count()
>= HEADER_MATCH_MIN
}
fn parse_row_segments(texts: &[String]) -> KseiRawRow {
let mut row = KseiRawRow {
date: String::new(),
share_code: String::new(),
issuer_name: String::new(),
investor_name: String::new(),
investor_type: String::new(),
local_foreign: String::new(),
nationality: String::new(),
domicile: String::new(),
holdings_scripless: String::new(),
holdings_scrip: String::new(),
total_holding_shares: String::new(),
percentage: String::new(),
};
let mut remaining: Vec<String> = texts
.iter()
.map(|text| normalize_spaces(text))
.filter(|text| !text.is_empty())
.collect();
if remaining.is_empty() {
return row;
}
if let Some((date, share_code)) = split_date_and_share(&remaining[0]) {
row.date = date;
row.share_code = share_code;
let _ = remaining.remove(0);
}
if row.date.is_empty() {
return row;
}
if row.share_code.is_empty()
&& remaining
.first()
.is_some_and(|segment| is_share_code_like(segment))
{
row.share_code = remaining.remove(0);
}
if remaining
.last()
.is_some_and(|segment| is_percentage_like(segment))
{
row.percentage = remaining.pop().unwrap_or_default();
}
if remaining
.last()
.is_some_and(|segment| is_id_number_like(segment))
{
row.total_holding_shares = remaining.pop().unwrap_or_default();
}
if remaining
.last()
.is_some_and(|segment| is_id_number_like(segment))
{
row.holdings_scrip = remaining.pop().unwrap_or_default();
}
if remaining
.last()
.is_some_and(|segment| is_id_number_like(segment))
{
row.holdings_scripless = remaining.pop().unwrap_or_default();
}
let mut geo_fields = Vec::new();
while remaining.len() > 2 {
let Some(candidate) = remaining.last().cloned() else {
break;
};
if row.local_foreign.is_empty() && is_locality_marker(&candidate) {
row.local_foreign = remaining.pop().unwrap_or_default();
continue;
}
if row.investor_type.is_empty() && is_investor_type_marker(&candidate) {
row.investor_type = remaining.pop().unwrap_or_default();
continue;
}
if geo_fields.len() < 2 {
geo_fields.push(remaining.pop().unwrap_or_default());
continue;
}
break;
}
geo_fields.reverse();
if let Some(first) = geo_fields.first() {
row.nationality = first.clone();
}
if geo_fields.len() > 1 {
row.domicile = geo_fields[1..].join(" ");
}
if let Some(first) = remaining.first() {
row.issuer_name = first.clone();
}
if remaining.len() > 1 {
row.investor_name = remaining[1..].join(" ");
}
row
}
fn split_date_and_share(segment: &str) -> Option<(String, String)> {
let trimmed = segment.trim();
if trimmed.len() < 11 {
return None;
}
let date = trimmed.get(..11)?.to_string();
if !is_ksei_date(&date) {
return None;
}
let share_code = trimmed.get(11..).unwrap_or_default().trim().to_string();
Some((date, share_code))
}
fn is_share_code_like(value: &str) -> bool {
let trimmed = value.trim();
(3..=8).contains(&trimmed.len())
&& trimmed
.chars()
.all(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit())
}
fn is_id_number_like(value: &str) -> bool {
let trimmed = value.trim();
!trimmed.is_empty()
&& trimmed != "-"
&& trimmed.chars().all(|ch| ch.is_ascii_digit() || ch == '.')
}
fn is_investor_type_marker(value: &str) -> bool {
let trimmed = value.trim();
!trimmed.is_empty() && trimmed.len() <= 4 && trimmed.chars().all(|ch| ch.is_ascii_uppercase())
}
fn is_locality_marker(value: &str) -> bool {
matches!(
value.trim().to_ascii_uppercase().as_str(),
"L" | "F" | "A" | "D" | "LOCAL" | "FOREIGN" | "ASING" | "DOMESTIC"
)
}
fn normalize_header_label(text: &str) -> String {
let mut normalized = String::new();
for ch in text.chars() {
if ch.is_ascii_alphanumeric() {
normalized.push(ch.to_ascii_uppercase());
}
}
normalized
}
fn y_bucket(y: f32) -> i32 {
@ -236,7 +384,7 @@ fn normalize_spaces(input: &str) -> String {
let mut prev_space = false;
for ch in input.chars() {
if ch == ' ' {
if ch.is_whitespace() {
if !prev_space {
out.push(' ');
}
@ -311,24 +459,37 @@ fn is_percentage_like(s: &str) -> bool {
#[cfg(test)]
mod tests {
use std::fs;
use std::path::Path;
use super::{check_mutool, parse_ksei_pdf, parse_stext_xml};
#[test]
fn test_parse_stext_xml_fixture_extracts_rows() {
let fixture_path = Path::new("tests/fixtures/ksei_stext_sample.xml");
let xml = fs::read_to_string(fixture_path).expect("failed to read stext fixture");
fn test_parse_stext_xml_live_like_lines_extract_rows() {
let xml = build_live_like_stext_xml();
let rows = parse_stext_xml(&xml).expect("failed to parse fixture XML");
let rows = parse_stext_xml(&xml).expect("failed to parse live-like fixture XML");
assert_eq!(rows.len(), 3);
let first = &rows[0];
assert_eq!(first.date, "27-Feb-2026");
assert_eq!(first.share_code, "BBCA");
assert_eq!(first.investor_name, "PT DWIMURIA INVESTAMA ANDALAN");
assert_eq!(first.percentage, "54,94");
assert_eq!(first.share_code, "AADI");
assert_eq!(first.issuer_name, "ADARO ANDALAN INDONESIA Tbk");
assert_eq!(first.investor_name, "ADARO STRATEGIC INVESTMENTS");
assert_eq!(first.investor_type, "CP");
assert_eq!(first.local_foreign, "D");
assert_eq!(first.nationality, "INDONESIA");
assert_eq!(first.holdings_scripless, "3.200.142.830");
assert_eq!(first.holdings_scrip, "0");
assert_eq!(first.total_holding_shares, "3.200.142.830");
assert_eq!(first.percentage, "66,18");
let last = &rows[2];
assert_eq!(last.share_code, "BBRI");
assert_eq!(last.investor_name, "PT NUSANTARA CAPITAL");
assert_eq!(last.investor_type, "ID");
assert_eq!(last.local_foreign, "A");
assert_eq!(last.nationality, "SINGAPORE");
assert_eq!(last.percentage, "15,00");
}
#[test]
@ -352,4 +513,97 @@ mod tests {
rows.len()
);
}
fn build_live_like_stext_xml() -> String {
let mut xml = String::from(r#"<?xml version="1.0"?><document>"#);
append_page(
&mut xml,
"page1",
&[
(31.56, 86.33, "DATE"),
(56.64, 86.33, "SHARE_CODE"),
(121.22, 86.33, "ISSUER_NAME"),
(267.17, 86.33, "INVESTOR_NAME"),
(390.89, 86.33, "INVESTOR_TYPE"),
(434.11, 86.33, "LOCAL_FOREIGN"),
(475.87, 86.33, "NATIONALITY"),
(525.19, 86.33, "DOMICILE"),
(574.63, 86.33, "HOLDINGS_SCRIPLESS"),
(629.98, 86.33, "HOLDINGS_SCRIP"),
(680.02, 86.33, "TOTAL_HOLDING_SHARES"),
(741.22, 86.33, "PERCENTAGE"),
(28.68, 91.01, "27-Feb-2026 AADI"),
(85.10, 91.01, "ADARO ANDALAN INDONESIA Tbk"),
(179.30, 91.01, "ADARO STRATEGIC INVESTMENTS"),
(381.41, 91.01, "CP"),
(424.75, 91.01, "D"),
(504.07, 91.01, "INDONESIA"),
(597.58, 91.01, "3.200.142.830"),
(630.10, 91.01, "0"),
(680.12, 91.01, "3.200.142.830"),
(741.30, 91.01, "66,18"),
(28.68, 96.01, "27-Feb-2026 AADI"),
(85.10, 96.01, "ADARO ANDALAN INDONESIA Tbk"),
(179.30, 96.01, "PUBLIC"),
(381.41, 96.01, "OT"),
(424.75, 96.01, "A"),
(504.07, 96.01, "SINGAPORE"),
(597.58, 96.01, "500.000.000"),
(630.10, 96.01, "0"),
(680.12, 96.01, "500.000.000"),
(741.30, 96.01, "10,34"),
],
);
append_page(
&mut xml,
"page2",
&[
(28.68, 20.00, "27-Feb-2026 BBRI"),
(85.10, 20.00, "BANK RAKYAT INDONESIA Tbk"),
(179.30, 20.00, "PT NUSANTARA CAPITAL"),
(381.41, 20.00, "ID"),
(424.75, 20.00, "A"),
(504.07, 20.00, "SINGAPORE"),
(597.58, 20.00, "1.250.000.000"),
(630.10, 20.00, "0"),
(680.12, 20.00, "1.250.000.000"),
(741.30, 20.00, "15,00"),
],
);
xml.push_str("</document>");
xml
}
fn append_page(xml: &mut String, id: &str, lines: &[(f32, f32, &str)]) {
xml.push_str(&format!(r#"<page id="{id}" width="792" height="612">"#));
for (x, y, text) in lines {
append_line(xml, *x, *y, text);
}
xml.push_str("</page>");
}
fn append_line(xml: &mut String, x: f32, y: f32, text: &str) {
let width = x + (text.len() as f32 * 2.0);
let height = y + 3.48;
xml.push_str(&format!(
r#"<line bbox="{x:.2} {y:.2} {width:.2} {height:.2}" text="{}"></line>"#,
escape_xml_attr(text)
));
}
fn escape_xml_attr(text: &str) -> String {
text.chars()
.map(|ch| match ch {
'&' => "&amp;".to_string(),
'<' => "&lt;".to_string(),
'>' => "&gt;".to_string(),
'"' => "&quot;".to_string(),
'\'' => "&apos;".to_string(),
other => other.to_string(),
})
.collect::<String>()
}
}

664
src/ownership/remote.rs Normal file
View file

@ -0,0 +1,664 @@
use std::fs;
use std::path::Path;
use serde::{Deserialize, Serialize};
use crate::curl_impersonate;
use crate::error::IdxError;
pub const IDX_ANNOUNCEMENT_LISTING_URL: &str = "https://www.idx.co.id/id/berita/pengumuman/";
const DEFAULT_IDX_ANNOUNCEMENT_API_URL: &str =
"https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement";
const IDX_ANNOUNCEMENT_API_ENV: &str = "IDX_OWNERSHIP_ANNOUNCEMENT_API_URL";
const IDX_ANNOUNCEMENT_LISTING_ENV: &str = "IDX_OWNERSHIP_ANNOUNCEMENT_PAGE_URL";
const IDX_ANNOUNCEMENT_PAGE_SIZE: usize = 10;
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
#[derive(Debug, Clone, Deserialize)]
pub struct AnnouncementPage {
#[serde(rename = "Items", default)]
pub items: Vec<AnnouncementItem>,
#[serde(rename = "ItemCount")]
pub item_count: Option<usize>,
#[serde(rename = "PageCount")]
pub page_count: Option<usize>,
}
#[derive(Debug, Clone, Deserialize)]
pub struct AnnouncementItem {
#[serde(rename = "PublishDate")]
pub publish_date: String,
#[serde(rename = "Title")]
pub title: String,
#[serde(rename = "AnnouncementType")]
pub announcement_type: Option<String>,
#[serde(rename = "Code")]
pub code: Option<String>,
#[serde(rename = "Attachments", default)]
pub attachments: Vec<AnnouncementAttachment>,
#[serde(rename = "PdfPath")]
pub pdf_path: Option<String>,
}
#[derive(Debug, Clone, Deserialize)]
pub struct AnnouncementAttachment {
#[serde(rename = "FullSavePath")]
pub full_save_path: String,
#[serde(rename = "OriginalFilename")]
pub original_filename: Option<String>,
#[serde(rename = "PDFFilename")]
pub pdf_filename: Option<String>,
#[serde(rename = "IsAttachment")]
pub is_attachment: Option<serde_json::Value>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum OwnershipReportFamily {
AboveOnePercent,
AboveFivePercent,
InvestorTypeBreakdown,
}
impl OwnershipReportFamily {
pub fn cli_name(self) -> &'static str {
match self {
Self::AboveOnePercent => "above1",
Self::AboveFivePercent => "above5",
Self::InvestorTypeBreakdown => "investor-type",
}
}
pub fn label(self) -> &'static str {
match self {
Self::AboveOnePercent => "Above 1%",
Self::AboveFivePercent => "Above 5%",
Self::InvestorTypeBreakdown => "Investor Type Breakdown",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct DiscoveredOwnershipPdf {
pub family: OwnershipReportFamily,
pub listing_page_url: String,
pub query_url: String,
pub pdf_url: String,
pub title: String,
pub publish_date: String,
pub code: Option<String>,
pub original_filename: Option<String>,
pub is_attachment: bool,
}
struct DiscoveryQuery {
family: OwnershipReportFamily,
keywords: &'static str,
title_needles: &'static [&'static str],
}
const DISCOVERY_QUERIES: &[DiscoveryQuery] = &[
DiscoveryQuery {
family: OwnershipReportFamily::AboveOnePercent,
keywords: "pemegang saham di atas 1",
title_needles: &["PEMEGANG SAHAM DI ATAS 1", "SHAREHOLDERS ABOVE 1"],
},
DiscoveryQuery {
family: OwnershipReportFamily::AboveFivePercent,
keywords: "pemegang saham di atas 5",
title_needles: &["PEMEGANG SAHAM DI ATAS 5", "SHAREHOLDERS ABOVE 5"],
},
DiscoveryQuery {
family: OwnershipReportFamily::InvestorTypeBreakdown,
keywords: "kepemilikan saham perusahaan tercatat",
title_needles: &[
"KEPEMILIKAN SAHAM PERUSAHAAN TERCATAT BERDASARKAN TIPE INVESTOR",
"DATA KSEI TERKAIT KEPEMILIKAN SAHAM PERUSAHAAN TERCATAT",
],
},
];
pub fn announcement_listing_url() -> String {
std::env::var(IDX_ANNOUNCEMENT_LISTING_ENV)
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| IDX_ANNOUNCEMENT_LISTING_URL.to_string())
}
pub fn announcement_api_url() -> String {
std::env::var(IDX_ANNOUNCEMENT_API_ENV)
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| DEFAULT_IDX_ANNOUNCEMENT_API_URL.to_string())
}
pub fn build_announcement_query_url(
keywords: &str,
page_number: usize,
page_size: usize,
) -> String {
format!(
"{}?keywords={}&pageNumber={page_number}&pageSize={page_size}&lang=id",
announcement_api_url(),
percent_encode(keywords),
)
}
pub fn parse_announcement_page(raw: &str) -> Result<AnnouncementPage, IdxError> {
let trimmed = raw.trim();
if trimmed.is_empty() {
return Err(IdxError::Http(
"IDX ownership discovery returned an empty announcement payload".to_string(),
));
}
let normalized = trimmed.to_ascii_lowercase();
if normalized.starts_with("<!doctype html") || normalized.starts_with("<html") {
return Err(IdxError::Http(
"IDX ownership discovery returned HTML instead of announcement JSON".to_string(),
));
}
serde_json::from_str(trimmed)
.map_err(|e| IdxError::ParseError(format!("failed to parse IDX announcement JSON: {e}")))
}
pub fn select_latest_ownership_reports(
page: &AnnouncementPage,
query_url: &str,
family: OwnershipReportFamily,
) -> Result<Vec<DiscoveredOwnershipPdf>, IdxError> {
let Some(query) = DISCOVERY_QUERIES
.iter()
.find(|query| query.family == family)
else {
return Err(IdxError::Http(format!(
"failed to discover IDX ownership reports from {query_url}: unknown report family"
)));
};
let mut matches: Vec<&AnnouncementItem> = page
.items
.iter()
.filter(|item| item_matches_family(item, query))
.collect();
matches.sort_by(|left, right| right.publish_date.cmp(&left.publish_date));
let Some(item) = matches.into_iter().next() else {
return Err(IdxError::Http(format!(
"failed to discover IDX ownership reports from {query_url}: no matching announcement found"
)));
};
let mut attachments = item_pdf_attachments(item)
.into_iter()
.map(|attachment| {
let is_attachment = attachment_is_attachment(&attachment);
let original_filename = attachment
.original_filename
.clone()
.or(attachment.pdf_filename.clone())
.map(|value| value.trim().to_string());
DiscoveredOwnershipPdf {
family,
listing_page_url: announcement_listing_url(),
query_url: query_url.to_string(),
pdf_url: attachment.full_save_path,
title: item.title.clone(),
publish_date: item.publish_date.clone(),
code: clean_option(item.code.as_deref()),
original_filename,
is_attachment,
}
})
.collect::<Vec<_>>();
attachments.sort_by(|left, right| {
left.is_attachment
.cmp(&right.is_attachment)
.then_with(|| left.original_filename.cmp(&right.original_filename))
});
if attachments.is_empty() {
return Err(IdxError::Http(format!(
"failed to discover IDX ownership reports from {query_url}: matching announcement had no PDF attachments"
)));
}
Ok(attachments)
}
pub fn discover_idx_ownership_reports(
family_filter: Option<OwnershipReportFamily>,
limit: usize,
) -> Result<Vec<DiscoveredOwnershipPdf>, IdxError> {
let mut discovered = Vec::new();
let mut errors = Vec::new();
for query in DISCOVERY_QUERIES
.iter()
.filter(|query| family_filter.is_none_or(|family| family == query.family))
{
let query_url = build_announcement_query_url(query.keywords, 1, IDX_ANNOUNCEMENT_PAGE_SIZE);
match fetch_text(
"IDX ownership announcement discovery",
&query_url,
&json_headers(),
)
.and_then(|raw| parse_announcement_page(&raw))
.and_then(|page| select_latest_ownership_reports(&page, &query_url, query.family))
{
Ok(mut reports) => discovered.append(&mut reports),
Err(err) => errors.push(err.to_string()),
}
}
discovered.sort_by(|left, right| {
right
.publish_date
.cmp(&left.publish_date)
.then_with(|| left.is_attachment.cmp(&right.is_attachment))
.then_with(|| left.original_filename.cmp(&right.original_filename))
});
if limit < discovered.len() {
discovered.truncate(limit);
}
if discovered.is_empty() {
let detail = if errors.is_empty() {
"no matching ownership reports found".to_string()
} else {
errors.join("; ")
};
return Err(IdxError::Http(format!(
"failed to discover IDX ownership reports from {}: {detail}",
announcement_listing_url()
)));
}
Ok(discovered)
}
pub fn download_idx_pdf(url: &str, target: &Path) -> Result<(), IdxError> {
let bytes = fetch_bytes("IDX ownership PDF download", url, &pdf_headers())?;
validate_pdf_payload(&bytes)?;
fs::write(target, &bytes).map_err(|e| {
IdxError::Io(format!(
"failed writing cached PDF {}: {e}",
target.display()
))
})?;
Ok(())
}
pub fn validate_pdf_payload(bytes: &[u8]) -> Result<(), IdxError> {
if bytes.is_empty() {
return Err(IdxError::Http(
"IDX ownership download returned an empty response".to_string(),
));
}
let trimmed = bytes
.iter()
.skip_while(|byte| byte.is_ascii_whitespace())
.copied()
.collect::<Vec<_>>();
if trimmed.starts_with(b"%PDF-") {
return Ok(());
}
let preview = String::from_utf8_lossy(&trimmed[..trimmed.len().min(256)]).to_ascii_lowercase();
if preview.contains("<!doctype html") || preview.contains("<html") {
return Err(IdxError::Http(
"IDX ownership download returned HTML instead of a PDF".to_string(),
));
}
Err(IdxError::Http(
"IDX ownership download did not look like a PDF".to_string(),
))
}
fn fetch_text(stage: &str, url: &str, headers: &[(String, String)]) -> Result<String, IdxError> {
let bytes = fetch_bytes(stage, url, headers)?;
String::from_utf8(bytes)
.map_err(|e| IdxError::Http(format!("failed to decode {stage} response as utf-8: {e}")))
}
fn fetch_bytes(stage: &str, url: &str, headers: &[(String, String)]) -> Result<Vec<u8>, IdxError> {
let mut args = vec![
"--silent".to_string(),
"--show-error".to_string(),
"--location".to_string(),
"--fail".to_string(),
"--compressed".to_string(),
];
for (name, value) in headers {
args.push("--header".to_string());
args.push(format!("{name}: {value}"));
}
args.push(url.to_string());
let output = curl_impersonate::run_owned(stage, &args)?;
Ok(output.stdout)
}
fn json_headers() -> Vec<(String, String)> {
vec![
("User-Agent".to_string(), USER_AGENT.to_string()),
(
"Accept".to_string(),
"application/json,text/plain,*/*".to_string(),
),
(
"Accept-Language".to_string(),
"id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7".to_string(),
),
("Referer".to_string(), announcement_listing_url()),
]
}
fn pdf_headers() -> Vec<(String, String)> {
vec![
("User-Agent".to_string(), USER_AGENT.to_string()),
(
"Accept".to_string(),
"application/pdf,application/octet-stream,*/*;q=0.8".to_string(),
),
(
"Accept-Language".to_string(),
"id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7".to_string(),
),
("Referer".to_string(), announcement_listing_url()),
]
}
fn item_matches_family(item: &AnnouncementItem, query: &DiscoveryQuery) -> bool {
let title = item.title.to_ascii_uppercase();
query
.title_needles
.iter()
.any(|needle| title.contains(needle))
}
fn item_pdf_attachments(item: &AnnouncementItem) -> Vec<AnnouncementAttachment> {
let attachments = if item.attachments.is_empty() {
item.pdf_path
.as_deref()
.and_then(parse_pdf_path)
.unwrap_or_default()
} else {
item.attachments.clone()
};
attachments
.into_iter()
.filter(|attachment| {
attachment
.full_save_path
.to_ascii_lowercase()
.ends_with(".pdf")
})
.collect()
}
fn attachment_label(attachment: &AnnouncementAttachment) -> String {
attachment
.original_filename
.as_deref()
.or(attachment.pdf_filename.as_deref())
.unwrap_or_default()
.trim()
.to_ascii_lowercase()
}
fn parse_pdf_path(raw: &str) -> Option<Vec<AnnouncementAttachment>> {
serde_json::from_str::<Vec<AnnouncementAttachment>>(raw).ok()
}
fn attachment_is_attachment(attachment: &AnnouncementAttachment) -> bool {
match attachment.is_attachment.as_ref() {
Some(serde_json::Value::Bool(value)) => *value,
Some(serde_json::Value::Number(value)) => value.as_i64() == Some(1),
Some(serde_json::Value::String(value)) => {
matches!(value.trim(), "1" | "true" | "TRUE" | "True")
}
_ => attachment_label(attachment).contains("lamp"),
}
}
fn clean_option(value: Option<&str>) -> Option<String> {
value
.map(str::trim)
.filter(|value| !value.is_empty())
.map(str::to_string)
}
fn percent_encode(value: &str) -> String {
let mut encoded = String::with_capacity(value.len());
for byte in value.bytes() {
match byte {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
encoded.push(byte as char)
}
b' ' => encoded.push_str("%20"),
other => encoded.push_str(&format!("%{other:02X}")),
}
}
encoded
}
#[cfg(test)]
mod tests {
use super::{
AnnouncementPage, IDX_ANNOUNCEMENT_LISTING_URL, OwnershipReportFamily,
build_announcement_query_url, parse_announcement_page, select_latest_ownership_reports,
validate_pdf_payload,
};
use crate::error::IdxError;
#[test]
fn builds_announcement_query_url() {
let url = build_announcement_query_url("pemegang saham di atas 5", 1, 10);
assert_eq!(
url,
"https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%205&pageNumber=1&pageSize=10&lang=id"
);
}
#[test]
fn parses_fixture_and_selects_latest_above_5_reports() {
let raw = include_str!("../../tests/fixtures/idx_announcement_kepemilikan.json");
let page = parse_announcement_page(raw).expect("announcement fixture should parse");
let discovered = select_latest_ownership_reports(
&page,
"https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%205&pageNumber=1&pageSize=10&lang=id",
OwnershipReportFamily::AboveFivePercent,
)
.expect("should select ownership reports");
assert_eq!(discovered.len(), 2);
assert_eq!(
discovered[0].family,
OwnershipReportFamily::AboveFivePercent
);
assert_eq!(discovered[0].listing_page_url, IDX_ANNOUNCEMENT_LISTING_URL);
assert_eq!(discovered[0].publish_date, "2026-03-27T16:34:20");
assert_eq!(
discovered[0].pdf_url,
"https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/5d31bb6f49_announcement.pdf"
);
assert_eq!(
discovered[0].original_filename.as_deref(),
Some("20260327_Semua Emiten Saham_Pengumuman Bursa_32055594.pdf")
);
assert!(!discovered[0].is_attachment);
assert_eq!(
discovered[1].pdf_url,
"https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/4f5c4efc6f_bf70f249ac_lamp1.pdf"
);
assert_eq!(
discovered[1].original_filename.as_deref(),
Some("20260327_Semua Emiten Saham_Pengumuman Bursa_32055594_lamp1.pdf")
);
assert!(discovered[1].is_attachment);
}
#[test]
fn selects_above_1_reports() {
let raw = r#"{
"Items": [
{
"PublishDate": "2026-03-10T12:09:09",
"Title": "Pemegang Saham di atas 1% (KSEI)",
"AnnouncementType": "",
"Code": " Semua Emiten Saham ",
"Attachments": [
{
"PDFFilename": "d67ebf37e6_10d4080288.pdf",
"FullSavePath": "https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/d67ebf37e6_10d4080288.pdf",
"IsAttachment": 0,
"OriginalFilename": "20260310_Semua Emiten Saham_Pengumuman Bursa_32052554.pdf"
},
{
"PDFFilename": "b9b638e5a8_8928aca255.pdf",
"FullSavePath": "https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/b9b638e5a8_8928aca255.pdf",
"IsAttachment": 1,
"OriginalFilename": "20260310_Semua Emiten Saham_Pengumuman Bursa_32052554_lamp1.pdf"
}
],
"PdfPath": ""
}
],
"ItemCount": 1,
"PageCount": 1
}"#;
let page = parse_announcement_page(raw).expect("above-1 fixture should parse");
let discovered = select_latest_ownership_reports(
&page,
"https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%201&pageNumber=1&pageSize=10&lang=id",
OwnershipReportFamily::AboveOnePercent,
)
.expect("should select above-1 reports");
assert_eq!(discovered.len(), 2);
assert_eq!(discovered[0].family, OwnershipReportFamily::AboveOnePercent);
assert_eq!(discovered[0].code.as_deref(), Some("Semua Emiten Saham"));
assert_eq!(
discovered[0].original_filename.as_deref(),
Some("20260310_Semua Emiten Saham_Pengumuman Bursa_32052554.pdf")
);
assert!(!discovered[0].is_attachment);
assert_eq!(
discovered[1].pdf_url,
"https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/b9b638e5a8_8928aca255.pdf"
);
assert!(discovered[1].is_attachment);
}
#[test]
fn selects_investor_type_breakdown_reports() {
let raw = r#"{
"Items": [
{
"PublishDate": "2026-03-02T16:14:37",
"Title": "Data KSEI terkait Kepemilikan Saham Perusahaan Tercatat Berdasarkan Tipe Investor per 27 Februari 2026",
"AnnouncementType": "",
"Code": " ",
"Attachments": [
{
"PDFFilename": "20260302_Pengumuman Bursa_32040089.pdf",
"FullSavePath": "https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/43025665c8_fda8953269.pdf",
"IsAttachment": 0,
"OriginalFilename": "20260302_Pengumuman Bursa_32040089.pdf"
},
{
"PDFFilename": "20260302_Pengumuman Bursa_32040089_lamp1.pdf",
"FullSavePath": "https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/db5b5a86e1_2e2b3d976a.pdf",
"IsAttachment": 1,
"OriginalFilename": "20260302_Pengumuman Bursa_32040089_lamp1.pdf"
}
],
"PdfPath": ""
}
],
"ItemCount": 1,
"PageCount": 1
}"#;
let page = parse_announcement_page(raw).expect("investor-type fixture should parse");
let discovered = select_latest_ownership_reports(
&page,
"https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=kepemilikan%20saham%20perusahaan%20tercatat&pageNumber=1&pageSize=10&lang=id",
OwnershipReportFamily::InvestorTypeBreakdown,
)
.expect("should select investor-type reports");
assert_eq!(discovered.len(), 2);
assert_eq!(
discovered[0].original_filename.as_deref(),
Some("20260302_Pengumuman Bursa_32040089.pdf")
);
assert_eq!(
discovered[1].original_filename.as_deref(),
Some("20260302_Pengumuman Bursa_32040089_lamp1.pdf")
);
}
#[test]
fn falls_back_to_pdf_path_when_attachments_are_missing() {
let raw = r#"{
"Items": [
{
"PublishDate": "2026-03-04T08:54:00",
"Title": "Pemegang Saham di atas 5% (KSEI)",
"AnnouncementType": "",
"Code": "Semua Emiten Saham",
"Attachments": [],
"PdfPath": "[{\"PDFFilename\":\"20260305_LKS_KSEI_000043_lamp1.pdf\",\"FullSavePath\":\"https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/ec835d7451_c30c886a1a.pdf\",\"IsAttachment\":\"1\",\"OriginalFilename\":\"20260305_LKS_KSEI_000043_lamp1.pdf\"}]"
}
],
"ItemCount": 1,
"PageCount": 1
}"#;
let page: AnnouncementPage = parse_announcement_page(raw).expect("fallback page parses");
let discovered = select_latest_ownership_reports(
&page,
"https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%205&pageNumber=1&pageSize=10&lang=id",
OwnershipReportFamily::AboveFivePercent,
)
.expect("fallback attachment should be parsed");
assert_eq!(discovered.len(), 1);
assert_eq!(
discovered[0].pdf_url,
"https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/ec835d7451_c30c886a1a.pdf"
);
assert_eq!(discovered[0].code.as_deref(), Some("Semua Emiten Saham"));
assert!(discovered[0].is_attachment);
}
#[test]
fn rejects_html_instead_of_announcement_json() {
let err = parse_announcement_page("<!doctype html><html><body>blocked</body></html>")
.expect_err("html response must fail");
assert!(matches!(err, IdxError::Http(_)));
}
#[test]
fn accepts_pdf_header() {
validate_pdf_payload(b"%PDF-1.7\n1 0 obj\n").expect("pdf header should pass");
}
#[test]
fn rejects_html_instead_of_pdf() {
let err = validate_pdf_payload(b"<!doctype html><html><body>blocked</body></html>")
.expect_err("html body must fail");
assert!(matches!(err, IdxError::Http(_)));
}
}