mirror of
https://github.com/0xrsydn/idx-cli.git
synced 2026-11-07 09:45:36 +00:00
fix(ownership): harden data-page discovery per review
Addresses review on #26: - strip ?query/#fragment before the .xlsx/.pdf extension check - skip page entries without a parseable as-of date instead of emitting an empty publish_date - warn on stderr when the data page fails or parses to zero entries while announcement results still exist, instead of silently degrading to announcement-only results Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017a5Cv5Z1NzdYKgKnTiAF4E
This commit is contained in:
parent
29e416109c
commit
bb39e98fab
2 changed files with 49 additions and 5 deletions
|
|
@ -307,6 +307,7 @@ pub fn discover_idx_ownership_reports(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let mut page_error = None;
|
||||||
match fetch_text(
|
match fetch_text(
|
||||||
"IDX share ownership data page discovery",
|
"IDX share ownership data page discovery",
|
||||||
&share_ownership_page_url(),
|
&share_ownership_page_url(),
|
||||||
|
|
@ -315,6 +316,15 @@ pub fn discover_idx_ownership_reports(
|
||||||
) {
|
) {
|
||||||
Ok(html) => {
|
Ok(html) => {
|
||||||
let mut page_reports = parse_share_ownership_page(&html, &share_ownership_page_url());
|
let mut page_reports = parse_share_ownership_page(&html, &share_ownership_page_url());
|
||||||
|
if page_reports.is_empty() {
|
||||||
|
// The page loaded (it has its file list) but nothing parsed:
|
||||||
|
// the payload shape changed. Say so instead of silently
|
||||||
|
// falling back to announcement-only results.
|
||||||
|
page_error = Some(format!(
|
||||||
|
"IDX share ownership data page {} loaded but no report entries could be parsed; the page layout may have changed",
|
||||||
|
share_ownership_page_url()
|
||||||
|
));
|
||||||
|
}
|
||||||
page_reports
|
page_reports
|
||||||
.retain(|report| family_filter.is_none_or(|family| family == report.family));
|
.retain(|report| family_filter.is_none_or(|family| family == report.family));
|
||||||
// Keep the full above-1% history (snapshot backfill needs it), but
|
// Keep the full above-1% history (snapshot backfill needs it), but
|
||||||
|
|
@ -333,7 +343,7 @@ pub fn discover_idx_ownership_reports(
|
||||||
});
|
});
|
||||||
discovered.extend(page_reports);
|
discovered.extend(page_reports);
|
||||||
}
|
}
|
||||||
Err(err) => errors.push(err.to_string()),
|
Err(err) => page_error = Some(err.to_string()),
|
||||||
}
|
}
|
||||||
|
|
||||||
discovered.sort_by(|left, right| {
|
discovered.sort_by(|left, right| {
|
||||||
|
|
@ -349,6 +359,15 @@ pub fn discover_idx_ownership_reports(
|
||||||
discovered.truncate(limit);
|
discovered.truncate(limit);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !discovered.is_empty()
|
||||||
|
&& let Some(err) = &page_error
|
||||||
|
{
|
||||||
|
// Announcement results still exist, but since June 2026 the newest
|
||||||
|
// above-1% reports are only on the data page: don't hide its failure.
|
||||||
|
crate::runtime::warn(err);
|
||||||
|
}
|
||||||
|
errors.extend(page_error);
|
||||||
|
|
||||||
if discovered.is_empty() {
|
if discovered.is_empty() {
|
||||||
let detail = if errors.is_empty() {
|
let detail = if errors.is_empty() {
|
||||||
"no matching ownership reports found".to_string()
|
"no matching ownership reports found".to_string()
|
||||||
|
|
@ -549,7 +568,8 @@ fn classify_page_entry(
|
||||||
return None;
|
return None;
|
||||||
};
|
};
|
||||||
|
|
||||||
let file_name = url.rsplit('/').next().unwrap_or_default().to_string();
|
let path = url.split(['?', '#']).next().unwrap_or_default();
|
||||||
|
let file_name = path.rsplit('/').next().unwrap_or_default().to_string();
|
||||||
let lower_name = file_name.to_ascii_lowercase();
|
let lower_name = file_name.to_ascii_lowercase();
|
||||||
let format = if lower_name.ends_with(".xlsx") {
|
let format = if lower_name.ends_with(".xlsx") {
|
||||||
"xlsx"
|
"xlsx"
|
||||||
|
|
@ -563,7 +583,9 @@ fn classify_page_entry(
|
||||||
} else {
|
} else {
|
||||||
OwnershipReportStatus::Unsupported
|
OwnershipReportStatus::Unsupported
|
||||||
};
|
};
|
||||||
let as_of_date = parse_indonesian_as_of_date(description).map(|date| date.to_string());
|
// Without an as-of date the entry cannot be ordered against other
|
||||||
|
// months, and the snapshot manifest needs a date, so skip it.
|
||||||
|
let as_of_date = parse_indonesian_as_of_date(description)?.to_string();
|
||||||
|
|
||||||
Some(DiscoveredOwnershipPdf {
|
Some(DiscoveredOwnershipPdf {
|
||||||
family,
|
family,
|
||||||
|
|
@ -572,12 +594,12 @@ fn classify_page_entry(
|
||||||
query_url: page_url.to_string(),
|
query_url: page_url.to_string(),
|
||||||
pdf_url: url.to_string(),
|
pdf_url: url.to_string(),
|
||||||
title: description.to_string(),
|
title: description.to_string(),
|
||||||
publish_date: as_of_date.clone().unwrap_or_default(),
|
publish_date: as_of_date.clone(),
|
||||||
code: None,
|
code: None,
|
||||||
original_filename: (!file_name.is_empty()).then_some(file_name),
|
original_filename: (!file_name.is_empty()).then_some(file_name),
|
||||||
is_attachment: true,
|
is_attachment: true,
|
||||||
format: format.to_string(),
|
format: format.to_string(),
|
||||||
as_of_date,
|
as_of_date: Some(as_of_date),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1140,6 +1162,24 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn share_ownership_page_handles_query_strings_and_dateless_entries() {
|
||||||
|
let html = r#"{Description:"Pemegang Saham di Atas 1% per 31 Agustus 2026",Prospectus:"https://www.idx.co.id/Media/x/peng-satu-persen.xlsx?download=1",ListingDate:r},{Description:"Pemegang Saham di Atas 1%",Prospectus:"https://www.idx.co.id/Media/y/no-date.xlsx",ListingDate:r}"#;
|
||||||
|
let reports = parse_share_ownership_page(html, "p");
|
||||||
|
assert_eq!(
|
||||||
|
reports.len(),
|
||||||
|
1,
|
||||||
|
"dateless entry skipped, query-string entry kept"
|
||||||
|
);
|
||||||
|
assert_eq!(reports[0].format, "xlsx");
|
||||||
|
assert_eq!(reports[0].status, OwnershipReportStatus::Supported);
|
||||||
|
assert_eq!(
|
||||||
|
reports[0].original_filename.as_deref(),
|
||||||
|
Some("peng-satu-persen.xlsx")
|
||||||
|
);
|
||||||
|
assert_eq!(reports[0].publish_date, "2026-08-31");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn share_ownership_page_without_payload_yields_nothing() {
|
fn share_ownership_page_without_payload_yields_nothing() {
|
||||||
assert!(parse_share_ownership_page("<html>Just a moment...</html>", "p").is_empty());
|
assert!(parse_share_ownership_page("<html>Just a moment...</html>", "p").is_empty());
|
||||||
|
|
|
||||||
|
|
@ -1313,6 +1313,10 @@ fn ownership_discover_supports_above1_family() {
|
||||||
.stdout(predicate::str::contains("Above 1%"))
|
.stdout(predicate::str::contains("Above 1%"))
|
||||||
.stdout(predicate::str::contains(
|
.stdout(predicate::str::contains(
|
||||||
"20260310_Semua Emiten Saham_Pengumuman Bursa_32052554_lamp1.pdf",
|
"20260310_Semua Emiten Saham_Pengumuman Bursa_32052554_lamp1.pdf",
|
||||||
|
))
|
||||||
|
.stderr(predicate::str::contains("warning:"))
|
||||||
|
.stderr(predicate::str::contains(
|
||||||
|
"IDX share ownership data page discovery",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue