use std::fs;
use std::path::Path;

use serde::{Deserialize, Serialize};

use crate::curl_impersonate;
use crate::error::IdxError;

pub const IDX_ANNOUNCEMENT_LISTING_URL: &str = "https://www.idx.co.id/id/berita/pengumuman/";
const DEFAULT_IDX_ANNOUNCEMENT_API_URL: &str =
    "https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement";
const IDX_ANNOUNCEMENT_API_ENV: &str = "IDX_OWNERSHIP_ANNOUNCEMENT_API_URL";
const IDX_ANNOUNCEMENT_LISTING_ENV: &str = "IDX_OWNERSHIP_ANNOUNCEMENT_PAGE_URL";
const IDX_ANNOUNCEMENT_PAGE_SIZE: usize = 10;
/// Since June 2026 the monthly above-1% list (XLSX) is published only here.
pub const IDX_SHARE_OWNERSHIP_PAGE_URL: &str =
    "https://www.idx.co.id/id/perusahaan-tercatat/data-kepemilikan-saham/";
const IDX_SHARE_OWNERSHIP_PAGE_ENV: &str = "IDX_OWNERSHIP_DATA_PAGE_URL";
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";

#[derive(Debug, Clone, Deserialize)]
pub struct AnnouncementPage {
    #[serde(rename = "Items", default)]
    pub items: Vec<AnnouncementItem>,
    #[serde(rename = "ItemCount")]
    pub item_count: Option<usize>,
    #[serde(rename = "PageCount")]
    pub page_count: Option<usize>,
}

#[derive(Debug, Clone, Deserialize)]
pub struct AnnouncementItem {
    #[serde(rename = "PublishDate")]
    pub publish_date: String,
    #[serde(rename = "Title")]
    pub title: String,
    #[serde(rename = "AnnouncementType")]
    pub announcement_type: Option<String>,
    #[serde(rename = "Code")]
    pub code: Option<String>,
    #[serde(rename = "Attachments", default)]
    pub attachments: Vec<AnnouncementAttachment>,
    #[serde(rename = "PdfPath")]
    pub pdf_path: Option<String>,
}

#[derive(Debug, Clone, Deserialize)]
pub struct AnnouncementAttachment {
    #[serde(rename = "FullSavePath")]
    pub full_save_path: String,
    #[serde(rename = "OriginalFilename")]
    pub original_filename: Option<String>,
    #[serde(rename = "PDFFilename")]
    pub pdf_filename: Option<String>,
    #[serde(rename = "IsAttachment")]
    pub is_attachment: Option<serde_json::Value>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum OwnershipReportFamily {
    AboveOnePercent,
    AboveFivePercent,
    InvestorTypeBreakdown,
}

impl OwnershipReportFamily {
    pub fn cli_name(self) -> &'static str {
        match self {
            Self::AboveOnePercent => "above1",
            Self::AboveFivePercent => "above5",
            Self::InvestorTypeBreakdown => "investor-type",
        }
    }

    pub fn label(self) -> &'static str {
        match self {
            Self::AboveOnePercent => "Above 1%",
            Self::AboveFivePercent => "Above 5%",
            Self::InvestorTypeBreakdown => "Investor Type Breakdown",
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum OwnershipReportStatus {
    Supported,
    AnnouncementOnly,
    Unsupported,
}

impl OwnershipReportStatus {
    pub fn label(self) -> &'static str {
        match self {
            Self::Supported => "supported",
            Self::AnnouncementOnly => "announcement_only",
            Self::Unsupported => "unsupported",
        }
    }

    fn sort_rank(self) -> u8 {
        match self {
            Self::Supported => 0,
            Self::AnnouncementOnly => 1,
            Self::Unsupported => 2,
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct DiscoveredOwnershipPdf {
    pub family: OwnershipReportFamily,
    pub status: OwnershipReportStatus,
    pub listing_page_url: String,
    pub query_url: String,
    pub pdf_url: String,
    pub title: String,
    pub publish_date: String,
    pub code: Option<String>,
    pub original_filename: Option<String>,
    pub is_attachment: bool,
    /// File format of `pdf_url`: `pdf` (announcement attachment) or `xlsx`
    /// (Data Kepemilikan Saham page).
    pub format: String,
    /// Report as-of date (`YYYY-MM-DD`) when the source states it. The data
    /// page shows no publish date, so its entries use this as `publish_date`.
    pub as_of_date: Option<String>,
}

struct DiscoveryQuery {
    family: OwnershipReportFamily,
    keywords: &'static str,
    title_needles: &'static [&'static str],
}

const DISCOVERY_QUERIES: &[DiscoveryQuery] = &[
    DiscoveryQuery {
        family: OwnershipReportFamily::AboveOnePercent,
        keywords: "pemegang saham di atas 1",
        title_needles: &["PEMEGANG SAHAM DI ATAS 1", "SHAREHOLDERS ABOVE 1"],
    },
    DiscoveryQuery {
        family: OwnershipReportFamily::AboveFivePercent,
        keywords: "pemegang saham di atas 5",
        title_needles: &["PEMEGANG SAHAM DI ATAS 5", "SHAREHOLDERS ABOVE 5"],
    },
    DiscoveryQuery {
        family: OwnershipReportFamily::InvestorTypeBreakdown,
        keywords: "kepemilikan saham perusahaan tercatat",
        title_needles: &[
            "KEPEMILIKAN SAHAM PERUSAHAAN TERCATAT BERDASARKAN TIPE INVESTOR",
            "DATA KSEI TERKAIT KEPEMILIKAN SAHAM PERUSAHAAN TERCATAT",
        ],
    },
];

pub fn announcement_listing_url() -> String {
    std::env::var(IDX_ANNOUNCEMENT_LISTING_ENV)
        .ok()
        .filter(|value| !value.trim().is_empty())
        .unwrap_or_else(|| IDX_ANNOUNCEMENT_LISTING_URL.to_string())
}

pub fn share_ownership_page_url() -> String {
    std::env::var(IDX_SHARE_OWNERSHIP_PAGE_ENV)
        .ok()
        .filter(|value| !value.trim().is_empty())
        .unwrap_or_else(|| IDX_SHARE_OWNERSHIP_PAGE_URL.to_string())
}

pub fn announcement_api_url() -> String {
    std::env::var(IDX_ANNOUNCEMENT_API_ENV)
        .ok()
        .filter(|value| !value.trim().is_empty())
        .unwrap_or_else(|| DEFAULT_IDX_ANNOUNCEMENT_API_URL.to_string())
}

pub fn build_announcement_query_url(
    keywords: &str,
    page_number: usize,
    page_size: usize,
) -> String {
    format!(
        "{}?keywords={}&pageNumber={page_number}&pageSize={page_size}&lang=id",
        announcement_api_url(),
        percent_encode(keywords),
    )
}

pub fn parse_announcement_page(raw: &str) -> Result<AnnouncementPage, IdxError> {
    let trimmed = raw.trim();
    if trimmed.is_empty() {
        return Err(IdxError::Http(
            "IDX ownership discovery returned an empty announcement payload; the IDX announcement API did not return any JSON items".to_string(),
        ));
    }
    let normalized = trimmed.to_ascii_lowercase();
    if normalized.starts_with("<!doctype html") || normalized.starts_with("<html") {
        return Err(IdxError::Http(
            "IDX ownership discovery returned HTML instead of announcement JSON; verify the announcement endpoint or try again later".to_string(),
        ));
    }

    serde_json::from_str(trimmed)
        .map_err(|e| IdxError::ParseError(format!("failed to parse IDX announcement JSON: {e}")))
}

pub fn select_latest_ownership_reports(
    page: &AnnouncementPage,
    query_url: &str,
    family: OwnershipReportFamily,
) -> Result<Vec<DiscoveredOwnershipPdf>, IdxError> {
    let Some(query) = DISCOVERY_QUERIES
        .iter()
        .find(|query| query.family == family)
    else {
        return Err(IdxError::Http(format!(
            "failed to discover {} IDX ownership reports from {query_url}: unknown report family",
            family.cli_name()
        )));
    };

    let mut matches: Vec<&AnnouncementItem> = page
        .items
        .iter()
        .filter(|item| item_matches_family(item, query))
        .collect();
    matches.sort_by(|left, right| right.publish_date.cmp(&left.publish_date));

    let Some(item) = matches.into_iter().next() else {
        return Err(IdxError::Http(format!(
            "failed to discover {} IDX ownership reports from {query_url}: no matching announcement found",
            family.cli_name()
        )));
    };

    let mut attachments = item_pdf_attachments(item)
        .into_iter()
        .map(|attachment| {
            let is_attachment = attachment_is_attachment(&attachment);
            let original_filename = attachment
                .original_filename
                .clone()
                .or(attachment.pdf_filename.clone())
                .map(|value| value.trim().to_string());
            let status = classify_report_status(family, is_attachment);

            DiscoveredOwnershipPdf {
                family,
                status,
                listing_page_url: announcement_listing_url(),
                query_url: query_url.to_string(),
                pdf_url: attachment.full_save_path,
                title: item.title.clone(),
                publish_date: item.publish_date.clone(),
                code: clean_option(item.code.as_deref()),
                original_filename,
                is_attachment,
                format: "pdf".to_string(),
                as_of_date: None,
            }
        })
        .collect::<Vec<_>>();

    attachments.sort_by(|left, right| {
        left.status
            .sort_rank()
            .cmp(&right.status.sort_rank())
            .then_with(|| right.is_attachment.cmp(&left.is_attachment))
            .then_with(|| left.original_filename.cmp(&right.original_filename))
    });

    if attachments.is_empty() {
        return Err(IdxError::Http(format!(
            "failed to discover {} IDX ownership reports from {query_url}: matching announcement had no PDF attachments",
            family.cli_name()
        )));
    }

    Ok(attachments)
}

pub fn discover_idx_ownership_reports(
    family_filter: Option<OwnershipReportFamily>,
    limit: usize,
) -> Result<Vec<DiscoveredOwnershipPdf>, IdxError> {
    let mut discovered = Vec::new();
    let mut errors = Vec::new();

    for query in DISCOVERY_QUERIES
        .iter()
        .filter(|query| family_filter.is_none_or(|family| family == query.family))
    {
        let query_url = build_announcement_query_url(query.keywords, 1, IDX_ANNOUNCEMENT_PAGE_SIZE);
        match fetch_text(
            "IDX ownership announcement discovery",
            &query_url,
            &json_headers(),
            &validate_json_payload,
        )
        .and_then(|raw| parse_announcement_page(&raw))
        .and_then(|page| select_latest_ownership_reports(&page, &query_url, query.family))
        {
            Ok(mut reports) => discovered.append(&mut reports),
            Err(err) => errors.push(err.to_string()),
        }
    }

    let mut page_error = None;
    match fetch_text(
        "IDX share ownership data page discovery",
        &share_ownership_page_url(),
        &page_headers(),
        &validate_share_ownership_page,
    ) {
        Ok(html) => {
            let mut page_reports = parse_share_ownership_page(&html, &share_ownership_page_url());
            if page_reports.is_empty() {
                // The page loaded (it has its file list) but nothing parsed:
                // the payload shape changed. Say so instead of silently
                // falling back to announcement-only results.
                page_error = Some(format!(
                    "IDX share ownership data page {} loaded but no report entries could be parsed; the page layout may have changed",
                    share_ownership_page_url()
                ));
            }
            page_reports
                .retain(|report| family_filter.is_none_or(|family| family == report.family));
            // Keep the full above-1% history (snapshot backfill needs it), but
            // only the newest entry of the unsupported daily/breakdown series.
            page_reports.sort_by(|left, right| right.publish_date.cmp(&left.publish_date));
            let mut seen_unsupported = Vec::new();
            page_reports.retain(|report| {
                if report.family == OwnershipReportFamily::AboveOnePercent {
                    return true;
                }
                if seen_unsupported.contains(&report.family) {
                    return false;
                }
                seen_unsupported.push(report.family);
                true
            });
            discovered.extend(page_reports);
        }
        Err(err) => page_error = Some(err.to_string()),
    }

    discovered.sort_by(|left, right| {
        right
            .publish_date
            .cmp(&left.publish_date)
            .then_with(|| left.status.sort_rank().cmp(&right.status.sort_rank()))
            .then_with(|| right.is_attachment.cmp(&left.is_attachment))
            .then_with(|| left.original_filename.cmp(&right.original_filename))
    });

    if limit < discovered.len() {
        discovered.truncate(limit);
    }

    if !discovered.is_empty()
        && let Some(err) = &page_error
    {
        // Announcement results still exist, but since June 2026 the newest
        // above-1% reports are only on the data page: don't hide its failure.
        crate::runtime::warn(err);
    }
    errors.extend(page_error);

    if discovered.is_empty() {
        let detail = if errors.is_empty() {
            "no matching ownership reports found".to_string()
        } else {
            errors.join("; ")
        };
        return Err(IdxError::Http(format!(
            "failed to discover IDX ownership reports from {} or {}: {detail}",
            announcement_listing_url(),
            share_ownership_page_url()
        )));
    }

    Ok(discovered)
}

pub fn download_idx_pdf(url: &str, target: &Path) -> Result<(), IdxError> {
    let bytes = fetch_bytes(
        "IDX ownership PDF download",
        url,
        &pdf_headers(),
        &validate_pdf_payload,
    )?;

    fs::write(target, &bytes).map_err(|e| {
        IdxError::Io(format!(
            "failed writing cached PDF {}: {e}",
            target.display()
        ))
    })?;

    Ok(())
}

pub fn download_idx_xlsx(url: &str, target: &Path) -> Result<(), IdxError> {
    let bytes = fetch_bytes(
        "IDX ownership XLSX download",
        url,
        &xlsx_headers(),
        &validate_xlsx_payload,
    )?;

    fs::write(target, &bytes).map_err(|e| {
        IdxError::Io(format!(
            "failed writing cached XLSX {}: {e}",
            target.display()
        ))
    })?;

    Ok(())
}

/// XLSX is a ZIP container; Cloudflare challenges come back as HTML.
pub fn validate_xlsx_payload(bytes: &[u8]) -> Result<(), IdxError> {
    if bytes.starts_with(b"PK\x03\x04") {
        return Ok(());
    }
    if looks_like_html(bytes) {
        return Err(IdxError::Http(
            "IDX ownership download returned HTML (likely a Cloudflare challenge) instead of an XLSX workbook".to_string(),
        ));
    }
    Err(IdxError::Http(
        "IDX ownership download did not look like an XLSX workbook".to_string(),
    ))
}

fn validate_json_payload(bytes: &[u8]) -> Result<(), IdxError> {
    if looks_like_html(bytes) {
        return Err(IdxError::Http(
            "IDX ownership discovery returned HTML (likely a Cloudflare challenge) instead of announcement JSON".to_string(),
        ));
    }
    Ok(())
}

fn validate_share_ownership_page(bytes: &[u8]) -> Result<(), IdxError> {
    if contains_bytes(bytes, b"Prospectus") {
        return Ok(());
    }
    Err(IdxError::Http(
        "IDX share ownership data page did not contain its file list (likely a Cloudflare challenge)".to_string(),
    ))
}

fn looks_like_html(bytes: &[u8]) -> bool {
    let head: Vec<u8> = bytes
        .iter()
        .skip_while(|byte| byte.is_ascii_whitespace())
        .take(256)
        .map(u8::to_ascii_lowercase)
        .collect();
    head.starts_with(b"<!doctype html") || head.starts_with(b"<html")
}

fn contains_bytes(haystack: &[u8], needle: &[u8]) -> bool {
    haystack
        .windows(needle.len())
        .any(|window| window == needle)
}

/// Parse the Data Kepemilikan Saham page. The page is server-rendered by
/// Nuxt and embeds its file list as JS object literals such as
/// `{Description:"Pemegang Saham di Atas 1% per 31 Agustus 2026",
///   Prospectus:"https:\u002F\u002Fwww.idx.co.id\u002FMedia\u002F...xlsx",...}`.
pub fn parse_share_ownership_page(html: &str, page_url: &str) -> Vec<DiscoveredOwnershipPdf> {
    let mut reports = Vec::new();
    let mut rest = html;

    while let Some(start) = find_js_key(rest, "Description") {
        let (description, after_description) = match read_js_string(&rest[start..]) {
            Some(value) => value,
            None => {
                rest = &rest[start..];
                continue;
            }
        };
        let object_tail = &rest[start + after_description..];
        // The URL belongs to the same object literal, so it must come before
        // the next Description key.
        let window_end = find_js_key(object_tail, "Description").unwrap_or(object_tail.len());
        let window = &object_tail[..window_end];
        rest = object_tail;

        let Some(url_start) = find_js_key(window, "Prospectus") else {
            continue;
        };
        let Some((url, _)) = read_js_string(&window[url_start..]) else {
            continue;
        };
        if let Some(report) = classify_page_entry(description.trim(), url.trim(), page_url) {
            reports.push(report);
        }
    }

    reports
}

/// Offset just past `Key:` / `"Key":` (the opening quote of the value).
fn find_js_key(haystack: &str, key: &str) -> Option<usize> {
    let bare = format!("{key}:\"");
    let quoted = format!("\"{key}\":\"");
    [bare, quoted]
        .iter()
        .filter_map(|needle| {
            haystack
                .find(needle.as_str())
                .map(|index| index + needle.len() - 1)
        })
        .min()
}

/// Read a JS/JSON string literal starting at the opening quote. Returns the
/// decoded value and the byte offset just past the closing quote.
fn read_js_string(input: &str) -> Option<(String, usize)> {
    let mut chars = input.char_indices();
    if chars.next()?.1 != '"' {
        return None;
    }
    let mut value = String::new();
    while let Some((index, ch)) = chars.next() {
        match ch {
            '"' => return Some((value, index + 1)),
            '\\' => match chars.next()?.1 {
                'u' => {
                    let hex: String = (0..4)
                        .filter_map(|_| chars.next().map(|(_, c)| c))
                        .collect();
                    value.push(char::from_u32(u32::from_str_radix(&hex, 16).ok()?)?);
                }
                'n' => value.push('\n'),
                't' => value.push('\t'),
                other => value.push(other),
            },
            _ => value.push(ch),
        }
    }
    None
}

fn classify_page_entry(
    description: &str,
    url: &str,
    page_url: &str,
) -> Option<DiscoveredOwnershipPdf> {
    let upper = description.to_uppercase();
    let family = if upper.contains("DI ATAS 1%") || upper.contains("ABOVE 1%") {
        OwnershipReportFamily::AboveOnePercent
    } else if upper.contains("DI ATAS 5%") || upper.contains("ABOVE 5%") {
        OwnershipReportFamily::AboveFivePercent
    } else if upper.contains("TIPE INVESTOR")
        || upper.contains("KLASIFIKASI")
        || upper.contains("INVESTOR TYPE")
        || upper.contains("CLASSIFICATION")
    {
        OwnershipReportFamily::InvestorTypeBreakdown
    } else {
        return None;
    };

    let path = url.split(['?', '#']).next().unwrap_or_default();
    let file_name = path.rsplit('/').next().unwrap_or_default().to_string();
    let lower_name = file_name.to_ascii_lowercase();
    let format = if lower_name.ends_with(".xlsx") {
        "xlsx"
    } else if lower_name.ends_with(".pdf") {
        "pdf"
    } else {
        return None;
    };
    let status = if family == OwnershipReportFamily::AboveOnePercent && format == "xlsx" {
        OwnershipReportStatus::Supported
    } else {
        OwnershipReportStatus::Unsupported
    };
    // Without an as-of date the entry cannot be ordered against other
    // months, and the snapshot manifest needs a date, so skip it.
    let as_of_date = parse_indonesian_as_of_date(description)?.to_string();

    Some(DiscoveredOwnershipPdf {
        family,
        status,
        listing_page_url: page_url.to_string(),
        query_url: page_url.to_string(),
        pdf_url: url.to_string(),
        title: description.to_string(),
        publish_date: as_of_date.clone(),
        code: None,
        original_filename: (!file_name.is_empty()).then_some(file_name),
        is_attachment: true,
        format: format.to_string(),
        as_of_date: Some(as_of_date),
    })
}

/// `"... per 31 Agustus 2026"` -> 2026-08-31 (Indonesian or English months).
fn parse_indonesian_as_of_date(description: &str) -> Option<chrono::NaiveDate> {
    let tokens: Vec<String> = description
        .split_whitespace()
        .map(|token| {
            token
                .trim_matches(|c: char| !c.is_alphanumeric())
                .to_lowercase()
        })
        .collect();
    tokens.windows(3).rev().find_map(|window| {
        let day = window[0].parse::<u32>().ok()?;
        let month = match window[1].as_str() {
            "januari" | "january" | "jan" => 1,
            "februari" | "february" | "feb" => 2,
            "maret" | "march" | "mar" => 3,
            "april" | "apr" => 4,
            "mei" | "may" => 5,
            "juni" | "june" | "jun" => 6,
            "juli" | "july" | "jul" => 7,
            "agustus" | "august" | "agu" | "aug" => 8,
            "september" | "sep" => 9,
            "oktober" | "october" | "okt" | "oct" => 10,
            "november" | "nov" => 11,
            "desember" | "december" | "des" | "dec" => 12,
            _ => return None,
        };
        let year = window[2].parse::<i32>().ok()?;
        chrono::NaiveDate::from_ymd_opt(year, month, day)
    })
}

pub fn validate_pdf_payload(bytes: &[u8]) -> Result<(), IdxError> {
    if bytes.is_empty() {
        return Err(IdxError::Http(
            "IDX ownership download returned an empty response".to_string(),
        ));
    }

    let trimmed = bytes
        .iter()
        .skip_while(|byte| byte.is_ascii_whitespace())
        .copied()
        .collect::<Vec<_>>();
    if trimmed.starts_with(b"%PDF-") {
        return Ok(());
    }

    let preview = String::from_utf8_lossy(&trimmed[..trimmed.len().min(256)]).to_ascii_lowercase();
    if preview.contains("<!doctype html") || preview.contains("<html") {
        return Err(IdxError::Http(
            "IDX ownership download returned HTML instead of a PDF/direct attachment".to_string(),
        ));
    }

    Err(IdxError::Http(
        "IDX ownership download did not look like a PDF/direct attachment".to_string(),
    ))
}

fn fetch_text(
    stage: &str,
    url: &str,
    headers: &[(String, String)],
    accept: &dyn Fn(&[u8]) -> Result<(), IdxError>,
) -> Result<String, IdxError> {
    let bytes = fetch_bytes(stage, url, headers, accept)?;
    String::from_utf8(bytes)
        .map_err(|e| IdxError::Http(format!("failed to decode {stage} response as utf-8: {e}")))
}

fn fetch_bytes(
    stage: &str,
    url: &str,
    headers: &[(String, String)],
    accept: &dyn Fn(&[u8]) -> Result<(), IdxError>,
) -> Result<Vec<u8>, IdxError> {
    let mut args = vec![
        "--silent".to_string(),
        "--show-error".to_string(),
        "--location".to_string(),
        "--fail".to_string(),
        "--compressed".to_string(),
    ];
    for (name, value) in headers {
        args.push("--header".to_string());
        args.push(format!("{name}: {value}"));
    }
    args.push(url.to_string());

    let output = curl_impersonate::run_owned_with_fallback(stage, &args, accept)?;
    Ok(output.stdout)
}

fn json_headers() -> Vec<(String, String)> {
    vec![
        ("User-Agent".to_string(), USER_AGENT.to_string()),
        (
            "Accept".to_string(),
            "application/json,text/plain,*/*".to_string(),
        ),
        (
            "Accept-Language".to_string(),
            "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7".to_string(),
        ),
        ("Referer".to_string(), announcement_listing_url()),
    ]
}

fn page_headers() -> Vec<(String, String)> {
    vec![
        ("User-Agent".to_string(), USER_AGENT.to_string()),
        (
            "Accept".to_string(),
            "text/html,application/xhtml+xml,*/*;q=0.8".to_string(),
        ),
        (
            "Accept-Language".to_string(),
            "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7".to_string(),
        ),
    ]
}

fn xlsx_headers() -> Vec<(String, String)> {
    vec![
        ("User-Agent".to_string(), USER_AGENT.to_string()),
        (
            "Accept".to_string(),
            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/octet-stream,*/*;q=0.8".to_string(),
        ),
        (
            "Accept-Language".to_string(),
            "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7".to_string(),
        ),
        ("Referer".to_string(), share_ownership_page_url()),
    ]
}

fn pdf_headers() -> Vec<(String, String)> {
    vec![
        ("User-Agent".to_string(), USER_AGENT.to_string()),
        (
            "Accept".to_string(),
            "application/pdf,application/octet-stream,*/*;q=0.8".to_string(),
        ),
        (
            "Accept-Language".to_string(),
            "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7".to_string(),
        ),
        ("Referer".to_string(), announcement_listing_url()),
    ]
}

fn item_matches_family(item: &AnnouncementItem, query: &DiscoveryQuery) -> bool {
    let title = item.title.to_ascii_uppercase();
    query
        .title_needles
        .iter()
        .any(|needle| title.contains(needle))
}

fn item_pdf_attachments(item: &AnnouncementItem) -> Vec<AnnouncementAttachment> {
    let attachments = if item.attachments.is_empty() {
        item.pdf_path
            .as_deref()
            .and_then(parse_pdf_path)
            .unwrap_or_default()
    } else {
        item.attachments.clone()
    };

    attachments
        .into_iter()
        .filter(|attachment| {
            attachment
                .full_save_path
                .to_ascii_lowercase()
                .ends_with(".pdf")
        })
        .collect()
}

fn attachment_label(attachment: &AnnouncementAttachment) -> String {
    attachment
        .original_filename
        .as_deref()
        .or(attachment.pdf_filename.as_deref())
        .unwrap_or_default()
        .trim()
        .to_ascii_lowercase()
}

fn classify_report_status(
    family: OwnershipReportFamily,
    is_attachment: bool,
) -> OwnershipReportStatus {
    match (family, is_attachment) {
        (OwnershipReportFamily::AboveOnePercent, true) => OwnershipReportStatus::Supported,
        (OwnershipReportFamily::AboveOnePercent, false) => OwnershipReportStatus::AnnouncementOnly,
        _ => OwnershipReportStatus::Unsupported,
    }
}

fn parse_pdf_path(raw: &str) -> Option<Vec<AnnouncementAttachment>> {
    serde_json::from_str::<Vec<AnnouncementAttachment>>(raw).ok()
}

fn attachment_is_attachment(attachment: &AnnouncementAttachment) -> bool {
    match attachment.is_attachment.as_ref() {
        Some(serde_json::Value::Bool(value)) => *value,
        Some(serde_json::Value::Number(value)) => value.as_i64() == Some(1),
        Some(serde_json::Value::String(value)) => {
            matches!(value.trim(), "1" | "true" | "TRUE" | "True")
        }
        _ => attachment_label(attachment).contains("lamp"),
    }
}

fn clean_option(value: Option<&str>) -> Option<String> {
    value
        .map(str::trim)
        .filter(|value| !value.is_empty())
        .map(str::to_string)
}

fn percent_encode(value: &str) -> String {
    let mut encoded = String::with_capacity(value.len());
    for byte in value.bytes() {
        match byte {
            b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
                encoded.push(byte as char)
            }
            b' ' => encoded.push_str("%20"),
            other => encoded.push_str(&format!("%{other:02X}")),
        }
    }
    encoded
}

#[cfg(test)]
mod tests {
    use super::{
        AnnouncementPage, IDX_ANNOUNCEMENT_LISTING_URL, OwnershipReportFamily,
        OwnershipReportStatus, build_announcement_query_url, parse_announcement_page,
        parse_indonesian_as_of_date, parse_share_ownership_page, select_latest_ownership_reports,
        validate_pdf_payload, validate_share_ownership_page, validate_xlsx_payload,
    };
    use crate::error::IdxError;

    #[test]
    fn builds_announcement_query_url() {
        let url = build_announcement_query_url("pemegang saham di atas 5", 1, 10);
        assert_eq!(
            url,
            "https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%205&pageNumber=1&pageSize=10&lang=id"
        );
    }

    #[test]
    fn parses_fixture_and_selects_latest_above_5_reports() {
        let raw = include_str!("../../tests/fixtures/idx_announcement_kepemilikan.json");
        let page = parse_announcement_page(raw).expect("announcement fixture should parse");
        let discovered = select_latest_ownership_reports(
            &page,
            "https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%205&pageNumber=1&pageSize=10&lang=id",
            OwnershipReportFamily::AboveFivePercent,
        )
        .expect("should select ownership reports");

        assert_eq!(discovered.len(), 2);
        assert_eq!(
            discovered[0].family,
            OwnershipReportFamily::AboveFivePercent
        );
        assert_eq!(discovered[0].listing_page_url, IDX_ANNOUNCEMENT_LISTING_URL);
        assert_eq!(discovered[0].publish_date, "2026-03-27T16:34:20");
        assert_eq!(
            discovered[0].pdf_url,
            "https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/4f5c4efc6f_bf70f249ac_lamp1.pdf"
        );
        assert_eq!(
            discovered[0].original_filename.as_deref(),
            Some("20260327_Semua Emiten Saham_Pengumuman Bursa_32055594_lamp1.pdf")
        );
        assert_eq!(discovered[0].status, OwnershipReportStatus::Unsupported);
        assert!(discovered[0].is_attachment);
        assert_eq!(
            discovered[1].pdf_url,
            "https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/5d31bb6f49_announcement.pdf"
        );
        assert_eq!(
            discovered[1].original_filename.as_deref(),
            Some("20260327_Semua Emiten Saham_Pengumuman Bursa_32055594.pdf")
        );
        assert!(!discovered[1].is_attachment);
    }

    #[test]
    fn selects_above_1_reports() {
        let raw = r#"{
          "Items": [
            {
              "PublishDate": "2026-03-10T12:09:09",
              "Title": "Pemegang Saham di atas 1% (KSEI)",
              "AnnouncementType": "",
              "Code": " Semua Emiten Saham ",
              "Attachments": [
                {
                  "PDFFilename": "d67ebf37e6_10d4080288.pdf",
                  "FullSavePath": "https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/d67ebf37e6_10d4080288.pdf",
                  "IsAttachment": 0,
                  "OriginalFilename": "20260310_Semua Emiten Saham_Pengumuman Bursa_32052554.pdf"
                },
                {
                  "PDFFilename": "b9b638e5a8_8928aca255.pdf",
                  "FullSavePath": "https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/b9b638e5a8_8928aca255.pdf",
                  "IsAttachment": 1,
                  "OriginalFilename": "20260310_Semua Emiten Saham_Pengumuman Bursa_32052554_lamp1.pdf"
                }
              ],
              "PdfPath": ""
            }
          ],
          "ItemCount": 1,
          "PageCount": 1
        }"#;

        let page = parse_announcement_page(raw).expect("above-1 fixture should parse");
        let discovered = select_latest_ownership_reports(
            &page,
            "https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%201&pageNumber=1&pageSize=10&lang=id",
            OwnershipReportFamily::AboveOnePercent,
        )
        .expect("should select above-1 reports");

        assert_eq!(discovered.len(), 2);
        assert_eq!(discovered[0].family, OwnershipReportFamily::AboveOnePercent);
        assert_eq!(discovered[0].status, OwnershipReportStatus::Supported);
        assert_eq!(discovered[0].code.as_deref(), Some("Semua Emiten Saham"));
        assert_eq!(
            discovered[0].original_filename.as_deref(),
            Some("20260310_Semua Emiten Saham_Pengumuman Bursa_32052554_lamp1.pdf")
        );
        assert!(discovered[0].is_attachment);
        assert_eq!(
            discovered[1].status,
            OwnershipReportStatus::AnnouncementOnly
        );
        assert!(!discovered[1].is_attachment);
    }

    #[test]
    fn selects_investor_type_breakdown_reports() {
        let raw = r#"{
          "Items": [
            {
              "PublishDate": "2026-03-02T16:14:37",
              "Title": "Data KSEI terkait Kepemilikan Saham Perusahaan Tercatat Berdasarkan Tipe Investor per 27 Februari 2026",
              "AnnouncementType": "",
              "Code": "  ",
              "Attachments": [
                {
                  "PDFFilename": "20260302_Pengumuman Bursa_32040089.pdf",
                  "FullSavePath": "https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/43025665c8_fda8953269.pdf",
                  "IsAttachment": 0,
                  "OriginalFilename": "20260302_Pengumuman Bursa_32040089.pdf"
                },
                {
                  "PDFFilename": "20260302_Pengumuman Bursa_32040089_lamp1.pdf",
                  "FullSavePath": "https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/db5b5a86e1_2e2b3d976a.pdf",
                  "IsAttachment": 1,
                  "OriginalFilename": "20260302_Pengumuman Bursa_32040089_lamp1.pdf"
                }
              ],
              "PdfPath": ""
            }
          ],
          "ItemCount": 1,
          "PageCount": 1
        }"#;

        let page = parse_announcement_page(raw).expect("investor-type fixture should parse");
        let discovered = select_latest_ownership_reports(
            &page,
            "https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=kepemilikan%20saham%20perusahaan%20tercatat&pageNumber=1&pageSize=10&lang=id",
            OwnershipReportFamily::InvestorTypeBreakdown,
        )
        .expect("should select investor-type reports");

        assert_eq!(discovered.len(), 2);
        assert_eq!(
            discovered[0].original_filename.as_deref(),
            Some("20260302_Pengumuman Bursa_32040089_lamp1.pdf")
        );
        assert_eq!(discovered[0].status, OwnershipReportStatus::Unsupported);
        assert_eq!(
            discovered[1].original_filename.as_deref(),
            Some("20260302_Pengumuman Bursa_32040089.pdf")
        );
    }

    #[test]
    fn falls_back_to_pdf_path_when_attachments_are_missing() {
        let raw = r#"{
          "Items": [
            {
              "PublishDate": "2026-03-04T08:54:00",
              "Title": "Pemegang Saham di atas 5% (KSEI)",
              "AnnouncementType": "",
              "Code": "Semua Emiten Saham",
              "Attachments": [],
              "PdfPath": "[{\"PDFFilename\":\"20260305_LKS_KSEI_000043_lamp1.pdf\",\"FullSavePath\":\"https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/ec835d7451_c30c886a1a.pdf\",\"IsAttachment\":\"1\",\"OriginalFilename\":\"20260305_LKS_KSEI_000043_lamp1.pdf\"}]"
            }
          ],
          "ItemCount": 1,
          "PageCount": 1
        }"#;

        let page: AnnouncementPage = parse_announcement_page(raw).expect("fallback page parses");
        let discovered = select_latest_ownership_reports(
            &page,
            "https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%205&pageNumber=1&pageSize=10&lang=id",
            OwnershipReportFamily::AboveFivePercent,
        )
        .expect("fallback attachment should be parsed");

        assert_eq!(discovered.len(), 1);
        assert_eq!(
            discovered[0].pdf_url,
            "https://www.idx.co.id/StaticData/NewsAndAnnouncement/ANNOUNCEMENTSTOCK/From_EREP/202603/ec835d7451_c30c886a1a.pdf"
        );
        assert_eq!(discovered[0].code.as_deref(), Some("Semua Emiten Saham"));
        assert!(discovered[0].is_attachment);
    }

    #[test]
    fn rejects_html_instead_of_announcement_json() {
        let err = parse_announcement_page("<!doctype html><html><body>blocked</body></html>")
            .expect_err("html response must fail");
        assert!(matches!(err, IdxError::Http(_)));
    }

    #[test]
    fn rejects_empty_announcement_payload() {
        let err = parse_announcement_page("   ").expect_err("empty payload must fail");
        assert!(matches!(err, IdxError::Http(_)));
    }

    #[test]
    fn select_latest_reports_errors_when_family_is_missing() {
        let raw = r#"{"Items":[],"ItemCount":0,"PageCount":0}"#;
        let page = parse_announcement_page(raw).expect("empty page parses");

        let err = select_latest_ownership_reports(
            &page,
            "https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%201&pageNumber=1&pageSize=10&lang=id",
            OwnershipReportFamily::AboveOnePercent,
        )
        .expect_err("missing announcement should fail");

        assert!(matches!(err, IdxError::Http(_)));
        assert!(err.to_string().contains("no matching announcement found"));
    }

    #[test]
    fn select_latest_reports_errors_when_matching_item_has_no_pdf() {
        let raw = r#"{
          "Items": [
            {
              "PublishDate": "2026-03-10T12:09:09",
              "Title": "Pemegang Saham di atas 1% (KSEI)",
              "AnnouncementType": "",
              "Code": "Semua Emiten Saham",
              "Attachments": [],
              "PdfPath": ""
            }
          ],
          "ItemCount": 1,
          "PageCount": 1
        }"#;
        let page = parse_announcement_page(raw).expect("page parses");

        let err = select_latest_ownership_reports(
            &page,
            "https://www.idx.co.id/primary/NewsAnnouncement/GetAllAnnouncement?keywords=pemegang%20saham%20di%20atas%201&pageNumber=1&pageSize=10&lang=id",
            OwnershipReportFamily::AboveOnePercent,
        )
        .expect_err("missing pdf attachments should fail");

        assert!(matches!(err, IdxError::Http(_)));
        assert!(err.to_string().contains("no PDF attachments"));
    }

    #[test]
    fn accepts_pdf_header() {
        validate_pdf_payload(b"%PDF-1.7\n1 0 obj\n").expect("pdf header should pass");
    }

    #[test]
    fn rejects_html_instead_of_pdf() {
        let err = validate_pdf_payload(b"<!doctype html><html><body>blocked</body></html>")
            .expect_err("html body must fail");
        assert!(matches!(err, IdxError::Http(_)));
        assert!(err.to_string().contains("PDF/direct attachment"));
    }

    #[test]
    fn parses_share_ownership_page_entries() {
        let page_url = "https://www.idx.co.id/id/perusahaan-tercatat/data-kepemilikan-saham/";
        let reports = parse_share_ownership_page(
            include_str!("../../tests/fixtures/idx_share_ownership_page_excerpt.html"),
            page_url,
        );
        assert_eq!(reports.len(), 7);

        let above1: Vec<_> = reports
            .iter()
            .filter(|r| r.family == OwnershipReportFamily::AboveOnePercent)
            .collect();
        assert_eq!(above1.len(), 3);
        assert!(
            above1
                .iter()
                .all(|r| r.status == OwnershipReportStatus::Supported)
        );
        assert!(above1.iter().all(|r| r.format == "xlsx"));

        let august = above1
            .iter()
            .find(|r| r.as_of_date.as_deref() == Some("2026-08-31"))
            .expect("August above-1% entry");
        assert_eq!(
            august.pdf_url,
            "https://www.idx.co.id/Media/fahlw1o2/peng-2026-08-00017-satu-persen.xlsx"
        );
        assert_eq!(august.publish_date, "2026-08-31");
        assert_eq!(august.listing_page_url, page_url);
        assert_eq!(
            august.original_filename.as_deref(),
            Some("peng-2026-08-00017-satu-persen.xlsx")
        );

        let above5 = reports
            .iter()
            .filter(|r| r.family == OwnershipReportFamily::AboveFivePercent)
            .count();
        let breakdown = reports
            .iter()
            .filter(|r| r.family == OwnershipReportFamily::InvestorTypeBreakdown)
            .count();
        assert_eq!((above5, breakdown), (2, 2));
        assert!(
            reports
                .iter()
                .filter(|r| r.family != OwnershipReportFamily::AboveOnePercent)
                .all(|r| r.status == OwnershipReportStatus::Unsupported)
        );
    }

    #[test]
    fn share_ownership_page_handles_query_strings_and_dateless_entries() {
        let html = r#"{Description:"Pemegang Saham di Atas 1% per 31 Agustus 2026",Prospectus:"https://www.idx.co.id/Media/x/peng-satu-persen.xlsx?download=1",ListingDate:r},{Description:"Pemegang Saham di Atas 1%",Prospectus:"https://www.idx.co.id/Media/y/no-date.xlsx",ListingDate:r}"#;
        let reports = parse_share_ownership_page(html, "p");
        assert_eq!(
            reports.len(),
            1,
            "dateless entry skipped, query-string entry kept"
        );
        assert_eq!(reports[0].format, "xlsx");
        assert_eq!(reports[0].status, OwnershipReportStatus::Supported);
        assert_eq!(
            reports[0].original_filename.as_deref(),
            Some("peng-satu-persen.xlsx")
        );
        assert_eq!(reports[0].publish_date, "2026-08-31");
    }

    #[test]
    fn share_ownership_page_without_payload_yields_nothing() {
        assert!(parse_share_ownership_page("<html>Just a moment...</html>", "p").is_empty());
        assert!(validate_share_ownership_page(b"<html>Just a moment...</html>").is_err());
    }

    #[test]
    fn parses_indonesian_and_english_as_of_dates() {
        let date = |y, m, d| chrono::NaiveDate::from_ymd_opt(y, m, d);
        assert_eq!(
            parse_indonesian_as_of_date("Pemegang Saham di Atas 1% per 31 Agustus 2026"),
            date(2026, 8, 31)
        );
        assert_eq!(
            parse_indonesian_as_of_date("Shareholders above 1% as of 29 May 2026"),
            date(2026, 5, 29)
        );
        assert_eq!(
            parse_indonesian_as_of_date("Pemegang Saham di Atas 1%"),
            None
        );
    }

    #[test]
    fn validates_xlsx_payloads() {
        assert!(validate_xlsx_payload(b"PK\x03\x04rest-of-zip").is_ok());
        let err = validate_xlsx_payload(b"<!DOCTYPE html><title>Just a moment...</title>")
            .expect_err("html challenge rejected");
        assert!(err.to_string().contains("Cloudflare"), "{err}");
        assert!(validate_xlsx_payload(b"%PDF-1.7").is_err());
    }
}