0xrsydn/idx-cli
1
mirror of https://github.com/0xrsydn/idx-cli.git synced 2026-11-07 09:45:36 +00:00
Code Issues Projects Releases 1 Packages Wiki Activity

feat(core): broker summaries from IPOT

Browse source 
Adds a BrokerSummaryProvider capability served by Indo Premier's public
broker summary page: the top 10 buyers and sellers of a stock by lots over
a date range, with totals. No login or key.

- api/ipot: pure parser (decimal scaling of `1.6 M`-style figures), a
  client that retries the server's spurious 404s, and a fixture mock.
- broker: Broker::summary/latest over the cache (7 days for closed ranges,
  5 minutes for today), range checks, and IDX member names bundled from
  GetBrokerCodeList.
- foreign_net_value is null for filtered summaries: the source does not
  apply the filters to it.

The endpoint's behavior was first mapped by fzn0x/idx (MIT).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V3syn2fN2LNo5pvTSqc6eh
This commit is contained in:
hermes 2026-10-18 04:28:38 +00:00
commit 2c93c73b6a
16 changed files with 1767 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

62
crates/idx-core/src/api/ipot/client.rs Normal file
View file

@ -0,0 +1,62 @@
use std::time::Duration;
use crate::error::IdxError;
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
/// Tries per request. In some stretches of seconds the server answers its
/// own nginx 404 page to a third or more of requests and then recovers, so a
/// 404 is retried (an unknown ticker is a 200, not a 404).
const ATTEMPTS: u32 = 5;
const FIRST_PAUSE: Duration = Duration::from_millis(250);
pub(super) struct IpotClient {
agent: ureq::Agent,
}
impl IpotClient {
pub(super) fn new() -> Self {
let agent: ureq::Agent = ureq::Agent::config_builder()
.timeout_connect(Some(Duration::from_secs(5)))
.timeout_recv_body(Some(Duration::from_secs(10)))
// A server that stalls before sending headers must not hold a
// call (and an MCP call slot) forever.
.timeout_global(Some(Duration::from_secs(20)))
.build()
.into();
Self { agent }
}
pub(super) fn fetch_html(&self, url: &str) -> Result<String, IdxError> {
let mut pause = FIRST_PAUSE;
let mut last = None;
for attempt in 1..=ATTEMPTS {
if attempt > 1 {
std::thread::sleep(pause);
pause *= 2;
}
let response = self
.agent
.get(url)
.header("User-Agent", USER_AGENT)
.header("Accept", "text/html")
.call();
match response {
Ok(ok) => {
return ok
.into_body()
.read_to_string()
.map_err(|e| IdxError::Http(format!("ipot broker summary: {e}")));
}
Err(ureq::Error::StatusCode(code)) if code == 404 || code >= 500 => {
last = Some(code);
}
Err(ureq::Error::StatusCode(429)) => return Err(IdxError::RateLimited),
Err(err) => return Err(IdxError::Http(format!("ipot broker summary: {err}"))),
}
}
Err(IdxError::Http(format!(
"ipot broker summary: status {} after {ATTEMPTS} attempts",
last.unwrap_or_default()
)))
}
}

101
crates/idx-core/src/api/ipot/mod.rs Normal file
View file

@ -0,0 +1,101 @@
//! Indo Premier Sekuritas (IPOT) broker summary: which brokers bought and
//! sold a stock. No login or key.
//!
//! `GET https://www.indopremier.com/module/saham/include/data-brokersummary.php`
//! `?code=BBCA&start=MM/DD/YYYY&end=MM/DD/YYYY&fd=all|F|D&board=all|RG|TN|NG`
//! answers an HTML fragment with the top 10 buyers and the top 10 sellers,
//! ranked by lots, and totals for the range. Figures of a million and more
//! are abbreviated to one decimal (`1.6 M`). A range without trades gives
//! blank rows and zero totals; an unknown code gives `Invalid stock code!`
//! with HTTP 200. The endpoint's behavior was first mapped by fzn0x/idx
//! (MIT).
mod client;
pub mod parse;
use client::IpotClient;
pub use parse::{BrokerSummaryRaw, IpotPage, parse_broker_summary, parse_figure};
use super::BrokerSummaryProvider;
use crate::broker::types::{BrokerBoard, BrokerInvestor, BrokerQuery};
use crate::error::IdxError;
const ENDPOINT: &str = "https://www.indopremier.com/module/saham/include/data-brokersummary.php";
/// The request URL for `code` (a bare IDX code such as `BBCA`).
pub fn broker_summary_url(code: &str, query: &BrokerQuery) -> String {
let investor = match query.investor {
BrokerInvestor::All => "all",
BrokerInvestor::Foreign => "F",
BrokerInvestor::Domestic => "D",
};
let board = match query.board {
BrokerBoard::All => "all",
BrokerBoard::Regular => "RG",
BrokerBoard::Cash => "TN",
BrokerBoard::Negotiated => "NG",
};
format!(
"{ENDPOINT}?code={code}&start={}&end={}&fd={investor}&board={board}",
query.from.format("%m/%d/%Y"),
query.to.format("%m/%d/%Y"),
)
}
/// A parsed page as a summary; an unknown code is `SymbolNotFound`.
pub(crate) fn summary_from_html(code: &str, html: &str) -> Result<BrokerSummaryRaw, IdxError> {
match parse_broker_summary(html)? {
IpotPage::Invalid => Err(IdxError::SymbolNotFound(code.to_string())),
IpotPage::Summary(raw) => Ok(raw),
}
}
pub struct IpotProvider {
client: IpotClient,
}
impl IpotProvider {
pub fn new() -> Self {
Self {
client: IpotClient::new(),
}
}
}
impl Default for IpotProvider {
fn default() -> Self {
Self::new()
}
}
impl BrokerSummaryProvider for IpotProvider {
fn broker_summary(
&self,
code: &str,
query: &BrokerQuery,
) -> Result<BrokerSummaryRaw, IdxError> {
let html = self.client.fetch_html(&broker_summary_url(code, query))?;
summary_from_html(code, &html)
}
}
#[cfg(test)]
mod tests {
use chrono::NaiveDate;
use super::*;
#[test]
fn test_broker_summary_url_maps_filters_to_source_codes() {
let query = BrokerQuery {
from: NaiveDate::from_ymd_opt(2026, 10, 12).unwrap(),
to: NaiveDate::from_ymd_opt(2026, 10, 16).unwrap(),
investor: BrokerInvestor::Foreign,
board: BrokerBoard::Regular,
};
assert_eq!(
broker_summary_url("BBCA", &query),
"https://www.indopremier.com/module/saham/include/data-brokersummary.php?code=BBCA&start=10/12/2026&end=10/16/2026&fd=F&board=RG"
);
}
}

317
crates/idx-core/src/api/ipot/parse.rs Normal file
View file

@ -0,0 +1,317 @@
//! Pure parsers for IPOT's broker summary fragment, tested against the
//! recorded pages in `tests/fixtures/ipot_brokersummary_*.html`.
//!
//! The fragment is a 9-column table: buyer code, lots, value, average, rank,
//! seller code, lots, value, average. Each code sits in a
//! `<span class="text-local|text-foreign|text-bumn">`; the footer holds
//! `T. Val`, `F. NVal`, `T.Lot` and `Avg`, and the date inputs echo the
//! range the source applied.
use chrono::NaiveDate;
use crate::broker::types::{BrokerFlow, BrokerKind};
use crate::error::IdxError;
/// What one page said.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum IpotPage {
/// `Invalid stock code!`: the source does not know the ticker.
Invalid,
Summary(BrokerSummaryRaw),
}
/// One parsed summary page, before the symbol and query are attached.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct BrokerSummaryRaw {
pub from: Option<NaiveDate>,
pub to: Option<NaiveDate>,
pub buyers: Vec<BrokerFlow>,
pub sellers: Vec<BrokerFlow>,
pub total_value: Option<i64>,
pub total_lots: Option<i64>,
pub average_price: Option<i64>,
pub foreign_net_value: Option<i64>,
pub rounded: bool,
}
/// A figure as published: `1.6 M` -> `(1_600_000, true)`, `687,067` ->
/// `(687_067, false)`, `-2.3 T` -> `(-2_300_000_000_000, true)`. Scaled in
/// decimal, so `1.6 M` is exact. Empty or unparsable -> `None`.
pub fn parse_figure(text: &str) -> Option<(i64, bool)> {
let text = text.trim().replace(',', "");
let (number, exponent) = match text.chars().last()?.to_ascii_uppercase() {
'K' => (&text[..text.len() - 1], 3),
'M' => (&text[..text.len() - 1], 6),
'B' => (&text[..text.len() - 1], 9),
'T' => (&text[..text.len() - 1], 12),
_ => (text.as_str(), 0),
};
let number = number.trim();
let (negative, digits) = match number.strip_prefix('-') {
Some(rest) => (true, rest),
None => (false, number),
};
let (whole, fraction) = digits.split_once('.').unwrap_or((digits, ""));
if whole.is_empty()
|| !whole.bytes().all(|b| b.is_ascii_digit())
|| !fraction.bytes().all(|b| b.is_ascii_digit())
|| (exponent > 0 && fraction.len() > exponent as usize)
{
return None;
}
let scale = 10_i64.checked_pow(exponent)?;
let mut value = whole.parse::<i64>().ok()?.checked_mul(scale)?;
if exponent == 0 {
// A plain decimal: round to whole units.
if fraction
.as_bytes()
.first()
.is_some_and(|digit| *digit >= b'5')
{
value = value.checked_add(1)?;
}
} else if !fraction.is_empty() {
let unit = 10_i64.checked_pow(exponent - fraction.len() as u32)?;
value = value.checked_add(fraction.parse::<i64>().ok()?.checked_mul(unit)?)?;
}
Some((if negative { -value } else { value }, exponent > 0))
}
struct Cell {
class: Option<String>,
text: String,
}
fn strip_tags(html: &str) -> String {
let mut out = String::with_capacity(html.len());
let mut in_tag = false;
for ch in html.chars() {
match ch {
'<' => in_tag = true,
'>' => in_tag = false,
_ if !in_tag => out.push(ch),
_ => {}
}
}
out.replace("&nbsp;", " ").trim().to_string()
}
fn cells(row: &str) -> Vec<Cell> {
let mut out = Vec::new();
let mut rest = row;
while let Some(start) = rest.find("<td") {
let after = &rest[start..];
let Some(open_end) = after.find('>') else {
break;
};
let inner_start = open_end + 1;
let Some(close) = after[inner_start..].find("</td>") else {
break;
};
let inner = &after[inner_start..inner_start + close];
let class = inner.find("class=\"").and_then(|at| {
let value = &inner[at + 7..];
value.find('"').map(|end| value[..end].to_string())
});
out.push(Cell {
class,
text: strip_tags(inner),
});
rest = &after[inner_start + close + 5..];
}
out
}
fn broker_kind(class: Option<&str>) -> Option<BrokerKind> {
match class?.strip_prefix("text-")? {
"local" => Some(BrokerKind::Local),
"foreign" => Some(BrokerKind::Foreign),
"bumn" => Some(BrokerKind::StateOwned),
_ => None,
}
}
/// The `MM/DD/YYYY` value of the input with this id.
fn input_date(html: &str, id: &str) -> Option<NaiveDate> {
let at = html.find(&format!("id=\"{id}\""))?;
let tag_start = html[..at].rfind('<')?;
let tag_end = at + html[at..].find('>')?;
let tag = &html[tag_start..tag_end];
let value = &tag[tag.find("value=\"")? + 7..];
NaiveDate::parse_from_str(&value[..value.find('"')?], "%m/%d/%Y").ok()
}
/// Text after `label` up to the next tag, e.g. `6.6 T` for `T. Val :`.
fn footer<'a>(html: &'a str, label: &str) -> Option<&'a str> {
let rest = &html[html.find(label)? + label.len()..];
Some(rest[..rest.find('<')?].trim())
}
pub fn parse_broker_summary(html: &str) -> Result<IpotPage, IdxError> {
if html.to_ascii_lowercase().contains("invalid stock code") {
return Ok(IpotPage::Invalid);
}
let (Some(body_start), Some(body_end)) = (html.find("<tbody>"), html.find("</tbody>")) else {
return Err(IdxError::ParseError(
"ipot broker summary: no table in the page".to_string(),
));
};
let mut raw = BrokerSummaryRaw {
from: input_date(html, "startDateBrokerSummary"),
to: input_date(html, "endDateBrokerSummary"),
..BrokerSummaryRaw::default()
};
let mut rounded = false;
let mut figure = |text: &str| {
parse_figure(text).map(|(value, abbreviated)| {
rounded |= abbreviated;
value
})
};
for (index, row) in html[body_start..body_end].split("</tr>").enumerate() {
let row = cells(row);
if row.len() < 9 {
continue;
}
let rank = row[4].text.parse::<u32>().unwrap_or(index as u32 + 1);
for (side, at) in [(&mut raw.buyers, 0), (&mut raw.sellers, 5)] {
let code = row[at].text.to_ascii_uppercase();
if code.is_empty() {
continue;
}
if code.len() != 2 || !code.bytes().all(|b| b.is_ascii_alphanumeric()) {
return Err(IdxError::ParseError(format!(
"ipot broker summary: unexpected broker code {code:?}"
)));
}
side.push(BrokerFlow {
rank,
code,
name: None,
kind: broker_kind(row[at].class.as_deref()),
lots: figure(&row[at + 1].text),
value: figure(&row[at + 2].text),
average_price: figure(&row[at + 3].text),
});
}
}
let mut total = |label: &str| footer(html, label).and_then(&mut figure);
raw.total_value = total("T. Val :");
raw.foreign_net_value = total("F. NVal :");
raw.total_lots = total("T.Lot :");
// The page prints `Avg : 0` when nothing traded; there is no average then.
raw.average_price = total("Avg :").filter(|_| raw.total_lots.is_some_and(|lots| lots > 0));
raw.rounded = rounded;
Ok(IpotPage::Summary(raw))
}
#[cfg(test)]
mod tests {
use super::*;
const DAY: &str = include_str!("../../../tests/fixtures/ipot_brokersummary_bbca_20261016.html");
const RANGE: &str =
include_str!("../../../tests/fixtures/ipot_brokersummary_bbca_20261012_20261016.html");
const NO_TRADES: &str =
include_str!("../../../tests/fixtures/ipot_brokersummary_bbca_20261017.html");
const INVALID: &str =
include_str!("../../../tests/fixtures/ipot_brokersummary_zzzz_invalid.html");
fn summary(html: &str) -> BrokerSummaryRaw {
match parse_broker_summary(html).unwrap() {
IpotPage::Summary(raw) => raw,
IpotPage::Invalid => panic!("expected a summary"),
}
}
fn date(y: i32, m: u32, d: u32) -> NaiveDate {
NaiveDate::from_ymd_opt(y, m, d).unwrap()
}
#[test]
fn test_parse_figure_scales_in_decimal() {
assert_eq!(parse_figure("1.6 M"), Some((1_600_000, true)));
assert_eq!(parse_figure("841.2 B"), Some((841_200_000_000, true)));
assert_eq!(parse_figure("-2.3 T"), Some((-2_300_000_000_000, true)));
assert_eq!(parse_figure("12 K"), Some((12_000, true)));
assert_eq!(parse_figure("687,067 "), Some((687_067, false)));
assert_eq!(parse_figure("5,191.5"), Some((5_192, false)));
assert_eq!(parse_figure("0"), Some((0, false)));
for bad in ["", " ", "n/a", "1.2.3 M", "M", "-", "1.6789 K"] {
assert_eq!(parse_figure(bad), None, "{bad:?}");
}
}
#[test]
fn test_parse_broker_summary_one_day() {
let raw = summary(DAY);
assert_eq!(
(raw.from, raw.to),
(Some(date(2026, 10, 16)), Some(date(2026, 10, 16)))
);
assert_eq!((raw.buyers.len(), raw.sellers.len()), (10, 10));
let top = &raw.buyers[0];
assert_eq!(
(top.rank, top.code.as_str(), top.kind),
(1, "MG", Some(BrokerKind::Local))
);
assert_eq!(
(top.lots, top.value, top.average_price),
(Some(1_600_000), Some(841_200_000_000), Some(5_191))
);
let seller = &raw.sellers[0];
assert_eq!(
(seller.code.as_str(), seller.kind, seller.value),
("CC", Some(BrokerKind::StateOwned), Some(2_200_000_000_000))
);
assert_eq!(raw.sellers[1].kind, Some(BrokerKind::Foreign));
let last = &raw.buyers[9];
assert_eq!((last.rank, last.code.as_str()), (10, "AK"));
assert_eq!(last.lots, Some(398_474), "small figures are exact");
assert_eq!(raw.total_value, Some(6_600_000_000_000));
assert_eq!(raw.foreign_net_value, Some(-2_300_000_000_000));
assert_eq!(raw.total_lots, Some(12_600_000));
assert_eq!(raw.average_price, Some(5_228));
assert!(raw.rounded);
}
#[test]
fn test_parse_broker_summary_range_echoes_its_dates() {
let raw = summary(RANGE);
assert_eq!(
(raw.from, raw.to),
(Some(date(2026, 10, 12)), Some(date(2026, 10, 16)))
);
assert_eq!(raw.buyers.len(), 10);
assert!(
raw.total_value
.is_some_and(|value| value > 6_600_000_000_000)
);
}
#[test]
fn test_parse_broker_summary_without_trades_is_empty() {
let raw = summary(NO_TRADES);
assert!(raw.buyers.is_empty() && raw.sellers.is_empty());
assert_eq!((raw.total_value, raw.total_lots), (Some(0), Some(0)));
assert_eq!(raw.average_price, None, "Avg : 0 means no average");
assert!(!raw.rounded);
assert_eq!(raw.to, Some(date(2026, 10, 17)));
}
#[test]
fn test_parse_broker_summary_rejects_unknown_codes_and_markup() {
assert_eq!(parse_broker_summary(INVALID).unwrap(), IpotPage::Invalid);
assert!(matches!(
parse_broker_summary("<html>maintenance</html>"),
Err(IdxError::ParseError(_))
));
let odd = DAY.replacen(">MG<", ">M G<", 1);
assert!(matches!(
parse_broker_summary(&odd),
Err(IdxError::ParseError(_))
));
}
}