0xrsydn/idx-cli
1
mirror of https://github.com/0xrsydn/idx-cli.git synced 2026-11-07 09:45:36 +00:00
Code Issues Projects Releases 1 Packages Wiki Activity

fix(core): parse IPOT pages with html5ever

Browse source 
Six rounds of the 0.4.0 review found crafted markup that the hand-written
reader read differently from HTML (cells in comments or attributes, stray
or spaced tags, a decoy date attribute or total, foreign content). Parse
the page with scraper (html5ever) instead: dates are the inputs' values,
rows are the single tbody's tr children with nine td each, totals are the
tfoot spans, all as a browser builds them. Rows that don't line up still
reject the page. scraper's CSS crates (selectors, cssparser) are MPL-2.0.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V3syn2fN2LNo5pvTSqc6eh
This commit is contained in:
hermes 2026-10-18 12:54:34 +00:00
commit 79b15d51b9
4 changed files with 517 additions and 321 deletions
Download patch file Download diff file Expand all files Collapse all files

266
Cargo.lock generated
View file

@ -380,6 +380,29 @@ dependencies = [
"typenum",
]
[[package]]
name = "cssparser"
version = "0.37.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c9cdaae01d5ed7882b04d795e7f752f46ff52d2fa3b50a20d28c464510bba98"
dependencies = [
"cssparser-macros",
"dtoa-short",
"itoa",
"phf",
"smallvec",
]
[[package]]
name = "cssparser-macros"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d045de693cb712d0b22c6a64be5b953f67b3ce00ab5ad3dd5d8b441886ab8e1a"
dependencies = [
"quote",
"syn 3.0.9",
]
[[package]]
name = "csv"
version = "1.4.0"
@ -444,6 +467,27 @@ dependencies = [
"powerfmt",
]
[[package]]
name = "derive_more"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134"
dependencies = [
"derive_more-impl",
]
[[package]]
name = "derive_more-impl"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb"
dependencies = [
"proc-macro2",
"quote",
"rustc_version",
"syn 2.0.117",
]
[[package]]
name = "difflib"
version = "0.4.0"
@ -501,12 +545,33 @@ dependencies = [
"litrs",
]
[[package]]
name = "dtoa"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590"
[[package]]
name = "dtoa-short"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"
dependencies = [
"dtoa",
]
[[package]]
name = "dyn-clone"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]]
name = "ego-tree"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b04dc5a38e4f151a79d9f2451ae6037fb6eaf5cba34771f44781f80e508498e3"
[[package]]
name = "encode_unicode"
version = "1.0.0"
@ -732,6 +797,16 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "html5ever"
version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46a1761807faccc9a19e86944bbf40610014066306f96edcdedc2fb714bcb7b8"
dependencies = [
"log",
"markup5ever",
]
[[package]]
name = "http"
version = "1.4.0"
@ -910,6 +985,7 @@ dependencies = [
"fastrand",
"quick-xml",
"rusqlite",
"scraper",
"serde",
"serde_json",
"sha2",
@ -1025,6 +1101,17 @@ version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "markup5ever"
version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7122d987ec5f704ee56f6e5b41a7d93722e9aae27ae07cafa4036c4d3f9757de"
dependencies = [
"log",
"tendril",
"web_atoms",
]
[[package]]
name = "memchr"
version = "2.8.0"
@ -1047,6 +1134,12 @@ dependencies = [
"simd-adler32",
]
[[package]]
name = "new_debug_unreachable"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
[[package]]
name = "nix"
version = "0.31.3"
@ -1149,6 +1242,59 @@ version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "phf"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
dependencies = [
"phf_macros",
"phf_shared",
"serde",
]
[[package]]
name = "phf_codegen"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
dependencies = [
"phf_generator",
"phf_shared",
]
[[package]]
name = "phf_generator"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
dependencies = [
"fastrand",
"phf_shared",
]
[[package]]
name = "phf_macros"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef"
dependencies = [
"phf_generator",
"phf_shared",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "phf_shared"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266"
dependencies = [
"siphasher",
]
[[package]]
name = "pin-project-lite"
version = "0.2.17"
@ -1176,6 +1322,12 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "precomputed-hash"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
[[package]]
name = "predicates"
version = "3.1.4"
@ -1377,6 +1529,21 @@ dependencies = [
"uuid",
]
[[package]]
name = "rustc-hash"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
[[package]]
name = "rustc_version"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
dependencies = [
"semver",
]
[[package]]
name = "rustix"
version = "1.1.4"
@ -1469,6 +1636,45 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "scraper"
version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bdd0be4d296f048bfb06dd01bbc80ef789ddd2e55583e8d2e6b804942abfabc2"
dependencies = [
"cssparser",
"ego-tree",
"html5ever",
"precomputed-hash",
"selectors",
"tendril",
]
[[package]]
name = "selectors"
version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8adfa1c298912827b8a28b223b3b874357397ae706e6190acd9bf28cee99114d"
dependencies = [
"bitflags",
"cssparser",
"derive_more",
"log",
"new_debug_unreachable",
"phf",
"phf_codegen",
"precomputed-hash",
"rustc-hash",
"servo_arc",
"smallvec",
]
[[package]]
name = "semver"
version = "1.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
[[package]]
name = "serde"
version = "1.0.228"
@ -1532,6 +1738,15 @@ dependencies = [
"serde",
]
[[package]]
name = "servo_arc"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930"
dependencies = [
"stable_deref_trait",
]
[[package]]
name = "sha2"
version = "0.10.9"
@ -1555,6 +1770,12 @@ version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
[[package]]
name = "siphasher"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256"
[[package]]
name = "slab"
version = "0.4.12"
@ -1573,6 +1794,30 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
[[package]]
name = "string_cache"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901"
dependencies = [
"new_debug_unreachable",
"parking_lot",
"phf_shared",
"precomputed-hash",
]
[[package]]
name = "string_cache_codegen"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69"
dependencies = [
"phf_generator",
"phf_shared",
"proc-macro2",
"quote",
]
[[package]]
name = "strsim"
version = "0.11.1"
@ -1618,6 +1863,15 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "tendril"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fed54709c5b3a53d09bb1c113ea4f5ceafd1e772ddcb0030a82e1d56c087b08"
dependencies = [
"new_debug_unreachable",
]
[[package]]
name = "termtree"
version = "0.5.1"
@ -2005,6 +2259,18 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "web_atoms"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba8b815c1b593dc0baf78dd0f4fc8fdb2de53198fb1163738093e9a311c33fb3"
dependencies = [
"phf",
"phf_codegen",
"string_cache",
"string_cache_codegen",
]
[[package]]
name = "webpki-roots"
version = "1.0.6"

1
crates/idx-core/Cargo.toml
View file

@ -34,3 +34,4 @@ rusqlite = { workspace = true, optional = true }
quick-xml = { version = "0.37", optional = true }
sha2 = { workspace = true, optional = true }
zip = { workspace = true, optional = true }
scraper = { version = "0.27", default-features = false }

558
crates/idx-core/src/api/ipot/parse.rs
View file

@ -5,9 +5,11 @@
//! seller code, lots, value, average. Each code sits in a
//! `<span class="text-local|text-foreign|text-bumn">`; the footer holds
//! `T. Val`, `F. NVal`, `T.Lot` and `Avg`, and the date inputs echo the
//! range the source applied.
//! range the source applied. The page is parsed with html5ever, so markup
//! reads as a browser builds it.
use chrono::NaiveDate;
use scraper::{ElementRef, Html, Selector};
use crate::broker::types::{BrokerFlow, BrokerKind};
use crate::error::IdxError;
@ -77,51 +79,6 @@ pub fn parse_figure(text: &str) -> Option<(i64, bool)> {
Some((if negative { -value } else { value }, exponent > 0))
}
struct Cell {
class: Option<String>,
text: String,
}
fn strip_tags(html: &str) -> String {
let mut out = String::with_capacity(html.len());
let mut in_tag = false;
for ch in html.chars() {
match ch {
'<' => in_tag = true,
'>' => in_tag = false,
_ if !in_tag => out.push(ch),
_ => {}
}
}
out.replace("&nbsp;", " ").trim().to_string()
}
fn cells(row: &str) -> Vec<Cell> {
let mut out = Vec::new();
let mut rest = row;
while let Some(start) = rest.find("<td") {
let after = &rest[start..];
let Some(open_end) = after.find('>') else {
break;
};
let inner_start = open_end + 1;
let Some(close) = after[inner_start..].find("</td>") else {
break;
};
let inner = &after[inner_start..inner_start + close];
let class = inner.find("class=\"").and_then(|at| {
let value = &inner[at + 7..];
value.find('"').map(|end| value[..end].to_string())
});
out.push(Cell {
class,
text: strip_tags(inner),
});
rest = &after[inner_start + close + 5..];
}
out
}
fn broker_kind(class: Option<&str>) -> Option<BrokerKind> {
match class?.strip_prefix("text-")? {
"local" => Some(BrokerKind::Local),
@ -131,174 +88,59 @@ fn broker_kind(class: Option<&str>) -> Option<BrokerKind> {
}
}
/// Whether `body` (lowercased) is exactly the markup IPOT serves, which
/// reads the same in any HTML parser:
///
/// ```text
/// body := ws (row ws)*
/// row := "<tr>" ws (cell ws){9} "</tr>"
/// cell := <td attrs> ws (<span attrs> text "</span>" | text) ws "</td>"
/// attrs := (" " name '="' value '"')* value without " < >
/// text := no "<", and no "&" except "&nbsp;"
/// ```
///
/// Anything else (comments, other tags, unclosed or stray tags, odd
/// spacing or quoting inside tags) rejects the page rather than risk
/// reading cells differently from how the page shows them.
fn canonical_rows(body: &str) -> bool {
let mut rest = body.trim_start();
while !rest.is_empty() {
let Some(after) = canonical_row(rest) else {
return false;
};
rest = after.trim_start();
}
true
/// The text an element shows, `&nbsp;` as a space, trimmed.
fn shown_text(element: ElementRef) -> String {
element
.text()
.collect::<String>()
.replace('\u{a0}', " ")
.trim()
.to_string()
}
fn canonical_row(row: &str) -> Option<&str> {
let mut rest = row.strip_prefix("<tr>")?;
for _ in 0..9 {
rest = canonical_cell(rest.trim_start())?;
}
rest.trim_start().strip_prefix("</tr>")
fn selector(css: &str) -> Selector {
Selector::parse(css).expect("a valid selector")
}
fn canonical_cell(cell: &str) -> Option<&str> {
let rest = open_tag(cell, "td")?.trim_start();
let rest = if rest.starts_with("<span") {
plain_text(open_tag(rest, "span")?)?
.strip_prefix("</span>")?
.trim_start()
} else {
plain_text(rest)?
};
rest.strip_prefix("</td>")
}
/// The rest after `<name attrs>`.
fn open_tag<'a>(markup: &'a str, name: &str) -> Option<&'a str> {
let mut rest = markup.strip_prefix('<')?.strip_prefix(name)?;
loop {
if let Some(after) = rest.strip_prefix('>') {
return Some(after);
}
rest = rest.strip_prefix(' ')?;
let name_len = rest
.bytes()
.take_while(|b| b.is_ascii_lowercase() || *b == b'-')
.count();
if name_len == 0 {
return None;
}
rest = rest[name_len..].strip_prefix("=\"")?;
let end = rest.find(['"', '<', '>'])?;
rest = rest[end..].strip_prefix('"')?;
/// The one element matching `css` within `scope`; `None` for none or more.
fn only<'a>(scope: ElementRef<'a>, css: &str) -> Option<ElementRef<'a>> {
let selector = selector(css);
let mut found = scope.select(&selector);
match (found.next(), found.next()) {
(Some(one), None) => Some(one),
_ => None,
}
}
/// The rest after text without markup or character references.
fn plain_text(text: &str) -> Option<&str> {
let end = text.find('<').unwrap_or(text.len());
let mut refs = text[..end].match_indices('&');
(!text[..end].contains('>') && refs.all(|(at, _)| text[at..].starts_with("&nbsp;")))
.then_some(&text[end..])
fn parse_error(what: &str) -> IdxError {
IdxError::ParseError(format!("ipot broker summary: {what}"))
}
/// Elements whose content is not markup; IPOT's page ends with one empty
/// script, after everything read here.
const RAW_TEXT: [&str; 11] = [
"<script",
"<style",
"<textarea",
"<template",
"<noscript",
"<title",
"<iframe",
"<xmp",
"<plaintext",
"<noembed",
"<noframes",
];
/// The page's markup as a browser reads it, up to the first raw-text
/// element: comments removed. A comment of an unusual form (`<!-->`,
/// `<!--->`, `--!>`), an unclosed one, or other `<!` / `<?` markup rejects
/// the page.
fn visible_markup(html: &str) -> Option<String> {
let mut out = String::with_capacity(html.len());
let mut rest = html;
while let Some(at) = rest.find("<!--") {
out.push_str(&rest[..at]);
let comment = &rest[at + "<!--".len()..];
let end = comment.find("-->")?;
let text = &comment[..end];
if text.starts_with('>') || text.starts_with("->") || text.contains("--!>") {
return None;
}
rest = &comment[end + "-->".len()..];
}
out.push_str(rest);
let lower = out.to_ascii_lowercase();
let cut = RAW_TEXT
.iter()
.filter_map(|tag| lower.find(tag))
.min()
.unwrap_or(lower.len());
let read = &lower[..cut];
let other = read
.match_indices("<!")
.any(|(at, _)| !read[at..].starts_with("<!doctype"));
if other || read.contains("<?") {
return None;
}
out.truncate(cut);
Some(out)
}
/// The `MM/DD/YYYY` value of the input with this id.
fn input_date(html: &str, id: &str) -> Option<NaiveDate> {
let at = html.find(&format!("id=\"{id}\""))?;
let tag_start = html[..at].rfind('<')?;
let tag_end = at + html[at..].find('>')?;
let tag = &html[tag_start..tag_end];
let value = &tag[tag.find("value=\"")? + 7..];
NaiveDate::parse_from_str(&value[..value.find('"')?], "%m/%d/%Y").ok()
}
/// Text after `label` up to the next tag, e.g. `6.6 T` for `T. Val :`.
fn footer<'a>(html: &'a str, label: &str) -> Option<&'a str> {
let rest = &html[html.find(label)? + label.len()..];
Some(rest[..rest.find('<')?].trim())
/// The `MM/DD/YYYY` value of the input with this id, the first one in the
/// page as `getElementById` finds it.
fn input_date(page: &Html, id: &str) -> Option<NaiveDate> {
let value = page
.select(&selector(&format!("input#{id}")))
.next()?
.attr("value")?;
NaiveDate::parse_from_str(value.trim(), "%m/%d/%Y").ok()
}
/// The page is read as a browser builds it from the fragment (html5ever),
/// so markup it would not show as table cells or footer text, such as
/// comments, attributes or other elements' content, is never read as such.
pub fn parse_broker_summary(html: &str) -> Result<IpotPage, IdxError> {
let page = visible_markup(html).ok_or_else(|| {
IdxError::ParseError("ipot broker summary: unexpected markup in the page".to_string())
})?;
let html = page.as_str();
if html.to_ascii_lowercase().contains("invalid stock code") {
let page = Html::parse_fragment(html);
let shown = shown_text(page.root_element()).to_ascii_lowercase();
if shown.contains("invalid stock code") {
return Ok(IpotPage::Invalid);
}
// Tags are matched case-insensitively on a lowercased copy; ASCII
// lowercasing keeps byte offsets, so slices of it index `html` too.
let lower = html.to_ascii_lowercase();
let body = lower.find("<tbody>").and_then(|start| {
lower[start..]
.find("</tbody>")
.map(|len| (start + "<tbody>".len(), start + len))
});
let single = lower.matches("<tbody").count() == 1 && lower.matches("</tbody").count() == 1;
let Some((body_start, body_end)) =
body.filter(|(start, end)| single && canonical_rows(&lower[*start..*end]))
else {
return Err(IdxError::ParseError(
"ipot broker summary: no single plain table in the page".to_string(),
));
};
let table = only(page.root_element(), "table")
.ok_or_else(|| parse_error("no single table in the page"))?;
let body = only(table, "tbody").ok_or_else(|| parse_error("no single table body"))?;
let mut raw = BrokerSummaryRaw {
from: input_date(&html[..body_start], "startDateBrokerSummary"),
to: input_date(&html[..body_start], "endDateBrokerSummary"),
from: input_date(&page, "startDateBrokerSummary"),
to: input_date(&page, "endDateBrokerSummary"),
..BrokerSummaryRaw::default()
};
let mut rounded = false;
@ -308,61 +150,61 @@ pub fn parse_broker_summary(html: &str) -> Result<IpotPage, IdxError> {
value
})
};
// The body is whole rows (see `canonical_rows`); ranks must run in
// order, and a broker after a side's blank rows rejects the page.
// Rows must be whole and ranked in order: a skipped or merged row would
// shift brokers between ranks. A broker after a side's blank rows
// rejects the page too.
let span = selector("span");
let (mut buyers_ended, mut sellers_ended) = (false, false);
let mut rank = 0;
let mut at = body_start;
while at < body_end {
let rest = &lower[at..body_end];
if rest.trim_start().is_empty() {
break;
}
let end = rest
.find("</tr>")
.map_or(rest.len(), |end| end + "</tr>".len());
let original = &html[at..at + end];
at += end;
rank += 1;
let row = cells(original);
if row.len() != 9 || row[4].text.parse::<u32>().ok() != Some(rank) {
return Err(IdxError::ParseError(format!(
"ipot broker summary: unexpected row {rank} ({} cells, rank {:?})",
row.len(),
row.get(4).map(|cell| cell.text.as_str())
)));
for (index, row) in body.child_elements().enumerate() {
let rank = index as u32 + 1;
let cells: Vec<ElementRef> = row.child_elements().collect();
let whole = row.value().name() == "tr"
&& cells.len() == 9
&& cells.iter().all(|cell| cell.value().name() == "td");
if !whole || shown_text(cells[4]).parse::<u32>().ok() != Some(rank) {
return Err(parse_error(&format!("unexpected row {rank}")));
}
for (side, at, ended) in [
(&mut raw.buyers, 0, &mut buyers_ended),
(&mut raw.sellers, 5, &mut sellers_ended),
] {
let code = row[at].text.to_ascii_uppercase();
let code = shown_text(cells[at]).to_ascii_uppercase();
if code.is_empty() {
*ended = true;
continue;
}
if *ended {
return Err(IdxError::ParseError(format!(
"ipot broker summary: broker {code:?} after a blank row"
)));
return Err(parse_error(&format!("broker {code:?} after a blank row")));
}
if code.len() != 2 || !code.bytes().all(|b| b.is_ascii_alphanumeric()) {
return Err(IdxError::ParseError(format!(
"ipot broker summary: unexpected broker code {code:?}"
)));
return Err(parse_error(&format!("unexpected broker code {code:?}")));
}
let class = cells[at]
.select(&span)
.next()
.and_then(|span| span.attr("class"));
side.push(BrokerFlow {
rank,
code,
name: None,
kind: broker_kind(row[at].class.as_deref()),
lots: figure(&row[at + 1].text),
value: figure(&row[at + 2].text),
average_price: figure(&row[at + 3].text),
kind: broker_kind(class),
lots: figure(&shown_text(cells[at + 1])),
value: figure(&shown_text(cells[at + 2])),
average_price: figure(&shown_text(cells[at + 3])),
});
}
}
let mut total = |label: &str| footer(&html[body_end..], label).and_then(&mut figure);
// The footer's figures, e.g. `T. Val : 6.6 T`, each in its own span.
let footer: Vec<String> = table
.select(&selector("tfoot span"))
.map(shown_text)
.collect();
let mut total = |label: &str| {
footer
.iter()
.find_map(|text| text.strip_prefix(label))
.and_then(&mut figure)
};
raw.total_value = total("T. Val :");
raw.foreign_net_value = total("F. NVal :");
raw.total_lots = total("T.Lot :");
@ -482,102 +324,184 @@ mod tests {
));
}
#[test]
fn test_parse_broker_summary_rejects_rows_that_would_shift_ranks() {
let first_cell = "<td align=\"center\"><span class=\"text-local\">MG</span></td>";
let short_row = DAY.replacen(first_cell, "", 1);
let skipped_rank = DAY.replacen(
"<td align=\"center\">1</td>",
"<td align=\"center\">2</td>",
1,
);
let gap = DAY.replacen(">XL<", "><", 1);
// Rank 1 without its cell ends parses to no cells at all.
/// The BBCA October 16 page with `edit` applied around its rows: `rows`
/// holds where each `</tr>` of the body starts.
fn mutated(edit: impl Fn(&str, &[usize]) -> String) -> String {
let body = DAY.find("<tbody>").unwrap();
let first_row_end = body + DAY[body..].find("</tr>").unwrap();
let unclosed = DAY[..body].to_string()
+ &DAY[body..first_row_end].replace("</td>", "")
+ &DAY[first_row_end..];
// Rank 9 left open and rank 10 without cell ends: rank 10 would
// vanish into rank 9.
let rows: Vec<usize> = DAY[body..]
.match_indices("</tr>")
.map(|(at, _)| body + at)
.collect();
let merged = DAY[..rows[8]].to_string()
+ &DAY[rows[8] + "</tr>".len()..rows[9]].replace("</td>", "")
+ &DAY[rows[9]..];
// An extra row in uppercase tags before rank 5.
let rank5 = DAY[..rows[4]].rfind("<tr>").unwrap();
let extra_row = DAY[rank5..rows[4] + "</tr>".len()]
.replace("<tr>", "<TR>")
.replace("<td", "<TD");
let uppercase = DAY[..rank5].to_string() + &extra_row + &DAY[rank5..];
// A stray tbody end, and rank 5's cells hidden in a comment or an
// attribute.
let early_end = DAY[..rows[3] + "</tr>".len()].to_string()
+ "</TbOdY>"
+ &DAY[rows[3] + "</tr>".len()..];
let rank5_cells = &DAY[rank5 + "<tr>".len()..rows[4]];
let hidden = |wrap: &dyn Fn(&str) -> String| {
DAY[..rank5].to_string() + &wrap(rank5_cells) + &DAY[rows[4] + "</tr>".len()..]
};
let commented = hidden(&|cells| format!("<tr><!--{cells}--></tr>"));
let in_attribute = hidden(&|cells| format!("<tr data-x='{cells}'></tr>"));
let after_quoted_end = hidden(&|cells| format!("<tr data-x='>{cells}'></tr>"));
// Closing tags with a space, a cell under another tag name, and a
// character reference a browser would decode.
edit(DAY, &rows)
}
/// Rank 5's row (`<tr>` to `</tr>`), and its cells alone.
fn rank5(page: &str, rows: &[usize]) -> (usize, usize, usize) {
let start = page[..rows[4]].rfind("<tr>").unwrap();
(start, start + "<tr>".len(), rows[4])
}
#[test]
fn test_parse_broker_summary_rejects_rows_that_do_not_line_up() {
let first_cell = "<td align=\"center\"><span class=\"text-local\">MG</span></td>";
let lots = "<span class=\"text-up\">828,163 </span>";
let spaced_row_end = DAY.replacen(lots, &format!("</tr >{lots}"), 1);
let spaced_cell_end = DAY.replacen(lots, &format!("</td >{lots}"), 1);
let other_tag = DAY.replacen(
let hide_rank5 = |wrap: fn(&str) -> String| {
mutated(|page, rows| {
let (start, cells, end) = rank5(page, rows);
page[..start].to_string() + &wrap(&page[cells..end]) + &page[end + 5..]
})
};
let pages = [
("a missing cell", DAY.replacen(first_cell, "", 1)),
(
"a rank out of order",
DAY.replacen(
"<td align=\"center\">1</td>",
"<td align=\"center\">2</td>",
1,
),
),
("a broker after a blank row", DAY.replacen(">XL<", "><", 1)),
(
"a duplicate rank-5 row",
mutated(|page, rows| {
let (start, _, end) = rank5(page, rows);
page[..start].to_string()
+ &page[start..end + 5].to_uppercase()
+ &page[start..]
}),
),
(
"a table body ended early",
mutated(|page, rows| {
page[..rows[3] + 5].to_string() + "</TbOdY>" + &page[rows[3] + 5..]
}),
),
(
"cells in a comment",
hide_rank5(|cells| format!("<tr><!--{cells}--></tr>")),
),
(
"cells in an attribute",
hide_rank5(|cells| format!("<tr data-x='{cells}'></tr>")),
),
(
"cells after a quoted '>'",
hide_rank5(|cells| format!("<tr data-x='>{cells}'></tr>")),
),
(
"a row ended mid-cell",
DAY.replacen(lots, &format!("</tr >{lots}"), 1),
),
(
"a cell under another tag",
DAY.replacen(
"<td align=\"center\">5</td>",
"<td-x align=\"center\">5</td>",
1,
);
let reference = DAY.replacen(">828,163 <", ">8&#50;8,163 <", 1);
// A '>' a browser shows as text, the table inside a comment, and an
// abrupt `<!-->` that a browser closes at once, so rows 9 and 10 stay
// visible.
let angle = DAY.replacen(">828,163 <", ">8>28,163 <", 1);
let tbody_start = DAY.find("<tbody>").unwrap();
let tbody_end = DAY.find("</tbody>").unwrap() + "</tbody>".len();
let commented_table = DAY[..tbody_start].to_string()
+ "<!--"
+ &DAY[tbody_start..tbody_end]
+ "-->"
+ &DAY[tbody_end..];
let row8_end = rows[7] + "</tr>".len();
let abrupt = DAY[..row8_end].to_string()
+ "<!-->"
+ &DAY[row8_end..rows[9] + "</tr>".len()]
+ "-->"
+ &DAY[rows[9] + "</tr>".len()..];
for page in [
short_row,
skipped_rank,
gap,
unclosed,
merged,
uppercase,
early_end,
commented,
in_attribute,
after_quoted_end,
spaced_row_end,
spaced_cell_end,
other_tag,
reference,
angle,
commented_table,
abrupt,
"</tbody><tbody>".to_string(),
] {
),
),
(
"the table in a comment",
DAY.replacen("<tbody>", "<!--<tbody>", 1)
.replacen("</tbody>", "</tbody>-->", 1),
),
(
"a table inside the table",
DAY.replacen("<tbody>", "<table></table><tbody>", 1),
),
(
"the body in svg",
DAY.replacen("<tbody>", "<svg><tbody>", 1).replacen(
"</tbody>",
"</tbody></svg>",
1,
),
),
("no table", "</tbody><tbody>".to_string()),
];
for (what, page) in pages {
assert!(
matches!(parse_broker_summary(&page), Err(IdxError::ParseError(_))),
"{}",
&page[..page.len().min(80)]
"{what}"
);
}
}
#[test]
fn test_parse_broker_summary_reads_what_a_browser_shows() {
let original = summary(DAY);
// Markup a browser repairs or decodes to the same table.
let same = [
(
"rank 1 without cell ends",
mutated(|page, rows| {
let start = page[..rows[0]].rfind("<tr>").unwrap();
page[..start].to_string()
+ &page[start..rows[0]].replace("</td>", "")
+ &page[rows[0]..]
}),
),
(
"rank 9 left open, rank 10 without cell ends",
mutated(|page, rows| {
page[..rows[8]].to_string()
+ &page[rows[8] + 5..rows[9]].replace("</td>", "")
+ &page[rows[9]..]
}),
),
(
"a character reference",
DAY.replacen(">828,163 <", ">8&#50;8,163 <", 1),
),
(
"an abrupt comment",
mutated(|page, rows| {
page[..rows[7] + 5].to_string() + "<!-->" + &page[rows[7] + 5..]
}),
),
(
"a decoy attribute before the date",
DAY.replacen(
"id=\"startDateBrokerSummary\"",
"id=\"startDateBrokerSummary\" data-value=\"01/01/2001\"",
1,
),
),
(
"a decoy total outside the footer",
DAY.replacen(
"</tbody>",
"</tbody><div data-note=\"T. Val : 1 <x>\"></div>",
1,
),
),
// The HTML standard keeps a table inside a select since its 2025
// select parsing change; html5ever and current browsers agree.
(
"the table in a select",
DAY.replacen("<table", "<select><table", 1).replacen(
"</table>",
"</table></select>",
1,
),
),
(
"a script after the table",
DAY.replacen("</tbody>", "</tbody><script></script>", 1),
),
];
for (what, page) in same {
assert_eq!(summary(&page), original, "{what}");
}
// Markup a browser shows differently: the figure it shows, not ours.
let angle = summary(&DAY.replacen(">828,163 <", ">8>28,163 <", 1));
assert_eq!(angle.buyers[4].lots, None);
let lots = "<span class=\"text-up\">828,163 </span>";
let early_cell_end = summary(&DAY.replacen(lots, &format!("</td >{lots}"), 1));
assert_eq!(early_cell_end.buyers[4].lots, None);
let commented_date =
summary(&DAY.replacen("value=\"10/16/2026\"", "value=\"10/<!--x-->16/2026\"", 1));
assert_eq!(commented_date.from, None);
}
}

7
docs/ARCHITECTURE.md
View file

@ -92,7 +92,12 @@ WIB as the fallback.
takes `code`, `start`/`end` (`MM/DD/YYYY`), `fd` (`all|F|D`) and `board`
(`all|RG|TN|NG`). It needs no login and answers an HTML fragment: the top 10
buyers and top 10 sellers ranked by lots (gross, so a broker can be on both
sides), plus range totals. `api/ipot/parse.rs` parses it purely.
sides), plus range totals. `api/ipot/parse.rs` parses it purely, with
html5ever (`scraper`), so it reads the cells, date inputs and footer a
browser would build: a hand-written reader kept disagreeing with HTML on
crafted markup in the 0.4.0 review. A table whose rows don't line up
(missing cells, ranks out of order, a broker after a blank row, a second
body) rejects the page, and the dates it echoes must match the request.
- Figures of a million and more arrive abbreviated (`1.6 M`); they are scaled
in decimal and the summary's `rounded` flag says so. `F. NVal` ignores the
investor and board filters, so `foreign_net_value` is null for a summary of