fix(core): close the round-12 audit of the IPOT parser

From Codex's exhaustive audit of every allowed element, attribute and
returned value:
- class tokens split on ASCII whitespace as HTML does, so a no-break
  space stays in the token (and fails the class allowlist);
- a date input's line breaks are dropped before parsing, as a text input
  drops them;
- ancestor placement compares class token sets, not exact strings;
- the echoed range needs both date inputs, start then end in document
  order, each with IPOT's id and name; otherwise no dates;
- a side with a blank broker code may show no figures.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V3syn2fN2LNo5pvTSqc6eh
This commit is contained in:
hermes 2026-10-18 14:15:51 +00:00
commit bf77073877

View file

@ -93,6 +93,14 @@ fn clean(text: &str) -> String {
text.replace('\u{a0}', " ").trim().to_string() text.replace('\u{a0}', " ").trim().to_string()
} }
/// Class tokens as HTML splits them: on ASCII whitespace only, so a
/// no-break space stays part of a token.
fn class_tokens(value: &str) -> impl Iterator<Item = &str> {
value
.split(['\t', '\n', '\x0c', '\r', ' '])
.filter(|token| !token.is_empty())
}
fn selector(css: &str) -> Selector { fn selector(css: &str) -> Selector {
Selector::parse(css).expect("a valid selector") Selector::parse(css).expect("a valid selector")
} }
@ -150,7 +158,7 @@ fn cell(td: ElementRef) -> Option<(String, Option<String>)> {
Node::Element(element) if element.name() == "span" && class.is_none() => { Node::Element(element) if element.name() == "span" && class.is_none() => {
let span = ElementRef::wrap(child)?; let span = ElementRef::wrap(child)?;
// One class token, as the browser splits it. // One class token, as the browser splits it.
let mut tokens = span.attr("class")?.split_whitespace(); let mut tokens = class_tokens(span.attr("class")?);
let token = tokens let token = tokens
.next() .next()
.filter(|token| token.starts_with("text-") && tokens.next().is_none())?; .filter(|token| token.starts_with("text-") && tokens.next().is_none())?;
@ -257,10 +265,7 @@ fn known_markup(page: &Html) -> bool {
.iter() .iter()
.filter(|(on, pinned, _)| *on == name && *pinned == attr) .filter(|(on, pinned, _)| *on == name && *pinned == attr)
.all(|(_, _, only)| *only == value) .all(|(_, _, only)| *only == value)
&& (attr != "class" && (attr != "class" || class_tokens(value).all(|class| CLASSES.contains(&class)))
|| value
.split_whitespace()
.all(|class| CLASSES.contains(&class)))
}) })
}) })
} }
@ -271,7 +276,12 @@ fn placed(element: ElementRef, path: &[(&str, &str)]) -> bool {
let ancestors: Vec<ElementRef> = element.ancestors().filter_map(ElementRef::wrap).collect(); let ancestors: Vec<ElementRef> = element.ancestors().filter_map(ElementRef::wrap).collect();
ancestors.len() == path.len() + 1 ancestors.len() == path.len() + 1
&& ancestors.iter().zip(path).all(|(ancestor, (name, class))| { && ancestors.iter().zip(path).all(|(ancestor, (name, class))| {
ancestor.value().name() == *name && ancestor.attr("class") == Some(*class) let mut found: Vec<&str> =
class_tokens(ancestor.attr("class").unwrap_or_default()).collect();
let mut wanted: Vec<&str> = class_tokens(class).collect();
found.sort_unstable();
wanted.sort_unstable();
ancestor.value().name() == *name && found == wanted
}) })
} }
@ -283,12 +293,43 @@ const INPUT_PATH: [(&str, &str); 3] = [
]; ];
/// The `MM/DD/YYYY` value of the one input with this id, where IPOT puts it. /// The `MM/DD/YYYY` value of the one input with this id, where IPOT puts it.
fn input_date(page: &Html, id: &str) -> Option<NaiveDate> { fn input_date(input: ElementRef) -> Option<NaiveDate> {
let input = only(page.root_element(), &format!("input#{id}")) // A text input drops line breaks from its value before showing it.
.filter(|input| placed(*input, &INPUT_PATH))?; let value = input.attr("value")?.replace(['\r', '\n'], "");
NaiveDate::parse_from_str(input.attr("value")?.trim(), "%m/%d/%Y").ok() NaiveDate::parse_from_str(value.trim(), "%m/%d/%Y").ok()
} }
/// The range the page echoes: IPOT's start and end date inputs, in that
/// order, each with its id and name where IPOT puts it. Either missing,
/// repeated or out of place gives no dates.
fn echoed_dates(page: &Html) -> (Option<NaiveDate>, Option<NaiveDate>) {
let inputs: Vec<ElementRef> = page
.root_element()
.descendent_elements()
.filter(|element| {
element.value().name() == "input"
&& DATE_INPUTS
.iter()
.any(|(id, _)| element.attr("id") == Some(*id))
})
.collect();
let in_place = inputs.len() == DATE_INPUTS.len()
&& inputs.iter().zip(DATE_INPUTS).all(|(input, (id, name))| {
input.attr("id") == Some(id)
&& input.attr("name") == Some(name)
&& placed(*input, &INPUT_PATH)
});
match inputs[..] {
[start, end] if in_place => (input_date(start), input_date(end)),
_ => (None, None),
}
}
const DATE_INPUTS: [(&str, &str); 2] = [
("startDateBrokerSummary", "startDate"),
("endDateBrokerSummary", "endDate"),
];
/// The only child element of `parent`, when it is a `name`. /// The only child element of `parent`, when it is a `name`.
fn sole_child<'a>(parent: ElementRef<'a>, name: &str) -> Option<ElementRef<'a>> { fn sole_child<'a>(parent: ElementRef<'a>, name: &str) -> Option<ElementRef<'a>> {
let mut children = parent.child_elements(); let mut children = parent.child_elements();
@ -373,9 +414,10 @@ pub fn parse_broker_summary(html: &str) -> Result<IpotPage, IdxError> {
if names != ["thead", "tbody", "tfoot"] { if names != ["thead", "tbody", "tfoot"] {
return Err(parse_error("unexpected table parts")); return Err(parse_error("unexpected table parts"));
} }
let (from, to) = echoed_dates(&page);
let mut raw = BrokerSummaryRaw { let mut raw = BrokerSummaryRaw {
from: input_date(&page, "startDateBrokerSummary"), from,
to: input_date(&page, "endDateBrokerSummary"), to,
..BrokerSummaryRaw::default() ..BrokerSummaryRaw::default()
}; };
let mut rounded = false; let mut rounded = false;
@ -409,6 +451,15 @@ pub fn parse_broker_summary(html: &str) -> Result<IpotPage, IdxError> {
] { ] {
let code = cells[at].0.to_ascii_uppercase(); let code = cells[at].0.to_ascii_uppercase();
if code.is_empty() { if code.is_empty() {
// A blank side shows no figures either.
if cells[at + 1..at + 4]
.iter()
.any(|(text, _)| !text.is_empty())
{
return Err(parse_error(&format!(
"figures without a broker in row {rank}"
)));
}
*ended = true; *ended = true;
continue; continue;
} }
@ -881,6 +932,49 @@ mod tests {
Some(original.clone()), Some(original.clone()),
"class spacing shows the same" "class spacing shows the same"
); );
// The round-12 audit: class tokens split on ASCII space only; a
// date's line breaks dropped as an input drops them; ancestor
// classes as a set; dates by id, name and order; no figures beside
// a blank code.
let nbsp_class = DAY.replacen(
"class=\"text-foreign\">KZ",
"class=\"text-foreign&nbsp;\">KZ",
1,
);
assert!(
read(&nbsp_class).is_none(),
"a no-break space is part of a class token"
);
let broken_date = DAY.replacen("value=\"10/16/2026\"", "value=\"1&#10;0/16/20&#13;26\"", 1);
assert_eq!(
read(&broken_date),
Some(original.clone()),
"line breaks in a date"
);
let reordered = DAY.replacen(
"class=\"col col-xs-5 col-sm-3\"",
"class=\" col-sm-3 col-xs-5\tcol \"",
1,
);
assert_eq!(
read(&reordered),
Some(original.clone()),
"ancestor classes in another order"
);
let swapped = RANGE
.replacen("startDateBrokerSummary", "TMP", 1)
.replacen("endDateBrokerSummary", "startDateBrokerSummary", 1)
.replacen("TMP", "endDateBrokerSummary", 1);
let raw = read(&swapped).unwrap();
assert_eq!((raw.from, raw.to), (None, None), "date inputs swapped");
// Rank 10's buyer AK, in the last row, so no broker follows it.
let last_ak = DAY.rfind(">AK<").unwrap();
let blank_code_with_figures =
DAY[..last_ak].to_string() + "><" + &DAY[last_ak + ">AK<".len()..];
assert!(
read(&blank_code_with_figures).is_none(),
"figures beside a blank broker code"
);
let script_text = DAY.replacen("></script>", ">/* invalid stock code */</script>", 1); let script_text = DAY.replacen("></script>", ">/* invalid stock code */</script>", 1);
assert_eq!( assert_eq!(
read(&script_text), read(&script_text),