diff --git a/crates/idx-core/src/api/ipot/parse.rs b/crates/idx-core/src/api/ipot/parse.rs index 516d4ae..b14e673 100644 --- a/crates/idx-core/src/api/ipot/parse.rs +++ b/crates/idx-core/src/api/ipot/parse.rs @@ -131,23 +131,78 @@ fn broker_kind(class: Option<&str>) -> Option { } } -/// Whether `body` (lowercased) holds only `tr`, `td` and `span` tags, each -/// closed before the next `<`: no comments, other elements, or markup -/// inside attributes that the row checks could miscount. -fn plain_markup(body: &str) -> bool { - body.match_indices('<').all(|(at, _)| { - let tag = &body[at + 1..]; - let Some(end) = tag.find('>') else { +/// Whether `body` (lowercased) is exactly the markup IPOT serves, which +/// reads the same in any HTML parser: +/// +/// ```text +/// body := ws (row ws)* +/// row := "" ws (cell ws){9} "" +/// cell := ws ( text "" | text) ws "" +/// attrs := (" " name '="' value '"')* value without " < > +/// text := no "<", and no "&" except " " +/// ``` +/// +/// Anything else (comments, other tags, unclosed or stray tags, odd +/// spacing or quoting inside tags) rejects the page rather than risk +/// reading cells differently from how the page shows them. +fn canonical_rows(body: &str) -> bool { + let mut rest = body.trim_start(); + while !rest.is_empty() { + let Some(after) = canonical_row(rest) else { return false; }; - let inner = &tag[..end]; - let name: String = inner - .trim_start_matches('/') - .chars() - .take_while(char::is_ascii_alphanumeric) - .collect(); - !inner.contains('<') && matches!(name.as_str(), "tr" | "td" | "span") - }) + rest = after.trim_start(); + } + true +} + +fn canonical_row(row: &str) -> Option<&str> { + let mut rest = row.strip_prefix("")?; + for _ in 0..9 { + rest = canonical_cell(rest.trim_start())?; + } + rest.trim_start().strip_prefix("") +} + +fn canonical_cell(cell: &str) -> Option<&str> { + let rest = open_tag(cell, "td")?.trim_start(); + let rest = if rest.starts_with("")? + .trim_start() + } else { + plain_text(rest)? + }; + rest.strip_prefix("") +} + +/// The rest after ``. +fn open_tag<'a>(markup: &'a str, name: &str) -> Option<&'a str> { + let mut rest = markup.strip_prefix('<')?.strip_prefix(name)?; + loop { + if let Some(after) = rest.strip_prefix('>') { + return Some(after); + } + rest = rest.strip_prefix(' ')?; + let name_len = rest + .bytes() + .take_while(|b| b.is_ascii_lowercase() || *b == b'-') + .count(); + if name_len == 0 { + return None; + } + rest = rest[name_len..].strip_prefix("=\"")?; + let end = rest.find(['"', '<', '>'])?; + rest = rest[end..].strip_prefix('"')?; + } +} + +/// The rest after text without markup or character references. +fn plain_text(text: &str) -> Option<&str> { + let end = text.find('<').unwrap_or(text.len()); + let mut refs = text[..end].match_indices('&'); + refs.all(|(at, _)| text[at..].starts_with(" ")) + .then_some(&text[end..]) } /// The `MM/DD/YYYY` value of the input with this id. @@ -180,7 +235,7 @@ pub fn parse_broker_summary(html: &str) -> Result { }); let single = lower.matches(" Result { value }) }; - // A skipped or merged row would shift brokers between ranks, so the body - // must be whole rows only, each one `` of nine closed cells, ranked - // in order, and a broker after a side's blank rows rejects the page. + // The body is whole rows (see `canonical_rows`); ranks must run in + // order, and a broker after a side's blank rows rejects the page. let (mut buyers_ended, mut sellers_ended) = (false, false); let mut rank = 0; let mut at = body_start; @@ -212,17 +266,11 @@ pub fn parse_broker_summary(html: &str) -> Result { let end = rest .find("") .map_or(rest.len(), |end| end + "".len()); - let piece = &rest[..end]; let original = &html[at..at + end]; at += end; rank += 1; - let whole_row = piece.trim_start().starts_with("") - && piece.matches("").count() == 9; let row = cells(original); - if !whole_row || row.len() != 9 || row[4].text.parse::().ok() != Some(rank) { + if row.len() != 9 || row[4].text.parse::().ok() != Some(rank) { return Err(IdxError::ParseError(format!( "ipot broker summary: unexpected row {rank} ({} cells, rank {:?})", row.len(), @@ -421,6 +469,18 @@ mod tests { }; let commented = hidden(&|cells| format!("")); let in_attribute = hidden(&|cells| format!("")); + let after_quoted_end = hidden(&|cells| format!("")); + // Closing tags with a space, a cell under another tag name, and a + // character reference a browser would decode. + let lots = "828,163 "; + let spaced_row_end = DAY.replacen(lots, &format!("{lots}"), 1); + let spaced_cell_end = DAY.replacen(lots, &format!("{lots}"), 1); + let other_tag = DAY.replacen( + "5", + "5", + 1, + ); + let reference = DAY.replacen(">828,163 <", ">828,163 <", 1); for page in [ short_row, skipped_rank, @@ -431,6 +491,11 @@ mod tests { early_end, commented, in_attribute, + after_quoted_end, + spaced_row_end, + spaced_cell_end, + other_tag, + reference, "".to_string(), ] { assert!(