fix(bot): correct combat estimates and enforce client and runner failures

jev.py

@@ -44,6 +44,7 @@ from __future__ import annotations
 
 import http.client
 import json
+import math
 import os
 import time
 import urllib.error
@@ -381,10 +382,20 @@ class JevClient:
         elapsed = time.monotonic() - started
         try:
             data = json.loads(raw)
-        except json.JSONDecodeError as exc:
+        except (json.JSONDecodeError, UnicodeDecodeError) as exc:
             raise JevError(f"non-JSON response: {raw[:200]!r}") from exc
 
         response = self._parse(data, elapsed)
+        if set(response.answers) != set(questions):
+            raise JevError("response question IDs do not match the request")
+        for qid, answer in response.answers.items():
+            question = questions[qid]
+            if answer.kind != question.get("type"):
+                raise JevError(f"answer type does not match question {qid!r}")
+            if isinstance(answer, ChoiceAnswer) and set(answer.probabilities) != set(question["criteria"]):
+                raise JevError(f"answer options do not match question {qid!r}")
+            if isinstance(answer, ScoreAnswer) and len(answer.legend) != len(question["criteria"]):
+                raise JevError(f"answer levels do not match question {qid!r}")
         _trace({
             "model": response.model,
             "latency_s": round(elapsed, 3),
@@ -399,34 +410,69 @@ class JevClient:
 
     @staticmethod
     def _parse(data: dict, elapsed: float) -> JevResponse:
+        def number(value, field: str, *, probability: bool = False) -> float:
+            if isinstance(value, bool) or not isinstance(value, (int, float)):
+                raise JevError(f"missing or non-numeric {field}")
+            if not math.isfinite(value) or (probability and not 0 <= value <= 1):
+                raise JevError(f"invalid {field}: expected a finite {'probability' if probability else 'number'}")
+            return float(value)
+
+        def probabilities(item: dict) -> dict[str, float]:
+            values = item.get("probabilities")
+            if not isinstance(values, dict) or not values:
+                raise JevError("missing or invalid probability distribution")
+            if not all(isinstance(k, str) for k in values):
+                raise JevError("probability keys must be strings")
+            return {k: number(v, f"probabilities[{k!r}]", probability=True) for k, v in values.items()}
+
+        if not isinstance(data, dict):
+            raise JevError("expected a response object")
+        raw_answers = data.get("answers")
+        if not isinstance(raw_answers, dict) or not raw_answers:
+            raise JevError("missing or invalid answers object")
         answers: dict[str, Answer] = {}
-        for qid, item in (data.get("answers") or {}).items():
+        for qid, item in raw_answers.items():
+            if not isinstance(qid, str) or not isinstance(item, dict):
+                raise JevError("invalid question ID or answer object")
             kind = item.get("type")
             if kind == "noul":
-                answers[qid] = NoulAnswer(noul=float(item.get("noul", 0.0)))
+                answers[qid] = NoulAnswer(number(item.get("noul"), "noul", probability=True))
             elif kind == "choice":
+                values = probabilities(item)
+                chosen = item.get("choice")
+                if not isinstance(chosen, str) or chosen not in values:
+                    raise JevError(f"invalid choice for question {qid!r}")
                 answers[qid] = ChoiceAnswer(
-                    choice=item.get("choice"),
-                    probabilities=item.get("probabilities") or {},
-                    confidence=float(item.get("confidence", 0.0)),
+                    choice=chosen, probabilities=values,
+                    confidence=number(item.get("confidence"), "confidence", probability=True),
                 )
             elif kind == "score":
+                values = probabilities(item)
+                legend = item.get("legend")
+                if not isinstance(legend, dict) or not 2 <= len(legend) <= 10:
+                    raise JevError(f"invalid score legend for question {qid!r}")
+                levels = {str(i) for i in range(len(legend))}
+                if set(legend) != levels or set(values) != levels:
+                    raise JevError(f"invalid score levels for question {qid!r}")
+                value = number(item.get("score"), "score")
+                if not 0 <= value <= len(legend) - 1:
+                    raise JevError(f"score outside its levels for question {qid!r}")
                 answers[qid] = ScoreAnswer(
-                    score=float(item.get("score", 0.0)),
-                    legend=item.get("legend") or {},
-                    probabilities=item.get("probabilities") or {},
-                    confidence=float(item.get("confidence", 0.0)),
+                    score=value, legend=legend, probabilities=values,
+                    confidence=number(item.get("confidence"), "confidence", probability=True),
                 )
             else:
                 raise JevError(f"unknown answer type {kind!r} for question {qid!r}")
 
-        usage = data.get("usage") or {}
+        usage = data.get("usage", {})
+        if not isinstance(usage, dict):
+            raise JevError("invalid token usage object")
+        counts = [usage.get(key, 0) for key in ("input_tokens", "output_tokens")]
+        if any(isinstance(n, bool) or not isinstance(n, int) or n < 0 for n in counts):
+            raise JevError("invalid token usage counts")
         return JevResponse(
-            answers=answers,
-            model=data.get("model", "?"),
-            input_tokens=int(usage.get("input_tokens", 0)),
-            output_tokens=int(usage.get("output_tokens", 0)),
-            latency_s=elapsed,
+            answers=answers, model=data.get("model", "?"),
+            input_tokens=counts[0], output_tokens=counts[1], latency_s=elapsed,
         )