diff --git a/jev.py b/jev.py index fc59f7d..7f6ff84 100644 --- a/jev.py +++ b/jev.py @@ -16,6 +16,27 @@ Constraints baked into this client (all measured, see DESIGN.md): * Because answers to questions in one call are independent, never make a question depend on another question's answer. Use a second call. +Question shapes follow the documented "advanced: structure" rules: the +`instructions` of every primitive, each Choice option description, each Score +level description and the Noul `criteria.true` / `criteria.false` entries all +accept a string, an object, or an array. Build them with `ask()` and `entry()`: + + noul(ask("Would `offered.card0` make this deck stronger?", + focus="Judge deck fit, not raw power.", + inspect="`offered.card0`, `deck`"), + true=entry("It adds damage, block or scaling this deck lacks", + examples=["A 2-cost attack that scales with Strength"]), + false=entry("It is off-plan, redundant, or too slow to matter", + not_for="A card that is merely different")) + + choice(ask("Which card best improves this deck, or is skipping better?"), + {f"card{i['index']}": entry(f"{i['name']}: {i['description']}", + not_for="...") for i in cards} + | {"skip": entry("Take nothing; keep the deck lean")}) + +A threshold tuned on a Noul is never reused on a Choice: state each gate's +floor and margin at the call site (`gate_choice`). + The API key is read from the sops-nix secrets path and is never logged. """ @@ -51,6 +72,13 @@ TRACE_PATH = os.environ.get("JEV_TRACE") # low-stakes binary (end the turn? use a potion?) is recoverable in this game. NOUL_MARGIN = 0.15 +# Choice gates are stated per call site, because a threshold tuned on a Noul +# must not be carried over to a Choice: the two answer different questions and +# are not on a comparable scale. These are the DEFAULTS for a low-stakes Choice +# (a card play). Irreversible decisions (events) pass their own stricter pair. +CHOICE_TOP_MIN = 0.45 +CHOICE_MARGIN_MIN = 0.20 + class JevError(RuntimeError): """Raised for transport, auth, and protocol failures.""" @@ -60,21 +88,91 @@ class JevError(RuntimeError): # Question builders # -------------------------------------------------------------------------- -def noul(instructions: Any, criteria: dict | None = None) -> dict: - """Yes/no question. Returns only P(yes); there is no confidence field.""" +def entry(what: Any = None, *, not_for: Any = None, examples: Any = None, + signals: Any = None, **more: Any) -> Any: + """ + One structured description entry, per the documented shapes. + + `instructions`, Choice option descriptions, Score level descriptions and + Noul `criteria.true` / `criteria.false` all accept a string, an object, or + an array. Use the object form when a description needs several kinds of + guidance: what it covers, what belongs elsewhere, and examples. + + Returns the bare string when only `what` is given, so a simple description + stays simple. The documented field names are used consistently across + entries so the model can compare them directly. + """ + fields = {"what": what, "not_for": not_for, "examples": examples, + "signals": signals, **more} + present = {k: v for k, v in fields.items() if v is not None} + if not present: + return None + if set(present) == {"what"} and isinstance(present["what"], str): + return present["what"] + return present + + +def ask(question: Any, *, focus: Any = None, inspect: Any = None, + compare: Any = None, **more: Any) -> Any: + """ + Structured instructions: the question plus named guidance fields. + + `focus` says what to judge, `inspect` / `compare` point at the exact state + paths the answer depends on (backticked dot-and-index paths). Use it when a + bare sentence would blur several instructions together; a short, + unambiguous question stays a string. + """ + fields = {"focus": focus, "inspect": inspect, "compare": compare, **more} + present = {k: v for k, v in fields.items() if v is not None} + if not present: + return question + return {"question": question, **present} + + +def noul_criteria(true: Any = None, false: Any = None) -> dict | None: + """ + Contrastive Noul criteria: what a yes means, what a no means. + + The instruction and the criteria must ask for the same thing — a Noul whose + `true` maps to "no" is a documented failure mode. + """ + criteria = {k: v for k, v in (("true", true), ("false", false)) if v is not None} + return criteria or None + + +def noul(instructions: Any, criteria: dict | None = None, *, + true: Any = None, false: Any = None) -> dict: + """ + Yes/no question. Returns only P(yes); there is no confidence field. + + Pass `true=` / `false=` for the contrastive criteria, or `criteria=` for a + prebuilt mapping. + """ q: dict[str, Any] = {"type": "noul", "instructions": instructions} - if criteria is not None: - q["criteria"] = criteria + built = criteria if criteria is not None else noul_criteria(true, false) + if built is not None: + q["criteria"] = built return q def choice(instructions: Any, criteria: dict) -> dict: - """Pick one option from a map of option -> description (max 255).""" + """ + Pick one option from a map of option -> description (max 255). + + Each description is an `entry()`. Give the model the full list rather than + a shortlist, and add an explicit "none of the above"-style option whenever + the set might not cover every input. + """ return {"type": "choice", "instructions": instructions, "criteria": criteria} def score(instructions: Any, criteria: list) -> dict: - """Position on an ordered list of levels.""" + """ + Position on an ordered list of levels, each an `entry()`. + + Use a Score when the answer is a position on a spectrum. The returned + `score` is a fractional position on the level scale, not an index. + """ return {"type": "score", "instructions": instructions, "criteria": criteria} @@ -121,6 +219,15 @@ class ScoreAnswer: def kind(self) -> str: return "score" + @property + def level(self) -> int: + """Nearest level index. `score` is a FRACTIONAL position on the scale.""" + return int(round(self.score)) + + def level_entry(self) -> Any: + """The legend entry for the nearest level, keyed by index as a string.""" + return self.legend.get(str(self.level)) + Answer = NoulAnswer | ChoiceAnswer | ScoreAnswer @@ -284,7 +391,7 @@ class JevClient: "input_tokens": response.input_tokens, "output_tokens": response.output_tokens, "questions": questions, - "answers": {qid: _answer_record(a) for qid, a in response.answers.items()}, + "answers": {qid: answer_record(a) for qid, a in response.answers.items()}, }) return response @@ -327,9 +434,14 @@ class JevClient: # Trace log (one JSON line per call, path from $JEV_TRACE) # -------------------------------------------------------------------------- -def _answer_record(a: Answer) -> dict: +def answer_record(a: Answer) -> dict: + """Machine-readable form of one answer, for a decision log. + + Carries the gate outcome for every primitive, so a gate can be analysed + offline from the log alone instead of being recomputed from the value. + """ if isinstance(a, NoulAnswer): - return {"kind": "noul", "noul": a.noul, "yes": a.yes} + return {"kind": "noul", "noul": a.noul, "yes": a.yes, "gated": gate(a)} if isinstance(a, ChoiceAnswer): return { "kind": "choice", @@ -364,6 +476,21 @@ def _trace(record: dict) -> None: # Confidence gate # -------------------------------------------------------------------------- +def gate_choice(answer: ChoiceAnswer, top_min: float, margin_min: float) -> bool: + """ + Gate a Choice with thresholds stated for THIS decision. + + A threshold tuned on a Noul must not be carried over to a Choice: the two + answer different questions and are not on a comparable scale. So each call + site names the floor and the margin that decision actually needs. + + The margin is the scale-free signal; `confidence` is peakedness and falls as + the option count rises, so it is never the primary gate. + """ + top = answer.probabilities.get(answer.choice, 0.0) + return top >= top_min and margin(answer) >= margin_min + + def gate(answer: Answer, threshold: float = 0.6) -> bool: """ True when the answer is safe to act on automatically. @@ -381,12 +508,7 @@ def gate(answer: Answer, threshold: float = 0.6) -> bool: if isinstance(answer, NoulAnswer): return abs(answer.noul - 0.5) >= NOUL_MARGIN if isinstance(answer, ChoiceAnswer): - top = answer.probabilities.get(answer.choice, 0.0) - runner = max( - (v for k, v in answer.probabilities.items() if k != answer.choice), - default=0.0, - ) - return top >= 0.45 and (top - runner) >= 0.20 + return gate_choice(answer, CHOICE_TOP_MIN, CHOICE_MARGIN_MIN) return answer.confidence >= threshold