From 10e2393c6489c9e5c84c5c8d71c3ac7773db8cf1 Mon Sep 17 00:00:00 2001 From: TimothyVang Date: Sat, 2 May 2026 08:40:53 -0500 Subject: [PATCH 1/2] =?UTF-8?q?test(graph):=20RED=20=E2=80=94=20Comprehens?= =?UTF-8?q?ionMismatch=20schema=20+=20gate=20emission=20[W2.B.3]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds tests/graph/test_comprehension_mismatch.py asserting the W2.B.3 contract: - ComprehensionMismatch Pydantic v2 model in verdict.schemas.comprehension_mismatch with fields case_id / clarify_iterations_spent / disagreeing_keys / per_executor / event_id (ULID) / event_type=\"comprehension_check\" / timestamp_utc (TZ-aware required, naive rejected per CLAUDE.md §3 forensic doctrine); - ExecutorEchoDiff submodel preserving each branch's parsed_* values verbatim for forensic auditability; - empty disagreeing_keys is a contradiction in terms — rejected; - comprehension_gate_node populates state[\"pending_ledger_entry\"] with a structured ComprehensionMismatch payload only on the terminal-concede branch (after MAX_CLARIFY_ITERATIONS rounds), not on intermediate clarify rounds; - 4-way-consensus path emits NO ledger entry. RED: ModuleNotFoundError on verdict.schemas.comprehension_mismatch. GREEN follows. --- tests/graph/test_comprehension_mismatch.py | 233 +++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 tests/graph/test_comprehension_mismatch.py diff --git a/tests/graph/test_comprehension_mismatch.py b/tests/graph/test_comprehension_mismatch.py new file mode 100644 index 0000000..dfd02ba --- /dev/null +++ b/tests/graph/test_comprehension_mismatch.py @@ -0,0 +1,233 @@ +"""W2.B.3 — `ComprehensionMismatch` ledger entry on disagreement. + +When the comprehension_gate concedes after MAX_CLARIFY_ITERATIONS, it +must emit a structured per-executor diff that downstream LedgerEmitter +(W2.C.3) writes as `event_type="comprehension_check"` with payload +schema `ComprehensionMismatch`. + +The schema invariants tested here: + +- Pydantic v2 model in `verdict.schemas.comprehension_mismatch` with the + three CONSENSUS_KEYS captured per executor; +- per-executor diff structure: which echoes disagreed on which key; +- ULID-shaped event_id, UTC-Z timestamp; +- the gate node populates `state["pending_ledger_entry"]` with the + structured payload so the next ledger super-step picks it up. + +The ledger writer itself (write+fsync+verify-readback) lives in W2.G.1; +this task is purely about the EVENT TYPE + PAYLOAD SCHEMA contract that +W2.G.1 will consume. +""" + +from __future__ import annotations + +import re +from datetime import datetime, timezone + +import pytest +from pydantic import ValidationError + +from verdict.graph.nodes import ( + MAX_CLARIFY_ITERATIONS, + comprehension_gate_node, +) +from verdict.schemas.comprehension_mismatch import ( + ComprehensionMismatch, + ExecutorEchoDiff, +) + + +def _echo(branch: str, pos: list[str], neg: list[str], succ_hash: str) -> dict: + return { + "branch_name": branch, + "parsed_positive_hypothesis_ids": pos, + "parsed_negative_hypothesis_ids": neg, + "parsed_success_criteria_hash": succ_hash, + } + + +# --------------------------------------------------------------------------- +# Schema-level tests +# --------------------------------------------------------------------------- + + +def test_comprehension_mismatch_schema_round_trip() -> None: + """ComprehensionMismatch round-trips through Pydantic.""" + payload = ComprehensionMismatch( + case_id="case_001", + clarify_iterations_spent=2, + disagreeing_keys=["parsed_negative_hypothesis_ids"], + per_executor=[ + ExecutorEchoDiff( + branch_name="vol_exec", + parsed_positive_hypothesis_ids=["H1", "H2"], + parsed_negative_hypothesis_ids=["N1"], + parsed_success_criteria_hash="abc123", + ), + ExecutorEchoDiff( + branch_name="hay_exec", + parsed_positive_hypothesis_ids=["H1", "H2"], + parsed_negative_hypothesis_ids=["N1"], + parsed_success_criteria_hash="abc123", + ), + ExecutorEchoDiff( + branch_name="pls_exec", + parsed_positive_hypothesis_ids=["H1", "H2"], + parsed_negative_hypothesis_ids=["N1"], + parsed_success_criteria_hash="abc123", + ), + ExecutorEchoDiff( + branch_name="mft_exec", + parsed_positive_hypothesis_ids=["H1", "H2"], + parsed_negative_hypothesis_ids=["DIFFERENT"], + parsed_success_criteria_hash="abc123", + ), + ], + timestamp_utc=datetime(2026, 5, 2, 12, 0, 0, tzinfo=timezone.utc), + ) + assert payload.event_type == "comprehension_check" + assert payload.timestamp_utc.tzinfo is not None # always TZ-aware + dumped = payload.model_dump(mode="json") + # UTC Z-suffix per CLAUDE.md §3 forensic doctrine. + assert dumped["timestamp_utc"].endswith("Z") or "+00:00" in dumped["timestamp_utc"] + # Round-trip + reloaded = ComprehensionMismatch.model_validate(dumped) + assert reloaded == payload + + +def test_comprehension_mismatch_rejects_naive_timestamp() -> None: + """Naive datetime is forensically unsound (CLAUDE.md §3 — UTC Z).""" + with pytest.raises(ValidationError): + ComprehensionMismatch( + case_id="case_001", + clarify_iterations_spent=2, + disagreeing_keys=["parsed_positive_hypothesis_ids"], + per_executor=[ + ExecutorEchoDiff( + branch_name="vol_exec", + parsed_positive_hypothesis_ids=["H1"], + parsed_negative_hypothesis_ids=["N1"], + parsed_success_criteria_hash="abc", + ) + ], + timestamp_utc=datetime(2026, 5, 2, 12, 0, 0), # NAIVE + ) + + +def test_comprehension_mismatch_rejects_empty_disagreeing_keys() -> None: + """A `comprehension_check` mismatch event with NO disagreeing keys is + a contradiction in terms — the gate should never emit it.""" + with pytest.raises(ValidationError): + ComprehensionMismatch( + case_id="case_001", + clarify_iterations_spent=2, + disagreeing_keys=[], # empty — invalid + per_executor=[ + ExecutorEchoDiff( + branch_name="vol_exec", + parsed_positive_hypothesis_ids=["H1"], + parsed_negative_hypothesis_ids=["N1"], + parsed_success_criteria_hash="abc", + ) + ], + timestamp_utc=datetime(2026, 5, 2, 12, 0, 0, tzinfo=timezone.utc), + ) + + +def test_comprehension_mismatch_event_id_ulid_shape() -> None: + """`event_id` defaults to a ULID — 26 chars Crockford base32 per + LedgerEntry contract (ARCHITECTURE.md §5). + """ + payload = ComprehensionMismatch( + case_id="case_001", + clarify_iterations_spent=2, + disagreeing_keys=["parsed_positive_hypothesis_ids"], + per_executor=[ + ExecutorEchoDiff( + branch_name="vol_exec", + parsed_positive_hypothesis_ids=["H1"], + parsed_negative_hypothesis_ids=["N1"], + parsed_success_criteria_hash="abc", + ) + ], + timestamp_utc=datetime(2026, 5, 2, 12, 0, 0, tzinfo=timezone.utc), + ) + assert re.match(r"^[0-9A-HJKMNP-TV-Z]{26}$", payload.event_id), ( + f"event_id={payload.event_id!r} is not a ULID" + ) + + +# --------------------------------------------------------------------------- +# Gate -> ledger-payload integration +# --------------------------------------------------------------------------- + + +def test_gate_emits_pending_ledger_entry_on_concede() -> None: + """When the gate concedes after MAX_CLARIFY_ITERATIONS rounds, it + populates `pending_ledger_entry` with a ComprehensionMismatch + payload that downstream LedgerEmitter (W2.C.3) will write. + """ + pos, neg, hsh = ["H1"], ["N1"], "abc" + echoes = [ + _echo("vol_exec", pos, neg, hsh), + _echo("hay_exec", pos, neg, hsh), + _echo("pls_exec", pos, neg, hsh), + _echo("mft_exec", pos, ["DIFFERENT"], hsh), + ] + state = { + "case_id": "case_test_001", + "comprehension_echoes": echoes, + "clarify_iterations": MAX_CLARIFY_ITERATIONS, + } + update = comprehension_gate_node(state) + + assert update["comprehension_consensus"] is False + assert "pending_ledger_entry" in update, ( + "gate must populate pending_ledger_entry on concede so the " + "ledger emitter can write the mismatch event" + ) + payload = update["pending_ledger_entry"] + # Validate as the schema directly to enforce the contract. + parsed = ComprehensionMismatch.model_validate(payload) + assert parsed.event_type == "comprehension_check" + assert parsed.case_id == "case_test_001" + assert "parsed_negative_hypothesis_ids" in parsed.disagreeing_keys + # All 4 executor echoes preserved verbatim for forensic auditability. + assert {d.branch_name for d in parsed.per_executor} == { + "vol_exec", "hay_exec", "pls_exec", "mft_exec" + } + + +def test_gate_omits_ledger_entry_on_consensus() -> None: + """Happy path: 4-way consensus -> no ledger event needed.""" + pos, neg, hsh = ["H1"], ["N1"], "abc" + echoes = [ + _echo(b, pos, neg, hsh) + for b in ("vol_exec", "hay_exec", "pls_exec", "mft_exec") + ] + state = {"case_id": "case_001", "comprehension_echoes": echoes, "clarify_iterations": 0} + update = comprehension_gate_node(state) + assert update["comprehension_consensus"] is True + assert "pending_ledger_entry" not in update + + +def test_gate_omits_ledger_entry_during_clarify() -> None: + """Mid-clarify (budget not exhausted) -> no mismatch ledger event yet. + Ledger entries are emitted only on terminal disagreement, not on + every re-prompt round. + """ + pos, neg, hsh = ["H1"], ["N1"], "abc" + echoes = [ + _echo("vol_exec", pos, neg, hsh), + _echo("hay_exec", pos, neg, hsh), + _echo("pls_exec", pos, neg, hsh), + _echo("mft_exec", pos, ["DIFFERENT"], hsh), + ] + state = { + "case_id": "case_001", + "comprehension_echoes": echoes, + "clarify_iterations": 0, # budget remaining + } + update = comprehension_gate_node(state) + assert update["clarify_iterations"] == 1 + assert "pending_ledger_entry" not in update From 8a02b3753eb8ea7e985bc23309f2b5325ba5310a Mon Sep 17 00:00:00 2001 From: TimothyVang Date: Sat, 2 May 2026 08:42:35 -0500 Subject: [PATCH 2/2] feat(ledger): ComprehensionMismatch event with per-executor diff [W2.B.3] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the ledger payload schema for the comprehension_check event type emitted when comprehension_gate concedes after MAX_CLARIFY_ITERATIONS rounds of persistent executor disagreement. Files: - verdict/schemas/comprehension_mismatch.py — Pydantic v2 model ComprehensionMismatch (event_id ULID + UTC-Z timestamp + case_id + clarify_iterations_spent + disagreeing_keys >=1 + per_executor list of ExecutorEchoDiff). The TZ-aware timestamp validator rejects naive datetimes per CLAUDE.md §3 forensic doctrine; the JSON serializer emits the canonical Z suffix the SANS examiner format expects. - verdict/schemas/ulid.py — minimal pure-Python ULID generator (10-char Crockford-base32 timestamp + 16-char randomness, 26 chars total). Avoids a python-ulid direct dep; license-clean per §3.8. - verdict/graph/nodes.py — comprehension_gate_node now builds a ComprehensionMismatch on the terminal-concede branch and stores its model_dump(mode=\"json\") in state[\"pending_ledger_entry\"] for the downstream LedgerEmitter (W2.C.3). Every echo received is preserved verbatim — chain-of-custody per CLAUDE.md §9. Empty/malformed echo cases populate placeholder ExecutorEchoDiffs rather than dropping them silently (ARCHITECTURE.md §1 empty-set rule). - tests/graph/test_comprehension_mismatch.py — schema round-trip, naive-tz rejection, ULID shape, gate-emits-on-concede, gate-omits- on-consensus, gate-omits-during-clarify. Tests: 25 passed (W2.B.1 + W2.B.2 + W2.B.3). Ruff: clean. --- tests/graph/test_comprehension_mismatch.py | 8 +-- verdict/graph/nodes.py | 72 ++++++++++++++++++- verdict/schemas/comprehension_mismatch.py | 81 ++++++++++++++++++++++ verdict/schemas/ulid.py | 47 +++++++++++++ 4 files changed, 203 insertions(+), 5 deletions(-) create mode 100644 verdict/schemas/comprehension_mismatch.py create mode 100644 verdict/schemas/ulid.py diff --git a/tests/graph/test_comprehension_mismatch.py b/tests/graph/test_comprehension_mismatch.py index dfd02ba..1a79cf6 100644 --- a/tests/graph/test_comprehension_mismatch.py +++ b/tests/graph/test_comprehension_mismatch.py @@ -22,7 +22,7 @@ from __future__ import annotations import re -from datetime import datetime, timezone +from datetime import UTC, datetime import pytest from pydantic import ValidationError @@ -83,7 +83,7 @@ def test_comprehension_mismatch_schema_round_trip() -> None: parsed_success_criteria_hash="abc123", ), ], - timestamp_utc=datetime(2026, 5, 2, 12, 0, 0, tzinfo=timezone.utc), + timestamp_utc=datetime(2026, 5, 2, 12, 0, 0, tzinfo=UTC), ) assert payload.event_type == "comprehension_check" assert payload.timestamp_utc.tzinfo is not None # always TZ-aware @@ -130,7 +130,7 @@ def test_comprehension_mismatch_rejects_empty_disagreeing_keys() -> None: parsed_success_criteria_hash="abc", ) ], - timestamp_utc=datetime(2026, 5, 2, 12, 0, 0, tzinfo=timezone.utc), + timestamp_utc=datetime(2026, 5, 2, 12, 0, 0, tzinfo=UTC), ) @@ -150,7 +150,7 @@ def test_comprehension_mismatch_event_id_ulid_shape() -> None: parsed_success_criteria_hash="abc", ) ], - timestamp_utc=datetime(2026, 5, 2, 12, 0, 0, tzinfo=timezone.utc), + timestamp_utc=datetime(2026, 5, 2, 12, 0, 0, tzinfo=UTC), ) assert re.match(r"^[0-9A-HJKMNP-TV-Z]{26}$", payload.event_id), ( f"event_id={payload.event_id!r} is not a ULID" diff --git a/verdict/graph/nodes.py b/verdict/graph/nodes.py index 52e691b..d29efee 100644 --- a/verdict/graph/nodes.py +++ b/verdict/graph/nodes.py @@ -23,9 +23,14 @@ from __future__ import annotations +from datetime import UTC, datetime from typing import Any from verdict.graph.state import GraphState +from verdict.schemas.comprehension_mismatch import ( + ComprehensionMismatch, + ExecutorEchoDiff, +) # Budget invariants per CLAUDE.md §8 / ARCHITECTURE.md §2. PIVOT_MAX = 15 @@ -197,19 +202,84 @@ def comprehension_gate_node(state: GraphState) -> dict[str, Any]: f"{len(distinct_branches)} of {EXECUTOR_FANOUT_SIZE} executor " "branches echoed within budget" ) + # No-quorum: the disagreement is on EVERY consensus key by + # definition (we cannot prove agreement we never observed). + disagreeing_keys_for_ledger = list(CONSENSUS_KEYS) elif disagreeing: # Surface the most-actionable disagreeing key. hint = ( "comprehension_persistent_mismatch: executors disagreed on " f"{disagreeing[0]} after {MAX_CLARIFY_ITERATIONS} clarify rounds" ) + disagreeing_keys_for_ledger = disagreeing else: - # Should be unreachable, but emit a sane default. + # Should be unreachable (consensus_reached would be True), but + # emit a sane default rather than a crash. hint = "comprehension_persistent_mismatch: unspecified" + disagreeing_keys_for_ledger = list(CONSENSUS_KEYS) + + # Build the ComprehensionMismatch ledger payload for the downstream + # LedgerEmitter (W2.C.3). Every echo we received is preserved + # verbatim — chain-of-custody per CLAUDE.md §9. + case_id = state.get("case_id") or "" # type: ignore[assignment] + per_executor: list[ExecutorEchoDiff] = [] + for echo in echoes: + try: + per_executor.append( + ExecutorEchoDiff( + branch_name=echo.get("branch_name") or "unknown", + parsed_positive_hypothesis_ids=list( + echo.get("parsed_positive_hypothesis_ids") or [] + ), + parsed_negative_hypothesis_ids=list( + echo.get("parsed_negative_hypothesis_ids") or [] + ), + parsed_success_criteria_hash=str( + echo.get("parsed_success_criteria_hash") or "" + ) or "missing", + ) + ) + except Exception: + # An echo so malformed it cannot even be captured for the + # ledger is itself the disagreement — record a placeholder + # rather than dropping it (silent drops violate + # ARCHITECTURE.md §1 empty-set rule). + per_executor.append( + ExecutorEchoDiff( + branch_name=str(echo.get("branch_name") or "malformed"), + parsed_positive_hypothesis_ids=[], + parsed_negative_hypothesis_ids=[], + parsed_success_criteria_hash="malformed_echo", + ) + ) + + if not per_executor: + # No echoes at all — record one synthetic entry so the schema's + # `min_length=1` is satisfied without dropping the event. + per_executor.append( + ExecutorEchoDiff( + branch_name="no_branches_echoed", + parsed_positive_hypothesis_ids=[], + parsed_negative_hypothesis_ids=[], + parsed_success_criteria_hash="no_echoes", + ) + ) + + pending = ComprehensionMismatch( + case_id=case_id or "unknown", + timestamp_utc=datetime.now(tz=UTC), + clarify_iterations_spent=clarify_iterations, + disagreeing_keys=disagreeing_keys_for_ledger, + per_executor=per_executor, + ) return { "comprehension_consensus": False, "contested_hint": hint, + # `pending_ledger_entry` carries a model_dump (mode="json") so + # the LedgerEmitter (W2.C.3) hashes the same bytes Pydantic + # would emit — preserves HMAC reproducibility. + "pending_ledger_entry": pending.model_dump(mode="json"), } diff --git a/verdict/schemas/comprehension_mismatch.py b/verdict/schemas/comprehension_mismatch.py new file mode 100644 index 0000000..9df360e --- /dev/null +++ b/verdict/schemas/comprehension_mismatch.py @@ -0,0 +1,81 @@ +"""ComprehensionMismatch — ledger payload for executor disagreement. + +Emitted by `comprehension_gate_node` when the gate concedes after +`MAX_CLARIFY_ITERATIONS=2` rounds of persistent disagreement among the +4 executor branches. The downstream `LedgerEmitter` (W2.C.3) writes +this as a `LedgerEntry(event_type="comprehension_check")`. + +The schema preserves each executor's full parsed echo so a SANS judge +or human IR lead can reconstruct exactly which branch disagreed on +which key — chain-of-custody discipline per CLAUDE.md §9. + +References: +- ARCHITECTURE.md §2 (comprehension-gate clarify budget) +- ARCHITECTURE.md §5 (LedgerEntry contract) +- BUILD_PLAN W2.B.3 (ComprehensionMismatch event with per-executor diff) +""" + +from __future__ import annotations + +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator + +from verdict.schemas.ulid import new_ulid + + +class ExecutorEchoDiff(BaseModel): + """One executor branch's parsed plan-comprehension echo, captured + verbatim for forensic audit. Field names mirror + `verdict.graph.nodes.CONSENSUS_KEYS`. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + branch_name: str = Field(min_length=1) + parsed_positive_hypothesis_ids: list[str] + parsed_negative_hypothesis_ids: list[str] + parsed_success_criteria_hash: str = Field(min_length=1) + + +class ComprehensionMismatch(BaseModel): + """Structured per-executor diff written to the ledger when the + comprehension_gate concedes. + + `event_type` is fixed at `comprehension_check` — the LedgerEntry + `event_type` Literal that ARCHITECTURE.md §5 enumerates. The judge + ledger walker uses this discriminator. + """ + + model_config = ConfigDict(extra="forbid") + + event_id: str = Field(default_factory=new_ulid) + event_type: Literal["comprehension_check"] = "comprehension_check" + case_id: str = Field(min_length=1) + timestamp_utc: datetime + clarify_iterations_spent: int = Field(ge=0) + disagreeing_keys: list[str] = Field(min_length=1) + per_executor: list[ExecutorEchoDiff] = Field(min_length=1) + + @field_validator("timestamp_utc") + @classmethod + def _require_tz(cls, v: datetime) -> datetime: + """Reject naive datetimes — CLAUDE.md §3 demands UTC Z forensic + timestamps; a naive datetime would silently downcast and break + chain-of-custody reconstruction.""" + if v.tzinfo is None: + raise ValueError("timestamp_utc must be TZ-aware (UTC)") + return v + + @field_serializer("timestamp_utc") + def _serialize_ts(self, v: datetime) -> str: + """Serialize as ISO-8601 with explicit `Z` suffix — required by + CLAUDE.md §3 forensic-doctrine timestamp rule. + """ + # `isoformat()` on a UTC dt produces `+00:00`; replace with `Z` + # to match the canonical SANS examiner format. + s = v.isoformat() + if s.endswith("+00:00"): + return s[:-6] + "Z" + return s diff --git a/verdict/schemas/ulid.py b/verdict/schemas/ulid.py new file mode 100644 index 0000000..f8e73a3 --- /dev/null +++ b/verdict/schemas/ulid.py @@ -0,0 +1,47 @@ +"""ULID generation — minimal pure-Python implementation. + +ULID (https://github.com/ulid/spec) is the canonical event_id shape for +LedgerEntry per ARCHITECTURE.md §5: 26-character Crockford-base32, +lexicographically sortable by timestamp, with 80 bits of randomness. + +We avoid adding a third-party `python-ulid` direct dep — pyproject is +already MIT/Apache-only and CLAUDE.md §3.8 wants every direct dep +license-checked. This implementation is ~25 lines of stdlib + os.urandom. + +If a downstream consumer wants the canonical implementation, swap this +module for `python-ulid` (MIT) — the public API matches. +""" + +from __future__ import annotations + +import os +import time + +# Crockford base32 alphabet (excludes I, L, O, U to avoid ambiguity). +_CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" +assert len(_CROCKFORD) == 32 + + +def _encode_crockford(value: int, length: int) -> str: + """Encode `value` (an int) as Crockford base32 of fixed `length`.""" + if value < 0: + raise ValueError("ULID component must be non-negative") + out = [""] * length + for i in range(length - 1, -1, -1): + out[i] = _CROCKFORD[value & 0b11111] + value >>= 5 + if value: + raise ValueError(f"value too large for {length} crockford chars") + return "".join(out) + + +def new_ulid() -> str: + """Generate a fresh 26-char ULID. + + Layout: 10 chars timestamp (48-bit ms) + 16 chars randomness (80 bits). + Lexicographically sortable: a ULID minted later sorts after one minted + earlier (within ms granularity). + """ + ts_ms = int(time.time() * 1000) # 48-bit unix ms + rand_bits = int.from_bytes(os.urandom(10), "big") # 80 bits + return _encode_crockford(ts_ms, 10) + _encode_crockford(rand_bits, 16)