Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
216 changes: 155 additions & 61 deletions pipelock_verify/_canonical.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,49 +26,128 @@
import json
from typing import Any

# ActionRecord fields in Go struct-tag order. Each tuple is
# (json_name, has_omitempty). Source of truth:
# Field specs are (json_name, has_omitempty, nested_kind). nested_kind is one
# of None, "action_record", "redaction", "shield", or "taint_source" and tells
# the orderer to recurse into a nested object (or, for "taint_source", into
# each element of a nested array) so the nested object's keys are reordered to
# the Go struct order too. Go re-marshals nested structs in declaration order,
# so a verifier that leaves nested object keys in input order recomputes a
# different signing hash whenever the input keys are not already Go-ordered.
#
# ActionRecord fields in Go struct-tag order. Source of truth:
# https://github.com/luckyPipewrench/pipelock/blob/main/internal/receipt/action.go
_ACTION_RECORD_FIELDS: list[tuple[str, bool]] = [
("version", False),
("action_id", False),
("action_type", False),
("timestamp", False),
("principal", False),
("actor", False),
("delegation_chain", False),
("target", False),
("intent", True),
("data_classes_in", True),
("data_classes_out", True),
("side_effect_class", False),
("reversibility", False),
("policy_hash", False),
("verdict", False),
("transport", False),
("method", True),
("layer", True),
("pattern", True),
("request_id", True),
("chain_prev_hash", False),
("chain_seq", False),
("venue", True),
("jurisdiction", True),
("rulebook_id", True),
("remedy_class", True),
("contestation_window", True),
("precedent_refs", True),
# The list MUST match that struct's field set, declaration order, and omitempty
# semantics EXACTLY — including parent_action_id, the taint block, the contract
# block, severity, redaction, and shield. Any omitted or reordered field breaks
# signature verification for receipts that carry it.
_ACTION_RECORD_FIELDS: list[tuple[str, bool, str | None]] = [
("version", False, None),
("action_id", False, None),
("parent_action_id", True, None),
("action_type", False, None),
("timestamp", False, None),
("principal", False, None),
("actor", False, None),
("delegation_chain", False, None),
("target", False, None),
("intent", True, None),
("data_classes_in", True, None),
("data_classes_out", True, None),
("side_effect_class", False, None),
("reversibility", False, None),
("policy_hash", False, None),
("verdict", False, None),
("session_taint_level", True, None),
("session_contaminated", True, None),
("recent_taint_sources", True, "taint_source"),
("session_task_id", True, None),
("session_task_label", True, None),
("authority_kind", True, None),
("taint_decision", True, None),
("taint_decision_reason", True, None),
("task_override_applied", True, None),
("contract_winning_source", True, None),
("contract_live_verdict", True, None),
("contract_policy_sources", True, None),
("contract_rule_id", True, None),
("active_manifest_hash", True, None),
("contract_hash", True, None),
("contract_selector_id", True, None),
("contract_generation", True, None),
("transport", False, None),
("method", True, None),
("layer", True, None),
("pattern", True, None),
("severity", True, None),
("redaction", True, "redaction"),
("shield", True, "shield"),
("request_id", True, None),
("chain_prev_hash", False, None),
("chain_seq", False, None),
("venue", True, None),
("jurisdiction", True, None),
("rulebook_id", True, None),
("remedy_class", True, None),
("contestation_window", True, None),
("precedent_refs", True, None),
]

# Receipt envelope fields in Go struct-tag order. Source of truth:
# https://github.com/luckyPipewrench/pipelock/blob/main/internal/receipt/receipt.go
_RECEIPT_FIELDS: list[tuple[str, bool]] = [
("version", False),
("action_record", False),
("signature", False),
("signer_key", False),
_RECEIPT_FIELDS: list[tuple[str, bool, str | None]] = [
("version", False, None),
("action_record", False, "action_record"),
("signature", False, None),
("signer_key", False, None),
]

# RedactionSummary fields, Go struct order (receipt.RedactionSummary).
_REDACTION_FIELDS: list[tuple[str, bool, str | None]] = [
("profile", True, None),
("provider", True, None),
("parser", True, None),
("total_redactions", True, None),
("by_class", True, None),
("cache_boundary_kept", True, None),
]

# ShieldSummary fields, Go struct order (receipt.ShieldSummary).
_SHIELD_FIELDS: list[tuple[str, bool, str | None]] = [
("pipeline", True, None),
("total_rewrites", True, None),
("extension_probes", True, None),
("tracking_beacons", True, None),
("agent_traps", True, None),
("fingerprint_shim_injected", True, None),
("svg_foreign_objects", True, None),
("svg_event_handlers", True, None),
("svg_external_references", True, None),
("svg_hidden_text", True, None),
("svg_animation_injections", True, None),
("body_bytes", True, None),
("scanned_bytes", True, None),
("partial", True, None),
("adaptive_signals_recorded", True, None),
("adaptive_signal_max_per_body", True, None),
]

# TaintSourceRef fields, Go struct order (session.TaintSourceRef).
_TAINT_SOURCE_FIELDS: list[tuple[str, bool, str | None]] = [
("url", False, None),
("kind", False, None),
("level", False, None),
("timestamp", False, None),
("receipt_id", True, None),
("match_reason", True, None),
]

# Maps a nested_kind tag to its field list (object-valued nests only).
_NESTED_OBJECT_FIELDS: dict[str, list[tuple[str, bool, str | None]]] = {
"action_record": _ACTION_RECORD_FIELDS,
"redaction": _REDACTION_FIELDS,
"shield": _SHIELD_FIELDS,
}


def _is_go_zero(value: Any) -> bool:
"""Return True if ``value`` matches Go's ``omitempty`` zero value.
Expand Down Expand Up @@ -97,40 +176,55 @@ def _is_go_zero(value: Any) -> bool:
return False


def _order_action_record(ar: dict[str, Any]) -> dict[str, Any]:
"""Return a new dict with ActionRecord fields in Go's canonical order.
def _normalize_maps(value: Any) -> Any:
"""Recursively sort the keys of any plain map, matching Go's json.Marshal.

Go marshals a ``map[string]T`` with keys in sorted order. Struct fields are
emitted in declaration order (handled by the field-list orderers), but a
value that is a free-form map — the only one in the schema is
``redaction.by_class`` — must have its keys sorted here or it canonicalizes
differently than Go whenever the input keys are not already sorted. Mirrors
the TypeScript ``normalizeMaps`` and Rust ``normalize_maps`` helpers.
"""
if isinstance(value, list):
return [_normalize_maps(item) for item in value]
if isinstance(value, dict):
return {key: _normalize_maps(value[key]) for key in sorted(value)}
return value


def _order_object(
obj: dict[str, Any],
fields: list[tuple[str, bool, str | None]],
) -> dict[str, Any]:
"""Return a new dict with ``obj``'s keys in Go's canonical struct order.

Missing fields are skipped. ``omitempty`` fields with zero values are
dropped. Unknown fields are ignored: Go's ``json.Unmarshal`` would drop
them, and the re-serialized canonical form should not include them.
"""
ordered: dict[str, Any] = {}
for name, omitempty in _ACTION_RECORD_FIELDS:
if name not in ar:
continue
value = ar[name]
if omitempty and _is_go_zero(value):
continue
ordered[name] = value
return ordered


def _order_receipt(receipt: dict[str, Any]) -> dict[str, Any]:
"""Return a new dict with Receipt fields in Go's canonical order.
Nested fields recurse so nested object keys are reordered to Go order too:

The nested ``action_record`` is also reordered, so this function alone
is enough to produce canonical bytes for the full envelope (used when
computing chain prev_hash linkage).
* ``"action_record"``/``"redaction"``/``"shield"`` reorder an object value.
* ``"taint_source"`` reorders each object element of an array value.
"""
ordered: dict[str, Any] = {}
for name, omitempty in _RECEIPT_FIELDS:
if name not in receipt:
for name, omitempty, nested in fields:
if name not in obj:
continue
value = receipt[name]
value = obj[name]
if omitempty and _is_go_zero(value):
continue
if name == "action_record" and isinstance(value, dict):
value = _order_action_record(value)
if nested == "taint_source" and isinstance(value, list):
value = [
_order_object(item, _TAINT_SOURCE_FIELDS) if isinstance(item, dict) else item
for item in value
]
elif nested in _NESTED_OBJECT_FIELDS and isinstance(value, dict):
value = _order_object(value, _NESTED_OBJECT_FIELDS[nested])
else:
# Non-struct value: sort any free-form map keys to match Go.
value = _normalize_maps(value)
ordered[name] = value
return ordered

Expand Down Expand Up @@ -164,7 +258,7 @@ def canonicalize_action_record(ar: dict[str, Any]) -> bytes:
The returned bytes are what Go's ``ActionRecord.Canonical()`` produces
and what gets SHA-256-hashed before Ed25519 signing.
"""
return _to_canonical_bytes(_order_action_record(ar))
return _to_canonical_bytes(_order_object(ar, _ACTION_RECORD_FIELDS))


def canonicalize_receipt(receipt: dict[str, Any]) -> bytes:
Expand All @@ -173,4 +267,4 @@ def canonicalize_receipt(receipt: dict[str, Any]) -> bytes:
The returned bytes are what Go's ``json.Marshal(receipt)`` produces
and what gets SHA-256-hashed to form the next receipt's ``chain_prev_hash``.
"""
return _to_canonical_bytes(_order_receipt(receipt))
return _to_canonical_bytes(_order_object(receipt, _RECEIPT_FIELDS))
83 changes: 83 additions & 0 deletions pipelock_verify/_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from __future__ import annotations

import json
import re
from datetime import datetime
from typing import Any
Expand Down Expand Up @@ -64,3 +65,85 @@ def _is_valid_rfc3339(value: Any) -> bool:

class InvalidReceiptError(Exception):
"""Raised when a receipt cannot be parsed as JSON at all."""


class DuplicateKeyError(InvalidReceiptError):
"""Raised when a receipt JSON object contains a duplicate key.

``json.loads`` silently keeps the last value for a duplicate key, so
``{"verdict": "allow", "verdict": "block"}`` would decode as ``"block"``
with no error. That is a parser-differential smuggling vector: a display or
log layer reading the first occurrence sees a value different from the one
the signature was verified against. The verify path rejects such input
before signature verification.
"""


def _reject_duplicate_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
"""``object_pairs_hook`` that rejects any duplicate key in an object.

``json.loads`` invokes this for every object at every nesting depth with
the full ordered list of (key, value) pairs *before* it collapses
duplicates, so a duplicate at any depth is caught.
"""
seen: set[str] = set()
for key, _ in pairs:
if key in seen:
raise DuplicateKeyError(f"duplicate object key: {key!r}")
seen.add(key)
return dict(pairs)


# Shared cross-language receipt-nesting cap. All four reference verifiers
# enforce the same value explicitly and reject input nested beyond this depth.
# Receipts nest ~4 levels, so honest input is never affected.
_MAX_NESTING_DEPTH = 128


def _exceeds_max_depth(text: str) -> bool:
"""Return True if text nests objects/arrays beyond _MAX_NESTING_DEPTH.

A cheap string-aware scan (string contents are skipped) run before
json.loads so deeply nested ARRAYS — which never reach object_pairs_hook and
would otherwise raise CPython's RecursionError — are rejected at the same
depth as the other verifiers, rather than at the interpreter's stack limit.
"""
depth = 0
in_string = False
escaped = False
for ch in text:
if in_string:
if escaped:
escaped = False
elif ch == "\\":
escaped = True
elif ch == '"':
in_string = False
continue
if ch == '"':
in_string = True
elif ch in "[{":
depth += 1
if depth > _MAX_NESTING_DEPTH:
return True
elif ch in "]}":
depth -= 1
return False


def loads_no_duplicate_keys(text: str | bytes) -> Any:
"""``json.loads`` that rejects duplicate object keys at any nesting depth.

Also rejects input nested beyond the shared cross-language depth cap. That
bound is enforced before json.loads so deeply nested arrays cannot make
CPython raise ``RecursionError`` (which is not a ``JSONDecodeError`` and
would otherwise escape the verify path's handling and crash the caller); a
``RecursionError`` backstop covers any residual case.
"""
scan_text = text.decode("utf-8") if isinstance(text, bytes) else text
if _exceeds_max_depth(scan_text):
raise InvalidReceiptError("JSON nesting too deep")
try:
return json.loads(text, object_pairs_hook=_reject_duplicate_pairs)
except RecursionError as exc:
raise InvalidReceiptError("JSON nesting too deep") from exc
7 changes: 4 additions & 3 deletions pipelock_verify/_verify.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from ._canonical import canonicalize_action_record, canonicalize_receipt
from ._common import InvalidReceiptError as InvalidReceiptError
from ._common import _is_valid_rfc3339 as _is_valid_rfc3339
from ._common import loads_no_duplicate_keys

# Wire format constants — keep in sync with internal/receipt/receipt.go.
_RECEIPT_VERSION = 1
Expand Down Expand Up @@ -199,7 +200,7 @@ def _parse_receipt(source: str | bytes | dict[str, Any]) -> dict[str, Any]:
raise InvalidReceiptError(
f"unsupported source type {type(source).__name__}; expected str, bytes, or dict"
)
parsed = json.loads(source)
parsed = loads_no_duplicate_keys(source)
if not isinstance(parsed, dict):
raise InvalidReceiptError("receipt must be a JSON object")
return parsed
Expand Down Expand Up @@ -228,7 +229,7 @@ def _extract_receipt(parsed: dict[str, Any]) -> dict[str, Any] | None:
if isinstance(detail, dict):
return detail
if isinstance(detail, (str, bytes)):
decoded = json.loads(detail)
decoded = loads_no_duplicate_keys(detail)
if not isinstance(decoded, dict):
raise InvalidReceiptError("flight-recorder detail JSON did not decode to an object")
return decoded
Expand All @@ -255,7 +256,7 @@ def _read_jsonl(path: Path) -> list[dict[str, Any]]:
if not line:
continue
try:
parsed = json.loads(line)
parsed = loads_no_duplicate_keys(line)
except json.JSONDecodeError as exc:
raise json.JSONDecodeError(f"line {lineno}: {exc.msg}", exc.doc, exc.pos) from exc
if not isinstance(parsed, dict):
Expand Down
Loading
Loading