harness: use a conforming RFC 8785 canonicalizer

docs/vector-format.md

@@ -29,7 +29,7 @@ The manifest (`vectors/v0/manifest.json`) carries artefact metadata: `schema_ver
 Per AlgoVoi artefact `context.verification_recipe`:
 
 1. Extract the `jws` field; the remaining object is the unsigned vector.
-2. Compute RFC 8785 canonical JSON (`sort_keys=True`, minimal separators) over the unsigned vector.
+2. Compute RFC 8785 canonical JSON over the unsigned vector, using a conforming JCS implementation (the harness uses `rfc8785`, matching the `canonicalizer` declared in `manifest.json`). Note that `json.dumps(sort_keys=True, separators=(",", ":"))` is not equivalent: it escapes non-ASCII by default, serializes JSON numbers differently, and sorts by code point rather than by UTF-16 code unit.
 3. Split `jws` on `.` into `[header_b64, payload_b64, signature_b64]`.
 4. Confirm `base64url_decode(payload_b64)` equals the canonical bytes from step 2.
 5. Resolve `signer_did` to the Ed25519 public key via JWKS (`kid: d0481df4cbbda8e8aba86709419884ef`).

harness/jws.py

@@ -7,6 +7,7 @@
 from pathlib import Path
 from typing import Any
 
+import rfc8785
 from nacl.exceptions import BadSignatureError
 from nacl.signing import VerifyKey
 
@@ -24,10 +25,18 @@ def unsigned_vector(vector: dict[str, Any]) -> dict[str, Any]:
 
 
 def canonical_vector_bytes(vector: dict[str, Any]) -> bytes:
-    """RFC8785-compatible canonical form (JCS subset: sorted keys, minimal separators)."""
-    return json.dumps(unsigned_vector(vector), sort_keys=True, separators=(",", ":")).encode(
-        "utf-8"
-    )
+    """RFC 8785 (JCS) canonical form of the unsigned vector.
+
+    manifest.json declares `"canonicalizer": "rfc8785@0.1.4"`, so this uses
+    that implementation rather than a `json.dumps` approximation. The two
+    agree on the current v0 vectors, which are pure ASCII with string
+    amounts, and diverge as soon as a vector is not: `json.dumps` defaults
+    to `ensure_ascii=True` and escapes non-ASCII, RFC 8785 requires UTF-8
+    output. They also disagree on JSON number forms and on sort order for
+    characters outside the BMP, because RFC 8785 sorts by UTF-16 code unit.
+    See tests/test_canonicalization.py.
+    """
+    return rfc8785.dumps(unsigned_vector(vector))
 
 
 def verify_vector_jws(vector: dict[str, Any], *, jwk_x: str = ALGOVOI_JWK_X) -> tuple[bool, str]:

harness/requirements.txt

@@ -1,3 +1,4 @@
 PyNaCl>=1.5.0
 pytest>=8.0.0
 httpx>=0.27.0
+rfc8785>=0.1.4

tests/test_canonicalization.py

@@ -0,0 +1,74 @@
+"""Canonicalization conformance.
+
+manifest.json declares `"canonicalizer": "rfc8785@0.1.4"`. These tests pin
+the harness to that, so a vector signed by a conforming RFC 8785 signer
+verifies here.
+
+Every case below passes with rfc8785 and fails with the
+`json.dumps(sort_keys=True, separators=(",", ":"))` approximation. The
+current v0 vectors are unaffected either way, because they are pure ASCII
+and carry amounts as strings rather than JSON numbers. The divergence
+shows up on the first vector that is not, and it surfaces as
+"payload bytes != canonical unsigned vector", which reads like a bad
+signature rather than a canonicalizer mismatch.
+"""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from harness.jws import canonical_vector_bytes
+
+
+def canonical(obj: dict) -> bytes:
+    # canonical_vector_bytes strips "jws"; nothing here uses that key.
+    return canonical_vector_bytes(obj)
+
+
+def approximation(obj: dict) -> bytes:
+    """What the harness used before: not RFC 8785."""
+    return json.dumps(obj, sort_keys=True, separators=(",", ":")).encode("utf-8")
+
+
+@pytest.mark.parametrize(
+    "obj,expected",
+    [
+        # RFC 8785 section 3.2.4: strings are serialized as UTF-8, not escaped.
+        ({"note": "café"}, b'{"note":"caf\xc3\xa9"}'),
+        ({"merchant": "München GmbH"}, b'{"merchant":"M\xc3\xbcnchen GmbH"}'),
+        # RFC 8785 section 3.2.3: JSON number serialization (ECMAScript).
+        ({"amount": 1e16}, b'{"amount":10000000000000000}'),
+        ({"amount": -0.0}, b'{"amount":0}'),
+    ],
+)
+def test_canonical_form_is_rfc8785(obj: dict, expected: bytes) -> None:
+    assert canonical(obj) == expected
+    assert approximation(obj) != expected, "case no longer distinguishes the two"
+
+
+def test_property_names_sort_by_utf16_code_unit() -> None:
+    """RFC 8785 section 3.2.3 sorts property names by UTF-16 code unit.
+
+    U+1F600 is a surrogate pair (0xD83D 0xDE00) so it sorts before
+    U+FFFD, which is the opposite of Python's default code point order.
+    """
+    out = canonical({"�": 1, "\U0001f600": 2}).decode("utf-8")
+    assert out.index("\U0001f600") < out.index("�")
+
+
+def test_existing_v0_vectors_are_unaffected() -> None:
+    """The vendored vectors canonicalize identically under both.
+
+    This is why the mismatch has not bitten yet, and it is worth keeping
+    as a regression test: it shows the change is signature-preserving for
+    everything currently in the corpus.
+    """
+    from pathlib import Path
+
+    from harness.jws import load_all_vectors, unsigned_vector
+
+    vectors_dir = Path(__file__).resolve().parent.parent / "vectors" / "v0"
+    for vector in load_all_vectors(vectors_dir):
+        assert canonical_vector_bytes(vector) == approximation(unsigned_vector(vector))