From 3d973e6a24c34e49e80b36d95cffd464ccbd3837 Mon Sep 17 00:00:00 2001 From: Tobias Oberstein Date: Sat, 13 Jun 2026 15:55:11 +0200 Subject: [PATCH 1/6] add new dev branch with audit file --- .audit/oberstet_fix_1849.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .audit/oberstet_fix_1849.md diff --git a/.audit/oberstet_fix_1849.md b/.audit/oberstet_fix_1849.md new file mode 100644 index 000000000..3a09db20e --- /dev/null +++ b/.audit/oberstet_fix_1849.md @@ -0,0 +1,8 @@ +- [ ] I did **not** use any AI-assistance tools to help create this pull request. +- [x] I **did** use AI-assistance tools to *help* create this pull request. +- [x] I have read, understood and followed the projects' [AI Policy](https://github.com/crossbario/autobahn-python/blob/main/AI_POLICY.md) when creating code, documentation etc. for this pull request. + +Submitted by: @oberstet +Date: 2025-06-13 +Related issue(s): #1849 +Branch: oberstet:fix_1849 From 0d15d6c52d3108595adcc01263dec9ba5258e77a Mon Sep 17 00:00:00 2001 From: Tobias Oberstein Date: Sat, 13 Jun 2026 17:07:29 +0200 Subject: [PATCH 2/6] Switch WAMP ubjson serializer from py-ubjson to bjdata (#1849) Fixes #1849: py-ubjson is unmaintained and ships no wheels, so `pip install --only-binary :all:` for autobahn failed. The WAMP "ubjson" serializer is now backed by bjdata (Binary JData), a maintained, wheel-shipping successor. - serializer.py / message.py: import bjdata as the "ubjson" backend (serializer id unchanged for transport negotiation). - pyproject.toml: drop the unconditional py-ubjson dependency; bjdata is an OPTIONAL dependency in the `serialization` extra (it pulls in numpy, which we keep out of a minimal install). A minimal `pip install autobahn` now installs cleanly from wheels only. - PyPy: set PYBJDATA_NO_EXTENSION=1 in the test recipes and CI (serdes) to use bjdata's JIT-friendly pure-Python path (also avoids its numpy-ABI-fragile C extension). WIRE-LEVEL CHANGE: bjdata's octet encoding is NOT identical to the prior py-ubjson/UBJSON bytes (unsigned-int markers, little-endian). The wamp-proto UBJSON test vectors will be regenerated in a follow-up wamp-proto PR after the next autobahn-python release; until then the serdes byte-vector conformance suite excludes "ubjson" (round-trip + cross-serializer coverage is retained via test_wamp_serializer.py). See the changelog. Note: This work was completed with AI assistance (Claude Code). --- .github/workflows/main.yml | 4 ++++ docs/changelog.rst | 6 ++++++ examples/serdes/tests/conftest.py | 19 +++++++++++++++++-- justfile | 16 ++++++++++++++++ pyproject.toml | 16 ++++++++++++---- src/autobahn/wamp/message.py | 4 ++-- src/autobahn/wamp/serializer.py | 17 +++++++++++++---- .../wamp/test/test_wamp_serializer.py | 8 +++++--- 8 files changed, 75 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ce1a7f26f..75e2b45c4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -190,6 +190,10 @@ jobs: # Create output directory for test results mkdir -p test-results/serdes-${{ matrix.python-env }} + # On PyPy, use bjdata's pure-Python path (JIT-friendly; avoids the + # numpy-ABI-fragile C extension). See autobahn #1849. + if [[ "${{ matrix.python-env }}" == pypy* ]]; then export PYBJDATA_NO_EXTENSION=1; fi + # Run tests and generate reports VENV_PYTHON=$(just --quiet _get-venv-python ${{ matrix.python-env }}) ${VENV_PYTHON} -m pytest -v \ diff --git a/docs/changelog.rst b/docs/changelog.rst index e190d82f5..831c87e69 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,6 +8,12 @@ Changelog 26.6.1 ------ +**WAMP Serialization** + +* The WAMP ``ubjson`` serializer is now backed by ``bjdata`` (Binary JData) instead of the unmaintained, wheel-less ``py-ubjson``. ``py-ubjson`` is removed as a dependency, which fixes installation via wheels only (``pip install --only-binary :all:``) (#1849) +* ⚠️ **Wire-level change to watch out for:** bjdata's octet-level encoding is NOT identical to the previous py-ubjson/UBJSON bytes (different integer markers, little-endian). The WAMP serializer id remains ``ubjson`` for transport negotiation. The ``wamp-proto`` UBJSON test vectors will be regenerated in a follow-up PR after this release; until then the ``ubjson`` serializer is excluded from the byte-vector conformance suite (round-trip and cross-serializer coverage retained) (#1849) +* ``bjdata`` is an OPTIONAL dependency (it pulls in numpy): the ``ubjson`` serializer now requires the ``autobahn[serialization]`` extra, keeping numpy out of a minimal install. On PyPy, set ``PYBJDATA_NO_EXTENSION=1`` to use the JIT-friendly pure-Python path (#1849) + **FlatBuffers** * Bump vendored FlatBuffers from v25.9.23 to v25.12.19, restoring the version-sync with zlmdb 26.6.1 (#1853) diff --git a/examples/serdes/tests/conftest.py b/examples/serdes/tests/conftest.py index 34832742e..1710a8020 100644 --- a/examples/serdes/tests/conftest.py +++ b/examples/serdes/tests/conftest.py @@ -12,6 +12,21 @@ from .utils import load_test_vector, get_serializer_ids +# The WAMP "ubjson" serializer is now backed by bjdata (autobahn #1849). Every +# test in this conformance suite is built on the wamp-proto canonical byte vectors +# (serialize-to / deserialize-from / cross-serializer all reference the stored +# ``bytes_hex``). bjdata's octet-level encoding intentionally differs from the +# legacy UBJSON bytes in those vectors, so "ubjson" is excluded from the byte-vector +# conformance suite until the wamp-proto UBJSON vectors are regenerated (a follow-up +# wamp-proto PR after the next autobahn-python release). bjdata round-trip +# correctness is covered by src/autobahn/wamp/test/test_wamp_serializer.py. +_VECTOR_EXCLUDED_SERIALIZERS = ("ubjson",) + + +def _conformance_serializer_ids(): + return [s for s in get_serializer_ids() if s not in _VECTOR_EXCLUDED_SERIALIZERS] + + @pytest.fixture(scope="session") def wamp_test_vector_publish(): """Load PUBLISH test vector""" @@ -48,12 +63,12 @@ def pytest_generate_tests(metafunc): This generates test parameters for serializer_id based on available serializers. """ if "serializer_id" in metafunc.fixturenames: - serializer_ids = get_serializer_ids() + serializer_ids = _conformance_serializer_ids() metafunc.parametrize("serializer_id", serializer_ids) if "serializer_pair" in metafunc.fixturenames: # Generate all unique pairs of serializers for cross-serializer tests - serializer_ids = get_serializer_ids() + serializer_ids = _conformance_serializer_ids() pairs = [] for i, ser1 in enumerate(serializer_ids): for ser2 in serializer_ids[i + 1 :]: diff --git a/justfile b/justfile index 3eb233c2a..a7dc58441 100644 --- a/justfile +++ b/justfile @@ -746,6 +746,10 @@ check-coverage-twisted venv="" use_nvx="": (install-tools venv) (install-dev ven VENV_PATH="{{ VENV_DIR }}/${VENV_NAME}" VENV_PYTHON=$(just --quiet _get-venv-python "${VENV_NAME}") + # On PyPy, use bjdata's pure-Python path (JIT-friendly; avoids the + # numpy-ABI-fragile C extension). See autobahn #1849. + if [[ "${VENV_NAME}" == pypy* ]]; then export PYBJDATA_NO_EXTENSION=1; fi + # Handle NVX configuration USE_NVX="{{ use_nvx }}" if [ "${USE_NVX}" = "1" ]; then @@ -1093,6 +1097,10 @@ test-twisted venv="" use_nvx="": (install-tools venv) (install-dev venv) VENV_PATH="{{ VENV_DIR }}/${VENV_NAME}" VENV_PYTHON=$(just --quiet _get-venv-python "${VENV_NAME}") + # On PyPy, use bjdata's pure-Python path (JIT-friendly; avoids the + # numpy-ABI-fragile C extension). See autobahn #1849. + if [[ "${VENV_NAME}" == pypy* ]]; then export PYBJDATA_NO_EXTENSION=1; fi + # Handle NVX configuration USE_NVX="{{ use_nvx }}" if [ "${USE_NVX}" = "1" ]; then @@ -1134,6 +1142,10 @@ test-asyncio venv="" use_nvx="": (install-tools venv) (install-dev venv) VENV_PATH="{{ VENV_DIR }}/${VENV_NAME}" VENV_PYTHON=$(just --quiet _get-venv-python "${VENV_NAME}") + # On PyPy, use bjdata's pure-Python path (JIT-friendly; avoids the + # numpy-ABI-fragile C extension). See autobahn #1849. + if [[ "${VENV_NAME}" == pypy* ]]; then export PYBJDATA_NO_EXTENSION=1; fi + # Handle NVX configuration USE_NVX="{{ use_nvx }}" if [ "${USE_NVX}" = "1" ]; then @@ -1162,6 +1174,10 @@ test-serdes venv="": (install-tools venv) (install-dev venv) fi VENV_PYTHON=$(just --quiet _get-venv-python "${VENV_NAME}") + # On PyPy, use bjdata's pure-Python path (JIT-friendly; avoids the + # numpy-ABI-fragile C extension). See autobahn #1849. + if [[ "${VENV_NAME}" == pypy* ]]; then export PYBJDATA_NO_EXTENSION=1; fi + echo "==> Running WAMP message serdes conformance tests in ${VENV_NAME}..." echo "==> Test vectors loaded from: wamp-proto/testsuite/" ${VENV_PYTHON} -m pytest -v \ diff --git a/pyproject.toml b/pyproject.toml index 0156c0204..c85918480 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,10 @@ dependencies = [ "u-msgpack-python>=2.1; platform_python_implementation != 'CPython'", "ujson>=4.0.2", # Binary wheels for both CPython and PyPy "cbor2>=5.2.0", # Binary wheels + pure Python fallback - "py-ubjson>=0.16.1", # Pure Python implementation (set PYUBJSON_NO_EXTENSION=1 to skip C extension build) + # NOTE: the WAMP "ubjson" serializer (bjdata) is an OPTIONAL extra, NOT a base + # dependency: bjdata pulls in numpy, which we don't want in a minimal install. + # See the [serialization] extra below. This also fixes #1849 (the old, wheel-less + # py-ubjson made `pip install --only-binary :all:` fail). # flatbuffers is vendored - no external dependency needed ] @@ -78,9 +81,14 @@ compress = [ # Users who need snappy: pip install python-snappy ] -# Backwards compatibility - serialization now included by default in base install -# All WAMP serializers (JSON, MessagePack, CBOR, UBJSON, Flatbuffers) are always available -serialization = [] +# Optional binary-JSON ("ubjson") WAMP serializer, backed by bjdata. +# JSON, MessagePack, CBOR and (vendored) FlatBuffers are always available in the +# base install; the "ubjson" serializer requires this extra because bjdata pulls +# in numpy, which we keep out of a minimal autobahn install. +# On PyPy, set PYBJDATA_NO_EXTENSION=1 to use the JIT-friendly pure-Python path. +serialization = [ + "bjdata>=0.6.0", +] # TLS transport encryption, WAMP-cryptosign end-to-end encryption and authentication encryption = [ diff --git a/src/autobahn/wamp/message.py b/src/autobahn/wamp/message.py index 80fd845ec..1e71ccd82 100644 --- a/src/autobahn/wamp/message.py +++ b/src/autobahn/wamp/message.py @@ -774,9 +774,9 @@ def _deserialize_payload(self, data_bytes, ser_id): # msgpack supports memoryview (zero-copy) return msgpack.unpackb(data_bytes) elif ser_id == "ubjson": - import ubjson + # The WAMP "ubjson" serializer is backed by bjdata (see serializer.py) + import bjdata as ubjson - # ubjson supports memoryview (zero-copy) return ubjson.loadb(data_bytes) else: # Fallback to CBOR for unknown serializers diff --git a/src/autobahn/wamp/serializer.py b/src/autobahn/wamp/serializer.py index 3ec31902e..3972b58c6 100644 --- a/src/autobahn/wamp/serializer.py +++ b/src/autobahn/wamp/serializer.py @@ -909,11 +909,20 @@ def __init__(self, batched=False): __all__.append("CBORSerializer") -# UBJSON serialization depends on the `py-ubjson` package being available -# https://pypi.python.org/pypi/py-ubjson -# https://github.com/Iotic-Labs/py-ubjson +# The WAMP "ubjson" serializer is backed by the `bjdata` package (Binary JData), +# a maintained, wheel-shipping successor of the (unmaintained, wheel-less) +# `py-ubjson` package. +# https://pypi.org/project/bjdata/ https://github.com/NeuroJSON/pybj +# +# NOTE: bjdata's on-the-wire (octet-level) encoding is NOT identical to the older +# py-ubjson/UBJSON bytes (e.g. unsigned-integer markers, little-endian). The WAMP +# serializer id remains "ubjson" for transport negotiation. See the changelog. +# +# `bjdata` is an OPTIONAL dependency (it pulls in numpy): install it via the +# `autobahn[serialization]` extra. On PyPy, set PYBJDATA_NO_EXTENSION=1 to use the +# JIT-friendly pure-Python path instead of the C extension. try: - import ubjson + import bjdata as ubjson except ImportError: pass else: diff --git a/src/autobahn/wamp/test/test_wamp_serializer.py b/src/autobahn/wamp/test/test_wamp_serializer.py index e6be642bc..5b6045f51 100644 --- a/src/autobahn/wamp/test/test_wamp_serializer.py +++ b/src/autobahn/wamp/test/test_wamp_serializer.py @@ -229,9 +229,11 @@ def create_serializers(decimal_support=False): _serializers.append(serializer.MsgPackSerializer()) _serializers.append(serializer.MsgPackSerializer(batched=True)) - # roundtrip error - _serializers.append(serializer.UBJSONSerializer()) - _serializers.append(serializer.UBJSONSerializer(batched=True)) + # UBJSON (bjdata-backed) is optional: only present when the + # `autobahn[serialization]` extra (bjdata) is installed. + if hasattr(serializer, "UBJSONSerializer"): + _serializers.append(serializer.UBJSONSerializer()) + _serializers.append(serializer.UBJSONSerializer(batched=True)) # FIXME: implement full FlatBuffers serializer for WAMP # WAMP-FlatBuffers currently only supports Python 3 From 3b70863b5db1cb19e4cf9b009ca7985edf9e62ca Mon Sep 17 00:00:00 2001 From: Tobias Oberstein Date: Sat, 13 Jun 2026 17:35:30 +0200 Subject: [PATCH 3/6] Docs: correct ubjson/bjdata packaging notes (sdist-only); drop py-ubjson refs (#1849) Follow-up wording/doc fixes for the py-ubjson -> bjdata switch: - README.md / docs/installation.rst: bjdata (like py-ubjson) is published sdist-only with no PyPI wheels and builds an optional C extension; document the optional `autobahn[serialization]` extra, PYBJDATA_NO_EXTENSION=1, and steer wheels-only/cross-arch users to cbor/msgpack. UBJSON is no longer described as "included by default". - docs/changelog.rst: reframe the #1849 fix as "the binary-JSON dependency is now optional, so a base autobahn install is wheel-clean" (bjdata itself ships no wheels). - serializer.py: drop the inaccurate "wheel-shipping" wording and state the sdist/C-extension reality. Note: This work was completed with AI assistance (Claude Code). --- README.md | 14 +++++++------- docs/changelog.rst | 5 +++-- docs/installation.rst | 2 +- src/autobahn/wamp/serializer.py | 15 +++++++++------ 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 8b3ddcc0f..bd0699948 100644 --- a/README.md +++ b/README.md @@ -343,14 +343,14 @@ masking) and UTF-8 validation. ### WAMP Serializers -**As of v25.11.1, all WAMP serializers are included by default** - batteries included! +**The JSON, MessagePack, CBOR and FlatBuffers serializers are included by default** - batteries included! UBJSON is available via the optional `autobahn[serialization]` extra. -Autobahn|Python now ships with full support for all WAMP serializers out-of-the-box: +Autobahn|Python ships with the following WAMP serializers: - **JSON** (standard library) - always available - **MessagePack** - high-performance binary serialization - **CBOR** - IETF standard binary serialization (RFC 8949) -- **UBJSON** - Universal Binary JSON +- **UBJSON** - Universal Binary JSON *(optional: `pip install autobahn[serialization]`)* - **Flatbuffers** - Google's zero-copy serialization (vendored) #### Architecture & Performance @@ -363,12 +363,12 @@ The serializer dependencies are optimized for both **CPython** and **PyPy**: | **msgpack** | Binary wheel (C extension) | u-msgpack-python (pure Python) | Native + Universal | PyPy JIT makes pure Python faster than C | | **ujson** | Binary wheel | Binary wheel | Native | Available for both implementations | | **cbor2** | Binary wheel | Pure Python fallback | Native + Universal | Binary wheels + py3-none-any | -| **ubjson** | Pure Python | Pure Python | Source | Set `PYUBJSON_NO_EXTENSION=1` to skip C build | +| **ubjson** *(optional)* | C ext (from sdist) | Pure Python | Source only — no wheels | Optional `autobahn[serialization]` extra (bjdata, pulls numpy); set `PYBJDATA_NO_EXTENSION=1` to skip the C build (recommended on PyPy) | | **flatbuffers** | Vendored | Vendored | Included | Always available, no external dependency | **Key Design Principles:** -1. **Batteries Included**: All serializers available without extra install steps +1. **Batteries Included**: Core serializers (JSON, MessagePack, CBOR, FlatBuffers) available without extra install steps; UBJSON via the optional `autobahn[serialization]` extra 2. **PyPy Optimization**: Pure Python implementations leverage PyPy's JIT for superior performance 3. **Binary Wheels**: Native wheels for all major platforms (Linux x86_64/ARM64, macOS x86_64/ARM64, Windows x86_64) 4. **Zero System Pollution**: All dependencies install cleanly via wheels or pure Python @@ -429,7 +429,7 @@ All dependencies follow these design principles: ### WAMP Serializers (Batteries Included) -All serializers are now **included by default** in the base installation: +All serializers **except UBJSON** are included by default in the base installation; UBJSON is an optional extra (`pip install autobahn[serialization]`): | Serializer | Purpose | CPython | PyPy | Wheel Coverage | Notes | |------------|---------|---------|------|----------------|-------| @@ -437,7 +437,7 @@ All serializers are now **included by default** in the base installation: | **msgpack** | MessagePack serialization | msgpack (binary wheel) | u-msgpack-python (pure Python) | ✅ Excellent | 50+ wheels for CPython; PyPy JIT optimized | | **ujson** | Fast JSON (optional) | Binary wheel | Binary wheel | ✅ Excellent | 30+ wheels; both implementations | | **cbor2** | CBOR serialization (RFC 8949) | Binary wheel | Pure Python fallback | ✅ Excellent | 30+ binary wheels + universal fallback | -| **py-ubjson** | UBJSON serialization | Pure Python | Pure Python | ✅ Good | Optional C extension (can skip with `PYUBJSON_NO_EXTENSION=1`) | +| **bjdata** | UBJSON serialization *(optional)* | C ext (from sdist) | Pure Python | ⚠️ sdist only — no wheels | `autobahn[serialization]` extra; pulls numpy; set `PYBJDATA_NO_EXTENSION=1` to skip the C build (recommended on PyPy). For wheels-only/cross-arch installs prefer cbor/msgpack | | **flatbuffers** | Google Flatbuffers | **Vendored** | **Vendored** | ✅ Perfect | Included in our wheel, zero external dependency | ### Optional: Twisted Framework diff --git a/docs/changelog.rst b/docs/changelog.rst index 831c87e69..0a8648dc7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,9 +10,10 @@ Changelog **WAMP Serialization** -* The WAMP ``ubjson`` serializer is now backed by ``bjdata`` (Binary JData) instead of the unmaintained, wheel-less ``py-ubjson``. ``py-ubjson`` is removed as a dependency, which fixes installation via wheels only (``pip install --only-binary :all:``) (#1849) +* ``py-ubjson`` (unmaintained, sdist-only) is no longer an unconditional dependency. A base ``pip install autobahn`` — and the wheels-only / cross-arch case from #1849 (``pip download --only-binary :all: --platform ...``) — now resolves entirely from binary wheels (#1849) +* The WAMP ``ubjson`` serializer is now backed by the maintained ``bjdata`` (Binary JData) package, provided as the OPTIONAL ``autobahn[serialization]`` extra (it also pulls in numpy), keeping both out of a minimal install (#1849) +* ``bjdata`` is itself published sdist-only (no PyPI wheels) and builds an optional C extension at install time. On CPython that gives a native speedup; on PyPy set ``PYBJDATA_NO_EXTENSION=1`` for the JIT-friendly pure-Python path (this also installs without a C compiler / numpy headers). For wheels-only, cross-arch or no-compiler deployments, prefer the always-available ``cbor`` or ``msgpack`` binary serializers (#1849) * ⚠️ **Wire-level change to watch out for:** bjdata's octet-level encoding is NOT identical to the previous py-ubjson/UBJSON bytes (different integer markers, little-endian). The WAMP serializer id remains ``ubjson`` for transport negotiation. The ``wamp-proto`` UBJSON test vectors will be regenerated in a follow-up PR after this release; until then the ``ubjson`` serializer is excluded from the byte-vector conformance suite (round-trip and cross-serializer coverage retained) (#1849) -* ``bjdata`` is an OPTIONAL dependency (it pulls in numpy): the ``ubjson`` serializer now requires the ``autobahn[serialization]`` extra, keeping numpy out of a minimal install. On PyPy, set ``PYBJDATA_NO_EXTENSION=1`` to use the JIT-friendly pure-Python path (#1849) **FlatBuffers** diff --git a/docs/installation.rst b/docs/installation.rst index 3b1659123..dbcfae319 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -152,7 +152,7 @@ Install Variants * - ``scram`` - Install WAMP-SCRAM authentication packages. * - ``serialization`` - - Backwards-compatible no-op; WAMP serializers are included by default. + - Install ``bjdata`` to enable the optional UBJSON WAMP serializer (pulls in numpy; published sdist-only, builds a C extension - set ``PYBJDATA_NO_EXTENSION=1`` to skip it / on PyPy). JSON, MessagePack, CBOR and FlatBuffers are always available without this extra. * - ``nvx`` - Backwards-compatible no-op; NVX acceleration is included in binary wheels where supported. * - ``all`` diff --git a/src/autobahn/wamp/serializer.py b/src/autobahn/wamp/serializer.py index 3972b58c6..3aad60598 100644 --- a/src/autobahn/wamp/serializer.py +++ b/src/autobahn/wamp/serializer.py @@ -910,17 +910,20 @@ def __init__(self, batched=False): # The WAMP "ubjson" serializer is backed by the `bjdata` package (Binary JData), -# a maintained, wheel-shipping successor of the (unmaintained, wheel-less) -# `py-ubjson` package. +# a maintained successor of the (unmaintained) `py-ubjson` package. # https://pypi.org/project/bjdata/ https://github.com/NeuroJSON/pybj # +# `bjdata` is an OPTIONAL dependency: install it via the `autobahn[serialization]` +# extra. Like py-ubjson, bjdata is published as an sdist only (no binary wheels on +# PyPI) and builds an optional C extension at install time (and pulls in numpy). +# On CPython the C extension gives a native speedup; on PyPy the JIT-compiled +# pure-Python path is faster, so set PYBJDATA_NO_EXTENSION=1 there (this also lets +# the package install without a C compiler / numpy headers). For wheels-only or +# cross-arch deployments, prefer the always-available `cbor`/`msgpack` serializers. +# # NOTE: bjdata's on-the-wire (octet-level) encoding is NOT identical to the older # py-ubjson/UBJSON bytes (e.g. unsigned-integer markers, little-endian). The WAMP # serializer id remains "ubjson" for transport negotiation. See the changelog. -# -# `bjdata` is an OPTIONAL dependency (it pulls in numpy): install it via the -# `autobahn[serialization]` extra. On PyPy, set PYBJDATA_NO_EXTENSION=1 to use the -# JIT-friendly pure-Python path instead of the C extension. try: import bjdata as ubjson except ImportError: From e034744fc7f2b7b3eb1074c679bfa4a0aa8c1232 Mon Sep 17 00:00:00 2001 From: Tobias Oberstein Date: Sat, 13 Jun 2026 19:44:37 +0200 Subject: [PATCH 4/6] Make bjdata (ubjson serializer) CPython-only; PyPy can't build it (#1849) bjdata's sdist build pulls numpy as an unconditional build dependency (`oldest-supported-numpy` in build-system.requires, plus a top-level `from numpy import get_include` in setup.py) even though it skips its C extension on PyPy. On PyPy that numpy pin (1.23.2) has no wheel and fails to compile, so `pip install autobahn[serialization]` cannot be installed on PyPy at all -- this broke the wstest CI jobs. Reported upstream as NeuroJSON/pybj#6. - pyproject.toml: restrict bjdata to CPython (`bjdata>=0.6.0; platform_python_implementation == 'CPython'`). On PyPy the UBJSON serializer is unavailable (graceful try/except ImportError); use cbor/msgpack instead. - Remove the now-moot PyPy PYBJDATA_NO_EXTENSION lines from justfile and CI: the flag can't help, because the numpy build dependency is installed under PEP 517 build isolation before bjdata's setup.py is ever run. - Update README, docs/installation.rst, docs/changelog.rst and the serializer.py comment to document bjdata as CPython-only, linking NeuroJSON/pybj#6. Note: This work was completed with AI assistance (Claude Code). --- .github/workflows/main.yml | 4 ---- README.md | 4 ++-- docs/changelog.rst | 2 +- docs/installation.rst | 2 +- justfile | 16 ---------------- pyproject.toml | 5 ++++- src/autobahn/wamp/serializer.py | 15 ++++++++------- 7 files changed, 16 insertions(+), 32 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 75e2b45c4..ce1a7f26f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -190,10 +190,6 @@ jobs: # Create output directory for test results mkdir -p test-results/serdes-${{ matrix.python-env }} - # On PyPy, use bjdata's pure-Python path (JIT-friendly; avoids the - # numpy-ABI-fragile C extension). See autobahn #1849. - if [[ "${{ matrix.python-env }}" == pypy* ]]; then export PYBJDATA_NO_EXTENSION=1; fi - # Run tests and generate reports VENV_PYTHON=$(just --quiet _get-venv-python ${{ matrix.python-env }}) ${VENV_PYTHON} -m pytest -v \ diff --git a/README.md b/README.md index bd0699948..433fe4416 100644 --- a/README.md +++ b/README.md @@ -363,7 +363,7 @@ The serializer dependencies are optimized for both **CPython** and **PyPy**: | **msgpack** | Binary wheel (C extension) | u-msgpack-python (pure Python) | Native + Universal | PyPy JIT makes pure Python faster than C | | **ujson** | Binary wheel | Binary wheel | Native | Available for both implementations | | **cbor2** | Binary wheel | Pure Python fallback | Native + Universal | Binary wheels + py3-none-any | -| **ubjson** *(optional)* | C ext (from sdist) | Pure Python | Source only — no wheels | Optional `autobahn[serialization]` extra (bjdata, pulls numpy); set `PYBJDATA_NO_EXTENSION=1` to skip the C build (recommended on PyPy) | +| **ubjson** *(optional)* | C ext (from sdist) | ❌ n/a (CPython only) | Source only — no wheels | Optional `autobahn[serialization]` extra (bjdata, pulls numpy). CPython-only: bjdata can't install on PyPy ([NeuroJSON/pybj#6](https://github.com/NeuroJSON/pybj/issues/6)); on PyPy use cbor/msgpack. On CPython without a compiler set `PYBJDATA_NO_EXTENSION=1` | | **flatbuffers** | Vendored | Vendored | Included | Always available, no external dependency | **Key Design Principles:** @@ -437,7 +437,7 @@ All serializers **except UBJSON** are included by default in the base installati | **msgpack** | MessagePack serialization | msgpack (binary wheel) | u-msgpack-python (pure Python) | ✅ Excellent | 50+ wheels for CPython; PyPy JIT optimized | | **ujson** | Fast JSON (optional) | Binary wheel | Binary wheel | ✅ Excellent | 30+ wheels; both implementations | | **cbor2** | CBOR serialization (RFC 8949) | Binary wheel | Pure Python fallback | ✅ Excellent | 30+ binary wheels + universal fallback | -| **bjdata** | UBJSON serialization *(optional)* | C ext (from sdist) | Pure Python | ⚠️ sdist only — no wheels | `autobahn[serialization]` extra; pulls numpy; set `PYBJDATA_NO_EXTENSION=1` to skip the C build (recommended on PyPy). For wheels-only/cross-arch installs prefer cbor/msgpack | +| **bjdata** | UBJSON serialization *(optional)* | C ext (from sdist) | ❌ n/a (CPython only) | ⚠️ sdist only — no wheels | `autobahn[serialization]` extra; pulls numpy. CPython-only: can't install on PyPy ([NeuroJSON/pybj#6](https://github.com/NeuroJSON/pybj/issues/6)). On CPython without a compiler set `PYBJDATA_NO_EXTENSION=1`. For wheels-only/cross-arch/PyPy installs prefer cbor/msgpack | | **flatbuffers** | Google Flatbuffers | **Vendored** | **Vendored** | ✅ Perfect | Included in our wheel, zero external dependency | ### Optional: Twisted Framework diff --git a/docs/changelog.rst b/docs/changelog.rst index 0a8648dc7..c9ea74e3f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,7 +12,7 @@ Changelog * ``py-ubjson`` (unmaintained, sdist-only) is no longer an unconditional dependency. A base ``pip install autobahn`` — and the wheels-only / cross-arch case from #1849 (``pip download --only-binary :all: --platform ...``) — now resolves entirely from binary wheels (#1849) * The WAMP ``ubjson`` serializer is now backed by the maintained ``bjdata`` (Binary JData) package, provided as the OPTIONAL ``autobahn[serialization]`` extra (it also pulls in numpy), keeping both out of a minimal install (#1849) -* ``bjdata`` is itself published sdist-only (no PyPI wheels) and builds an optional C extension at install time. On CPython that gives a native speedup; on PyPy set ``PYBJDATA_NO_EXTENSION=1`` for the JIT-friendly pure-Python path (this also installs without a C compiler / numpy headers). For wheels-only, cross-arch or no-compiler deployments, prefer the always-available ``cbor`` or ``msgpack`` binary serializers (#1849) +* ``bjdata`` is published sdist-only (no PyPI wheels) and is currently **CPython-only**: on PyPy its sdist build pulls an unbuildable numpy (upstream ``NeuroJSON/pybj#6``), so the ``ubjson`` serializer is unavailable on PyPy - use ``cbor``/``msgpack`` there. On CPython without a compiler, set ``PYBJDATA_NO_EXTENSION=1`` for a pure-Python build. For wheels-only or cross-arch deployments, also prefer ``cbor``/``msgpack`` (#1849) * ⚠️ **Wire-level change to watch out for:** bjdata's octet-level encoding is NOT identical to the previous py-ubjson/UBJSON bytes (different integer markers, little-endian). The WAMP serializer id remains ``ubjson`` for transport negotiation. The ``wamp-proto`` UBJSON test vectors will be regenerated in a follow-up PR after this release; until then the ``ubjson`` serializer is excluded from the byte-vector conformance suite (round-trip and cross-serializer coverage retained) (#1849) **FlatBuffers** diff --git a/docs/installation.rst b/docs/installation.rst index dbcfae319..1488eef50 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -152,7 +152,7 @@ Install Variants * - ``scram`` - Install WAMP-SCRAM authentication packages. * - ``serialization`` - - Install ``bjdata`` to enable the optional UBJSON WAMP serializer (pulls in numpy; published sdist-only, builds a C extension - set ``PYBJDATA_NO_EXTENSION=1`` to skip it / on PyPy). JSON, MessagePack, CBOR and FlatBuffers are always available without this extra. + - Install ``bjdata`` to enable the optional UBJSON WAMP serializer. CPython-only - it cannot install on PyPy (upstream ``NeuroJSON/pybj#6``); on CPython without a compiler set ``PYBJDATA_NO_EXTENSION=1`` for a pure-Python build. JSON, MessagePack, CBOR and FlatBuffers are always available without this extra (use them on PyPy). * - ``nvx`` - Backwards-compatible no-op; NVX acceleration is included in binary wheels where supported. * - ``all`` diff --git a/justfile b/justfile index a7dc58441..3eb233c2a 100644 --- a/justfile +++ b/justfile @@ -746,10 +746,6 @@ check-coverage-twisted venv="" use_nvx="": (install-tools venv) (install-dev ven VENV_PATH="{{ VENV_DIR }}/${VENV_NAME}" VENV_PYTHON=$(just --quiet _get-venv-python "${VENV_NAME}") - # On PyPy, use bjdata's pure-Python path (JIT-friendly; avoids the - # numpy-ABI-fragile C extension). See autobahn #1849. - if [[ "${VENV_NAME}" == pypy* ]]; then export PYBJDATA_NO_EXTENSION=1; fi - # Handle NVX configuration USE_NVX="{{ use_nvx }}" if [ "${USE_NVX}" = "1" ]; then @@ -1097,10 +1093,6 @@ test-twisted venv="" use_nvx="": (install-tools venv) (install-dev venv) VENV_PATH="{{ VENV_DIR }}/${VENV_NAME}" VENV_PYTHON=$(just --quiet _get-venv-python "${VENV_NAME}") - # On PyPy, use bjdata's pure-Python path (JIT-friendly; avoids the - # numpy-ABI-fragile C extension). See autobahn #1849. - if [[ "${VENV_NAME}" == pypy* ]]; then export PYBJDATA_NO_EXTENSION=1; fi - # Handle NVX configuration USE_NVX="{{ use_nvx }}" if [ "${USE_NVX}" = "1" ]; then @@ -1142,10 +1134,6 @@ test-asyncio venv="" use_nvx="": (install-tools venv) (install-dev venv) VENV_PATH="{{ VENV_DIR }}/${VENV_NAME}" VENV_PYTHON=$(just --quiet _get-venv-python "${VENV_NAME}") - # On PyPy, use bjdata's pure-Python path (JIT-friendly; avoids the - # numpy-ABI-fragile C extension). See autobahn #1849. - if [[ "${VENV_NAME}" == pypy* ]]; then export PYBJDATA_NO_EXTENSION=1; fi - # Handle NVX configuration USE_NVX="{{ use_nvx }}" if [ "${USE_NVX}" = "1" ]; then @@ -1174,10 +1162,6 @@ test-serdes venv="": (install-tools venv) (install-dev venv) fi VENV_PYTHON=$(just --quiet _get-venv-python "${VENV_NAME}") - # On PyPy, use bjdata's pure-Python path (JIT-friendly; avoids the - # numpy-ABI-fragile C extension). See autobahn #1849. - if [[ "${VENV_NAME}" == pypy* ]]; then export PYBJDATA_NO_EXTENSION=1; fi - echo "==> Running WAMP message serdes conformance tests in ${VENV_NAME}..." echo "==> Test vectors loaded from: wamp-proto/testsuite/" ${VENV_PYTHON} -m pytest -v \ diff --git a/pyproject.toml b/pyproject.toml index c85918480..07ce47f77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,7 +87,10 @@ compress = [ # in numpy, which we keep out of a minimal autobahn install. # On PyPy, set PYBJDATA_NO_EXTENSION=1 to use the JIT-friendly pure-Python path. serialization = [ - "bjdata>=0.6.0", + # CPython-only: on PyPy, bjdata's sdist build pulls an unbuildable numpy and + # cannot be pip-installed (upstream NeuroJSON/pybj#6). On PyPy the UBJSON + # serializer is therefore unavailable - use cbor/msgpack instead. + "bjdata>=0.6.0; platform_python_implementation == 'CPython'", ] # TLS transport encryption, WAMP-cryptosign end-to-end encryption and authentication diff --git a/src/autobahn/wamp/serializer.py b/src/autobahn/wamp/serializer.py index 3aad60598..084accfd4 100644 --- a/src/autobahn/wamp/serializer.py +++ b/src/autobahn/wamp/serializer.py @@ -913,13 +913,14 @@ def __init__(self, batched=False): # a maintained successor of the (unmaintained) `py-ubjson` package. # https://pypi.org/project/bjdata/ https://github.com/NeuroJSON/pybj # -# `bjdata` is an OPTIONAL dependency: install it via the `autobahn[serialization]` -# extra. Like py-ubjson, bjdata is published as an sdist only (no binary wheels on -# PyPI) and builds an optional C extension at install time (and pulls in numpy). -# On CPython the C extension gives a native speedup; on PyPy the JIT-compiled -# pure-Python path is faster, so set PYBJDATA_NO_EXTENSION=1 there (this also lets -# the package install without a C compiler / numpy headers). For wheels-only or -# cross-arch deployments, prefer the always-available `cbor`/`msgpack` serializers. +# `bjdata` is an OPTIONAL, CPython-only dependency: install it via the +# `autobahn[serialization]` extra. It is published as an sdist only (no PyPI +# wheels) and builds a C extension at install time (and pulls in numpy); on +# CPython without a compiler, set PYBJDATA_NO_EXTENSION=1 for a pure-Python build. +# bjdata cannot currently be installed on PyPy (its sdist build pulls an +# unbuildable numpy - see NeuroJSON/pybj#6), so the UBJSON serializer is +# unavailable on PyPy; use the always-available `cbor`/`msgpack` serializers there +# (and generally for wheels-only / cross-arch deployments). # # NOTE: bjdata's on-the-wire (octet-level) encoding is NOT identical to the older # py-ubjson/UBJSON bytes (e.g. unsigned-integer markers, little-endian). The WAMP From fbf52f28af361ee1f2fcb04b3123d06f36958d5a Mon Sep 17 00:00:00 2001 From: Tobias Oberstein Date: Sun, 14 Jun 2026 21:02:30 +0200 Subject: [PATCH 5/6] test: bump .proto submodule to wamp-proto master (#557 merged) wamp-proto#557 (the canonical WAMP conformance test-vector suite) was squash-merged to wamp-proto master as d8389b8. Bump the .proto submodule to pick it up. This brings in the BJData (bjdata) UBJSON byte vectors, so the serdes conformance suite can be re-enabled for the "ubjson" serializer (follow-up commit). The serdes suite still passes (446) with "ubjson" currently excluded. Refs: #1849. Note: This work was completed with AI assistance (Claude Code). --- .proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.proto b/.proto index 2fcf8091d..d8389b83b 160000 --- a/.proto +++ b/.proto @@ -1 +1 @@ -Subproject commit 2fcf8091dea4c68870ca05b244b7ce316d336b51 +Subproject commit d8389b83b18da834f20fdae6ea5ec57b0c4da4e5 From 76f8d6a0f58be8e5c3c55ea191c20b86447c8e35 Mon Sep 17 00:00:00 2001 From: Tobias Oberstein Date: Sun, 14 Jun 2026 21:18:57 +0200 Subject: [PATCH 6/6] test: re-enable ubjson in serdes conformance suite against bjdata vectors With the wamp-proto vectors now carrying both the legacy py-ubjson and the new bjdata/BJData byte representations for "ubjson" (.proto bumped in the previous commit), re-enable the serializer in the byte-vector conformance suite: - conftest: _VECTOR_EXCLUDED_SERIALIZERS is now empty (no serializer excluded). - utils.require_decodable(): new helper implementing the "at least one must match" rule for the deserialize direction. A serializer's variant list may hold encodings produced by other implementations/versions that this backend cannot decode, or decodes to a valid-but-wrong message (py-ubjson bytes read as little-endian bjdata). It keeps variants whose decode re-serializes to a canonical variant (cosmetic variants + this backend's own encoding), falling back to decode-without-error for serializers whose re-serialization is not byte-canonical (flatbuffers). At least one variant must qualify. - The 25 per-message test files use require_decodable() at their decode sites (deserialize-from-bytes, roundtrip, cross-serializer source) instead of iterating/indexing the raw variant list. Serdes suite: 529 passed (up from 446 with ubjson excluded). On PyPy, bjdata is unavailable so the ubjson serializer is simply not registered and these tests do not run. Refs: #1849. Note: This work was completed with AI assistance (Claude Code). --- examples/serdes/tests/conftest.py | 17 +++--- examples/serdes/tests/test_abort.py | 5 +- examples/serdes/tests/test_authenticate.py | 5 +- examples/serdes/tests/test_call.py | 5 +- examples/serdes/tests/test_cancel.py | 5 +- examples/serdes/tests/test_challenge.py | 5 +- examples/serdes/tests/test_error.py | 5 +- examples/serdes/tests/test_event.py | 5 +- examples/serdes/tests/test_eventreceived.py | 5 +- examples/serdes/tests/test_goodbye.py | 5 +- examples/serdes/tests/test_hello.py | 5 +- examples/serdes/tests/test_interrupt.py | 5 +- examples/serdes/tests/test_invocation.py | 5 +- examples/serdes/tests/test_publish.py | 5 +- examples/serdes/tests/test_published.py | 5 +- examples/serdes/tests/test_register.py | 5 +- examples/serdes/tests/test_registered.py | 5 +- examples/serdes/tests/test_result.py | 5 +- examples/serdes/tests/test_subscribe.py | 5 +- examples/serdes/tests/test_subscribed.py | 5 +- examples/serdes/tests/test_unregister.py | 5 +- examples/serdes/tests/test_unregistered.py | 5 +- examples/serdes/tests/test_unsubscribe.py | 5 +- examples/serdes/tests/test_unsubscribed.py | 5 +- examples/serdes/tests/test_welcome.py | 5 +- examples/serdes/tests/test_yield.py | 5 +- examples/serdes/tests/utils.py | 57 +++++++++++++++++++++ 27 files changed, 140 insertions(+), 59 deletions(-) diff --git a/examples/serdes/tests/conftest.py b/examples/serdes/tests/conftest.py index 1710a8020..0c3057ba0 100644 --- a/examples/serdes/tests/conftest.py +++ b/examples/serdes/tests/conftest.py @@ -12,15 +12,14 @@ from .utils import load_test_vector, get_serializer_ids -# The WAMP "ubjson" serializer is now backed by bjdata (autobahn #1849). Every -# test in this conformance suite is built on the wamp-proto canonical byte vectors -# (serialize-to / deserialize-from / cross-serializer all reference the stored -# ``bytes_hex``). bjdata's octet-level encoding intentionally differs from the -# legacy UBJSON bytes in those vectors, so "ubjson" is excluded from the byte-vector -# conformance suite until the wamp-proto UBJSON vectors are regenerated (a follow-up -# wamp-proto PR after the next autobahn-python release). bjdata round-trip -# correctness is covered by src/autobahn/wamp/test/test_wamp_serializer.py. -_VECTOR_EXCLUDED_SERIALIZERS = ("ubjson",) +# The WAMP "ubjson" serializer is backed by bjdata (autobahn #1849). The +# wamp-proto canonical vectors (via the .proto submodule) now carry BOTH the +# legacy py-ubjson bytes and the new bjdata/BJData bytes for "ubjson" (each +# tagged with a "note"), matched with "at least one must match" semantics. The +# deserialize-direction tests use utils.require_decodable() to skip byte +# variants this backend cannot decode (the two encodings are not mutually +# decodable), so no serializer needs to be excluded from the byte-vector suite. +_VECTOR_EXCLUDED_SERIALIZERS = () def _conformance_serializer_ids(): diff --git a/examples/serdes/tests/test_abort.py b/examples/serdes/tests/test_abort.py index cab25cd02..7976f0762 100644 --- a/examples/serdes/tests/test_abort.py +++ b/examples/serdes/tests/test_abort.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -62,7 +63,7 @@ def test_abort_deserialize_from_bytes(serializer_id, abort_samples, create_seria byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -129,7 +130,7 @@ def test_abort_roundtrip(serializer_id, abort_samples, create_serializer): if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_authenticate.py b/examples/serdes/tests/test_authenticate.py index 097b8dfc5..97298b068 100644 --- a/examples/serdes/tests/test_authenticate.py +++ b/examples/serdes/tests/test_authenticate.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -64,7 +65,7 @@ def test_authenticate_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -133,7 +134,7 @@ def test_authenticate_roundtrip(serializer_id, authenticate_samples, create_seri if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_call.py b/examples/serdes/tests/test_call.py index d33f1ec48..5f0814981 100644 --- a/examples/serdes/tests/test_call.py +++ b/examples/serdes/tests/test_call.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -62,7 +63,7 @@ def test_call_deserialize_from_bytes(serializer_id, call_samples, create_seriali byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -149,7 +150,7 @@ def test_call_roundtrip(serializer_id, call_samples, create_serializer): if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_cancel.py b/examples/serdes/tests/test_cancel.py index 39a8dfe29..a3267a4fe 100644 --- a/examples/serdes/tests/test_cancel.py +++ b/examples/serdes/tests/test_cancel.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -64,7 +65,7 @@ def test_cancel_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -131,7 +132,7 @@ def test_cancel_roundtrip(serializer_id, cancel_samples, create_serializer): if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_challenge.py b/examples/serdes/tests/test_challenge.py index a642d6bb5..0492f37f3 100644 --- a/examples/serdes/tests/test_challenge.py +++ b/examples/serdes/tests/test_challenge.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -64,7 +65,7 @@ def test_challenge_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -133,7 +134,7 @@ def test_challenge_roundtrip(serializer_id, challenge_samples, create_serializer if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_error.py b/examples/serdes/tests/test_error.py index 33a60905f..fe9475a83 100644 --- a/examples/serdes/tests/test_error.py +++ b/examples/serdes/tests/test_error.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -62,7 +63,7 @@ def test_error_deserialize_from_bytes(serializer_id, error_samples, create_seria byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -135,7 +136,7 @@ def test_error_roundtrip(serializer_id, error_samples, create_serializer): if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_event.py b/examples/serdes/tests/test_event.py index 1294f4aee..fa9f9a139 100644 --- a/examples/serdes/tests/test_event.py +++ b/examples/serdes/tests/test_event.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, validate_message_with_code, @@ -73,7 +74,7 @@ def test_event_deserialize_from_bytes(serializer_id, event_samples, create_seria byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -223,7 +224,7 @@ def test_event_cross_serializer_preservation(serializer_pair, event_samples): pytest.skip(f"No byte variants for {ser1_id} or {ser2_id}") # Take the first canonical byte representation from ser1 - variant1 = ser1_variants[0] + variant1 = require_decodable(ser1, ser1_variants)[0] if "bytes_hex" in variant1: bytes1 = bytes_from_hex(variant1["bytes_hex"]) elif "bytes" in variant1: diff --git a/examples/serdes/tests/test_eventreceived.py b/examples/serdes/tests/test_eventreceived.py index 684a01ed4..8e0f27d46 100644 --- a/examples/serdes/tests/test_eventreceived.py +++ b/examples/serdes/tests/test_eventreceived.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -64,7 +65,7 @@ def test_eventreceived_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -135,7 +136,7 @@ def test_eventreceived_roundtrip( if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_goodbye.py b/examples/serdes/tests/test_goodbye.py index 65cb9faad..70945b5a3 100644 --- a/examples/serdes/tests/test_goodbye.py +++ b/examples/serdes/tests/test_goodbye.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -64,7 +65,7 @@ def test_goodbye_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -131,7 +132,7 @@ def test_goodbye_roundtrip(serializer_id, goodbye_samples, create_serializer): if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_hello.py b/examples/serdes/tests/test_hello.py index 99d52886d..e44daf132 100644 --- a/examples/serdes/tests/test_hello.py +++ b/examples/serdes/tests/test_hello.py @@ -20,6 +20,7 @@ ) from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -95,7 +96,7 @@ def test_hello_deserialize_from_bytes(serializer_id, hello_samples, create_seria byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -165,7 +166,7 @@ def test_hello_roundtrip(serializer_id, hello_samples, create_serializer): if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_interrupt.py b/examples/serdes/tests/test_interrupt.py index 751dc2981..600ee5386 100644 --- a/examples/serdes/tests/test_interrupt.py +++ b/examples/serdes/tests/test_interrupt.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -64,7 +65,7 @@ def test_interrupt_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -133,7 +134,7 @@ def test_interrupt_roundtrip(serializer_id, interrupt_samples, create_serializer if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_invocation.py b/examples/serdes/tests/test_invocation.py index 712d4b42e..81d2b8e65 100644 --- a/examples/serdes/tests/test_invocation.py +++ b/examples/serdes/tests/test_invocation.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -64,7 +65,7 @@ def test_invocation_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -136,7 +137,7 @@ def test_invocation_roundtrip(serializer_id, invocation_samples, create_serializ if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_publish.py b/examples/serdes/tests/test_publish.py index 7c75b2adf..088c64395 100644 --- a/examples/serdes/tests/test_publish.py +++ b/examples/serdes/tests/test_publish.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, validate_message_with_code, @@ -75,7 +76,7 @@ def test_publish_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -224,7 +225,7 @@ def test_publish_cross_serializer_preservation(serializer_pair, publish_samples) pytest.skip(f"No byte variants for {ser1_id} or {ser2_id}") # Take the first canonical byte representation from ser1 - variant1 = ser1_variants[0] + variant1 = require_decodable(ser1, ser1_variants)[0] if "bytes_hex" in variant1: bytes1 = bytes_from_hex(variant1["bytes_hex"]) elif "bytes" in variant1: diff --git a/examples/serdes/tests/test_published.py b/examples/serdes/tests/test_published.py index 10629a428..b9af8ab0d 100644 --- a/examples/serdes/tests/test_published.py +++ b/examples/serdes/tests/test_published.py @@ -15,6 +15,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -65,7 +66,7 @@ def test_published_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -137,7 +138,7 @@ def test_published_roundtrip(serializer_id, published_samples, create_serializer if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_register.py b/examples/serdes/tests/test_register.py index 897229456..e372593ac 100644 --- a/examples/serdes/tests/test_register.py +++ b/examples/serdes/tests/test_register.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -64,7 +65,7 @@ def test_register_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -134,7 +135,7 @@ def test_register_roundtrip(serializer_id, register_samples, create_serializer): if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_registered.py b/examples/serdes/tests/test_registered.py index 189448cfd..4f81b886f 100644 --- a/examples/serdes/tests/test_registered.py +++ b/examples/serdes/tests/test_registered.py @@ -15,6 +15,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -65,7 +66,7 @@ def test_registered_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -137,7 +138,7 @@ def test_registered_roundtrip(serializer_id, registered_samples, create_serializ if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_result.py b/examples/serdes/tests/test_result.py index 3d2d99cff..571de7dfc 100644 --- a/examples/serdes/tests/test_result.py +++ b/examples/serdes/tests/test_result.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -64,7 +65,7 @@ def test_result_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -149,7 +150,7 @@ def test_result_roundtrip(serializer_id, result_samples, create_serializer): if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_subscribe.py b/examples/serdes/tests/test_subscribe.py index a377fba89..25b15fb5f 100644 --- a/examples/serdes/tests/test_subscribe.py +++ b/examples/serdes/tests/test_subscribe.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, validate_message_with_code, @@ -74,7 +75,7 @@ def test_subscribe_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -161,7 +162,7 @@ def test_subscribe_roundtrip(serializer_id, subscribe_samples, create_serializer if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_subscribed.py b/examples/serdes/tests/test_subscribed.py index 463130d87..a77401c3d 100644 --- a/examples/serdes/tests/test_subscribed.py +++ b/examples/serdes/tests/test_subscribed.py @@ -15,6 +15,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -65,7 +66,7 @@ def test_subscribed_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -137,7 +138,7 @@ def test_subscribed_roundtrip(serializer_id, subscribed_samples, create_serializ if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_unregister.py b/examples/serdes/tests/test_unregister.py index 302aaefea..64f841780 100644 --- a/examples/serdes/tests/test_unregister.py +++ b/examples/serdes/tests/test_unregister.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -64,7 +65,7 @@ def test_unregister_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -136,7 +137,7 @@ def test_unregister_roundtrip(serializer_id, unregister_samples, create_serializ if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_unregistered.py b/examples/serdes/tests/test_unregistered.py index 5350d97d2..ed4b3920d 100644 --- a/examples/serdes/tests/test_unregistered.py +++ b/examples/serdes/tests/test_unregistered.py @@ -15,6 +15,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -65,7 +66,7 @@ def test_unregistered_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -134,7 +135,7 @@ def test_unregistered_roundtrip(serializer_id, unregistered_samples, create_seri if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_unsubscribe.py b/examples/serdes/tests/test_unsubscribe.py index ac43750f2..e592a99e4 100644 --- a/examples/serdes/tests/test_unsubscribe.py +++ b/examples/serdes/tests/test_unsubscribe.py @@ -15,6 +15,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -65,7 +66,7 @@ def test_unsubscribe_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -137,7 +138,7 @@ def test_unsubscribe_roundtrip(serializer_id, unsubscribe_samples, create_serial if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_unsubscribed.py b/examples/serdes/tests/test_unsubscribed.py index a674baae8..53c3b1a4b 100644 --- a/examples/serdes/tests/test_unsubscribed.py +++ b/examples/serdes/tests/test_unsubscribed.py @@ -15,6 +15,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -65,7 +66,7 @@ def test_unsubscribed_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -134,7 +135,7 @@ def test_unsubscribed_roundtrip(serializer_id, unsubscribed_samples, create_seri if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_welcome.py b/examples/serdes/tests/test_welcome.py index 92e408853..2c3b12c25 100644 --- a/examples/serdes/tests/test_welcome.py +++ b/examples/serdes/tests/test_welcome.py @@ -15,6 +15,7 @@ from autobahn.wamp.role import RoleBrokerFeatures, RoleDealerFeatures from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -90,7 +91,7 @@ def test_welcome_deserialize_from_bytes( byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -161,7 +162,7 @@ def test_welcome_roundtrip(serializer_id, welcome_samples, create_serializer): if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/test_yield.py b/examples/serdes/tests/test_yield.py index 2585ece57..c8a41a79e 100644 --- a/examples/serdes/tests/test_yield.py +++ b/examples/serdes/tests/test_yield.py @@ -14,6 +14,7 @@ from autobahn.wamp.serializer import create_transport_serializer from .utils import ( + require_decodable, load_test_vector, bytes_from_hex, matches_any_byte_representation, @@ -62,7 +63,7 @@ def test_yield_deserialize_from_bytes(serializer_id, yield_samples, create_seria byte_variants = sample["serializers"][serializer_id] # Try deserializing each byte variant - for variant in byte_variants: + for variant in require_decodable(serializer, byte_variants): # Get bytes if "bytes_hex" in variant: test_bytes = bytes_from_hex(variant["bytes_hex"]) @@ -129,7 +130,7 @@ def test_yield_roundtrip(serializer_id, yield_samples, create_serializer): if not byte_variants: continue - variant = byte_variants[0] + variant = require_decodable(serializer, byte_variants)[0] if "bytes_hex" in variant: original_bytes = bytes_from_hex(variant["bytes_hex"]) elif "bytes" in variant: diff --git a/examples/serdes/tests/utils.py b/examples/serdes/tests/utils.py index de08e2b31..ccd394154 100644 --- a/examples/serdes/tests/utils.py +++ b/examples/serdes/tests/utils.py @@ -188,6 +188,63 @@ def matches_any_byte_representation( return False +def require_decodable(serializer: Any, variants: List[Dict[str, str]]) -> List[Dict[str, str]]: + """ + Return the subset of byte variants that ``serializer`` can actually decode. + + This implements the "at least one" semantics for the deserialize direction. + A serializer's variant list may include encodings produced by other WAMP + implementations or library versions (for example, the legacy py-ubjson bytes + vs. the new bjdata/BJData bytes for the "ubjson" serializer, which are not + mutually decodable). Feeding a foreign encoding to this serializer either + raises, or - worse - decodes to a *valid but wrong* message. Both cases are + skipped rather than treated as failures. + + A variant is considered "this serializer's own" iff it decodes AND + re-serializing the decoded message reproduces one of the canonical variants + in the list. This keeps purely cosmetic variants (e.g. spaced vs. compact + JSON, which all decode to the same message) while dropping foreign encodings. + + At least one variant MUST qualify, otherwise this is a genuine conformance + failure and an AssertionError is raised. + + Args: + serializer: A transport serializer (with ``serialize``/``unserialize``) + variants: List of byte representations (each with 'bytes' or 'bytes_hex') + + Returns: + The variants (in original order) that ``serializer`` decodes consistently + """ + # Tier 1: variants whose decode re-serializes to one of the canonical + # variants (cosmetic variants + this backend's own encoding). Tier 2 (fallback): + # variants that merely decode without raising - used for serializers whose + # re-serialization is not byte-canonical (e.g. flatbuffers), where Tier 1 + # would be empty even though the bytes are this backend's own. + consistent = [] + decodes = [] + for variant in variants: + if "bytes_hex" in variant: + data = bytes_from_hex(variant["bytes_hex"]) + elif "bytes" in variant: + data = variant["bytes"].encode("utf-8") + else: + continue + try: + msg = serializer.unserialize(data)[0] + except Exception: + continue + decodes.append(variant) + try: + reserialized, _ = serializer.serialize(msg) + except Exception: + continue + if matches_any_byte_representation(reserialized, variants): + consistent.append(variant) + result = consistent or decodes + assert result, "no byte variant could be deserialized by this serializer" + return result + + def validates_with_any_code(msg: Any, validation_codes: List[str]) -> Optional[str]: """ Try to validate message with any of the validation code blocks.