77``bq-trace-id`` = trace_id, ``bq-message-id`` = meta.id, plus ``bq-schema-version`` /
88``bq-source-lang`` / ``bq-attempts`` — so a Java/.NET/... peer can route on ``bq-job``
99without parsing the body. Consuming receives one message at a time
10- (``receive`` -> process -> ``acknowledge``); the authoritative attempt count is the
11- envelope's ``bq-attempts`` (the body), cross-checked against the broker's native
12- ``redelivery_count`` (0-based) and reconciled onto the envelope as
13- ``attempts = max(body, redelivery_count)`` — no -1, because Pulsar's redelivery count is
14- 0-based (0 on first delivery). A message left un-acked is redelivered (at-least-once).
10+ (``receive`` -> process -> ``acknowledge``). Per §5.2 / §5.7 the ``bq-job`` property is the
11+ routing key (the body's URN is used only when ``bq-job`` is absent or blank) and the
12+ ``bq-attempts`` property is the authoritative attempt count (the body's ``attempts`` is used
13+ only when the property is absent or not a plain non-negative ASCII-decimal integer that fits
14+ a 32-bit int). The result is then raised to the broker's native ``redelivery_count``
15+ (0-based): ``attempts = max(bq-attempts, redelivery_count)`` — no -1, because Pulsar's
16+ redelivery count is 0-based (0 on first delivery). The routing URN is handed to the app through
17+ ``ReceivedMessage.route_urn`` — the body's ``job`` / ``urn`` is never rewritten, so retries and
18+ the DLQ keep the producer's original body. When ``bq-job`` and the body name different URNs
19+ the property wins and the optional ``on_routing_mismatch(property_urn, body_urn, envelope)``
20+ hook is called (for an accepted envelope, after attempts are reconciled). A message left un-acked is redelivered (at-least-once).
1521
1622This implements §5 of the broker-bindings contract. The envelope is unchanged
1723(``schema_version`` stays 1); Apache Pulsar is purely additive.
2430
2531from __future__ import annotations
2632
27- from typing import Any , Dict , Optional
33+ import re
34+ from typing import Any , Callable , Dict , Optional , Tuple
2835
2936from .codec import EnvelopeCodec , parse_envelope
3037from .transport import ReceivedMessage , Transport
3138
39+ #: ``bq-attempts`` is valid only as a run of ASCII digits (no sign, no whitespace) ...
40+ _ATTEMPTS_PATTERN = re .compile (r"[0-9]+" )
41+ #: ... that fits a signed 32-bit int, matching the Java and .NET consumers (GR-5).
42+ _MAX_ATTEMPTS = 2 ** 31 - 1
43+
3244
3345class PulsarTransport (Transport ):
3446 def __init__ (
@@ -40,6 +52,7 @@ def __init__(
4052 consumer_type : Any = None ,
4153 topic_prefix : str = "" ,
4254 receive_timeout_millis : int = 1000 ,
55+ on_routing_mismatch : Optional [Callable [[str , str , Dict [str , Any ]], None ]] = None ,
4356 ** subscribe_options : Any ,
4457 ) -> None :
4558 self ._url = url or "pulsar://localhost:6650"
@@ -48,6 +61,7 @@ def __init__(
4861 self ._topic_prefix = topic_prefix
4962 self ._receive_timeout_millis = receive_timeout_millis
5063 self ._subscribe_options = subscribe_options
64+ self ._on_routing_mismatch = on_routing_mismatch
5165 self ._producers : Dict [str , Any ] = {}
5266 self ._consumers : Dict [str , Any ] = {}
5367
@@ -105,9 +119,12 @@ def _default_shared_type() -> Any:
105119 return pulsar .ConsumerType .Shared # pragma: no cover - needs Pulsar
106120
107121 @staticmethod
108- def _projection (body : str ) -> Dict [str , str ]:
122+ def _projection (body : str , route_urn : str = "" ) -> Dict [str , str ]:
109123 """Native Pulsar message properties (string->string) — a redundant, routable view of
110- the body: bq-job/bq-trace-id/bq-message-id + bq-schema-version/lang/attempts. §5.2."""
124+ the body: bq-job/bq-trace-id/bq-message-id + bq-schema-version/lang/attempts. §5.2.
125+
126+ A non-empty ``route_urn`` pins ``bq-job`` (a retry / release re-publish keeps the original
127+ delivery's routing URN); blank derives it from the body."""
111128 env = parse_envelope (body )
112129 if not env :
113130 return {}
@@ -125,25 +142,80 @@ def _projection(body: str) -> Dict[str, str]:
125142 if meta .get ("lang" ):
126143 props ["bq-source-lang" ] = str (meta ["lang" ])
127144 props ["bq-attempts" ] = str (int (env .get ("attempts" , 0 ) or 0 ))
145+ if route_urn :
146+ props ["bq-job" ] = route_urn
128147 return props
129148
130149 @staticmethod
131- def _reconcile (body : str , redelivery_count : Any ) -> str :
132- """Set attempts to max(current, redelivery_count). Pulsar's redelivery count is
133- 0-based (0 on first delivery), so it maps directly to attempts with no -1. The
134- runtime retries by republishing with attempts+1 in the body (redelivery count back to
135- 0), so a republished message must not have its higher body count lowered."""
150+ def _parse_attempts (raw : Any ) -> Optional [int ]:
151+ """The ``bq-attempts`` value as an int, or ``None`` when absent or invalid (§5.5)."""
152+ if not isinstance (raw , str ) or _ATTEMPTS_PATTERN .fullmatch (raw ) is None :
153+ return None
154+ value = int (raw )
155+ return value if value <= _MAX_ATTEMPTS else None
156+
157+ @staticmethod
158+ def _reconcile (
159+ body : str ,
160+ redelivery_count : Any ,
161+ properties : Optional [Dict [str , str ]] = None ,
162+ on_routing_mismatch : Optional [Callable [[str , str , Dict [str , Any ]], None ]] = None ,
163+ ) -> Tuple [str , str ]:
164+ """Apply §5.2 / §5.7 to a received body; returns ``(body, route_urn)``.
165+
166+ * Routing: a non-blank ``bq-job`` property is the URN the app routes on. It is returned
167+ as ``route_urn`` (set on :attr:`ReceivedMessage.route_urn`); the body's ``job`` /
168+ ``urn`` is never rewritten, so the handler, retries and the dead-letter record keep the
169+ producer's original body. A body with no URN gets no ``route_urn`` — it stays
170+ non-conformant and is rejected whatever ``bq-job`` says.
171+ * Attempts: a valid ``bq-attempts`` property replaces the body's value (the body's is
172+ kept when it is absent or invalid), then the result is raised to
173+ ``redelivery_count``. Pulsar's count is 0-based (0 on first delivery), so it maps
174+ directly with no -1; the max never lowers a runtime-incremented counter (the runtime
175+ retries by republishing with attempts+1, which resets the redelivery count to 0). The
176+ value is written into the body only when the body already holds an integer
177+ ``attempts`` and the value really changes, so a missing / non-integer ``attempts`` is
178+ still rejected by ``EnvelopeCodec.accepts``.
179+ * ``on_routing_mismatch(property_urn, body_urn, envelope)`` is called when the property
180+ and the body name different URNs — only for an envelope the consumer accepts and after
181+ attempts are reconciled (as in the Java and .NET consumers), so it sees what the
182+ handler will get. The hook is optional, separate from any error handling, and any
183+ exception it raises is swallowed — it never affects the message flow.
184+ """
185+ properties = properties or {}
186+ env = parse_envelope (body )
187+ if not env :
188+ return body , ""
189+
190+ routed = str (properties .get ("bq-job" ) or "" ).strip ()
191+ body_urn = EnvelopeCodec .urn (env )
192+ route_urn = routed if routed and body_urn else ""
193+
194+ current = env .get ("attempts" )
195+ has_int_attempts = isinstance (current , int ) and not isinstance (current , bool )
196+ attempts = current if has_int_attempts else 0
197+ from_property = PulsarTransport ._parse_attempts (properties .get ("bq-attempts" ))
198+ if from_property is not None :
199+ attempts = from_property
136200 try :
137201 rc = int (redelivery_count )
138- except (ValueError , TypeError ):
139- return body
140- if rc <= 0 :
141- return body
142- env = EnvelopeCodec .decode (body )
143- if not env or rc <= int (env .get ("attempts" , 0 ) or 0 ):
144- return body
145- env ["attempts" ] = rc
146- return EnvelopeCodec .encode (env )
202+ except (TypeError , ValueError ):
203+ rc = 0
204+ if rc > attempts :
205+ attempts = rc
206+
207+ out = body
208+ if has_int_attempts and current != attempts :
209+ env ["attempts" ] = attempts
210+ out = EnvelopeCodec .encode (env )
211+
212+ if route_urn and route_urn != body_urn and on_routing_mismatch is not None :
213+ if EnvelopeCodec .accepts (env ):
214+ try :
215+ on_routing_mismatch (route_urn , body_urn , dict (env ))
216+ except Exception : # noqa: BLE001 - a diagnostic hook must never break consumption
217+ pass
218+ return out , route_urn
147219
148220 @staticmethod
149221 def _redelivery_count (message : Any ) -> int :
@@ -156,6 +228,12 @@ def _redelivery_count(message: Any) -> int:
156228 except (TypeError , ValueError ):
157229 return 0
158230
231+ @staticmethod
232+ def _properties (message : Any ) -> Dict [str , str ]:
233+ getter = getattr (message , "properties" , None )
234+ value = getter () if callable (getter ) else getter
235+ return dict (value ) if isinstance (value , dict ) else {}
236+
159237 @staticmethod
160238 def _payload (message : Any ) -> str :
161239 getter = getattr (message , "data" , None )
@@ -167,7 +245,14 @@ def _payload(message: Any) -> str:
167245 # -- Transport ----------------------------------------------------------
168246
169247 def publish (self , queue : str , body : str ) -> None :
170- self ._producer (queue ).send (body .encode ("utf-8" ), properties = self ._projection (body ))
248+ self .publish_routed (queue , body , "" )
249+
250+ def publish_routed (self , queue : str , body : str , route_urn : str ) -> None :
251+ """:meth:`publish` with ``bq-job`` pinned to ``route_urn`` (:class:`RoutedPublisher`);
252+ a blank ``route_urn`` derives it from the body."""
253+ self ._producer (queue ).send (
254+ body .encode ("utf-8" ), properties = self ._projection (body , route_urn )
255+ )
171256
172257 def pop (self , queue : str , timeout : float = 1.0 ) -> Optional [ReceivedMessage ]:
173258 wait = self ._receive_timeout_millis
@@ -182,8 +267,13 @@ def pop(self, queue: str, timeout: float = 1.0) -> Optional[ReceivedMessage]:
182267 raise
183268 if message is None : # pragma: no cover - defensive (real client raises instead)
184269 return None
185- body = self ._reconcile (self ._payload (message ), self ._redelivery_count (message ))
186- return ReceivedMessage (body = body , queue = queue , handle = message )
270+ body , route_urn = self ._reconcile (
271+ self ._payload (message ),
272+ self ._redelivery_count (message ),
273+ self ._properties (message ),
274+ self ._on_routing_mismatch ,
275+ )
276+ return ReceivedMessage (body = body , queue = queue , handle = message , route_urn = route_urn )
187277
188278 def ack (self , message : ReceivedMessage ) -> None :
189279 if message .handle is None :
0 commit comments