From 92a064740c8c87ad4d2b92186df9152afb2401fe Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Mon, 26 Oct 2026 18:38:03 +0300 Subject: [PATCH 1/3] Fix topic writer batch sizes with message metadata --- CHANGELOG.md | 2 + docs/topic.rst | 4 + tests/test_errors.py | 15 ++- ydb/_topic_writer/topic_writer.py | 101 +++++------------ .../topic_writer_asyncio_test.py | 21 +++- ydb/_topic_writer/topic_writer_test.py | 105 ++++++++---------- ydb/connection.py | 5 +- 7 files changed, 121 insertions(+), 132 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44a1c411f..c5473223a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Fix topic writer batching to include message metadata and protobuf overhead, and stop retrying gRPC message size errors + ## 3.33.2 ## * Fix sync `QuerySessionPool` and `QuerySession.create()` hanging for 600 seconds and then failing with a non-retriable `TimeoutError` when a new session's attach stream fails before its first message (e.g. during a node restart); the attach error is now raised immediately, so retriable errors such as `Unavailable` are retried diff --git a/docs/topic.rst b/docs/topic.rst index bd5455691..48dc1ec83 100644 --- a/docs/topic.rst +++ b/docs/topic.rst @@ -213,6 +213,10 @@ Sending Messages metadata_items={"key": "value"}, )) +The writer splits buffered messages into gRPC requests using their serialized size, +including message metadata. A message that cannot fit in a request on its own fails +with ``ydb.BadRequest``. gRPC message size errors are reported without retrying. + **Write and wait for server acknowledgment:** .. code-block:: python diff --git a/tests/test_errors.py b/tests/test_errors.py index 1afc7b899..695fcaea4 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -15,13 +15,24 @@ def _make_rpc_error(code, details): return error -def test_resource_exhausted_large_message_returns_bad_request(): +@pytest.mark.parametrize( + "details", + [ + "CLIENT: Sent message larger than max (101289032 vs. 64000000)", + "SERVER: Received message larger than max (101289032 vs. 64000000)", + "Received message larger than max (101289032 vs. 64000000)", + "grpc: received message larger than max (101289032 vs. 64000000)", + ], +) +def test_resource_exhausted_large_message_returns_bad_request(details): rpc_error = _make_rpc_error( grpc.StatusCode.RESOURCE_EXHAUSTED, - "CLIENT: Sent message larger than max (101289032 vs. 64000000)", + details, ) result = _rpc_error_handler("test_rpc", rpc_error) assert isinstance(result, ydb.issues.BadRequest) + assert details in str(result) + assert not ydb._errors.check_retriable_error(result, ydb.RetrySettings(), 0).is_retriable def test_resource_exhausted_other_reason_returns_connection_lost(): diff --git a/ydb/_topic_writer/topic_writer.py b/ydb/_topic_writer/topic_writer.py index 23e4cd5a2..b5a2c1383 100644 --- a/ydb/_topic_writer/topic_writer.py +++ b/ydb/_topic_writer/topic_writer.py @@ -14,7 +14,7 @@ from .._grpc.grpcwrapper.ydb_topic import TransactionIdentity from .._grpc.grpcwrapper.common_utils import IToProto from .._grpc.grpcwrapper.ydb_topic_public_types import PublicCodec -from .. import connection +from .. import connection, issues Message = typing.Union["PublicMessage", "PublicMessage.SimpleSourceType"] @@ -257,7 +257,7 @@ def messages_to_proto_requests( tx_identity: Optional[TransactionIdentity], ) -> List[StreamWriteMessage.FromClient]: - groups = _split_messages_for_send(messages) + groups = _split_messages_for_send(messages, tx_identity) res = [] # type: List[StreamWriteMessage.FromClient] for group in groups: @@ -272,55 +272,43 @@ def messages_to_proto_requests( return res -_max_int = 2**63 - 1 - -_message_data_overhead = ( - StreamWriteMessage.FromClient( - StreamWriteMessage.WriteRequest( - messages=[ - StreamWriteMessage.WriteRequest.MessageData( - seq_no=_max_int, - created_at=datetime.datetime(3000, 1, 1, 1, 1, 1, 1), - data=bytes(1), - metadata_items={}, - uncompressed_size=_max_int, - partitioning=StreamWriteMessage.PartitioningMessageGroupID( - message_group_id="a" * 100, - ), - ), - ], - codec=20000, - tx_identity=None, - ) - ) - .to_proto() - .ByteSize() -) +def _protobuf_field_size(size: int) -> int: + """Size of a length-delimited protobuf field with a one-byte tag.""" + return 1 + max(1, (size.bit_length() + 6) // 7) + size def _split_messages_for_send( messages: List[InternalMessage], + tx_identity: Optional[TransactionIdentity] = None, ) -> List[List[InternalMessage]]: - codec_groups: List[List[InternalMessage]] = [] - for _, group_iter in itertools.groupby(messages, lambda x: x.codec): - codec_groups.append(list(group_iter)) - + limit = connection._DEFAULT_MAX_GRPC_MESSAGE_SIZE res: List[List[InternalMessage]] = [] - for codec_group in codec_groups: - group_by_size = _split_messages_by_size_with_default_overhead(codec_group) - res.extend(group_by_size) + for codec, codec_messages in itertools.groupby(messages, lambda msg: msg.codec): + base_size = ( + StreamWriteMessage.WriteRequest(messages=[], codec=codec, tx_identity=tx_identity).to_proto().ByteSize() + ) + group: List[InternalMessage] = [] + group_size = base_size + for msg in codec_messages: + message_size = _protobuf_field_size(msg.to_message_data().to_proto().ByteSize()) + request_size = _protobuf_field_size(group_size + message_size) + if group and request_size > limit: + res.append(group) + group = [] + group_size = base_size + request_size = _protobuf_field_size(group_size + message_size) + if request_size > limit: + raise issues.BadRequest( + "Topic message with seqno %s exceeds gRPC request size limit (%d vs. %d bytes)" + % (msg.seq_no, request_size, limit) + ) + group.append(msg) + group_size += message_size + if group: + res.append(group) return res -def _split_messages_by_size_with_default_overhead( - messages: List[InternalMessage], -) -> List[List[InternalMessage]]: - def get_message_size(msg: InternalMessage): - return len(msg.data) + _message_data_overhead - - return _split_messages_by_size(messages, connection._DEFAULT_MAX_GRPC_MESSAGE_SIZE, get_message_size) - - def internal_message_size_bytes(msg: InternalMessage) -> int: """Approximate size in bytes for buffer accounting (data + metadata + overhead). @@ -328,32 +316,3 @@ def internal_message_size_bytes(msg: InternalMessage) -> int: """ meta_len = sum(len(k) + len(v) for k, v in msg.metadata_items.items()) if msg.metadata_items else 0 return msg.uncompressed_size + meta_len + 64 # 64 bytes overhead per message (seq_no, timestamps, etc.) - - -def _split_messages_by_size( - messages: List[InternalMessage], - split_size: int, - get_msg_size: typing.Callable[[InternalMessage], int], -) -> List[List[InternalMessage]]: - res: List[List[InternalMessage]] = [] - group: List[InternalMessage] = [] - group_size = 0 - - for msg in messages: - msg_size = get_msg_size(msg) - - if len(group) == 0: - group.append(msg) - group_size += msg_size - elif group_size + msg_size <= split_size: - group.append(msg) - group_size += msg_size - else: - res.append(group) - group = [msg] - group_size = msg_size - - if len(group) > 0: - res.append(group) - - return res diff --git a/ydb/_topic_writer/topic_writer_asyncio_test.py b/ydb/_topic_writer/topic_writer_asyncio_test.py index 5c77fc2f1..6e6d2bbd0 100644 --- a/ydb/_topic_writer/topic_writer_asyncio_test.py +++ b/ydb/_topic_writer/topic_writer_asyncio_test.py @@ -17,7 +17,7 @@ import grpc import pytest -from .. import aio +from .. import aio, connection from .. import StatusCode, issues from .._grpc.grpcwrapper.ydb_topic import ( Codec, @@ -460,6 +460,25 @@ async def test_reconnect_and_resent_non_acked_messages_on_retriable_error( second_writer.from_server.put_nowait(self.make_default_ack_message(seq_no=2)) await reconnector.close(flush=True) + async def test_received_message_too_large_fails_pending_ack(self, reconnector, get_stream_writer): + message = PublicMessage(data=b"test", seqno=1, created_at=datetime.datetime.now(datetime.timezone.utc)) + futures = await reconnector.write_with_ack_future([message]) + stream_writer = get_stream_writer() + await wait_for_fast(stream_writer.from_client.get()) + rpc_error = grpc.aio.AioRpcError( + grpc.StatusCode.RESOURCE_EXHAUSTED, + initial_metadata=grpc.aio.Metadata(), + trailing_metadata=grpc.aio.Metadata(), + details="SERVER: Received message larger than max (1001 vs. 1000)", + ) + stream_writer.from_server.put_nowait(connection._rpc_error_handler("test", rpc_error)) + + with pytest.raises(issues.BadRequest, match="Received message larger than max"): + await wait_for_fast(futures[0]) + assert isinstance(await wait_for_fast(reconnector.wait_stop()), issues.BadRequest) + with pytest.raises(issues.BadRequest): + await reconnector.close(flush=False) + async def test_reconnect_on_cancelled_error_from_receive(self, default_driver, default_settings, monkeypatch): stream_creates = 0 stream_2_created = asyncio.Event() diff --git a/ydb/_topic_writer/topic_writer_test.py b/ydb/_topic_writer/topic_writer_test.py index 5b857ab75..68ed57d7d 100644 --- a/ydb/_topic_writer/topic_writer_test.py +++ b/ydb/_topic_writer/topic_writer_test.py @@ -1,17 +1,18 @@ import asyncio +import datetime import threading -from typing import List from unittest import mock import pytest +from .. import connection, issues +from .._grpc.grpcwrapper.ydb_topic import StreamWriteMessage, TransactionIdentity from .._grpc.grpcwrapper.ydb_topic_public_types import PublicCodec from .topic_writer import ( InternalMessage, PublicMessage, PublicWriterSettings, TopicWriterBufferFullError, - _split_messages_by_size, _split_messages_for_send, messages_to_proto_requests, ) @@ -19,57 +20,26 @@ from .topic_writer_sync import WriterSync -@pytest.mark.parametrize( - "messages,split_size,expected", - [ - ( - [1, 2, 3], - 0, - [[1], [2], [3]], - ), - ( - [1, 2, 3], - 1, - [[1], [2], [3]], - ), - ( - [1, 2, 3], - 3, - [[1, 2], [3]], - ), - ( - [1, 2, 3], - 100, - [[1, 2, 3]], - ), - ( - [100, 2, 3], - 100, - [[100], [2, 3]], - ), - ( - [], - 100, - [], - ), - ( - [], - 100, - [], - ), - ], -) -def test_split_messages_by_size(messages: List[int], split_size: int, expected: List[List[int]]): - res = _split_messages_by_size(messages, split_size, lambda x: x) # noqa - assert res == expected - - def _make_msg(data: bytes, codec: PublicCodec = PublicCodec.RAW, seqno: int = 1) -> InternalMessage: - msg = InternalMessage(PublicMessage(data=data, seqno=seqno)) + msg = InternalMessage(PublicMessage(data=data, seqno=seqno, created_at=datetime.datetime(2026, 1, 1))) msg.codec = codec return msg +def _request_size(messages, tx_identity=None): + return ( + StreamWriteMessage.FromClient( + StreamWriteMessage.WriteRequest( + messages=[msg.to_message_data() for msg in messages], + codec=messages[0].codec, + tx_identity=tx_identity, + ) + ) + .to_proto() + .ByteSize() + ) + + class TestSplitMessagesForSend: def test_empty(self): assert _split_messages_for_send([]) == [] @@ -97,13 +67,9 @@ def test_alternating_codecs_each_run_is_own_group(self): assert result == [[r1], [g1], [r2]] def test_size_limit_splits_same_codec_group(self, monkeypatch): - from .topic_writer import _message_data_overhead - from .. import connection - - # patch limit so that one 5-byte message fits but two do not - monkeypatch.setattr(connection, "_DEFAULT_MAX_GRPC_MESSAGE_SIZE", _message_data_overhead + 5) m1 = _make_msg(b"hello", PublicCodec.RAW, seqno=1) m2 = _make_msg(b"x", PublicCodec.RAW, seqno=2) + monkeypatch.setattr(connection, "_DEFAULT_MAX_GRPC_MESSAGE_SIZE", _request_size([m1])) result = _split_messages_for_send([m1, m2]) assert result == [[m1], [m2]] @@ -124,6 +90,34 @@ def test_codec_split_takes_priority_over_size(self): class TestMessagesToProtoRequests: + @pytest.mark.parametrize("body_size", [0, 100, 127, 128, 16383, 16384]) + @pytest.mark.parametrize("limit_delta", [-1, 0, 1]) + @pytest.mark.parametrize("with_tx", [False, True]) + def test_serialized_size_controls_splitting(self, monkeypatch, body_size, limit_delta, with_tx): + tx_identity = TransactionIdentity(tx_id="tx" * 100, session_id="session" * 100) if with_tx else None + messages = [_make_msg(b"x" * body_size, seqno=i) for i in range(1, 4)] + for message in messages: + message.metadata_items = {"key-\N{SNOWMAN}": b"m" * 200, "empty": b""} + limit = _request_size(messages, tx_identity) + limit_delta + monkeypatch.setattr(connection, "_DEFAULT_MAX_GRPC_MESSAGE_SIZE", limit) + + requests = messages_to_proto_requests(messages, tx_identity) + + assert [len(request.value.messages) for request in requests] == ([2, 1] if limit_delta < 0 else [3]) + assert all(request.to_proto().ByteSize() <= limit for request in requests) + assert [message.seq_no for request in requests for message in request.value.messages] == [1, 2, 3] + assert all(request.value.tx_identity == tx_identity for request in requests) + + @pytest.mark.parametrize("with_metadata", [False, True]) + def test_oversized_single_message_is_rejected(self, monkeypatch, with_metadata): + message = _make_msg(b"x" * 200) + if with_metadata: + message.metadata_items = {"key": b"m" * 300} + monkeypatch.setattr(connection, "_DEFAULT_MAX_GRPC_MESSAGE_SIZE", _request_size([message]) - 1) + + with pytest.raises(issues.BadRequest, match="size limit"): + messages_to_proto_requests([message], None) + def test_empty(self): assert messages_to_proto_requests([], tx_identity=None) == [] @@ -156,12 +150,9 @@ def test_different_codecs_produce_separate_requests(self): assert requests[1].value.codec == PublicCodec.GZIP def test_size_exceeded_produces_multiple_requests(self, monkeypatch): - from .topic_writer import _message_data_overhead - from .. import connection - - monkeypatch.setattr(connection, "_DEFAULT_MAX_GRPC_MESSAGE_SIZE", _message_data_overhead + 5) m1 = _make_msg(b"hello", PublicCodec.RAW, seqno=1) m2 = _make_msg(b"x", PublicCodec.RAW, seqno=2) + monkeypatch.setattr(connection, "_DEFAULT_MAX_GRPC_MESSAGE_SIZE", _request_size([m1])) requests = messages_to_proto_requests([m1, m2], tx_identity=None) assert len(requests) == 2 assert requests[0].value.messages[0].seq_no == 1 diff --git a/ydb/connection.py b/ydb/connection.py index 052b563ef..c99788a5e 100644 --- a/ydb/connection.py +++ b/ydb/connection.py @@ -101,7 +101,10 @@ def _rpc_error_handler( elif rpc_error.code() == grpc.StatusCode.CANCELLED: return issues.Cancelled(rpc_error.details()) elif rpc_error.code() == grpc.StatusCode.RESOURCE_EXHAUSTED: - if "Sent message larger than max" in rpc_error.details(): + if any( + text in rpc_error.details().lower() + for text in ("sent message larger than max", "received message larger than max") + ): return issues.BadRequest(rpc_error.details()) elif use_unavailable and rpc_error.code() == grpc.StatusCode.UNAVAILABLE: return issues.Unavailable(rpc_error.details()) From a6b241bd6140b8b33b5b5e363f4598341d5e29ca Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 27 Oct 2026 11:47:53 +0300 Subject: [PATCH 2/3] Limit topic writer changes to request size accounting --- CHANGELOG.md | 2 +- docs/topic.rst | 3 +-- tests/test_errors.py | 15 ++----------- ydb/_topic_writer/topic_writer.py | 11 ++-------- .../topic_writer_asyncio_test.py | 21 +------------------ ydb/_topic_writer/topic_writer_test.py | 12 ++++++----- ydb/connection.py | 5 +---- 7 files changed, 15 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5473223a..8d323b5fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -* Fix topic writer batching to include message metadata and protobuf overhead, and stop retrying gRPC message size errors +* Fix topic writer batching to include message metadata and protobuf overhead in request size calculations ## 3.33.2 ## * Fix sync `QuerySessionPool` and `QuerySession.create()` hanging for 600 seconds and then failing with a non-retriable `TimeoutError` when a new session's attach stream fails before its first message (e.g. during a node restart); the attach error is now raised immediately, so retriable errors such as `Unavailable` are retried diff --git a/docs/topic.rst b/docs/topic.rst index 48dc1ec83..4bf3a63ad 100644 --- a/docs/topic.rst +++ b/docs/topic.rst @@ -214,8 +214,7 @@ Sending Messages )) The writer splits buffered messages into gRPC requests using their serialized size, -including message metadata. A message that cannot fit in a request on its own fails -with ``ydb.BadRequest``. gRPC message size errors are reported without retrying. +including message metadata. **Write and wait for server acknowledgment:** diff --git a/tests/test_errors.py b/tests/test_errors.py index 695fcaea4..1afc7b899 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -15,24 +15,13 @@ def _make_rpc_error(code, details): return error -@pytest.mark.parametrize( - "details", - [ - "CLIENT: Sent message larger than max (101289032 vs. 64000000)", - "SERVER: Received message larger than max (101289032 vs. 64000000)", - "Received message larger than max (101289032 vs. 64000000)", - "grpc: received message larger than max (101289032 vs. 64000000)", - ], -) -def test_resource_exhausted_large_message_returns_bad_request(details): +def test_resource_exhausted_large_message_returns_bad_request(): rpc_error = _make_rpc_error( grpc.StatusCode.RESOURCE_EXHAUSTED, - details, + "CLIENT: Sent message larger than max (101289032 vs. 64000000)", ) result = _rpc_error_handler("test_rpc", rpc_error) assert isinstance(result, ydb.issues.BadRequest) - assert details in str(result) - assert not ydb._errors.check_retriable_error(result, ydb.RetrySettings(), 0).is_retriable def test_resource_exhausted_other_reason_returns_connection_lost(): diff --git a/ydb/_topic_writer/topic_writer.py b/ydb/_topic_writer/topic_writer.py index b5a2c1383..729f0de8f 100644 --- a/ydb/_topic_writer/topic_writer.py +++ b/ydb/_topic_writer/topic_writer.py @@ -14,7 +14,7 @@ from .._grpc.grpcwrapper.ydb_topic import TransactionIdentity from .._grpc.grpcwrapper.common_utils import IToProto from .._grpc.grpcwrapper.ydb_topic_public_types import PublicCodec -from .. import connection, issues +from .. import connection Message = typing.Union["PublicMessage", "PublicMessage.SimpleSourceType"] @@ -291,17 +291,10 @@ def _split_messages_for_send( group_size = base_size for msg in codec_messages: message_size = _protobuf_field_size(msg.to_message_data().to_proto().ByteSize()) - request_size = _protobuf_field_size(group_size + message_size) - if group and request_size > limit: + if group and _protobuf_field_size(group_size + message_size) > limit: res.append(group) group = [] group_size = base_size - request_size = _protobuf_field_size(group_size + message_size) - if request_size > limit: - raise issues.BadRequest( - "Topic message with seqno %s exceeds gRPC request size limit (%d vs. %d bytes)" - % (msg.seq_no, request_size, limit) - ) group.append(msg) group_size += message_size if group: diff --git a/ydb/_topic_writer/topic_writer_asyncio_test.py b/ydb/_topic_writer/topic_writer_asyncio_test.py index 6e6d2bbd0..5c77fc2f1 100644 --- a/ydb/_topic_writer/topic_writer_asyncio_test.py +++ b/ydb/_topic_writer/topic_writer_asyncio_test.py @@ -17,7 +17,7 @@ import grpc import pytest -from .. import aio, connection +from .. import aio from .. import StatusCode, issues from .._grpc.grpcwrapper.ydb_topic import ( Codec, @@ -460,25 +460,6 @@ async def test_reconnect_and_resent_non_acked_messages_on_retriable_error( second_writer.from_server.put_nowait(self.make_default_ack_message(seq_no=2)) await reconnector.close(flush=True) - async def test_received_message_too_large_fails_pending_ack(self, reconnector, get_stream_writer): - message = PublicMessage(data=b"test", seqno=1, created_at=datetime.datetime.now(datetime.timezone.utc)) - futures = await reconnector.write_with_ack_future([message]) - stream_writer = get_stream_writer() - await wait_for_fast(stream_writer.from_client.get()) - rpc_error = grpc.aio.AioRpcError( - grpc.StatusCode.RESOURCE_EXHAUSTED, - initial_metadata=grpc.aio.Metadata(), - trailing_metadata=grpc.aio.Metadata(), - details="SERVER: Received message larger than max (1001 vs. 1000)", - ) - stream_writer.from_server.put_nowait(connection._rpc_error_handler("test", rpc_error)) - - with pytest.raises(issues.BadRequest, match="Received message larger than max"): - await wait_for_fast(futures[0]) - assert isinstance(await wait_for_fast(reconnector.wait_stop()), issues.BadRequest) - with pytest.raises(issues.BadRequest): - await reconnector.close(flush=False) - async def test_reconnect_on_cancelled_error_from_receive(self, default_driver, default_settings, monkeypatch): stream_creates = 0 stream_2_created = asyncio.Event() diff --git a/ydb/_topic_writer/topic_writer_test.py b/ydb/_topic_writer/topic_writer_test.py index 68ed57d7d..33f965ce0 100644 --- a/ydb/_topic_writer/topic_writer_test.py +++ b/ydb/_topic_writer/topic_writer_test.py @@ -5,7 +5,7 @@ import pytest -from .. import connection, issues +from .. import connection from .._grpc.grpcwrapper.ydb_topic import StreamWriteMessage, TransactionIdentity from .._grpc.grpcwrapper.ydb_topic_public_types import PublicCodec from .topic_writer import ( @@ -109,14 +109,16 @@ def test_serialized_size_controls_splitting(self, monkeypatch, body_size, limit_ assert all(request.value.tx_identity == tx_identity for request in requests) @pytest.mark.parametrize("with_metadata", [False, True]) - def test_oversized_single_message_is_rejected(self, monkeypatch, with_metadata): - message = _make_msg(b"x" * 200) + def test_oversized_message_is_sent_on_its_own(self, monkeypatch, with_metadata): + message = _make_msg(b"x" * 200, seqno=2) if with_metadata: message.metadata_items = {"key": b"m" * 300} monkeypatch.setattr(connection, "_DEFAULT_MAX_GRPC_MESSAGE_SIZE", _request_size([message]) - 1) - with pytest.raises(issues.BadRequest, match="size limit"): - messages_to_proto_requests([message], None) + requests = messages_to_proto_requests([_make_msg(b"before"), message, _make_msg(b"after", seqno=3)], None) + + assert [[msg.seq_no for msg in request.value.messages] for request in requests] == [[1], [2], [3]] + assert requests[1].to_proto().ByteSize() > connection._DEFAULT_MAX_GRPC_MESSAGE_SIZE def test_empty(self): assert messages_to_proto_requests([], tx_identity=None) == [] diff --git a/ydb/connection.py b/ydb/connection.py index c99788a5e..052b563ef 100644 --- a/ydb/connection.py +++ b/ydb/connection.py @@ -101,10 +101,7 @@ def _rpc_error_handler( elif rpc_error.code() == grpc.StatusCode.CANCELLED: return issues.Cancelled(rpc_error.details()) elif rpc_error.code() == grpc.StatusCode.RESOURCE_EXHAUSTED: - if any( - text in rpc_error.details().lower() - for text in ("sent message larger than max", "received message larger than max") - ): + if "Sent message larger than max" in rpc_error.details(): return issues.BadRequest(rpc_error.details()) elif use_unavailable and rpc_error.code() == grpc.StatusCode.UNAVAILABLE: return issues.Unavailable(rpc_error.details()) From 901791f3ae3c71f9888f6e512dfddd97fca612fd Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Tue, 27 Oct 2026 13:58:32 +0300 Subject: [PATCH 3/3] Build topic protobuf requests in a single pass --- ydb/_grpc/grpcwrapper/common_utils.py | 6 +- ydb/_topic_common/test_helpers.py | 4 +- ydb/_topic_writer/topic_writer.py | 64 +++++------ .../topic_writer_asyncio_test.py | 37 ++++++- ydb/_topic_writer/topic_writer_test.py | 101 +++++++----------- 5 files changed, 105 insertions(+), 107 deletions(-) diff --git a/ydb/_grpc/grpcwrapper/common_utils.py b/ydb/_grpc/grpcwrapper/common_utils.py index 51f7b349a..e72102787 100644 --- a/ydb/_grpc/grpcwrapper/common_utils.py +++ b/ydb/_grpc/grpcwrapper/common_utils.py @@ -146,7 +146,7 @@ class IGrpcWrapperAsyncIO(abc.ABC): async def receive(self, timeout: Optional[int] = None) -> Any: ... @abc.abstractmethod - def write(self, wrap_message: IToProto): ... + def write(self, wrap_message: Union[IToProto, Message]): ... @abc.abstractmethod def close(self): ... @@ -253,8 +253,8 @@ async def get_response(): # print("rekby, grpc, received", grpc_message) return self.convert_server_grpc_to_wrapper(grpc_message) - def write(self, wrap_message: IToProto) -> None: - grpc_message = wrap_message.to_proto() + def write(self, wrap_message: Union[IToProto, Message]) -> None: + grpc_message = wrap_message if isinstance(wrap_message, Message) else wrap_message.to_proto() # print("rekby, grpc, send", grpc_message) self.from_client_grpc.put_nowait(grpc_message) diff --git a/ydb/_topic_common/test_helpers.py b/ydb/_topic_common/test_helpers.py index eb2bfc36e..5df3c80b1 100644 --- a/ydb/_topic_common/test_helpers.py +++ b/ydb/_topic_common/test_helpers.py @@ -2,6 +2,8 @@ import time import typing +from google.protobuf.message import Message + from .._grpc.grpcwrapper.common_utils import IToProto, IGrpcWrapperAsyncIO @@ -26,7 +28,7 @@ async def receive(self, timeout: typing.Optional[int] = None) -> typing.Any: raise item return item - def write(self, wrap_message: IToProto): + def write(self, wrap_message: typing.Union[IToProto, Message]): if self._closed: raise Exception("write to closed StreamMock") self.from_client.put_nowait(wrap_message) diff --git a/ydb/_topic_writer/topic_writer.py b/ydb/_topic_writer/topic_writer.py index 729f0de8f..18e790fe4 100644 --- a/ydb/_topic_writer/topic_writer.py +++ b/ydb/_topic_writer/topic_writer.py @@ -16,6 +16,11 @@ from .._grpc.grpcwrapper.ydb_topic_public_types import PublicCodec from .. import connection +if typing.TYPE_CHECKING: + from .._grpc.v4.protos import ydb_topic_pb2 +else: + from .._grpc.common.protos import ydb_topic_pb2 + Message = typing.Union["PublicMessage", "PublicMessage.SimpleSourceType"] @@ -255,53 +260,40 @@ def default_serializer_message_content(data: Any) -> bytes: def messages_to_proto_requests( messages: List[InternalMessage], tx_identity: Optional[TransactionIdentity], -) -> List[StreamWriteMessage.FromClient]: - - groups = _split_messages_for_send(messages, tx_identity) - - res = [] # type: List[StreamWriteMessage.FromClient] - for group in groups: - req = StreamWriteMessage.FromClient( - StreamWriteMessage.WriteRequest( - messages=list(map(InternalMessage.to_message_data, group)), - codec=group[0].codec, - tx_identity=tx_identity, - ) - ) - res.append(req) - return res - - -def _protobuf_field_size(size: int) -> int: - """Size of a length-delimited protobuf field with a one-byte tag.""" - return 1 + max(1, (size.bit_length() + 6) // 7) + size +) -> List[ydb_topic_pb2.StreamWriteMessage.FromClient]: + tx_proto = tx_identity.to_proto() if tx_identity is not None else None + def new_request(codec: int) -> ydb_topic_pb2.StreamWriteMessage.FromClient: + request = ydb_topic_pb2.StreamWriteMessage.FromClient() + request.write_request.codec = codec + if tx_proto is not None: + request.write_request.tx.CopyFrom(tx_proto) + return request -def _split_messages_for_send( - messages: List[InternalMessage], - tx_identity: Optional[TransactionIdentity] = None, -) -> List[List[InternalMessage]]: limit = connection._DEFAULT_MAX_GRPC_MESSAGE_SIZE - res: List[List[InternalMessage]] = [] + res: List[ydb_topic_pb2.StreamWriteMessage.FromClient] = [] for codec, codec_messages in itertools.groupby(messages, lambda msg: msg.codec): - base_size = ( - StreamWriteMessage.WriteRequest(messages=[], codec=codec, tx_identity=tx_identity).to_proto().ByteSize() - ) - group: List[InternalMessage] = [] + request = new_request(codec) + base_size = request.write_request.ByteSize() group_size = base_size for msg in codec_messages: - message_size = _protobuf_field_size(msg.to_message_data().to_proto().ByteSize()) - if group and _protobuf_field_size(group_size + message_size) > limit: - res.append(group) - group = [] + message_proto = msg.to_message_data().to_proto() + message_size = _protobuf_field_size(message_proto.ByteSize()) + if request.write_request.messages and _protobuf_field_size(group_size + message_size) > limit: + res.append(request) + request = new_request(codec) group_size = base_size - group.append(msg) + request.write_request.messages.append(message_proto) group_size += message_size - if group: - res.append(group) + res.append(request) return res +def _protobuf_field_size(size: int) -> int: + """Size of a length-delimited protobuf field with a one-byte tag.""" + return 1 + max(1, (size.bit_length() + 6) // 7) + size + + def internal_message_size_bytes(msg: InternalMessage) -> int: """Approximate size in bytes for buffer accounting (data + metadata + overhead). diff --git a/ydb/_topic_writer/topic_writer_asyncio_test.py b/ydb/_topic_writer/topic_writer_asyncio_test.py index 5c77fc2f1..f2775226c 100644 --- a/ydb/_topic_writer/topic_writer_asyncio_test.py +++ b/ydb/_topic_writer/topic_writer_asyncio_test.py @@ -26,7 +26,7 @@ UpdateTokenRequest, UpdateTokenResponse, ) -from .._grpc.grpcwrapper.common_utils import AsyncQueueToSyncIteratorAsyncIO, ServerStatus +from .._grpc.grpcwrapper.common_utils import AsyncQueueToSyncIteratorAsyncIO, GrpcWrapperAsyncIO, ServerStatus from .topic_writer import ( InternalMessage, PublicMessage, @@ -177,7 +177,7 @@ async def test_write_a_message(self, writer_and_stream: WriterWithMockedStream): ) sent_message = await writer_and_stream.stream.from_client.get() - assert expected_message == sent_message + assert expected_message.to_proto() == sent_message async def test_write_a_message_with_tx(self, writer_and_stream: WriterWithMockedStream): writer_and_stream.writer._tx_identity = FAKE_TRANSACTION_IDENTITY @@ -214,7 +214,7 @@ async def test_write_a_message_with_tx(self, writer_and_stream: WriterWithMocked ) sent_message = await writer_and_stream.stream.from_client.get() - assert expected_message == sent_message + assert expected_message.to_proto() == sent_message async def test_update_token(self, stream: StreamMock): writer = await self.get_started_writer(stream, update_token_interval=0.1, get_token_function=lambda: "foo-bar") @@ -238,6 +238,37 @@ async def test_update_token(self, stream: StreamMock): await writer.close() + async def test_messages_are_converted_once_before_grpc(self): + stream = GrpcWrapperAsyncIO(lambda message: message) + writer = WriterAsyncIOStream() + writer._stream = stream + messages = [ + InternalMessage( + PublicMessage( + data=b"payload", + metadata_items={"key": b"value"}, + seqno=seqno, + created_at=datetime.datetime(2026, 1, 1), + ) + ) + for seqno in (1, 2) + ] + + message_type = StreamWriteMessage.WriteRequest.MessageData + try: + with mock.patch.object( + message_type, "to_proto", autospec=True, side_effect=message_type.to_proto + ) as convert: + writer.write(messages) + assert convert.call_count == len(messages) + + request = stream.from_client_grpc.get_nowait() + assert [message.seq_no for message in request.write_request.messages] == [1, 2] + assert request.write_request.messages[0].metadata_items[0].value == b"value" + assert stream.from_client_grpc.empty() + finally: + await writer.close() + async def test_init_timeout_parameter(self, stream): """Test that WriterAsyncIOStream._start calls stream.receive with timeout=10""" writer_id = 1 diff --git a/ydb/_topic_writer/topic_writer_test.py b/ydb/_topic_writer/topic_writer_test.py index 33f965ce0..018823f3a 100644 --- a/ydb/_topic_writer/topic_writer_test.py +++ b/ydb/_topic_writer/topic_writer_test.py @@ -13,7 +13,6 @@ PublicMessage, PublicWriterSettings, TopicWriterBufferFullError, - _split_messages_for_send, messages_to_proto_requests, ) from .topic_writer_asyncio import WriterAsyncIOReconnector @@ -40,56 +39,27 @@ def _request_size(messages, tx_identity=None): ) -class TestSplitMessagesForSend: - def test_empty(self): - assert _split_messages_for_send([]) == [] - - def test_single_message(self): - msg = _make_msg(b"hello") - assert _split_messages_for_send([msg]) == [[msg]] - - def test_same_codec_kept_together(self): - msgs = [_make_msg(b"a", PublicCodec.RAW, i) for i in range(1, 4)] - assert _split_messages_for_send(msgs) == [msgs] - - def test_different_codecs_split_into_separate_groups(self): - raw = _make_msg(b"a", PublicCodec.RAW, seqno=1) - gzip = _make_msg(b"b", PublicCodec.GZIP, seqno=2) - result = _split_messages_for_send([raw, gzip]) - assert result == [[raw], [gzip]] - - def test_alternating_codecs_each_run_is_own_group(self): - # RAW, GZIP, RAW must produce 3 separate groups, not 2 - r1 = _make_msg(b"a", PublicCodec.RAW, seqno=1) - g1 = _make_msg(b"b", PublicCodec.GZIP, seqno=2) - r2 = _make_msg(b"c", PublicCodec.RAW, seqno=3) - result = _split_messages_for_send([r1, g1, r2]) - assert result == [[r1], [g1], [r2]] - - def test_size_limit_splits_same_codec_group(self, monkeypatch): - m1 = _make_msg(b"hello", PublicCodec.RAW, seqno=1) - m2 = _make_msg(b"x", PublicCodec.RAW, seqno=2) - monkeypatch.setattr(connection, "_DEFAULT_MAX_GRPC_MESSAGE_SIZE", _request_size([m1])) - result = _split_messages_for_send([m1, m2]) - assert result == [[m1], [m2]] - - def test_size_limit_not_exceeded_keeps_group(self): - m1 = _make_msg(b"hello", PublicCodec.RAW, seqno=1) - m2 = _make_msg(b"world", PublicCodec.RAW, seqno=2) - result = _split_messages_for_send([m1, m2]) - assert result == [[m1, m2]] - - def test_codec_split_takes_priority_over_size(self): - # tiny messages but different codecs — must still split by codec - r = _make_msg(b"a", PublicCodec.RAW, seqno=1) - g = _make_msg(b"b", PublicCodec.GZIP, seqno=2) - result = _split_messages_for_send([r, g]) - assert len(result) == 2 - assert result[0][0].codec == PublicCodec.RAW - assert result[1][0].codec == PublicCodec.GZIP - - class TestMessagesToProtoRequests: + def test_alternating_codecs_keep_message_order(self): + messages = [ + _make_msg(b"a", PublicCodec.RAW, seqno=1), + _make_msg(b"b", PublicCodec.GZIP, seqno=2), + _make_msg(b"c", PublicCodec.RAW, seqno=3), + ] + + requests = messages_to_proto_requests(messages, None) + + assert [request.write_request.codec for request in requests] == [ + PublicCodec.RAW, + PublicCodec.GZIP, + PublicCodec.RAW, + ] + assert [[message.seq_no for message in request.write_request.messages] for request in requests] == [ + [1], + [2], + [3], + ] + @pytest.mark.parametrize("body_size", [0, 100, 127, 128, 16383, 16384]) @pytest.mark.parametrize("limit_delta", [-1, 0, 1]) @pytest.mark.parametrize("with_tx", [False, True]) @@ -103,10 +73,13 @@ def test_serialized_size_controls_splitting(self, monkeypatch, body_size, limit_ requests = messages_to_proto_requests(messages, tx_identity) - assert [len(request.value.messages) for request in requests] == ([2, 1] if limit_delta < 0 else [3]) - assert all(request.to_proto().ByteSize() <= limit for request in requests) - assert [message.seq_no for request in requests for message in request.value.messages] == [1, 2, 3] - assert all(request.value.tx_identity == tx_identity for request in requests) + assert [len(request.write_request.messages) for request in requests] == ([2, 1] if limit_delta < 0 else [3]) + assert all(request.ByteSize() <= limit for request in requests) + assert [message.seq_no for request in requests for message in request.write_request.messages] == [1, 2, 3] + if tx_identity is not None: + assert all(request.write_request.tx == tx_identity.to_proto() for request in requests) + else: + assert all(not request.write_request.HasField("tx") for request in requests) @pytest.mark.parametrize("with_metadata", [False, True]) def test_oversized_message_is_sent_on_its_own(self, monkeypatch, with_metadata): @@ -117,8 +90,8 @@ def test_oversized_message_is_sent_on_its_own(self, monkeypatch, with_metadata): requests = messages_to_proto_requests([_make_msg(b"before"), message, _make_msg(b"after", seqno=3)], None) - assert [[msg.seq_no for msg in request.value.messages] for request in requests] == [[1], [2], [3]] - assert requests[1].to_proto().ByteSize() > connection._DEFAULT_MAX_GRPC_MESSAGE_SIZE + assert [[msg.seq_no for msg in request.write_request.messages] for request in requests] == [[1], [2], [3]] + assert requests[1].ByteSize() > connection._DEFAULT_MAX_GRPC_MESSAGE_SIZE def test_empty(self): assert messages_to_proto_requests([], tx_identity=None) == [] @@ -131,25 +104,25 @@ def test_single_message_produces_one_request(self): def test_request_codec_matches_messages(self): raw = _make_msg(b"a", PublicCodec.RAW, seqno=1) requests = messages_to_proto_requests([raw], tx_identity=None) - assert requests[0].value.codec == PublicCodec.RAW + assert requests[0].write_request.codec == PublicCodec.RAW gzip = _make_msg(b"b", PublicCodec.GZIP, seqno=1) requests = messages_to_proto_requests([gzip], tx_identity=None) - assert requests[0].value.codec == PublicCodec.GZIP + assert requests[0].write_request.codec == PublicCodec.GZIP def test_same_codec_produces_single_request(self): msgs = [_make_msg(b"x", PublicCodec.RAW, seqno=i) for i in range(1, 4)] requests = messages_to_proto_requests(msgs, tx_identity=None) assert len(requests) == 1 - assert len(requests[0].value.messages) == 3 + assert len(requests[0].write_request.messages) == 3 def test_different_codecs_produce_separate_requests(self): raw = _make_msg(b"a", PublicCodec.RAW, seqno=1) gzip = _make_msg(b"b", PublicCodec.GZIP, seqno=2) requests = messages_to_proto_requests([raw, gzip], tx_identity=None) assert len(requests) == 2 - assert requests[0].value.codec == PublicCodec.RAW - assert requests[1].value.codec == PublicCodec.GZIP + assert requests[0].write_request.codec == PublicCodec.RAW + assert requests[1].write_request.codec == PublicCodec.GZIP def test_size_exceeded_produces_multiple_requests(self, monkeypatch): m1 = _make_msg(b"hello", PublicCodec.RAW, seqno=1) @@ -157,14 +130,14 @@ def test_size_exceeded_produces_multiple_requests(self, monkeypatch): monkeypatch.setattr(connection, "_DEFAULT_MAX_GRPC_MESSAGE_SIZE", _request_size([m1])) requests = messages_to_proto_requests([m1, m2], tx_identity=None) assert len(requests) == 2 - assert requests[0].value.messages[0].seq_no == 1 - assert requests[1].value.messages[0].seq_no == 2 + assert requests[0].write_request.messages[0].seq_no == 1 + assert requests[1].write_request.messages[0].seq_no == 2 def test_messages_order_preserved_within_request(self): msgs = [_make_msg(f"msg{i}".encode(), PublicCodec.RAW, seqno=i) for i in range(1, 5)] requests = messages_to_proto_requests(msgs, tx_identity=None) assert len(requests) == 1 - seq_nos = [m.seq_no for m in requests[0].value.messages] + seq_nos = [m.seq_no for m in requests[0].write_request.messages] assert seq_nos == [1, 2, 3, 4]