diff --git a/CHANGELOG.md b/CHANGELOG.md index 44a1c411..8d323b5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Fix topic writer batching to include message metadata and protobuf overhead in request size calculations + ## 3.33.2 ## * Fix sync `QuerySessionPool` and `QuerySession.create()` hanging for 600 seconds and then failing with a non-retriable `TimeoutError` when a new session's attach stream fails before its first message (e.g. during a node restart); the attach error is now raised immediately, so retriable errors such as `Unavailable` are retried diff --git a/docs/topic.rst b/docs/topic.rst index bd545569..4bf3a63a 100644 --- a/docs/topic.rst +++ b/docs/topic.rst @@ -213,6 +213,9 @@ Sending Messages metadata_items={"key": "value"}, )) +The writer splits buffered messages into gRPC requests using their serialized size, +including message metadata. + **Write and wait for server acknowledgment:** .. code-block:: python diff --git a/ydb/_grpc/grpcwrapper/common_utils.py b/ydb/_grpc/grpcwrapper/common_utils.py index 51f7b349..e7210278 100644 --- a/ydb/_grpc/grpcwrapper/common_utils.py +++ b/ydb/_grpc/grpcwrapper/common_utils.py @@ -146,7 +146,7 @@ class IGrpcWrapperAsyncIO(abc.ABC): async def receive(self, timeout: Optional[int] = None) -> Any: ... @abc.abstractmethod - def write(self, wrap_message: IToProto): ... + def write(self, wrap_message: Union[IToProto, Message]): ... @abc.abstractmethod def close(self): ... @@ -253,8 +253,8 @@ async def get_response(): # print("rekby, grpc, received", grpc_message) return self.convert_server_grpc_to_wrapper(grpc_message) - def write(self, wrap_message: IToProto) -> None: - grpc_message = wrap_message.to_proto() + def write(self, wrap_message: Union[IToProto, Message]) -> None: + grpc_message = wrap_message if isinstance(wrap_message, Message) else wrap_message.to_proto() # print("rekby, grpc, send", grpc_message) self.from_client_grpc.put_nowait(grpc_message) diff --git a/ydb/_topic_common/test_helpers.py b/ydb/_topic_common/test_helpers.py index eb2bfc36..5df3c80b 100644 --- a/ydb/_topic_common/test_helpers.py +++ b/ydb/_topic_common/test_helpers.py @@ -2,6 +2,8 @@ import time import typing +from google.protobuf.message import Message + from .._grpc.grpcwrapper.common_utils import IToProto, IGrpcWrapperAsyncIO @@ -26,7 +28,7 @@ async def receive(self, timeout: typing.Optional[int] = None) -> typing.Any: raise item return item - def write(self, wrap_message: IToProto): + def write(self, wrap_message: typing.Union[IToProto, Message]): if self._closed: raise Exception("write to closed StreamMock") self.from_client.put_nowait(wrap_message) diff --git a/ydb/_topic_writer/topic_writer.py b/ydb/_topic_writer/topic_writer.py index 23e4cd5a..18e790fe 100644 --- a/ydb/_topic_writer/topic_writer.py +++ b/ydb/_topic_writer/topic_writer.py @@ -16,6 +16,11 @@ from .._grpc.grpcwrapper.ydb_topic_public_types import PublicCodec from .. import connection +if typing.TYPE_CHECKING: + from .._grpc.v4.protos import ydb_topic_pb2 +else: + from .._grpc.common.protos import ydb_topic_pb2 + Message = typing.Union["PublicMessage", "PublicMessage.SimpleSourceType"] @@ -255,70 +260,38 @@ def default_serializer_message_content(data: Any) -> bytes: def messages_to_proto_requests( messages: List[InternalMessage], tx_identity: Optional[TransactionIdentity], -) -> List[StreamWriteMessage.FromClient]: - - groups = _split_messages_for_send(messages) - - res = [] # type: List[StreamWriteMessage.FromClient] - for group in groups: - req = StreamWriteMessage.FromClient( - StreamWriteMessage.WriteRequest( - messages=list(map(InternalMessage.to_message_data, group)), - codec=group[0].codec, - tx_identity=tx_identity, - ) - ) - res.append(req) +) -> List[ydb_topic_pb2.StreamWriteMessage.FromClient]: + tx_proto = tx_identity.to_proto() if tx_identity is not None else None + + def new_request(codec: int) -> ydb_topic_pb2.StreamWriteMessage.FromClient: + request = ydb_topic_pb2.StreamWriteMessage.FromClient() + request.write_request.codec = codec + if tx_proto is not None: + request.write_request.tx.CopyFrom(tx_proto) + return request + + limit = connection._DEFAULT_MAX_GRPC_MESSAGE_SIZE + res: List[ydb_topic_pb2.StreamWriteMessage.FromClient] = [] + for codec, codec_messages in itertools.groupby(messages, lambda msg: msg.codec): + request = new_request(codec) + base_size = request.write_request.ByteSize() + group_size = base_size + for msg in codec_messages: + message_proto = msg.to_message_data().to_proto() + message_size = _protobuf_field_size(message_proto.ByteSize()) + if request.write_request.messages and _protobuf_field_size(group_size + message_size) > limit: + res.append(request) + request = new_request(codec) + group_size = base_size + request.write_request.messages.append(message_proto) + group_size += message_size + res.append(request) return res -_max_int = 2**63 - 1 - -_message_data_overhead = ( - StreamWriteMessage.FromClient( - StreamWriteMessage.WriteRequest( - messages=[ - StreamWriteMessage.WriteRequest.MessageData( - seq_no=_max_int, - created_at=datetime.datetime(3000, 1, 1, 1, 1, 1, 1), - data=bytes(1), - metadata_items={}, - uncompressed_size=_max_int, - partitioning=StreamWriteMessage.PartitioningMessageGroupID( - message_group_id="a" * 100, - ), - ), - ], - codec=20000, - tx_identity=None, - ) - ) - .to_proto() - .ByteSize() -) - - -def _split_messages_for_send( - messages: List[InternalMessage], -) -> List[List[InternalMessage]]: - codec_groups: List[List[InternalMessage]] = [] - for _, group_iter in itertools.groupby(messages, lambda x: x.codec): - codec_groups.append(list(group_iter)) - - res: List[List[InternalMessage]] = [] - for codec_group in codec_groups: - group_by_size = _split_messages_by_size_with_default_overhead(codec_group) - res.extend(group_by_size) - return res - - -def _split_messages_by_size_with_default_overhead( - messages: List[InternalMessage], -) -> List[List[InternalMessage]]: - def get_message_size(msg: InternalMessage): - return len(msg.data) + _message_data_overhead - - return _split_messages_by_size(messages, connection._DEFAULT_MAX_GRPC_MESSAGE_SIZE, get_message_size) +def _protobuf_field_size(size: int) -> int: + """Size of a length-delimited protobuf field with a one-byte tag.""" + return 1 + max(1, (size.bit_length() + 6) // 7) + size def internal_message_size_bytes(msg: InternalMessage) -> int: @@ -328,32 +301,3 @@ def internal_message_size_bytes(msg: InternalMessage) -> int: """ meta_len = sum(len(k) + len(v) for k, v in msg.metadata_items.items()) if msg.metadata_items else 0 return msg.uncompressed_size + meta_len + 64 # 64 bytes overhead per message (seq_no, timestamps, etc.) - - -def _split_messages_by_size( - messages: List[InternalMessage], - split_size: int, - get_msg_size: typing.Callable[[InternalMessage], int], -) -> List[List[InternalMessage]]: - res: List[List[InternalMessage]] = [] - group: List[InternalMessage] = [] - group_size = 0 - - for msg in messages: - msg_size = get_msg_size(msg) - - if len(group) == 0: - group.append(msg) - group_size += msg_size - elif group_size + msg_size <= split_size: - group.append(msg) - group_size += msg_size - else: - res.append(group) - group = [msg] - group_size = msg_size - - if len(group) > 0: - res.append(group) - - return res diff --git a/ydb/_topic_writer/topic_writer_asyncio_test.py b/ydb/_topic_writer/topic_writer_asyncio_test.py index 5c77fc2f..f2775226 100644 --- a/ydb/_topic_writer/topic_writer_asyncio_test.py +++ b/ydb/_topic_writer/topic_writer_asyncio_test.py @@ -26,7 +26,7 @@ UpdateTokenRequest, UpdateTokenResponse, ) -from .._grpc.grpcwrapper.common_utils import AsyncQueueToSyncIteratorAsyncIO, ServerStatus +from .._grpc.grpcwrapper.common_utils import AsyncQueueToSyncIteratorAsyncIO, GrpcWrapperAsyncIO, ServerStatus from .topic_writer import ( InternalMessage, PublicMessage, @@ -177,7 +177,7 @@ async def test_write_a_message(self, writer_and_stream: WriterWithMockedStream): ) sent_message = await writer_and_stream.stream.from_client.get() - assert expected_message == sent_message + assert expected_message.to_proto() == sent_message async def test_write_a_message_with_tx(self, writer_and_stream: WriterWithMockedStream): writer_and_stream.writer._tx_identity = FAKE_TRANSACTION_IDENTITY @@ -214,7 +214,7 @@ async def test_write_a_message_with_tx(self, writer_and_stream: WriterWithMocked ) sent_message = await writer_and_stream.stream.from_client.get() - assert expected_message == sent_message + assert expected_message.to_proto() == sent_message async def test_update_token(self, stream: StreamMock): writer = await self.get_started_writer(stream, update_token_interval=0.1, get_token_function=lambda: "foo-bar") @@ -238,6 +238,37 @@ async def test_update_token(self, stream: StreamMock): await writer.close() + async def test_messages_are_converted_once_before_grpc(self): + stream = GrpcWrapperAsyncIO(lambda message: message) + writer = WriterAsyncIOStream() + writer._stream = stream + messages = [ + InternalMessage( + PublicMessage( + data=b"payload", + metadata_items={"key": b"value"}, + seqno=seqno, + created_at=datetime.datetime(2026, 1, 1), + ) + ) + for seqno in (1, 2) + ] + + message_type = StreamWriteMessage.WriteRequest.MessageData + try: + with mock.patch.object( + message_type, "to_proto", autospec=True, side_effect=message_type.to_proto + ) as convert: + writer.write(messages) + assert convert.call_count == len(messages) + + request = stream.from_client_grpc.get_nowait() + assert [message.seq_no for message in request.write_request.messages] == [1, 2] + assert request.write_request.messages[0].metadata_items[0].value == b"value" + assert stream.from_client_grpc.empty() + finally: + await writer.close() + async def test_init_timeout_parameter(self, stream): """Test that WriterAsyncIOStream._start calls stream.receive with timeout=10""" writer_id = 1 diff --git a/ydb/_topic_writer/topic_writer_test.py b/ydb/_topic_writer/topic_writer_test.py index 5b857ab7..018823f3 100644 --- a/ydb/_topic_writer/topic_writer_test.py +++ b/ydb/_topic_writer/topic_writer_test.py @@ -1,129 +1,98 @@ import asyncio +import datetime import threading -from typing import List from unittest import mock import pytest +from .. import connection +from .._grpc.grpcwrapper.ydb_topic import StreamWriteMessage, TransactionIdentity from .._grpc.grpcwrapper.ydb_topic_public_types import PublicCodec from .topic_writer import ( InternalMessage, PublicMessage, PublicWriterSettings, TopicWriterBufferFullError, - _split_messages_by_size, - _split_messages_for_send, messages_to_proto_requests, ) from .topic_writer_asyncio import WriterAsyncIOReconnector from .topic_writer_sync import WriterSync -@pytest.mark.parametrize( - "messages,split_size,expected", - [ - ( - [1, 2, 3], - 0, - [[1], [2], [3]], - ), - ( - [1, 2, 3], - 1, - [[1], [2], [3]], - ), - ( - [1, 2, 3], - 3, - [[1, 2], [3]], - ), - ( - [1, 2, 3], - 100, - [[1, 2, 3]], - ), - ( - [100, 2, 3], - 100, - [[100], [2, 3]], - ), - ( - [], - 100, - [], - ), - ( - [], - 100, - [], - ), - ], -) -def test_split_messages_by_size(messages: List[int], split_size: int, expected: List[List[int]]): - res = _split_messages_by_size(messages, split_size, lambda x: x) # noqa - assert res == expected - - def _make_msg(data: bytes, codec: PublicCodec = PublicCodec.RAW, seqno: int = 1) -> InternalMessage: - msg = InternalMessage(PublicMessage(data=data, seqno=seqno)) + msg = InternalMessage(PublicMessage(data=data, seqno=seqno, created_at=datetime.datetime(2026, 1, 1))) msg.codec = codec return msg -class TestSplitMessagesForSend: - def test_empty(self): - assert _split_messages_for_send([]) == [] - - def test_single_message(self): - msg = _make_msg(b"hello") - assert _split_messages_for_send([msg]) == [[msg]] - - def test_same_codec_kept_together(self): - msgs = [_make_msg(b"a", PublicCodec.RAW, i) for i in range(1, 4)] - assert _split_messages_for_send(msgs) == [msgs] - - def test_different_codecs_split_into_separate_groups(self): - raw = _make_msg(b"a", PublicCodec.RAW, seqno=1) - gzip = _make_msg(b"b", PublicCodec.GZIP, seqno=2) - result = _split_messages_for_send([raw, gzip]) - assert result == [[raw], [gzip]] - - def test_alternating_codecs_each_run_is_own_group(self): - # RAW, GZIP, RAW must produce 3 separate groups, not 2 - r1 = _make_msg(b"a", PublicCodec.RAW, seqno=1) - g1 = _make_msg(b"b", PublicCodec.GZIP, seqno=2) - r2 = _make_msg(b"c", PublicCodec.RAW, seqno=3) - result = _split_messages_for_send([r1, g1, r2]) - assert result == [[r1], [g1], [r2]] - - def test_size_limit_splits_same_codec_group(self, monkeypatch): - from .topic_writer import _message_data_overhead - from .. import connection - - # patch limit so that one 5-byte message fits but two do not - monkeypatch.setattr(connection, "_DEFAULT_MAX_GRPC_MESSAGE_SIZE", _message_data_overhead + 5) - m1 = _make_msg(b"hello", PublicCodec.RAW, seqno=1) - m2 = _make_msg(b"x", PublicCodec.RAW, seqno=2) - result = _split_messages_for_send([m1, m2]) - assert result == [[m1], [m2]] - - def test_size_limit_not_exceeded_keeps_group(self): - m1 = _make_msg(b"hello", PublicCodec.RAW, seqno=1) - m2 = _make_msg(b"world", PublicCodec.RAW, seqno=2) - result = _split_messages_for_send([m1, m2]) - assert result == [[m1, m2]] - - def test_codec_split_takes_priority_over_size(self): - # tiny messages but different codecs — must still split by codec - r = _make_msg(b"a", PublicCodec.RAW, seqno=1) - g = _make_msg(b"b", PublicCodec.GZIP, seqno=2) - result = _split_messages_for_send([r, g]) - assert len(result) == 2 - assert result[0][0].codec == PublicCodec.RAW - assert result[1][0].codec == PublicCodec.GZIP +def _request_size(messages, tx_identity=None): + return ( + StreamWriteMessage.FromClient( + StreamWriteMessage.WriteRequest( + messages=[msg.to_message_data() for msg in messages], + codec=messages[0].codec, + tx_identity=tx_identity, + ) + ) + .to_proto() + .ByteSize() + ) class TestMessagesToProtoRequests: + def test_alternating_codecs_keep_message_order(self): + messages = [ + _make_msg(b"a", PublicCodec.RAW, seqno=1), + _make_msg(b"b", PublicCodec.GZIP, seqno=2), + _make_msg(b"c", PublicCodec.RAW, seqno=3), + ] + + requests = messages_to_proto_requests(messages, None) + + assert [request.write_request.codec for request in requests] == [ + PublicCodec.RAW, + PublicCodec.GZIP, + PublicCodec.RAW, + ] + assert [[message.seq_no for message in request.write_request.messages] for request in requests] == [ + [1], + [2], + [3], + ] + + @pytest.mark.parametrize("body_size", [0, 100, 127, 128, 16383, 16384]) + @pytest.mark.parametrize("limit_delta", [-1, 0, 1]) + @pytest.mark.parametrize("with_tx", [False, True]) + def test_serialized_size_controls_splitting(self, monkeypatch, body_size, limit_delta, with_tx): + tx_identity = TransactionIdentity(tx_id="tx" * 100, session_id="session" * 100) if with_tx else None + messages = [_make_msg(b"x" * body_size, seqno=i) for i in range(1, 4)] + for message in messages: + message.metadata_items = {"key-\N{SNOWMAN}": b"m" * 200, "empty": b""} + limit = _request_size(messages, tx_identity) + limit_delta + monkeypatch.setattr(connection, "_DEFAULT_MAX_GRPC_MESSAGE_SIZE", limit) + + requests = messages_to_proto_requests(messages, tx_identity) + + assert [len(request.write_request.messages) for request in requests] == ([2, 1] if limit_delta < 0 else [3]) + assert all(request.ByteSize() <= limit for request in requests) + assert [message.seq_no for request in requests for message in request.write_request.messages] == [1, 2, 3] + if tx_identity is not None: + assert all(request.write_request.tx == tx_identity.to_proto() for request in requests) + else: + assert all(not request.write_request.HasField("tx") for request in requests) + + @pytest.mark.parametrize("with_metadata", [False, True]) + def test_oversized_message_is_sent_on_its_own(self, monkeypatch, with_metadata): + message = _make_msg(b"x" * 200, seqno=2) + if with_metadata: + message.metadata_items = {"key": b"m" * 300} + monkeypatch.setattr(connection, "_DEFAULT_MAX_GRPC_MESSAGE_SIZE", _request_size([message]) - 1) + + requests = messages_to_proto_requests([_make_msg(b"before"), message, _make_msg(b"after", seqno=3)], None) + + assert [[msg.seq_no for msg in request.write_request.messages] for request in requests] == [[1], [2], [3]] + assert requests[1].ByteSize() > connection._DEFAULT_MAX_GRPC_MESSAGE_SIZE + def test_empty(self): assert messages_to_proto_requests([], tx_identity=None) == [] @@ -135,43 +104,40 @@ def test_single_message_produces_one_request(self): def test_request_codec_matches_messages(self): raw = _make_msg(b"a", PublicCodec.RAW, seqno=1) requests = messages_to_proto_requests([raw], tx_identity=None) - assert requests[0].value.codec == PublicCodec.RAW + assert requests[0].write_request.codec == PublicCodec.RAW gzip = _make_msg(b"b", PublicCodec.GZIP, seqno=1) requests = messages_to_proto_requests([gzip], tx_identity=None) - assert requests[0].value.codec == PublicCodec.GZIP + assert requests[0].write_request.codec == PublicCodec.GZIP def test_same_codec_produces_single_request(self): msgs = [_make_msg(b"x", PublicCodec.RAW, seqno=i) for i in range(1, 4)] requests = messages_to_proto_requests(msgs, tx_identity=None) assert len(requests) == 1 - assert len(requests[0].value.messages) == 3 + assert len(requests[0].write_request.messages) == 3 def test_different_codecs_produce_separate_requests(self): raw = _make_msg(b"a", PublicCodec.RAW, seqno=1) gzip = _make_msg(b"b", PublicCodec.GZIP, seqno=2) requests = messages_to_proto_requests([raw, gzip], tx_identity=None) assert len(requests) == 2 - assert requests[0].value.codec == PublicCodec.RAW - assert requests[1].value.codec == PublicCodec.GZIP + assert requests[0].write_request.codec == PublicCodec.RAW + assert requests[1].write_request.codec == PublicCodec.GZIP def test_size_exceeded_produces_multiple_requests(self, monkeypatch): - from .topic_writer import _message_data_overhead - from .. import connection - - monkeypatch.setattr(connection, "_DEFAULT_MAX_GRPC_MESSAGE_SIZE", _message_data_overhead + 5) m1 = _make_msg(b"hello", PublicCodec.RAW, seqno=1) m2 = _make_msg(b"x", PublicCodec.RAW, seqno=2) + monkeypatch.setattr(connection, "_DEFAULT_MAX_GRPC_MESSAGE_SIZE", _request_size([m1])) requests = messages_to_proto_requests([m1, m2], tx_identity=None) assert len(requests) == 2 - assert requests[0].value.messages[0].seq_no == 1 - assert requests[1].value.messages[0].seq_no == 2 + assert requests[0].write_request.messages[0].seq_no == 1 + assert requests[1].write_request.messages[0].seq_no == 2 def test_messages_order_preserved_within_request(self): msgs = [_make_msg(f"msg{i}".encode(), PublicCodec.RAW, seqno=i) for i in range(1, 5)] requests = messages_to_proto_requests(msgs, tx_identity=None) assert len(requests) == 1 - seq_nos = [m.seq_no for m in requests[0].value.messages] + seq_nos = [m.seq_no for m in requests[0].write_request.messages] assert seq_nos == [1, 2, 3, 4]