Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
* Fix topic writer batching to include message metadata and protobuf overhead in request size calculations

## 3.33.2 ##
* Fix sync `QuerySessionPool` and `QuerySession.create()` hanging for 600 seconds and then failing with a non-retriable `TimeoutError` when a new session's attach stream fails before its first message (e.g. during a node restart); the attach error is now raised immediately, so retriable errors such as `Unavailable` are retried

Expand Down
3 changes: 3 additions & 0 deletions docs/topic.rst
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,9 @@ Sending Messages
metadata_items={"key": "value"},
))

The writer splits buffered messages into gRPC requests using their serialized size,
including message metadata.

**Write and wait for server acknowledgment:**

.. code-block:: python
Expand Down
6 changes: 3 additions & 3 deletions ydb/_grpc/grpcwrapper/common_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ class IGrpcWrapperAsyncIO(abc.ABC):
async def receive(self, timeout: Optional[int] = None) -> Any: ...

@abc.abstractmethod
def write(self, wrap_message: IToProto): ...
def write(self, wrap_message: Union[IToProto, Message]): ...

@abc.abstractmethod
def close(self): ...
Expand Down Expand Up @@ -253,8 +253,8 @@ async def get_response():
# print("rekby, grpc, received", grpc_message)
return self.convert_server_grpc_to_wrapper(grpc_message)

def write(self, wrap_message: IToProto) -> None:
grpc_message = wrap_message.to_proto()
def write(self, wrap_message: Union[IToProto, Message]) -> None:
grpc_message = wrap_message if isinstance(wrap_message, Message) else wrap_message.to_proto()
# print("rekby, grpc, send", grpc_message)
self.from_client_grpc.put_nowait(grpc_message)

Expand Down
4 changes: 3 additions & 1 deletion ydb/_topic_common/test_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
import time
import typing

from google.protobuf.message import Message

from .._grpc.grpcwrapper.common_utils import IToProto, IGrpcWrapperAsyncIO


Expand All @@ -26,7 +28,7 @@ async def receive(self, timeout: typing.Optional[int] = None) -> typing.Any:
raise item
return item

def write(self, wrap_message: IToProto):
def write(self, wrap_message: typing.Union[IToProto, Message]):
if self._closed:
raise Exception("write to closed StreamMock")
self.from_client.put_nowait(wrap_message)
Expand Down
124 changes: 34 additions & 90 deletions ydb/_topic_writer/topic_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@
from .._grpc.grpcwrapper.ydb_topic_public_types import PublicCodec
from .. import connection

if typing.TYPE_CHECKING:
from .._grpc.v4.protos import ydb_topic_pb2
else:
from .._grpc.common.protos import ydb_topic_pb2

Message = typing.Union["PublicMessage", "PublicMessage.SimpleSourceType"]


Expand Down Expand Up @@ -255,70 +260,38 @@ def default_serializer_message_content(data: Any) -> bytes:
def messages_to_proto_requests(
messages: List[InternalMessage],
tx_identity: Optional[TransactionIdentity],
) -> List[StreamWriteMessage.FromClient]:

groups = _split_messages_for_send(messages)

res = [] # type: List[StreamWriteMessage.FromClient]
for group in groups:
req = StreamWriteMessage.FromClient(
StreamWriteMessage.WriteRequest(
messages=list(map(InternalMessage.to_message_data, group)),
codec=group[0].codec,
tx_identity=tx_identity,
)
)
res.append(req)
) -> List[ydb_topic_pb2.StreamWriteMessage.FromClient]:
tx_proto = tx_identity.to_proto() if tx_identity is not None else None

def new_request(codec: int) -> ydb_topic_pb2.StreamWriteMessage.FromClient:
request = ydb_topic_pb2.StreamWriteMessage.FromClient()
request.write_request.codec = codec
if tx_proto is not None:
request.write_request.tx.CopyFrom(tx_proto)
return request

limit = connection._DEFAULT_MAX_GRPC_MESSAGE_SIZE
res: List[ydb_topic_pb2.StreamWriteMessage.FromClient] = []
for codec, codec_messages in itertools.groupby(messages, lambda msg: msg.codec):
request = new_request(codec)
base_size = request.write_request.ByteSize()
group_size = base_size
for msg in codec_messages:
message_proto = msg.to_message_data().to_proto()
message_size = _protobuf_field_size(message_proto.ByteSize())
if request.write_request.messages and _protobuf_field_size(group_size + message_size) > limit:
res.append(request)
request = new_request(codec)
group_size = base_size
request.write_request.messages.append(message_proto)
group_size += message_size
res.append(request)
return res


_max_int = 2**63 - 1

_message_data_overhead = (
StreamWriteMessage.FromClient(
StreamWriteMessage.WriteRequest(
messages=[
StreamWriteMessage.WriteRequest.MessageData(
seq_no=_max_int,
created_at=datetime.datetime(3000, 1, 1, 1, 1, 1, 1),
data=bytes(1),
metadata_items={},
uncompressed_size=_max_int,
partitioning=StreamWriteMessage.PartitioningMessageGroupID(
message_group_id="a" * 100,
),
),
],
codec=20000,
tx_identity=None,
)
)
.to_proto()
.ByteSize()
)


def _split_messages_for_send(
messages: List[InternalMessage],
) -> List[List[InternalMessage]]:
codec_groups: List[List[InternalMessage]] = []
for _, group_iter in itertools.groupby(messages, lambda x: x.codec):
codec_groups.append(list(group_iter))

res: List[List[InternalMessage]] = []
for codec_group in codec_groups:
group_by_size = _split_messages_by_size_with_default_overhead(codec_group)
res.extend(group_by_size)
return res


def _split_messages_by_size_with_default_overhead(
messages: List[InternalMessage],
) -> List[List[InternalMessage]]:
def get_message_size(msg: InternalMessage):
return len(msg.data) + _message_data_overhead

return _split_messages_by_size(messages, connection._DEFAULT_MAX_GRPC_MESSAGE_SIZE, get_message_size)
def _protobuf_field_size(size: int) -> int:
"""Size of a length-delimited protobuf field with a one-byte tag."""
return 1 + max(1, (size.bit_length() + 6) // 7) + size


def internal_message_size_bytes(msg: InternalMessage) -> int:
Expand All @@ -328,32 +301,3 @@ def internal_message_size_bytes(msg: InternalMessage) -> int:
"""
meta_len = sum(len(k) + len(v) for k, v in msg.metadata_items.items()) if msg.metadata_items else 0
return msg.uncompressed_size + meta_len + 64 # 64 bytes overhead per message (seq_no, timestamps, etc.)


def _split_messages_by_size(
messages: List[InternalMessage],
split_size: int,
get_msg_size: typing.Callable[[InternalMessage], int],
) -> List[List[InternalMessage]]:
res: List[List[InternalMessage]] = []
group: List[InternalMessage] = []
group_size = 0

for msg in messages:
msg_size = get_msg_size(msg)

if len(group) == 0:
group.append(msg)
group_size += msg_size
elif group_size + msg_size <= split_size:
group.append(msg)
group_size += msg_size
else:
res.append(group)
group = [msg]
group_size = msg_size

if len(group) > 0:
res.append(group)

return res
37 changes: 34 additions & 3 deletions ydb/_topic_writer/topic_writer_asyncio_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
UpdateTokenRequest,
UpdateTokenResponse,
)
from .._grpc.grpcwrapper.common_utils import AsyncQueueToSyncIteratorAsyncIO, ServerStatus
from .._grpc.grpcwrapper.common_utils import AsyncQueueToSyncIteratorAsyncIO, GrpcWrapperAsyncIO, ServerStatus
from .topic_writer import (
InternalMessage,
PublicMessage,
Expand Down Expand Up @@ -177,7 +177,7 @@ async def test_write_a_message(self, writer_and_stream: WriterWithMockedStream):
)

sent_message = await writer_and_stream.stream.from_client.get()
assert expected_message == sent_message
assert expected_message.to_proto() == sent_message

async def test_write_a_message_with_tx(self, writer_and_stream: WriterWithMockedStream):
writer_and_stream.writer._tx_identity = FAKE_TRANSACTION_IDENTITY
Expand Down Expand Up @@ -214,7 +214,7 @@ async def test_write_a_message_with_tx(self, writer_and_stream: WriterWithMocked
)

sent_message = await writer_and_stream.stream.from_client.get()
assert expected_message == sent_message
assert expected_message.to_proto() == sent_message

async def test_update_token(self, stream: StreamMock):
writer = await self.get_started_writer(stream, update_token_interval=0.1, get_token_function=lambda: "foo-bar")
Expand All @@ -238,6 +238,37 @@ async def test_update_token(self, stream: StreamMock):

await writer.close()

async def test_messages_are_converted_once_before_grpc(self):
stream = GrpcWrapperAsyncIO(lambda message: message)
writer = WriterAsyncIOStream()
writer._stream = stream
messages = [
InternalMessage(
PublicMessage(
data=b"payload",
metadata_items={"key": b"value"},
seqno=seqno,
created_at=datetime.datetime(2026, 1, 1),
)
)
for seqno in (1, 2)
]

message_type = StreamWriteMessage.WriteRequest.MessageData
try:
with mock.patch.object(
message_type, "to_proto", autospec=True, side_effect=message_type.to_proto
) as convert:
writer.write(messages)
assert convert.call_count == len(messages)

request = stream.from_client_grpc.get_nowait()
assert [message.seq_no for message in request.write_request.messages] == [1, 2]
assert request.write_request.messages[0].metadata_items[0].value == b"value"
assert stream.from_client_grpc.empty()
finally:
await writer.close()

async def test_init_timeout_parameter(self, stream):
"""Test that WriterAsyncIOStream._start calls stream.receive with timeout=10"""
writer_id = 1
Expand Down
Loading
Loading