diff --git a/.github/workflows/slo.yml b/.github/workflows/slo.yml index 8c066a3d..75ba92b4 100644 --- a/.github/workflows/slo.yml +++ b/.github/workflows/slo.yml @@ -41,6 +41,10 @@ jobs: command: "--write-rps 200 --write-threads 8 --read-threads 8" metrics_yaml_path: sdk-current/tests/slo/metrics-topic.yaml thresholds_yaml_path: sdk-current/tests/slo/thresholds-topic.yaml + - name: sync-topic-multiwriter + command: "--write-rps 200 --write-threads 2 --keys-per-writer 16 --read-threads 8" + metrics_yaml_path: sdk-current/tests/slo/metrics-topic.yaml + thresholds_yaml_path: sdk-current/tests/slo/thresholds-topic.yaml concurrency: group: slo-${{ github.ref }}-${{ matrix.sdk.name }} @@ -82,12 +86,26 @@ jobs: - name: Determine baseline commit id: baseline working-directory: sdk-current + env: + SLO_SCENARIO: ${{ matrix.sdk.name }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | set -euo pipefail BASELINE=$(git merge-base HEAD origin/main) - echo "sha=${BASELINE}" >> "$GITHUB_OUTPUT" - if git merge-base --is-ancestor "${BASELINE}" origin/main && \ + if [ "${SLO_SCENARIO}" = "sync-topic-multiwriter" ] && \ + ! git grep -q -F 'def multiwriter(' "${BASELINE}" -- ydb/topic.py; then + # Before the API reaches main, compare against the preceding SDK implementation. + # Harness-only commits must not turn this into a comparison with identical SDK code. + LAST_IMPLEMENTATION=$(git log -1 --format=%H "${PR_HEAD_SHA}" -- \ + ydb/_topic_writer ydb/topic.py ':(exclude)**/*_test.py') + BASELINE=$(git rev-parse "${LAST_IMPLEMENTATION}^") + if ! git grep -q -F 'def multiwriter(' "${BASELINE}" -- ydb/topic.py; then + echo "No earlier multiwriter implementation is available for comparison." >&2 + exit 1 + fi + BASELINE_REF="previous-multiwriter@${BASELINE:0:7}" + elif git merge-base --is-ancestor "${BASELINE}" origin/main && \ [ "$(git rev-parse origin/main)" = "${BASELINE}" ]; then BASELINE_REF="main" else @@ -98,6 +116,7 @@ jobs: BASELINE_REF="${BASELINE:0:7}" fi fi + echo "sha=${BASELINE}" >> "$GITHUB_OUTPUT" echo "ref=${BASELINE_REF}" >> "$GITHUB_OUTPUT" - name: Checkout baseline SDK version @@ -136,6 +155,7 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} workload_name: ${{ matrix.sdk.name }} workload_duration: "600" + fail_on_workload_error: "true" workload_current_ref: ${{ github.head_ref || github.ref_name }} workload_current_image: ydb-app-current workload_current_command: ${{ matrix.sdk.command }} @@ -149,3 +169,10 @@ jobs: # Ignored by action versions without ydb-slo-action#57; self-activates # once per-scenario thresholds land in the consumed tag. thresholds_yaml_path: ${{ matrix.sdk.thresholds_yaml_path }} + + - name: Inspect workload exit status + if: always() + run: | + docker inspect --format \ + '{{.Name}} status={{.State.Status}} exit={{.State.ExitCode}} oom_killed={{.State.OOMKilled}} memory_limit={{.HostConfig.Memory}} error={{.State.Error}}' \ + ydb-workload-current ydb-workload-baseline diff --git a/CHANGELOG.md b/CHANGELOG.md index 89b1dcb0..80858be0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +* Fix multiwriter admission during close and partition changes, and allow retrying a single write with its original sequence number after a buffer timeout +* Fix topic writer shutdown after a close timeout and make multiwriter flush and close wait for delivery after an acknowledgement waiter is cancelled +* Add a topic multi-partition writer (`topic_client.multiwriter(...)`) that routes messages across partitions by their `key`, with Kafka-hash and key-range partition choosers, and transparently resends in-flight messages to child partitions on an auto-partition split (no loss, no duplicates); expose partition `key_range` on `describe_topic` results * Fix automatic refresh of short-lived login tokens by using their JWT expiration time. ## 3.35.0 ## diff --git a/docker-compose.yml b/docker-compose.yml index 7ae4f549..0c6270d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,7 @@ version: "3.9" services: ydb: image: ydbplatform/local-ydb:trunk + command: ["--enable-feature-flag", "enable_topic_split_merge"] restart: always ports: - 2136:2136 @@ -11,4 +12,4 @@ services: - ./ydb_certs:/ydb_certs environment: - YDB_ENABLE_COLUMN_TABLES=true - - YDB_USE_IN_MEMORY_PDISKS=true \ No newline at end of file + - YDB_USE_IN_MEMORY_PDISKS=true diff --git a/docs/topic.rst b/docs/topic.rst index 4bf3a63a..e6b65fa6 100644 --- a/docs/topic.rst +++ b/docs/topic.rst @@ -170,6 +170,11 @@ Use ``topic_client.writer()`` as a context manager (recommended) or create it ma finally: writer.close() +With ``close(flush=True)``, the writer keeps reconnecting as needed until pending writes +finish. A transport disconnect during this wait does not stop recovery. If the close call +itself times out or is cancelled while waiting, the writer stops its background tasks and +closes the stream. A message whose acknowledgement was lost may already be on the server. + Writer Parameters ^^^^^^^^^^^^^^^^^ @@ -262,6 +267,125 @@ For high-throughput pipelines, buffer writes and gather futures: raise f.exception() +Writing by Key (Multiple Partitions) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A regular writer targets a single partition for its whole lifetime. To spread load across +all partitions of a topic from one logical writer — while keeping every message with the same +key on the same partition (so per-key ordering is preserved) — use ``topic_client.multiwriter()``. + +Each message carries a ``key``; the writer hashes it and routes the message to the owning +partition, maintaining a separate underlying writer per partition. This is the client-side +companion to auto-partitioning. + +**Synchronous:** + +.. code-block:: python + + with driver.topic_client.multiwriter("/local/my-topic") as writer: + writer.write(ydb.TopicWriterMessage(data="event", key="user-42")) + writer.write(ydb.TopicWriterMessage(data="event", key="user-7")) + +**Asynchronous:** + +.. code-block:: python + + async with driver.topic_client.multiwriter("/local/my-topic") as writer: + await writer.write(ydb.TopicWriterMessage(data="event", key="user-42")) + +**Partition choosers** decide how a key maps to a partition. By default the writer picks one +automatically after describing the topic — the key-range chooser for auto-partitioned topics, +the Kafka-hash chooser otherwise — so ``multiwriter(topic)`` works out of the box. You can also +set one explicitly: + +* :class:`~ydb.TopicWriterPartitionByKeyBound` — hashes the key and selects the partition whose + server-side key range owns it. Mirrors YDB auto-partitioning, so a key lands where the server + expects it. Used automatically for auto-partitioned topics. +* :class:`~ydb.TopicWriterPartitionByKeyKafka` — Kafka-compatible routing for topics + with a fixed partition count. Hashes the UTF-8 key bytes with 32-bit MurmurHash2 + and seed ``0x9747b28c``, then computes ``(hash & 0x7fffffff) % partitions_count``. + The result is an index into partition IDs sorted in ascending order. Compatibility + requires identical key bytes and partition IDs; changing the partition set can + route an existing key to a different partition. Missing and empty keys are hashed + as empty bytes; this differs from Kafka's handling of a missing key. + +.. code-block:: python + + writer = driver.topic_client.multiwriter( + "/local/my-topic", + partition_chooser=ydb.TopicWriterPartitionByKeyBound(), + producer_id_prefix="my-app", # each partition writer uses "-" + ) + +The multi-writer accepts the same ``codec``, ``encoders``, ``auto_seqno``, ``auto_created_at`` +and buffer-limit parameters as :meth:`writer`, and exposes ``write``, ``write_with_ack``, +``flush`` and ``close``. Its ``wait_init()`` method returns ``None`` once the topic has been +described and the partition set is known. + +When passed a list, the multi-writer accepts messages one at a time in input order. +Each message's key selects its partition, and the buffer limits apply per partition. +If admission fails for one message, the call raises and skips the rest of the list. +For example, a timeout waiting for buffer space raises ``TopicWriterBufferFullError``. +The writer continues delivering messages it has already accepted. The exception does not +report how many were accepted. Retrying the whole list with automatic sequence numbers +can duplicate those messages. + +With ``auto_seqno=False``, supply a positive ``seqno`` for every message. Numbers must strictly +increase across the whole multi-writer, including messages routed to different partitions. +Both automatic and manual sequence numbers are preserved on resend, including split and merge. +A single write rejected with ``TopicWriterBufferFullError`` can be retried with the same number +as long as it still exceeds the last accepted number. + +Cancelling or timing out a call that is waiting for initialization does not cancel the shared +initialization. Other waiters and subsequent writes can still complete. Closing the writer +cancels unfinished initialization and waits for its task to stop. Each ``DescribeTopic`` RPC +has a separate 30-second timeout; a caller's shorter wait timeout does not shorten that RPC. + +The multi-writer copies each message and its metadata when accepting a write. Later changes to +the original message do not change an accepted write or its retries. An automatically assigned +creation timestamp is also preserved across retries. + +If an accepted write fails permanently, its acknowledgement future, ``flush()`` and +``close(flush=True)`` report the error, including when it happened before the flush or close +call. Closing still releases the underlying writers before raising the error. Use +``close(flush=False)`` to abandon pending writes without waiting for delivery. +Once closing begins, new writes raise ``TopicWriterClosedError``. Calling ``close(flush=False)`` +also wakes a write waiting for buffer space. If the partition splits during that wait, an +open writer refreshes the route and retries the message. +Cancelling an individual acknowledgement future only ends that wait. Both ``flush()`` +and ``close(flush=True)`` still wait for the message if it moves to a new partition. + +.. note:: + + When an auto-partitioned partition is split (one into two) or merged (two into one), the + multi-writer re-describes the topic, routes subsequent keys to the new child partition(s), and + transparently resends the messages that were still in flight to the retired partition(s). + + Which of those messages to resend is decided by asking the server how far the retired + partition's producer actually got, not by looking at the acknowledgements the client happened + to receive: a message can be persisted and its acknowledgement lost together with the session + that a split tears down. Anything at or below that point is reported as written instead of + being sent again, so a split produces neither loss nor duplicates, and per-key ordering is + preserved throughout. + + Recovery also refreshes routing when a selected partition cannot initialize and follows + cascaded splits to their active leaf partitions. An incomplete set of child key ranges is + retried before updating routing. If recovery cannot find a complete topology or deliver an + accepted message within its retry limits, the affected messages fail explicitly and the + error is reported by ``flush()`` and ``close(flush=True)``. + + When multiple parents share a descendant, recovery stops all affected parents and reads + each producer's cut before sending any of their messages. It filters each parent's tail + against that parent's cut, then sends the combined remainder in increasing sequence-number + order. New writes cannot overtake this transfer. If a destination producer's existing history + conflicts with a carried sequence number, the affected message fails explicitly instead of + being renumbered or reported as already written. + + This client-side handling of merge does not enable server-side automatic merge. Its + availability depends on the target server. Recovery also follows intermediate inactive + partitions, including a merge followed by a split. + + Writer Backpressure ^^^^^^^^^^^^^^^^^^^ @@ -281,10 +405,9 @@ A message is counted as occupying the buffer from the moment it is passed to ``w until the server acknowledges it. Backpressure is active when **at least one** limit is set; setting both means either limit can trigger a wait (OR semantics). -The limits are **soft**: ``write()`` blocks only if the buffer is *already* at or above the -limit when the call starts. Once unblocked, the entire batch is admitted regardless of its -size. This means callers that batch multiple messages in a single ``write()`` call will never -deadlock even when the batch is larger than the limit. +The single-partition writer uses **soft limits**. Its ``write()`` blocks only if the buffer is +*already* at or above the limit when the call starts. Once unblocked, the writer admits the +entire batch, even if the batch itself exceeds the limit. **Blocking behavior (default)** diff --git a/examples/topic/multiwriter_example.py b/examples/topic/multiwriter_example.py new file mode 100644 index 00000000..9363d8d9 --- /dev/null +++ b/examples/topic/multiwriter_example.py @@ -0,0 +1,59 @@ +"""Examples for the topic multi-partition writer (write-by-key). + +The multi-writer routes each message to a partition by its ``key``, keeping every +message with the same key on the same partition (per-key ordering). By default it +picks a partition chooser automatically: the key-range chooser for +auto-partitioned topics, the Kafka-hash chooser otherwise. +""" + +import asyncio + +import ydb + + +def write_by_key_sync(db: ydb.Driver, topic_path: str): + with db.topic_client.multiwriter(topic_path, producer_id_prefix="my-app") as writer: + writer.write(ydb.TopicWriterMessage(data="event-a", key="user-42")) + writer.write(ydb.TopicWriterMessage(data="event-b", key="user-7")) + # messages with the same key always land on the same partition, in order + writer.write(ydb.TopicWriterMessage(data="event-c", key="user-42")) + writer.flush() + + +async def write_by_key_async(db: ydb.aio.Driver, topic_path: str): + async with db.topic_client.multiwriter(topic_path, producer_id_prefix="my-app") as writer: + await writer.write(ydb.TopicWriterMessage(data="event-a", key="user-42")) + # wait for the server ack of a specific message + await writer.write_with_ack(ydb.TopicWriterMessage(data="event-b", key="user-7")) + + +def write_by_key_with_explicit_chooser(db: ydb.Driver, topic_path: str): + # Force the Kafka-compatible hash chooser (murmur2(key) % partitions_count). + with db.topic_client.multiwriter( + topic_path, + partition_chooser=ydb.TopicWriterPartitionByKeyKafka(), + ) as writer: + writer.write(ydb.TopicWriterMessage(data="event", key="user-42")) + + +def run_sync(): + with ydb.Driver( + connection_string="grpc://localhost:2135?database=/local", + credentials=ydb.credentials.AnonymousCredentials(), + ) as db: + db.wait(timeout=5, fail_fast=True) + write_by_key_sync(db, "/local/topic") + + +async def run_async(): + async with ydb.aio.Driver( + connection_string="grpc://localhost:2135?database=/local", + credentials=ydb.credentials.AnonymousCredentials(), + ) as db: + await db.wait(timeout=5, fail_fast=True) + await write_by_key_async(db, "/local/topic") + + +if __name__ == "__main__": + run_sync() + asyncio.run(run_async()) diff --git a/tests/slo/Dockerfile b/tests/slo/Dockerfile index b45d2611..8abd04e0 100644 --- a/tests/slo/Dockerfile +++ b/tests/slo/Dockerfile @@ -15,9 +15,7 @@ # `workload_current_command` from the action is appended to the `*-run` call # as tuning flags (--read-rps, --write-rps, ...). # -# Notes: -# - OpenTelemetry 1.39.x requires Python >= 3.9. -# - gcc/libc6-dev are needed to build hdrhistogram from sdist. +# gcc/libc6-dev are needed to build hdrhistogram from sdist. FROM python:3.11-slim diff --git a/tests/slo/README.md b/tests/slo/README.md index 88d46e72..d5a51959 100644 --- a/tests/slo/README.md +++ b/tests/slo/README.md @@ -24,10 +24,13 @@ async (`ydb.aio`) path: | `async-query` | Query service | async | | `sync-topic` | Topic service | sync | | `async-topic` | Topic service | async | +| `sync-topic-multiwriter` | Topic service, writes by key | sync | > The `--async` CLI flag is kept as a manual override for `*-run` commands. > The bare `topic` label is still accepted as an alias for `sync-topic`. +Each SLO workflow run executes all six scenarios listed above. + ### Usage: Each workload type has 3 commands: @@ -267,7 +270,24 @@ When running `topic-run` (`sync-topic` / `async-topic`), the program creates `re - a **backward** seqno (already seen) is a **duplicate** — reconnect redelivery; with producer-id dedup it should stay near zero (informational); - **end-to-end latency** is `read_ts − write_ts` for the first delivery of each message (writer and reader share the process, so the timestamps are comparable). -Each message carries `writer_id:seqno:write_ts_ns:` followed by padding to the configured size. Topics are scoped per ref so the current and baseline containers (same cluster, run in parallel) don't share a topic. +Each message carries `writer_id:seqno:write_ts_ns:` followed by padding to the configured size. Topics are scoped per workload and per ref, so neither two workloads nor the current and baseline containers (same cluster, run in parallel) share a topic — sharing one would mix their producers into each other's delivery and ordering accounting. + +### Topic multi-writer workload + +`sync-topic-multiwriter` runs the same read side and the same accounting, but writes through `topic_client.multiwriter(...)`: each message carries a `key` and the writer picks the partition itself. That covers what the pinned-writer workload cannot — key routing, the per-partition sub-writer pool, and the recovery of both when nodes disappear under chaos. + +- `writeJob` — each thread owns one multi-writer and cycles through `--keys-per-writer` distinct keys. One payload stream per **key** (not per partition), because a key is what the writer keeps ordered, and two keys may share a partition. Use fewer write threads than the pinned workload: every thread opens a sub-writer per partition it touches. +- `readJob` — unchanged. A forward gap is still loss and a backward seqno is still a duplicate; the streams it validates are keys rather than partition-pinned producers. + +Enabled by `--use-multiwriter` on `topic-run`; the label sets it automatically. There is no async variant yet — the sync facade drives the same async implementation underneath, so the code under test is the same. + +When the merge-base SDK does not provide `multiwriter`, this scenario compares against the +parent of the latest SDK implementation commit in the PR. Changes limited to the workload or +its tests do not advance that baseline. The report labels it `previous-multiwriter@`. +If no earlier implementation exists, baseline selection fails explicitly. An SDK without the +API also fails workload initialization instead of running reader threads with no writers. + +> Local re-runs against an existing topic will report duplicates: per-key seqno counters restart at 1 with the process, while the reader still expects the sequence left by the previous run. Recreate the topic (`topic-cleanup` + `topic-create`) between local runs. CI is unaffected — every run gets its own topic. ## Collected metrics - `oks` - amount of OK requests diff --git a/tests/slo/docker-entrypoint.sh b/tests/slo/docker-entrypoint.sh index 55d933ec..edb5b6b1 100755 --- a/tests/slo/docker-entrypoint.sh +++ b/tests/slo/docker-entrypoint.sh @@ -19,9 +19,13 @@ set -e +WORKLOAD_ARGS="" case "${WORKLOAD_NAME:-sync-query}" in sync-table|sync-query|async-query) PREFIX=table ;; topic|sync-topic|async-topic) PREFIX=topic ;; + # Same topic workload, but writes go through the multi-partition writer (routing by key + # plus its sub-writer pool) instead of one writer pinned per partition. + sync-topic-multiwriter) PREFIX=topic; WORKLOAD_ARGS="--use-multiwriter" ;; *) echo "Unknown WORKLOAD_NAME: ${WORKLOAD_NAME}" >&2 exit 1 @@ -37,11 +41,15 @@ DURATION="${WORKLOAD_DURATION:-600}" # Scope the topic by ref so the current and baseline containers (same cluster, # run in parallel) don't share a topic — otherwise their readers/producers would # cross-contaminate delivery/ordering validation. +# The workload name is part of the path too: topic workloads differ in what they write +# (producer ids, key layout), so two of them sharing a topic would corrupt each other's +# delivery and ordering accounting. EXTRA_ARGS="" if [ "$PREFIX" = "topic" ]; then REF_RAW="${WORKLOAD_REF:-${REF:-main}}" SAFE_REF=$(printf '%s' "$REF_RAW" | tr -c 'a-zA-Z0-9_' '_') - EXTRA_ARGS="--path ${DATABASE%/}/slo_topic_${SAFE_REF}" + SAFE_WORKLOAD=$(printf '%s' "${WORKLOAD_NAME:-topic}" | tr -c 'a-zA-Z0-9_' '_') + EXTRA_ARGS="--path ${DATABASE%/}/slo_topic_${SAFE_WORKLOAD}_${SAFE_REF}" fi # Schema prep is idempotent at the SDK level for topics; for tables, a parallel @@ -53,4 +61,5 @@ exec python ./tests/slo/src \ "${PREFIX}-run" "$ENDPOINT" "$DATABASE" \ --time "$DURATION" \ $EXTRA_ARGS \ + $WORKLOAD_ARGS \ "$@" diff --git a/tests/slo/metrics-topic.yaml b/tests/slo/metrics-topic.yaml index f1fd913d..a34c563d 100644 --- a/tests/slo/metrics-topic.yaml +++ b/tests/slo/metrics-topic.yaml @@ -9,6 +9,14 @@ # *_error* -> warning_max 0.1 / critical_max 1.0 (fails the run if > 0) metrics: + # Reader/writer recovery is managed outside the logical operation measured by + # the generic attempts counters, so scrape-boundary deltas are not retries. + - name: read_retry_attempts + disabled: true + + - name: write_retry_attempts + disabled: true + - name: topic_e2e_latency_p50_ms unit: ms query: | diff --git a/tests/slo/requirements.txt b/tests/slo/requirements.txt index 68f516f1..56a006aa 100644 --- a/tests/slo/requirements.txt +++ b/tests/slo/requirements.txt @@ -2,8 +2,5 @@ requests==2.33.0 aiolimiter==1.1.0 hdrhistogram==0.10.3 -# OpenTelemetry (OTLP/HTTP exporter) -# NOTE: OpenTelemetry 1.39.1 requires Python >= 3.9. -opentelemetry-api==1.45.1 -opentelemetry-sdk==1.39.1 -opentelemetry-exporter-otlp-proto-http==1.39.1 +# The OTLP/HTTP exporter pulls in a compatible SDK, which pins its API version. +opentelemetry-exporter-otlp-proto-http==1.45.1 diff --git a/tests/slo/src/jobs/topic_multiwriter_jobs.py b/tests/slo/src/jobs/topic_multiwriter_jobs.py new file mode 100644 index 00000000..5ca97b61 --- /dev/null +++ b/tests/slo/src/jobs/topic_multiwriter_jobs.py @@ -0,0 +1,114 @@ +"""Topic SLO workload variant that writes through the multi-partition writer. + +The regular topic workload pins one writer per partition and drives it directly. This one +writes by key instead and lets the multi-writer decide the partition, so the chaos run +exercises the parts that only exist there: routing, the per-partition sub-writer pool, and +recovery of both when nodes go away underneath. + +Everything else is inherited: the reader side, the metrics, and the delivery accounting. +That works unchanged because the accounting only needs a stream whose sequence numbers grow +monotonically, and per-key ordering gives exactly that -- one payload stream per key rather +than per partition. +""" + +import logging +import threading +import time + +from core.metrics import OP_TYPE_WRITE, REF + +import ydb + +from .base import SyncRateLimiter +from .topic_jobs import TopicJobManager +from .topic_payload import encode_payload + +logger = logging.getLogger(__name__) + + +class TopicMultiWriterJobManager(TopicJobManager): + def __init__(self, driver, args, metrics): + super().__init__(driver, args, metrics) + if not callable(getattr(self.driver.topic_client, "multiwriter", None)): + raise RuntimeError("This SDK does not support the topic multiwriter workload") + self.keys_per_writer = max(1, int(getattr(self.args, "keys_per_writer", 8))) + + def _run_topic_write_jobs(self): + logger.info( + "Start topic multi-writer jobs: %d writers x %d keys", + self.args.write_threads, + self.keys_per_writer, + ) + + write_rps = int(getattr(self.args, "write_rps", 0)) + write_limiter = SyncRateLimiter(min_interval_s=0.0 if write_rps <= 0 else 1.0 / write_rps) + + futures = [] + for i in range(self.args.write_threads): + future = threading.Thread( + name=f"slo_topic_multiwrite_{i}", + target=self._run_topic_writes, + args=(i, write_limiter), + ) + future.start() + futures.append(future) + return futures + + def _stream_id(self, writer_id: int, key_index: int) -> int: + """Payload stream id for one key. + + The reader validates ordering per stream id, so it has to be one id per key: a key is + what the multi-writer keeps ordered, and two keys may share a partition. + """ + return writer_id * self.keys_per_writer + key_index + + def _run_topic_writes(self, writer_id, limiter): + start_time = time.time() + producer_id_prefix = f"{REF}-mw{writer_id}" + write_timeout = self.args.write_timeout / 1000 + keys = [f"slo-key-{self._stream_id(writer_id, j)}" for j in range(self.keys_per_writer)] + + logger.info("Start topic multi-writer %s (prefix %s, keys %s)", writer_id, producer_id_prefix, len(keys)) + + # Sequence numbers live across writer recreations so the reader never sees a stream + # restart, exactly as in the single-partition workload. + seqno = {key: 1 for key in keys} + next_key = 0 + + while time.time() - start_time < self.args.time: + try: + with self.driver.topic_client.multiwriter( + self.args.path, + producer_id_prefix=producer_id_prefix, + codec=ydb.TopicCodec.RAW, + ) as writer: + while time.time() - start_time < self.args.time: + with limiter: + key_index = next_key % len(keys) + key = keys[key_index] + next_key += 1 + + payload = encode_payload( + self._stream_id(writer_id, key_index), + seqno[key], + time.monotonic_ns(), + self.args.message_size, + ) + message = ydb.TopicWriterMessage(data=payload, key=key) + + ts = self.metrics.start((OP_TYPE_WRITE,)) + try: + writer.write_with_ack(message, timeout=write_timeout) + self.metrics.stop((OP_TYPE_WRITE,), ts) + # Advance only on success: a failed write retries the same + # seqno, so at worst it duplicates, never fakes a loss. + seqno[key] += 1 + except Exception as e: + self.metrics.stop((OP_TYPE_WRITE,), ts, error=e) + logger.error("Multi-writer write error (recreating writer): %s", e) + break # drop the possibly wedged writer and remake it + except Exception as e: + logger.error("Topic multi-writer %s recreate: %s", writer_id, e) + time.sleep(0.2) + + logger.info("Stop topic multi-writer %s", writer_id) diff --git a/tests/slo/src/options.py b/tests/slo/src/options.py index 200c3c55..2ef5eeae 100644 --- a/tests/slo/src/options.py +++ b/tests/slo/src/options.py @@ -130,6 +130,17 @@ def make_topic_run_parser(subparsers): help="Number of threads for topic writing", ) topic_parser.add_argument("--message-size", default=100, type=int, help="Topic message size in bytes") + topic_parser.add_argument( + "--use-multiwriter", + action="store_true", + help="Write by key through the multi-partition writer instead of one writer per partition", + ) + topic_parser.add_argument( + "--keys-per-writer", + default=8, + type=int, + help="Distinct routing keys each multi-writer thread cycles through (--use-multiwriter only)", + ) topic_parser.add_argument("--time", default=10, type=int, help="Time to run in seconds") topic_parser.add_argument( diff --git a/tests/slo/src/runners/topic_runner.py b/tests/slo/src/runners/topic_runner.py index c9a8bdaa..4ff0f127 100644 --- a/tests/slo/src/runners/topic_runner.py +++ b/tests/slo/src/runners/topic_runner.py @@ -3,6 +3,7 @@ from core.metrics import create_metrics from jobs.async_topic_jobs import AsyncTopicJobManager from jobs.topic_jobs import TopicJobManager +from jobs.topic_multiwriter_jobs import TopicMultiWriterJobManager import ydb import ydb.aio @@ -72,9 +73,11 @@ def run(self, args): assert self.driver is not None, "Driver is not initialized. Call set_driver() before run()." metrics = create_metrics(args.otlp_endpoint) - self.logger.info("Starting topic SLO tests") + use_multiwriter = getattr(args, "use_multiwriter", False) + self.logger.info("Starting topic SLO tests (multiwriter=%s)", use_multiwriter) - job_manager = TopicJobManager(self.driver, args, metrics) + manager_cls = TopicMultiWriterJobManager if use_multiwriter else TopicJobManager + job_manager = manager_cls(self.driver, args, metrics) job_manager.run_tests() self.logger.info("Topic SLO tests completed") @@ -85,6 +88,10 @@ def run(self, args): async def run_async(self, args): """Async version of topic SLO tests using ydb.aio.Driver""" assert self.driver is not None, "Driver is not initialized. Call set_driver() before run_async()." + if getattr(args, "use_multiwriter", False): + # Silently running the single-partition workload under a multiwriter label would + # make the SLO report compare something other than what it claims. + raise NotImplementedError("--use-multiwriter has no async workload yet; run it as a sync-* workload") metrics = create_metrics(args.otlp_endpoint) self.logger.info("Starting async topic SLO tests") diff --git a/tests/topics/test_topic_writer.py b/tests/topics/test_topic_writer.py index 035c3b80..c1e99bad 100644 --- a/tests/topics/test_topic_writer.py +++ b/tests/topics/test_topic_writer.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import datetime from typing import List # noqa: F401 import pytest @@ -324,3 +325,255 @@ class TestException(Exception): writer.write_with_ack("123") raise TestException() + + +@pytest.mark.asyncio +class TestTopicMultiWriterAsyncIO: + async def test_flush_and_close_report_encoder_errors(self, driver, database, topic_consumer): + path = database + "/mw-encoder-error" + await self._recreate(driver, path, topic_consumer) + + def failing_encoder(data): + raise ValueError("encoder failed") + + writer = driver.topic_client.multiwriter( + path, + codec=ydb.TopicCodec.GZIP, + encoders={ydb.TopicCodec.GZIP: failing_encoder}, + ) + try: + await writer.write(ydb.TopicWriterMessage(data=b"payload", key="key")) + with pytest.raises(ValueError, match="encoder failed"): + await asyncio.wait_for(writer.flush(), timeout=10) + with pytest.raises(ValueError, match="encoder failed"): + await asyncio.wait_for(writer.close(), timeout=10) + assert writer._closed + assert not writer._writers + finally: + await writer.close(flush=False) + + async def _recreate(self, driver, path, consumer, **kwargs): + try: + await driver.topic_client.drop_topic(path) + except ydb.SchemeError: + pass + await driver.topic_client.create_topic(path=path, consumers=[consumer], **kwargs) + + def _auto_partitioning(self): + return ydb.TopicAutoPartitioningSettings( + strategy=ydb.TopicAutoPartitioningStrategy.SCALE_UP, + up_utilization_percent=1, + down_utilization_percent=1, + stabilization_window=datetime.timedelta(seconds=1), + ) + + async def test_key_range_exposed_for_autopartitioned_topic(self, driver, database, topic_consumer): + path = database + "/mw-keyrange" + await self._recreate( + driver, + path, + topic_consumer, + min_active_partitions=2, + max_active_partitions=50, + auto_partitioning_settings=self._auto_partitioning(), + ) + desc = await driver.topic_client.describe_topic(path) + assert any(p.key_range is not None for p in desc.partitions) + + async def test_write_by_key_preserves_per_key_order(self, driver, database, topic_consumer): + path = database + "/mw-plain" + await self._recreate(driver, path, topic_consumer, min_active_partitions=3) + + keys = ["user-1", "user-2", "user-3", "user-4", "user-5"] + per_key = 8 + async with driver.topic_client.multiwriter(path, producer_id_prefix="mw") as writer: + await writer.wait_init() + assert isinstance(writer._chooser, ydb.TopicWriterPartitionByKeyKafka) + for i in range(per_key): + for key in keys: + await writer.write(ydb.TopicWriterMessage(data=("%s:%d" % (key, i)).encode(), key=key)) + await writer.flush() + + total = per_key * len(keys) + received = {key: [] for key in keys} + async with driver.topic_client.reader(path, consumer=topic_consumer) as reader: + for _ in range(total): + message = await asyncio.wait_for(reader.receive_message(), timeout=30) + key, index = message.data.decode().split(":") + received[key].append(int(index)) + reader.commit(message) + + for key in keys: + assert received[key] == list(range(per_key)), (key, received[key]) + + async def test_write_by_key_on_autopartitioned_topic(self, driver, database, topic_consumer): + path = database + "/mw-auto" + await self._recreate( + driver, + path, + topic_consumer, + min_active_partitions=2, + max_active_partitions=50, + auto_partitioning_settings=self._auto_partitioning(), + ) + + keys = ["alpha", "beta", "gamma", "delta"] + per_key = 5 + async with driver.topic_client.multiwriter(path, producer_id_prefix="mw") as writer: + await writer.wait_init() + # auto-partitioned topics report key ranges -> adaptive default picks the bound chooser + assert isinstance(writer._chooser, ydb.TopicWriterPartitionByKeyBound) + for i in range(per_key): + for key in keys: + # write_with_ack verifies the server accepts bound-routed writes + await writer.write_with_ack(ydb.TopicWriterMessage(data=("%s:%d" % (key, i)).encode(), key=key)) + + total = per_key * len(keys) + seen = 0 + async with driver.topic_client.reader(path, consumer=topic_consumer) as reader: + for _ in range(total): + message = await asyncio.wait_for(reader.receive_message(), timeout=30) + seen += 1 + reader.commit(message) + assert seen == total + + async def test_write_by_key_survives_partition_split(self, driver, database, topic_consumer): + path = database + "/mw-split" + await self._recreate( + driver, + path, + topic_consumer, + min_active_partitions=1, + max_active_partitions=4, + auto_partitioning_settings=ydb.TopicAutoPartitioningSettings( + strategy=ydb.TopicAutoPartitioningStrategy.SCALE_UP, + up_utilization_percent=90, + down_utilization_percent=1, + stabilization_window=datetime.timedelta(seconds=300), + ), + ) + + async def wait_for_partitions(expected): + deadline = asyncio.get_running_loop().time() + 30 + while True: + description = await driver.topic_client.describe_topic(path) + active = [partition for partition in description.partitions if partition.active] + if len(active) == expected: + return description + assert asyncio.get_running_loop().time() < deadline, ( + "topic did not reach %d active partitions" % expected + ) + await asyncio.sleep(0.1) + + description = await wait_for_partitions(1) + assert len(description.partitions) == 1 + + keys = ["k%d" % i for i in range(32)] + written = 0 + async with driver.topic_client.multiwriter(path, producer_id_prefix="mw-split") as writer: + await writer.wait_init() + assert isinstance(writer._chooser, ydb.TopicWriterPartitionByKeyBound) + + async def write_batch(count): + nonlocal written + for _ in range(count): + key = keys[written % len(keys)] + await writer.write(ydb.TopicWriterMessage(data=("%d:%s" % (written, key)).encode(), key=key)) + written += 1 + await writer.flush() + + await write_batch(50) + + await driver.topic_client.alter_topic(path, set_min_active_partitions=2) + description = await wait_for_partitions(2) + assert len(description.partitions) == 3 + root = next(partition for partition in description.partitions if not partition.parent_partition_ids) + active_ids = {partition.partition_id for partition in description.partitions if partition.active} + assert not root.active + assert set(root.child_partition_ids) == active_ids + await write_batch(100) + + await driver.topic_client.alter_topic(path, set_min_active_partitions=4) + description = await wait_for_partitions(4) + assert len(description.partitions) == 7 + await write_batch(100) + + total = written + seen = set() + last_by_key = {} + async with driver.topic_client.reader(path, consumer=topic_consumer) as reader: + while True: + try: + message = await asyncio.wait_for( + reader.receive_message(), + timeout=30 if len(seen) < total else 2, + ) + except asyncio.TimeoutError: + assert len(seen) == total, "reader did not receive all messages" + break + index_raw, key_raw = message.data.split(b":", 1) + index = int(index_raw) + key = key_raw.decode() + assert index not in seen, "resend produced duplicate message %d" % index + assert index > last_by_key.get(key, -1), "messages for key %s were reordered" % key + last_by_key[key] = index + seen.add(index) + reader.commit(message) + + assert seen == set(range(total)), "some messages were lost" + + +class TestTopicMultiWriterSync: + def test_flush_and_close_report_encoder_errors(self, driver_sync, database, topic_consumer): + path = database + "/mw-sync-encoder-error" + try: + driver_sync.topic_client.drop_topic(path) + except ydb.SchemeError: + pass + driver_sync.topic_client.create_topic(path=path, consumers=[topic_consumer]) + + def failing_encoder(data): + raise ValueError("encoder failed") + + writer = driver_sync.topic_client.multiwriter( + path, + codec=ydb.TopicCodec.GZIP, + encoders={ydb.TopicCodec.GZIP: failing_encoder}, + ) + try: + writer.write(ydb.TopicWriterMessage(data=b"payload", key="key"), timeout=10) + with pytest.raises(ValueError, match="encoder failed"): + writer.flush(timeout=10) + with pytest.raises(ValueError, match="encoder failed"): + writer.close(timeout=10) + assert writer._closed + finally: + writer.close(flush=False, timeout=10) + + def test_write_by_key_preserves_per_key_order(self, driver_sync, database, topic_consumer): + path = database + "/mw-sync" + try: + driver_sync.topic_client.drop_topic(path) + except ydb.SchemeError: + pass + driver_sync.topic_client.create_topic(path=path, consumers=[topic_consumer], min_active_partitions=3) + + keys = ["a", "b", "c"] + per_key = 6 + with driver_sync.topic_client.multiwriter(path, producer_id_prefix="mw-sync") as writer: + for i in range(per_key): + for key in keys: + writer.write(ydb.TopicWriterMessage(data=("%s:%d" % (key, i)).encode(), key=key)) + writer.flush() + + total = per_key * len(keys) + received = {key: [] for key in keys} + with driver_sync.topic_client.reader(path, consumer=topic_consumer) as reader: + for _ in range(total): + message = reader.receive_message(timeout=30) + key, index = message.data.decode().split(":") + received[key].append(int(index)) + reader.commit(message) + + for key in keys: + assert received[key] == list(range(per_key)), (key, received[key]) diff --git a/ydb/_grpc/grpcwrapper/ydb_topic.py b/ydb/_grpc/grpcwrapper/ydb_topic.py index 19b38bf0..3609225f 100644 --- a/ydb/_grpc/grpcwrapper/ydb_topic.py +++ b/ydb/_grpc/grpcwrapper/ydb_topic.py @@ -1657,6 +1657,38 @@ def to_public(self) -> ydb_topic_public_types.PublicDescribeTopicResult: topic_stats=topic_stats, ) + @dataclass + class PartitionKeyRange( + IFromProto[ + Optional["ydb_topic_pb2.PartitionKeyRange"], + Optional["DescribeTopicResult.PartitionKeyRange"], + ], + IToPublic, + ): + # Empty bytes mean an open bound: from_bound == b"" is the start of the + # key space (first partition), to_bound == b"" is the end (last partition). + from_bound: bytes + to_bound: bytes + + @staticmethod + def from_proto( + msg: Optional[ydb_topic_pb2.PartitionKeyRange], + ) -> Optional["DescribeTopicResult.PartitionKeyRange"]: + if msg is None: + return None + return DescribeTopicResult.PartitionKeyRange( + from_bound=msg.from_bound, + to_bound=msg.to_bound, + ) + + def to_public( + self, + ) -> ydb_topic_public_types.PublicDescribeTopicResult.PartitionKeyRange: + return ydb_topic_public_types.PublicDescribeTopicResult.PartitionKeyRange( + from_bound=self.from_bound, + to_bound=self.to_bound, + ) + @dataclass class PartitionInfo( IFromProto[ @@ -1670,6 +1702,7 @@ class PartitionInfo( child_partition_ids: List[int] parent_partition_ids: List[int] partition_stats: Optional["PartitionStats"] + key_range: Optional["DescribeTopicResult.PartitionKeyRange"] @staticmethod def from_proto( @@ -1678,12 +1711,17 @@ def from_proto( if msg is None: return None + key_range = None + if msg.HasField("key_range"): + key_range = DescribeTopicResult.PartitionKeyRange.from_proto(msg.key_range) + return DescribeTopicResult.PartitionInfo( partition_id=msg.partition_id, active=msg.active, child_partition_ids=list(msg.child_partition_ids), parent_partition_ids=list(msg.parent_partition_ids), partition_stats=PartitionStats.from_proto(msg.partition_stats), + key_range=key_range, ) def to_public( @@ -1692,12 +1730,16 @@ def to_public( partition_stats = None if self.partition_stats is not None: partition_stats = self.partition_stats.to_public() + key_range = None + if self.key_range is not None: + key_range = self.key_range.to_public() return ydb_topic_public_types.PublicDescribeTopicResult.PartitionInfo( partition_id=self.partition_id, active=self.active, child_partition_ids=self.child_partition_ids, parent_partition_ids=self.parent_partition_ids, partition_stats=partition_stats, + key_range=key_range, ) @dataclass diff --git a/ydb/_grpc/grpcwrapper/ydb_topic_public_types.py b/ydb/_grpc/grpcwrapper/ydb_topic_public_types.py index afb031d9..b6bda90c 100644 --- a/ydb/_grpc/grpcwrapper/ydb_topic_public_types.py +++ b/ydb/_grpc/grpcwrapper/ydb_topic_public_types.py @@ -242,6 +242,14 @@ class PublicDescribeTopicResult: auto_partitioning_settings: Optional["PublicAutoPartitioningSettings"] + @dataclass + class PartitionKeyRange: + from_bound: bytes + "Inclusive lower bound of the partition key range; empty bytes mean an open (leftmost) bound" + + to_bound: bytes + "Exclusive upper bound of the partition key range; empty bytes mean an open (rightmost) bound" + @dataclass class PartitionInfo: partition_id: int @@ -259,6 +267,9 @@ class PartitionInfo: partition_stats: Optional["PublicPartitionStats"] "Stats for partition, filled only when include_stats in request is true" + key_range: Optional["PublicDescribeTopicResult.PartitionKeyRange"] = None + "Key range owned by the partition; filled for auto-partitioned topics" + @dataclass class TopicStats: store_size_bytes: int diff --git a/ydb/_grpc/grpcwrapper/ydb_topic_test.py b/ydb/_grpc/grpcwrapper/ydb_topic_test.py index b9e30603..a9765b84 100644 --- a/ydb/_grpc/grpcwrapper/ydb_topic_test.py +++ b/ydb/_grpc/grpcwrapper/ydb_topic_test.py @@ -2,8 +2,11 @@ from google.protobuf.json_format import MessageToDict +# Same version dispatch the module under test uses: the CI matrix runs protobuf v3..v6. +from ydb._grpc.common.protos import ydb_topic_pb2 + from ydb._grpc.grpcwrapper.ydb_topic import OffsetsRange -from .ydb_topic import AlterTopicRequest +from .ydb_topic import AlterTopicRequest, DescribeTopicResult from .ydb_topic_public_types import ( AlterTopicRequestParams, PublicAlterConsumer, @@ -96,3 +99,44 @@ def test_alter_topic_request_from_public_to_proto(): } assert msg_dict == expected_dict + + +def test_partition_key_range_round_trip(): + """A bounded partition must survive proto -> internal -> public unchanged. + + The multi-partition writer routes by these bounds, and an empty bound is meaningful (it + marks an open end of the key space), so it has to stay distinguishable from an absent one. + """ + msg = ydb_topic_pb2.DescribeTopicResult.PartitionInfo( + partition_id=7, + active=True, + child_partition_ids=[8, 9], + parent_partition_ids=[3], + key_range=ydb_topic_pb2.PartitionKeyRange(from_bound=b"\x10", to_bound=b"\x80"), + ) + + internal = DescribeTopicResult.PartitionInfo.from_proto(msg) + assert internal.key_range.from_bound == b"\x10" + assert internal.key_range.to_bound == b"\x80" + + public = internal.to_public() + assert public.partition_id == 7 + assert public.child_partition_ids == [8, 9] + assert public.parent_partition_ids == [3] + assert public.key_range.from_bound == b"\x10" + assert public.key_range.to_bound == b"\x80" + + +def test_partition_without_key_range_stays_none(): + """Topics that are not auto-partitioned report no range at all. + + That is not the same as an open range: it is what tells the writer to route by hash + instead of by bounds, so it must not be turned into empty bounds along the way. + """ + msg = ydb_topic_pb2.DescribeTopicResult.PartitionInfo(partition_id=0, active=True) + + internal = DescribeTopicResult.PartitionInfo.from_proto(msg) + assert internal.key_range is None + assert internal.to_public().key_range is None + + assert DescribeTopicResult.PartitionKeyRange.from_proto(None) is None diff --git a/ydb/_topic_writer/topic_writer.py b/ydb/_topic_writer/topic_writer.py index 18e790fe..25ebfe60 100644 --- a/ydb/_topic_writer/topic_writer.py +++ b/ydb/_topic_writer/topic_writer.py @@ -47,6 +47,10 @@ class PublicWriterSettings: # Backpressure is enabled when at least one of the limits above is set. # None = wait indefinitely for buffer space; positive value = raise TopicWriterBufferFullError on timeout. buffer_wait_timeout_sec: Optional[float] = None + # Internal hook used by the multi-partition writer. Called with a connection error + # before the default retry classification; returning True force-stops the writer + # (used to catch OVERLOADED on a split partition). Not part of the public writer() API. + _on_check_retriable_error: Optional[typing.Callable[[BaseException], bool]] = None def __post_init__(self): if self.producer_id is None: @@ -127,6 +131,7 @@ class PublicMessage: created_at: Optional[datetime.datetime] data: "PublicMessage.SimpleSourceType" metadata_items: Optional[Dict[str, "PublicMessage.SimpleSourceType"]] + key: Optional[str] SimpleSourceType = Union[str, bytes] # Will be extend @@ -137,11 +142,15 @@ def __init__( metadata_items: Optional[Dict[str, "PublicMessage.SimpleSourceType"]] = None, seqno: Optional[int] = None, created_at: Optional[datetime.datetime] = None, + key: Optional[str] = None, ): self.seqno = seqno self.created_at = created_at self.data = data self.metadata_items = metadata_items + # Partitioning key: used only by the multi-partition writer to route the + # message to a partition. Ignored by the single-partition writer. + self.key = key @staticmethod def _create_message(data: Message) -> "PublicMessage": @@ -245,6 +254,17 @@ class TopicWriterBufferFullError(TopicWriterError): pass +class TopicWriterPartitionSplitError(TopicWriterRepeatableError): + """Raised internally when the partition targeted by the writer has split. + + Used by the multi-partition writer to stop a per-partition sub-writer so its + messages can be re-routed to the child partitions. + """ + + def __init__(self): + super().__init__("topic writer partition was split") + + def default_serializer_message_content(data: Any) -> bytes: if data is None: return bytes() diff --git a/ydb/_topic_writer/topic_writer_asyncio.py b/ydb/_topic_writer/topic_writer_asyncio.py index 81b9e719..4c102e20 100644 --- a/ydb/_topic_writer/topic_writer_asyncio.py +++ b/ydb/_topic_writer/topic_writer_asyncio.py @@ -19,6 +19,7 @@ InternalMessage, TopicWriterStopped, TopicWriterError, + TopicWriterPartitionSplitError, TopicWriterBufferFullError, internal_message_size_bytes, messages_to_proto_requests, @@ -185,6 +186,12 @@ async def flush(self): logger.debug("flush writer") return await self._reconnector.flush() + def _buffer_is_full(self) -> bool: + return self._reconnector.buffer_is_full() + + async def _wait_for_buffer_space(self) -> None: + await self._reconnector.wait_for_buffer_space() + async def wait_init(self) -> PublicWriterInitInfo: """ wait while real connection will be established to server. @@ -344,14 +351,14 @@ async def close(self, flush: bool): self._closed = True logger.debug("Close writer reconnector id=%s", self._id) - if flush: - await self.flush() - - self._stop(TopicWriterStopped()) - - for task in self._background_tasks: - task.cancel() - await asyncio.wait(self._background_tasks) + try: + if flush: + await self.flush() + finally: + self._stop(TopicWriterStopped()) + for task in self._background_tasks: + task.cancel() + await asyncio.wait(self._background_tasks) # if work was stopped before close by error - raise the error try: @@ -391,7 +398,9 @@ async def write_with_ack_future(self, messages: List[PublicMessage]) -> List[asy messages_future = [self._loop.create_future() for _ in internal_messages] if self._backpressure_enabled: - await self._acquire_buffer_space(internal_messages) + await self.wait_for_buffer_space() + self._buffer_bytes += sum(internal_message_size_bytes(m) for m in internal_messages) + self._buffer_messages += len(internal_messages) self._messages_future.extend(messages_future) @@ -402,24 +411,20 @@ async def write_with_ack_future(self, messages: List[PublicMessage]) -> List[asy return messages_future - async def _acquire_buffer_space(self, internal_messages: List[InternalMessage]) -> None: - """Wait until the buffer is below its limit, then admit the batch (soft-limit semantics). - - Blocking starts only when the buffer is already at or above the limit at call time. - Once unblocked, the entire batch is admitted regardless of its size, so callers that - batch messages never get a permanent deadlock. - """ + def buffer_is_full(self) -> bool: max_buf = self._settings.max_buffer_size_bytes max_msgs = self._settings.max_buffer_messages + return (max_buf is not None and self._buffer_bytes >= max_buf) or ( + max_msgs is not None and self._buffer_messages >= max_msgs + ) + + async def wait_for_buffer_space(self) -> None: + """A batch can enter as soon as the buffer is below both soft limits.""" timeout_sec = self._settings.buffer_wait_timeout_sec deadline = self._loop.time() + timeout_sec if timeout_sec is not None else None - while True: + while self.buffer_is_full(): self._buffer_updated.clear() - if (max_buf is None or self._buffer_bytes < max_buf) and ( - max_msgs is None or self._buffer_messages < max_msgs - ): - break self._check_stop() if deadline is not None: assert timeout_sec is not None @@ -428,7 +433,13 @@ async def _acquire_buffer_space(self, internal_messages: List[InternalMessage]) raise TopicWriterBufferFullError( "Topic writer buffer full: no free space within %.1f s" " (buffer_bytes=%d, max_bytes=%s, buffer_msgs=%d, max_msgs=%s)" - % (timeout_sec, self._buffer_bytes, max_buf, self._buffer_messages, max_msgs) + % ( + timeout_sec, + self._buffer_bytes, + self._settings.max_buffer_size_bytes, + self._buffer_messages, + self._settings.max_buffer_messages, + ) ) try: await asyncio.wait_for(self._buffer_updated.wait(), timeout=min(0.5, remaining)) @@ -438,9 +449,6 @@ async def _acquire_buffer_space(self, internal_messages: List[InternalMessage]) await self._buffer_updated.wait() self._check_stop() - new_bytes = sum(internal_message_size_bytes(m) for m in internal_messages) - self._buffer_bytes += new_bytes - self._buffer_messages += len(internal_messages) def _add_messages_to_send_queue(self, internal_messages: List[InternalMessage]): self._messages.extend(internal_messages) @@ -533,10 +541,22 @@ async def _connection_loop(self): done.pop().result() # need for raise exception - reason of stop task except (asyncio.CancelledError, issues.Error) as err: if isinstance(err, asyncio.CancelledError): - if self._closed: + # close(flush=True) still needs reconnects until pending writes are acked. + if self._stop_reason.done(): return err = issues.ConnectionLost("gRPC stream cancelled") + if self._settings._on_check_retriable_error is not None and self._settings._on_check_retriable_error( + err + ): + logger.debug( + "writer reconnector %s stop connection loop by on_check_retriable_error hook due to %s", + self._id, + err, + ) + self._stop(TopicWriterPartitionSplitError()) + return + err_info = check_retriable_error(err, retry_settings, attempt) if not err_info.is_retriable or self._tx is not None: # no retries in tx writer logger.debug("writer reconnector %s stop connection loop due to %s", self._id, err) @@ -770,12 +790,15 @@ def _stop(self, reason: BaseException): return self._stop_reason.set_exception(reason) + self._stop_reason.exception() # close() may itself have been cancelled. for f in self._messages_future: + if f.done(): + continue f.set_exception(reason) f.exception() # mark as retrieved so asyncio does not log "Future exception was never retrieved" - self._buffer_updated.set() # wake any tasks blocked in _acquire_buffer_space + self._buffer_updated.set() self._state_changed.set() logger.info("Stop topic writer %s: %s" % (self._id, reason)) diff --git a/ydb/_topic_writer/topic_writer_asyncio_test.py b/ydb/_topic_writer/topic_writer_asyncio_test.py index af741660..cb7bdfa8 100644 --- a/ydb/_topic_writer/topic_writer_asyncio_test.py +++ b/ydb/_topic_writer/topic_writer_asyncio_test.py @@ -8,6 +8,7 @@ import gzip import sys import typing +import weakref from concurrent.futures import ThreadPoolExecutor from queue import Queue, Empty from typing import List, Callable, Optional @@ -36,6 +37,9 @@ PublicWriteResult, TopicWriterError, TopicWriterBufferFullError, + TopicWriterClosedError, + TopicWriterPartitionSplitError, + TopicWriterStopped, ) from .._grpc.grpcwrapper.ydb_topic_public_types import PublicCodec from .._topic_common.test_helpers import StreamMock, wait_for_fast @@ -45,6 +49,15 @@ WriterAsyncIOReconnector, WriterAsyncIO, ) +from . import topic_writer_multi_asyncio +from .topic_writer_multi_asyncio import TopicWriterMultiAsyncIO, MultiWriterSettings +from .topic_writer_partition_chooser import ( + PublicPartitionByKeyKafka, + PublicPartitionByKeyBound, + PublicPartitionChooser, + PARTITION_KEY_METADATA_KEY, +) +from .._grpc.grpcwrapper.ydb_topic_public_types import PublicDescribeTopicResult from ..credentials import AnonymousCredentials @@ -528,6 +541,63 @@ async def create_mock(*args, **kwargs): finally: await reconnector.close(False) + async def test_close_with_flush_reconnects_after_transport_cancel( + self, default_driver, default_settings, get_stream_writer, monkeypatch + ): + original_receive = self.StreamWriterMock.receive + + async def cancellable_receive(stream): + response = await original_receive(stream) + if isinstance(response, asyncio.CancelledError): + raise response + return response + + monkeypatch.setattr(self.StreamWriterMock, "receive", cancellable_receive) + reconnector = WriterAsyncIOReconnector(default_driver, default_settings) + close_task = None + try: + first = get_stream_writer() + futures = await reconnector.write_with_ack_future([PublicMessage(data=b"pending", seqno=1)]) + await asyncio.wait_for(first.from_client.get(), timeout=1) + close_task = asyncio.create_task(reconnector.close(flush=True)) + await asyncio.sleep(0) + assert reconnector._closed and not close_task.done() + + first.from_server.put_nowait(asyncio.CancelledError()) + second = get_stream_writer() + resent = await asyncio.wait_for(second.from_client.get(), timeout=1) + assert [message.seq_no for message in resent] == [1] + second.from_server.put_nowait(self.make_default_ack_message(seq_no=1)) + + await asyncio.wait_for(close_task, timeout=1) + assert isinstance(futures[0].result(), PublicWriteResult.Written) + assert all(task.done() for task in reconnector._background_tasks) + finally: + reconnector._stop(TopicWriterStopped()) + tasks = list(reconnector._background_tasks) + if close_task is not None: + tasks.append(close_task) + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + reconnector._stop_reason.exception() + + async def test_close_timeout_stops_the_writer(self, default_driver, default_settings, get_stream_writer): + writer = WriterAsyncIO(default_driver, default_settings) + stream = get_stream_writer() + abandoned, pending = await writer.write_with_ack_future( + [PublicMessage(b"abandoned", seqno=1), PublicMessage(b"pending", seqno=2)] + ) + await asyncio.wait_for(stream.from_client.get(), timeout=1) + abandoned.cancel() + + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(writer.close(), timeout=0.01) + + assert stream._closed + assert all(task.done() for task in writer._reconnector._background_tasks) + assert isinstance(pending.exception(), TopicWriterStopped) + async def test_stop_on_unexpected_exception(self, reconnector: WriterAsyncIOReconnector, get_stream_writer): class TestException(Exception): pass @@ -552,6 +622,56 @@ async def wait_stop(): with pytest.raises(TestException): await reconnector.close(flush=False) + async def test_retriable_error_hook_stops_the_writer(self, default_driver, default_settings, get_stream_writer): + """The hook must be able to end the connection loop on an otherwise retriable error. + + OVERLOADED normally means "back off and retry", but for the multi-partition writer it is + also how a split announces itself: the partition went inactive and retrying against it + would spin forever. The hook lets the owner claim such an error and stop the writer with a + reason it can recognise, instead of the generic retry path swallowing it. + """ + seen = [] + + def hook(err): + seen.append(err) + return True + + settings = copy.deepcopy(default_settings) + settings._on_check_retriable_error = hook + reconnector = WriterAsyncIOReconnector(default_driver, settings) + + get_stream_writer().from_server.put_nowait(issues.Overloaded("Write to inactive partition 0")) + + with pytest.raises(TopicWriterPartitionSplitError): + + async def wait_stop(): + while True: + await reconnector.write_with_ack_future([PublicMessage(data="123", seqno=3)]) + await asyncio.sleep(0.01) + + await asyncio.wait_for(wait_stop(), 2) + + assert seen and isinstance(seen[0], issues.Overloaded) + + with pytest.raises(TopicWriterPartitionSplitError): + await reconnector.close(flush=False) + + async def test_retriable_error_hook_declining_keeps_default_retry( + self, default_driver, default_settings, get_stream_writer + ): + """A hook that declines must leave the normal retry policy untouched.""" + settings = copy.deepcopy(default_settings) + settings._on_check_retriable_error = lambda err: False + reconnector = WriterAsyncIOReconnector(default_driver, settings) + + get_stream_writer().from_server.put_nowait(issues.Overloaded("ordinary overload")) + await reconnector.write_with_ack_future([PublicMessage(data="123", seqno=3)]) + await asyncio.sleep(0.1) + + # Retriable error + declining hook -> the writer reconnected instead of stopping. + assert not reconnector._stop_reason.done() + await reconnector.close(flush=False) + async def test_wait_init(self, default_driver, default_settings, get_stream_writer): init_seqno = 100 expected_init_info = PublicWriterInitInfo(last_seqno=init_seqno, supported_codecs=[]) @@ -1011,10 +1131,9 @@ def default_settings(self) -> PublicWriterSettings: @pytest.fixture(autouse=True) def mock_reconnector_init(self, monkeypatch, reconnector): - def t(cls, driver, settings): - return reconnector - - monkeypatch.setattr(WriterAsyncIOReconnector, "__new__", t) + monkeypatch.setattr( + "ydb._topic_writer.topic_writer_asyncio.WriterAsyncIOReconnector", mock.Mock(return_value=reconnector) + ) @pytest.fixture def reconnector(self, monkeypatch) -> TestWriterAsyncIO.ReconnectorMock: @@ -1176,3 +1295,2554 @@ async def test_writer_create_failure_does_not_leak_grpc_thread(): finally: channel.close() server.stop() + + +class _PublicDescription: + def __init__(self, partitions): + self.partitions = partitions + + +class _MultiFakeDescribeDriver: + """Fake driver that answers DescribeTopic with a sequence of descriptions.""" + + _credentials = AnonymousCredentials() + + def __init__(self, descriptions): + self._descriptions = list(descriptions) + self.describe_calls = 0 + + async def __call__(self, request, stub, method, wrapper=None, *args, **kwargs): + idx = min(self.describe_calls, len(self._descriptions) - 1) + self.describe_calls += 1 + description = _PublicDescription(self._descriptions[idx]) + + class _Result: + def to_public(self): + return description + + return _Result() + + +_real_sleep = asyncio.sleep + + +async def _no_sleep(_delay): + """Run timer-driven loops at full speed. Holds the real sleep so patching + asyncio.sleep with this does not make it call itself.""" + await _real_sleep(0) + + +# Per-partition last persisted seqno seen by the fakes' wait_init(); tests mutate it. +_FAKE_LAST_SEQNO: dict = {} + + +class _FakeSubWriter: + """Stand-in for a per-partition WriterAsyncIO used by the multi-writer. + + Acks every write immediately. + """ + + def __init__(self, driver, settings): + self.settings = settings + self.partition_id = settings.partition_id + self.producer_id = settings.producer_id + self.split_hook = settings._on_check_retriable_error + self.messages: List = [] + self.closed = False + # The server keys persisted state by producer id, not by the partition a session happens + # to be pinned to: that is why an unpinned probe session can still report the last_seqno + # of a partition that has already gone inactive. Keying the fake on partition_id instead + # would make every probe read 0 and hide whether the dedup cut works at all. + self.producer_partition_id = int(settings.producer_id.rsplit("-", 1)[-1]) + + async def wait_init(self): + last_seqno = _FAKE_LAST_SEQNO.get(self.producer_partition_id, 0) + return PublicWriterInitInfo(last_seqno=last_seqno, supported_codecs=[]) + + def _buffer_is_full(self): + return False + + async def write_with_ack_future(self, message): + self.messages.append(message) + future = asyncio.get_running_loop().create_future() + future.set_result(PublicWriteResult.Written(offset=len(self.messages))) + return future + + async def flush(self): + pass + + async def close(self, flush=True): + self.closed = True + + +class _ControllableSubWriter(_FakeSubWriter): + """Sub-writer whose acks are resolved manually, to test split-resend.""" + + def __init__(self, driver, settings): + super().__init__(driver, settings) + self.pending: List = [] + + async def write_with_ack_future(self, message): + self.messages.append(message) + future = asyncio.get_running_loop().create_future() + self.pending.append(future) + return future + + def resolve_all(self): + for i, future in enumerate(self.pending): + if not future.done(): + future.set_result(PublicWriteResult.Written(offset=i)) + + +class _KeyMapChooser(PublicPartitionChooser): + """Deterministic chooser: routes by message key via a caller-controlled map.""" + + def __init__(self, mapping): + self._mapping = mapping + self.partitions = set() + + def add_partitions(self, partitions): + for p in partitions: + self.partitions.add(p.partition_id) + + def remove_partition(self, partition_id): + self.partitions.discard(partition_id) + + def choose_partition(self, message): + return self._mapping[message.key] + + +class _FlushControlledSubWriter(_FakeSubWriter): + """Sub-writer that acks buffered messages only when flush() is called.""" + + def __init__(self, driver, settings): + super().__init__(driver, settings) + self.pending: List = [] + + async def write_with_ack_future(self, message): + self.messages.append(message) + future = asyncio.get_running_loop().create_future() + self.pending.append(future) + return future + + async def flush(self): + for i, future in enumerate(self.pending): + if not future.done(): + future.set_result(PublicWriteResult.Written(offset=i)) + + +class _RaisingSubWriter(_FakeSubWriter): + """Sub-writer whose admission always fails.""" + + async def write_with_ack_future(self, message): + raise RuntimeError("admission failed") + + +class _CloseRaisesSubWriter(_ControllableSubWriter): + """Sub-writer whose close() re-raises the split stop reason (like a real hook-stopped writer).""" + + async def close(self, flush=True): + self.closed = True + raise TopicWriterPartitionSplitError() + + +class _SeqnoGuardSubWriter(_ControllableSubWriter): + """Sub-writer that enforces the real writer's explicit-seqno guard. + + With auto_seqno=False (what the multi-writer always uses) WriterAsyncIO seeds + _last_known_seq_no from the server's last_seqno at init and then rejects any message with + seq_no <= it -- see _prepare_internal_messages in topic_writer_asyncio.py. The plain fakes + accept every seqno, which hides resend bugs on that boundary. + """ + + def __init__(self, driver, settings): + super().__init__(driver, settings) + self._last_known_seqno: Optional[int] = None + + async def wait_init(self): + info = await super().wait_init() + if self._last_known_seqno is None: + self._last_known_seqno = info.last_seqno + return info + + async def write_with_ack_future(self, message): + if self._last_known_seqno is None: + self._last_known_seqno = _FAKE_LAST_SEQNO.get(self.partition_id, 0) + if message.seqno <= self._last_known_seqno: + raise TopicWriterError("Message seqno is duplicated: %s" % message.seqno) + self._last_known_seqno = message.seqno + return await super().write_with_ack_future(message) + + +# Partitions whose sub-writer never finishes init; models a writer opened against a partition +# that is already inactive (observed live: such a writer hangs in init forever). +_FAKE_HANGING_INIT_PARTITIONS: set = set() + + +class _HangingInitSubWriter(_ControllableSubWriter): + async def wait_init(self): + if self.partition_id in _FAKE_HANGING_INIT_PARTITIONS: + await asyncio.Event().wait() + return await super().wait_init() + + +class _AckOnCloseSubWriter(_ControllableSubWriter): + """Sub-writer whose outstanding acks land while the stream is being torn down. + + Models the server persisting (and acking) a message in the same moment the split is + detected -- the ack races the quiesce that the repartition performs before reading its + dedup cut. + """ + + async def close(self, flush=True): + self.resolve_all() + self.closed = True + + +def _retrieve_exceptions(futures) -> None: + """Consume results so asyncio does not warn about never-retrieved exceptions.""" + for future in futures: + if future.done() and not future.cancelled(): + future.exception() + + +def _multi_partition(partition_id, parents=None, children=None, from_bound=None, to_bound=None, active=True): + key_range = None + if from_bound is not None or to_bound is not None: + key_range = PublicDescribeTopicResult.PartitionKeyRange(from_bound=from_bound or b"", to_bound=to_bound or b"") + return PublicDescribeTopicResult.PartitionInfo( + partition_id=partition_id, + active=active, + child_partition_ids=children or [], + parent_partition_ids=parents or [], + partition_stats=None, + key_range=key_range, + ) + + +# Real YDB split topology (observed live against a cloud cluster): the split parent stays in the +# DescribeTopic result as an INACTIVE partition whose child_partition_ids point at the new leaves, +# and each child is active with parent_partition_ids == [parent]. Mocks below mirror that so the +# tests exercise the orchestrator's active/child filtering on realistic input. +def _split_parent(partition_id, children, parents=None): + return _multi_partition(partition_id, parents=parents, children=children, active=False) + + +@pytest.mark.asyncio +class TestTopicWriterMultiAsyncIO: + async def test_routes_messages_by_key(self): + driver = _MultiFakeDescribeDriver([[_multi_partition(0), _multi_partition(1), _multi_partition(2)]]) + settings = MultiWriterSettings( + topic="/local/topic", + producer_id_prefix="pfx", + partition_chooser=PublicPartitionByKeyKafka(), + ) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + + partitions_by_key = {"a": 1, "user-42": 1, "hello": 0, "мурмур2-хэш": 1, "0": 2, "zzz": 1} + for key in partitions_by_key: + await writer.write(PublicMessage(b"payload", key=key)) + + for key, partition_id in partitions_by_key.items(): + sub = writer._writers[partition_id] + assert sub.producer_id == "pfx-%d" % partition_id + assert any(m.key == key for m in sub.messages) + + assert sum(len(w.messages) for w in writer._writers.values()) == len(partitions_by_key) + await writer.close(flush=False) + + async def test_split_reroutes_to_child_partitions(self): + before = [_multi_partition(0), _multi_partition(1), _multi_partition(2)] + after = [ + _split_parent(0, children=[3, 4]), # split parent stays, inactive, with children + _multi_partition(1), + _multi_partition(2), + _multi_partition(3, parents=[0]), + _multi_partition(4, parents=[0]), + ] + driver = _MultiFakeDescribeDriver([before, after]) + settings = MultiWriterSettings( + topic="/local/topic", + producer_id_prefix="pfx", + partition_chooser=PublicPartitionByKeyKafka(), + ) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + + sub0 = await writer._get_or_create_writer(0) + await writer._on_partition_overloaded(0) + + assert sub0.closed + assert 0 not in writer._writers + assert 0 not in writer._partitions + assert set(writer._partitions) == {1, 2, 3, 4} + assert set(writer._chooser._partitions) == {1, 2, 3, 4} + await writer.close(flush=False) + + async def test_split_resends_unacked_messages_with_dedup_cut(self): + _FAKE_LAST_SEQNO.clear() + # Route all three keys to partition 0 initially; after the split, spread them + # across the two children. + mapping = {"a": 0, "b": 0, "c": 0} + chooser = _KeyMapChooser(mapping) + before = [_multi_partition(0), _multi_partition(1)] + after = [ + _split_parent(0, children=[2, 3]), # split parent stays, inactive, with children + _multi_partition(1), + _multi_partition(2, parents=[0]), + _multi_partition(3, parents=[0]), + ] + driver = _MultiFakeDescribeDriver([before, after]) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx", partition_chooser=chooser) + + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + + f_a = await writer.write_with_ack_future(PublicMessage(b"a", key="a")) # partition 0, seqno 1 + f_b = await writer.write_with_ack_future(PublicMessage(b"b", key="b")) # partition 0, seqno 2 + f_c = await writer.write_with_ack_future(PublicMessage(b"c", key="c")) # partition 0, seqno 3 + assert set(writer._inflight[0]) == {1, 2, 3} + + # Ack the first message before the split: seqno 1 is now the dedup cut (max_acked), + # and an acked message is already out of the in-flight set (never resent). + writer._writers[0].pending[0].set_result(PublicWriteResult.Written(offset=0)) + await asyncio.sleep(0) + assert f_a.done() and writer._max_acked.get(0) == 1 + assert set(writer._inflight[0]) == {2, 3} + + # Split: the un-acked b, c (seqno > cut) are re-routed to the children. + mapping.update({"a": 2, "b": 3, "c": 2}) + await writer._on_partition_overloaded(0) + + assert 0 not in writer._inflight + assert [m.key for m in writer._writers[3].messages] == ["b"] + assert [m.key for m in writer._writers[2].messages] == ["c"] + assert not f_b.done() and not f_c.done() + + writer._writers[2].resolve_all() + writer._writers[3].resolve_all() + await asyncio.sleep(0) + assert f_b.done() and f_c.done() + + await writer.close(flush=False) + + async def test_adaptive_default_chooser(self): + # A topic without key ranges -> Kafka hash chooser. + driver_plain = _MultiFakeDescribeDriver([[_multi_partition(0), _multi_partition(1)]]) + # A topic that reports key ranges (auto-partitioning) -> bound chooser. + driver_auto = _MultiFakeDescribeDriver( + [[_multi_partition(0, from_bound=b"", to_bound=b"\x80"), _multi_partition(1, from_bound=b"\x80")]] + ) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + plain = TopicWriterMultiAsyncIO(driver_plain, MultiWriterSettings(topic="/local/topic")) + await plain.wait_init() + assert isinstance(plain._chooser, PublicPartitionByKeyKafka) + await plain.close(flush=False) + + auto = TopicWriterMultiAsyncIO(driver_auto, MultiWriterSettings(topic="/local/topic")) + await auto.wait_init() + assert isinstance(auto._chooser, PublicPartitionByKeyBound) + await auto.close(flush=False) + + async def test_idle_writer_eviction(self): + driver = _MultiFakeDescribeDriver([[_multi_partition(0), _multi_partition(1)]]) + chooser = _KeyMapChooser({"a": 0, "b": 1}) + settings = MultiWriterSettings( + topic="/local/topic", producer_id_prefix="pfx", partition_chooser=chooser, writer_idle_timeout_sec=1000 + ) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + + f_a = await writer.write_with_ack_future(PublicMessage(b"a", key="a")) # partition 0 + f_b = await writer.write_with_ack_future(PublicMessage(b"b", key="b")) # partition 1 + assert set(writer._writers) == {0, 1} + sub0 = writer._writers[0] + + # ack partition 0 (it goes idle); leave partition 1 un-acked + sub0.resolve_all() + await asyncio.sleep(0) + assert not writer._inflight.get(0) and writer._inflight.get(1) + + # make both look old: only the idle partition 0 is evictable + old = writer._loop.time() - 5000 + writer._last_write_at[0] = old + writer._last_write_at[1] = old + await writer._evict_idle_writers() + + assert 0 not in writer._writers and sub0.closed # idle -> evicted + assert 1 in writer._writers # pending in-flight -> kept + assert writer._seqno == 2 # writer-wide cursor survives eviction (a -> 1, b -> 2) + + # writing to partition 0 again recreates a fresh sub-writer; numbering continues from + # the shared cursor rather than restarting for the re-opened partition + f_a2 = await writer.write_with_ack_future(PublicMessage(b"a2", key="a")) + assert 0 in writer._writers and writer._writers[0] is not sub0 + assert next(iter(writer._inflight[0])) == 3 + + writer._writers[0].resolve_all() + writer._writers[1].resolve_all() + await asyncio.sleep(0) + assert f_a.done() and f_b.done() and f_a2.done() + await writer.close(flush=False) + + async def test_split_hook_detects_overloaded_only(self): + # How the server signals a split (observed live): a split partition goes inactive, so the + # next write to it fails on the write stream with OVERLOADED (status_code 400060), + # message "Write to inactive partition N", surfaced by the SDK as issues.Overloaded. + # The hook triggers on that exception TYPE (the message text is not inspected); any other + # error is left to the writer's normal retry path. + driver = _MultiFakeDescribeDriver([[_multi_partition(0)]]) + settings = MultiWriterSettings(topic="/local/topic", partition_chooser=PublicPartitionByKeyKafka()) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + + observed: List[int] = [] + + async def fake_split(partition_id): + observed.append(partition_id) + + writer._on_partition_overloaded = fake_split + hook = writer._make_overloaded_hook(0) + + split_signal = issues.Overloaded("status is not ok: Write to inactive partition 0") + assert hook(split_signal) is True + await asyncio.sleep(0) + assert observed == [0] + assert hook(RuntimeError("some other error")) is False + + await writer.close(flush=False) + + async def test_repartition_tolerates_subwriter_close_raising(self): + # Regression: a hook-stopped sub-writer's close() re-raises TopicWriterPartitionSplitError. + # Repartition must swallow it and still migrate to the child (not fall back to recovery). + _FAKE_LAST_SEQNO.clear() + mapping = {"a": 0} + chooser = _KeyMapChooser(mapping) + before = [_multi_partition(0), _multi_partition(1)] + after = [ + _split_parent(0, children=[2, 3]), # split parent stays, inactive, with children + _multi_partition(1), + _multi_partition(2, parents=[0]), + _multi_partition(3, parents=[0]), + ] + driver = _MultiFakeDescribeDriver([before, after]) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx", partition_chooser=chooser) + + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _CloseRaisesSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + + f_a = await writer.write_with_ack_future(PublicMessage(b"a", key="a")) # partition 0 + mapping["a"] = 2 # after split, route to child 2 + + await writer._on_partition_overloaded(0) # must not raise despite close() raising + + assert 0 not in writer._partitions # retired cleanly (no recovery fallback) + assert [m.key for m in writer._writers[2].messages] == ["a"] # migrated to the child + assert 0 not in writer._inflight + + writer._writers[2].resolve_all() + await asyncio.sleep(0) + assert f_a.done() + + await writer.close(flush=False) + + async def test_merge_migrates_both_parents_to_shared_child(self): + _FAKE_LAST_SEQNO.clear() + # Two parents (0, 1) merge into one child (2), which lists both as parents. The bounds are + # real ones: a merge child owns the ranges of BOTH parents, so it covers strictly more of + # the key space than the parent whose OVERLOADED triggered the handler -- the child-range + # coverage check must accept that, not just an exact tiling. + mapping = {"x": 0, "y": 1} + chooser = _KeyMapChooser(mapping) + before = [ + _multi_partition(0, from_bound=b"", to_bound=b"\x80"), + _multi_partition(1, from_bound=b"\x80", to_bound=b""), + ] + after = [_multi_partition(2, parents=[0, 1], from_bound=b"", to_bound=b"")] + driver = _MultiFakeDescribeDriver([before, after]) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx", partition_chooser=chooser) + + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + + f_x = await writer.write_with_ack_future(PublicMessage(b"x", key="x")) # partition 0 + f_y = await writer.write_with_ack_future(PublicMessage(b"y", key="y")) # partition 1 + # One sequence for the whole writer, so the two partitions do not reuse a number and + # the merge can carry both into the shared child unchanged. + assert set(writer._inflight[0]) == {1} and set(writer._inflight[1]) == {2} + + # After the merge both keys route to the shared child 2. + mapping.update({"x": 2, "y": 2}) + # Overloaded fired for partition 0 only; the handler must retire partition 1 too. + await writer._on_partition_overloaded(0) + + assert set(writer._partitions) == {2} + assert writer._chooser.partitions == {2} + assert 0 not in writer._writers and 1 not in writer._writers + assert sorted(m.key for m in writer._writers[2].messages) == ["x", "y"] + assert 0 not in writer._inflight and 1 not in writer._inflight + + writer._writers[2].resolve_all() + await asyncio.sleep(0) + assert f_x.done() and f_y.done() + + await writer.close(flush=False) + + async def test_close_flushes_buffered_messages(self): + driver = _MultiFakeDescribeDriver([[_multi_partition(0), _multi_partition(1), _multi_partition(2)]]) + settings = MultiWriterSettings( + topic="/local/topic", producer_id_prefix="pfx", partition_chooser=PublicPartitionByKeyKafka() + ) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FlushControlledSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + futures = [ + await writer.write_with_ack_future(PublicMessage(("m%d" % i).encode(), key="k%d" % i)) for i in range(5) + ] + assert not any(f.done() for f in futures) # nothing acked yet + + await writer.close() # flush=True must deliver the buffered messages + + assert all(f.done() and not f.cancelled() and f.exception() is None for f in futures) + + async def test_adaptive_chooser_single_open_range_partition(self): + # A single auto-partitioned partition owns the fully-open range b""..b"". + driver = _MultiFakeDescribeDriver([[_multi_partition(0, from_bound=b"", to_bound=b"")]]) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(driver, MultiWriterSettings(topic="/local/topic")) + await writer.wait_init() + assert isinstance(writer._chooser, PublicPartitionByKeyBound) + await writer.close(flush=False) + + async def test_transient_overload_recovers_partition_in_place(self): + # DescribeTopic never shows children -> ordinary overload, not a repartition. + driver = _MultiFakeDescribeDriver([[_multi_partition(0), _multi_partition(1)]]) + chooser = _KeyMapChooser({"a": 0}) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx", partition_chooser=chooser) + with mock.patch( + "ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter + ), mock.patch("ydb._topic_writer.topic_writer_multi_asyncio._REPARTITION_DISCOVER_DELAY", 0), mock.patch( + "ydb._topic_writer.topic_writer_multi_asyncio._REPARTITION_DISCOVER_ATTEMPTS", 2 + ): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + + future = await writer.write_with_ack_future(PublicMessage(b"a", key="a")) + old_sub = writer._writers[0] + + await writer._on_partition_overloaded(0) + + # partition kept (not retired); a fresh sub-writer resends the message + assert 0 in writer._partitions + new_sub = writer._writers[0] + assert new_sub is not old_sub + assert old_sub.closed + assert [m.key for m in new_sub.messages] == ["a"] + assert not future.done() + + new_sub.resolve_all() + await asyncio.sleep(0) + assert future.done() + + await writer.close(flush=False) + + async def test_enqueue_failure_does_not_leak_inflight(self): + driver = _MultiFakeDescribeDriver([[_multi_partition(0), _multi_partition(1)]]) + chooser = _KeyMapChooser({"a": 0}) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx", partition_chooser=chooser) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _RaisingSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + + with pytest.raises(RuntimeError): + await writer.write_with_ack_future(PublicMessage(b"a", key="a")) + + assert not writer._inflight.get(0) # no leaked entry, no pending future + await writer.close(flush=False) + + @pytest.mark.parametrize("seqno", [-1, 5]) + async def test_explicit_seqno_rejected_without_leak(self, seqno): + _FAKE_LAST_SEQNO.clear() + driver = _MultiFakeDescribeDriver([[_multi_partition(0)]]) + settings = MultiWriterSettings(topic="/local/topic", partition_chooser=_KeyMapChooser({"a": 0})) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + futures = [] + try: + futures.append(await writer.write_with_ack_future(PublicMessage(b"first", key="a"))) + message = PublicMessage(b"invalid", key="a", seqno=seqno) + with pytest.raises(TopicWriterError, match="Explicit seqno"): + await writer.write_with_ack_future(message) + assert message.seqno == seqno + assert set(writer._inflight[0]) == {1} + futures.append(await writer.write_with_ack_future(PublicMessage(b"second", key="a"))) + assert [message.seqno for message in writer._writers[0].messages] == [1, 2] + writer._writers[0].resolve_all() + await asyncio.gather(*futures) + finally: + await writer.close(flush=False) + _retrieve_exceptions(futures) + + async def test_unusable_partition_fails_its_inflight_instead_of_stranding_it(self): + """When neither repartition nor recovery can serve a partition, its messages must fail. + + The children never complete the parent's range, so the topology is not committed; the + parent is already inactive, so re-opening a writer for it times out too. Nothing is left + that could ever ack these messages, and flush()/close(flush=True) wait on user futures + without a deadline -- so leaving them pending hangs the caller forever. + """ + _FAKE_LAST_SEQNO.clear() + _FAKE_HANGING_INIT_PARTITIONS.clear() + chooser = _KeyMapChooser({"a": 0}) + before = [_multi_partition(0, from_bound=b"", to_bound=b"")] + # Only one child of the split ever shows up -> coverage check refuses to retire parent 0. + partial = [ + _split_parent(0, children=[1, 2]), + _multi_partition(1, parents=[0], from_bound=b"", to_bound=b"\x80"), + ] + driver = _MultiFakeDescribeDriver([before, partial]) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx", partition_chooser=chooser) + + with mock.patch( + "ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _HangingInitSubWriter + ), mock.patch("ydb._topic_writer.topic_writer_multi_asyncio._WRITER_INIT_TIMEOUT", 0.1), mock.patch( + "ydb._topic_writer.topic_writer_multi_asyncio._REPARTITION_DISCOVER_DELAY", 0 + ), mock.patch( + "ydb._topic_writer.topic_writer_multi_asyncio._REPARTITION_DISCOVER_ATTEMPTS", 2 + ): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + future = await writer.write_with_ack_future(PublicMessage(b"a", key="a")) + + # Parent 0 has gone inactive: recovering it in place cannot finish init either. + _FAKE_HANGING_INIT_PARTITIONS.add(0) + try: + await writer._on_partition_overloaded(0) + + assert future.done(), "in-flight message left without an owner" + assert future.exception() is not None + assert not writer._inflight.get(0) + + # The caller must be able to finish; both would hang on a pending future. + with pytest.raises(TopicWriterError, match="active descendants"): + await asyncio.wait_for(writer.flush(), timeout=1) + with pytest.raises(TopicWriterError, match="active descendants"): + await asyncio.wait_for(writer.close(flush=True), timeout=1) + finally: + _FAKE_HANGING_INIT_PARTITIONS.clear() + await writer.close(flush=False) + _retrieve_exceptions([future]) + + async def test_repartition_tasks_are_coalesced_and_closed_with_the_writer(self): + """Repartition must be owned by the writer, not fire-and-forget. + + A burst of OVERLOADED on one partition otherwise starts several concurrent recoveries of + it, and any of them can outlive close() -- still describing the topic and opening + sub-writers for a multi-writer the caller believes is shut down. + """ + driver = _MultiFakeDescribeDriver([[_multi_partition(0), _multi_partition(1)]]) + settings = MultiWriterSettings( + topic="/local/topic", producer_id_prefix="pfx", partition_chooser=PublicPartitionByKeyKafka() + ) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + + started: List[int] = [] + + async def slow_repartition(partition_id): + started.append(partition_id) + await asyncio.sleep(30) + + writer._on_partition_overloaded = slow_repartition + hook = writer._make_overloaded_hook(0) + + for _ in range(5): # a burst for the same partition + assert hook(issues.Overloaded("Write to inactive partition 0")) is True + await asyncio.sleep(0) + + assert started == [0], "repeated signals for one partition must coalesce" + task = writer._repartition_tasks[0] + + await writer.close(flush=False) + assert task.done(), "close() must cancel and await the repartition task" + assert not writer._repartition_tasks + + # After close no further signal may start work. + assert hook(issues.Overloaded("again")) is True + await asyncio.sleep(0) + assert started == [0] + + @pytest.mark.parametrize("auto_seqno", [False, True]) + async def test_merge_rejects_conflicting_child_history_without_renumbering(self, auto_seqno): + _FAKE_LAST_SEQNO.clear() + mapping = {"x": 0, "y": 1} + driver = _MultiFakeDescribeDriver( + [ + [_multi_partition(0), _multi_partition(1)], + [_multi_partition(2, parents=[0, 1])], + ] + ) + settings = MultiWriterSettings( + topic="/local/topic", partition_chooser=_KeyMapChooser(mapping), auto_seqno=auto_seqno + ) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _SeqnoGuardSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + futures = [] + try: + for seqno, key in enumerate(["x", "y"], 1): + message = PublicMessage(key, key=key, seqno=None if auto_seqno else seqno) + futures.append(await writer.write_with_ack_future(message)) + entry = writer._inflight[0][1] + _FAKE_LAST_SEQNO[2] = 1 + mapping.update({"x": 2, "y": 2}) + await writer._on_partition_overloaded(0) + assert entry.message.seqno == 1 + assert [message.seqno for message in writer._writers[2].messages] == [2] + with pytest.raises(TopicWriterError, match="already persisted"): + futures[0].result() + writer._writers[2].resolve_all() + assert isinstance(await futures[1], PublicWriteResult.Written) + with pytest.raises(TopicWriterError, match="already persisted"): + await writer.flush() + finally: + await writer.close(flush=False) + _retrieve_exceptions(futures) + + async def test_message_persisted_with_a_lost_ack_is_not_resent_to_the_child(self): + """The dedup cut has to come from the server, not from the acks we happened to receive. + + A split kills the session, and a message the server already persisted can lose its ack on + the way back. Judging only by acks we saw, such a message looks unwritten, so it gets + resent to the child -- where the parent's producer id no longer covers it, because each + partition writes under its own. Nothing on the server can collapse the two, so the reader + sees the message twice. Reading the cut from the retiring producer instead closes that + window: the message is below it, and is reported written rather than resent. + """ + _FAKE_LAST_SEQNO.clear() + mapping = {"a": 0} + chooser = _KeyMapChooser(mapping) + before = [_multi_partition(0), _multi_partition(1)] + after = [ + _split_parent(0, children=[2, 3]), + _multi_partition(1), + _multi_partition(2, parents=[0]), + _multi_partition(3, parents=[0]), + ] + driver = _MultiFakeDescribeDriver([before, after]) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx", partition_chooser=chooser) + + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + future = await writer.write_with_ack_future(PublicMessage(b"a", key="a")) # seqno 1 + assert set(writer._inflight[0]) == {1} + assert writer._max_acked.get(0, 0) == 0 # no ack ever reached us + + # The server did persist it: partition 0's producer is at seqno 1. + _FAKE_LAST_SEQNO[0] = 1 + + mapping["a"] = 2 + await writer._on_partition_overloaded(0) + + resent = [m.seqno for m in writer._writers[2].messages] if 2 in writer._writers else [] + assert resent == [], "a message the server had already persisted was resent to the child" + assert future.done() and future.exception() is None, "the persisted message must resolve as written" + assert not writer._inflight.get(0) + + await writer.close(flush=False) + + async def test_cut_ignores_ancestor_history(self): + """The cut is the retiring partition's own high-water mark, nothing else. + + Two different things live in an ancestor's producer, and neither belongs in this number. + A merge child has two parents whose branches numbered independently, so a sibling's + history says nothing about messages that came down this branch. And producer ids are + `prefix-`, so with a stable prefix a retired ancestor still holds whatever a + *previous run* wrote there. + + Either one, folded into the maximum, marks unsent messages as already written and drops + them silently. Nothing needs to be read from them: a message only reaches this partition + by having a number above the cut that retired the previous one. + """ + _FAKE_LAST_SEQNO.clear() + _FAKE_LAST_SEQNO[1] = 5 # this branch numbered modestly + _FAKE_LAST_SEQNO[2] = 100 # the sibling branch ran far ahead + _FAKE_LAST_SEQNO[3] = 7 # the surviving partition itself + + # The topology comes from describe, as it does in production: p3 is the merge of p1 and + # p2, so both are reachable as its parents. + described = [ + _multi_partition(1, children=[3]), + _multi_partition(2, children=[3]), + _multi_partition(3, parents=[1, 2]), + ] + driver = _MultiFakeDescribeDriver([described]) + settings = MultiWriterSettings( + topic="/local/topic", producer_id_prefix="pfx", partition_chooser=_KeyMapChooser({}) + ) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + # This writer wrote to both branches before they merged. + for partition_id in (1, 2, 3): + await writer._get_or_create_writer(partition_id) + + assert await writer._max_seqno_cut(3) == 7, "an ancestor's history leaked into the cut" + + await writer.close(flush=False) + + async def test_a_message_survives_repeated_retirements(self): + """A message must not be written off by a partition it already escaped. + + This is the induction the cut relies on, end to end: seqno 6 outruns the parent's cut of + 5 and moves to the child, and when the child is retired in turn it has to move again. If + an earlier partition's number could still reach it, it would be dropped here. + """ + _FAKE_LAST_SEQNO.clear() + _FAKE_LAST_SEQNO[0] = 5 # parent persisted up to 5; our message will be 6 + mapping = {"a": 0} + chooser = _KeyMapChooser(mapping) + before = [_multi_partition(0), _multi_partition(1)] + after_first = [ + _split_parent(0, children=[2, 3]), + _multi_partition(1), + _multi_partition(2, parents=[0]), + _multi_partition(3, parents=[0]), + ] + after_second = [ + _split_parent(0, children=[2, 3]), + _split_parent(2, children=[4, 5], parents=[0]), + _multi_partition(1), + _multi_partition(3, parents=[0]), + _multi_partition(4, parents=[2]), + _multi_partition(5, parents=[2]), + ] + driver = _MultiFakeDescribeDriver([before, after_first, after_second]) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx", partition_chooser=chooser) + + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + future = await writer.write_with_ack_future(PublicMessage(b"a", key="a")) + seqno = next(iter(writer._inflight[0])) + assert seqno == 6, "the cursor starts above the parent's persisted history" + + mapping["a"] = 2 + await writer._on_partition_overloaded(0) + assert [m.seqno for m in writer._writers[2].messages] == [6] + + # The child retires too, and the message has to keep going. + mapping["a"] = 4 + await writer._on_partition_overloaded(2) + assert [m.seqno for m in writer._writers[4].messages] == [6] + assert not future.done() + + writer._writers[4].resolve_all() + await asyncio.sleep(0) + assert future.done() and future.exception() is None + await writer.close(flush=False) + + @pytest.mark.parametrize( + "auto_seqno,invalid_seqno,error", + [(True, 5, "Explicit seqno"), (False, None, "Empty seqno"), (False, 0, "Empty seqno")], + ) + async def test_batch_with_invalid_seqno_mode_is_rejected_before_admission(self, auto_seqno, invalid_seqno, error): + _FAKE_LAST_SEQNO.clear() + driver = _MultiFakeDescribeDriver([[_multi_partition(0)]]) + settings = MultiWriterSettings( + topic="/local/topic", partition_chooser=_KeyMapChooser({"a": 0}), auto_seqno=auto_seqno + ) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + try: + messages = [ + PublicMessage(b"valid", key="a", seqno=None if auto_seqno else 10), + PublicMessage(b"invalid", key="a", seqno=invalid_seqno), + ] + with pytest.raises(TopicWriterError, match=error): + await writer.write_with_ack_future(messages) + assert not writer._writers and not writer._inflight + assert writer._seqno == 0 + assert messages[0].seqno == (None if auto_seqno else 10) + await writer.write_with_ack(messages[0]) + assert writer._writers[0].messages[0].seqno == (1 if auto_seqno else 10) + finally: + await writer.close(flush=False) + + @pytest.mark.parametrize("auto_seqno", [False, True]) + async def test_recovery_after_lost_ack_resends_the_remaining_messages(self, auto_seqno): + """A transient overload plus one ack lost with the stream must not strand the partition. + + The server persisted seqno 1 but its ack never reached us, so the message is still + in-flight. _recover_partition resends the whole in-flight set to a fresh sub-writer for + the same partition -- whose init reports last_seqno=1, so the real writer rejects seqno 1 + with "Message seqno is duplicated" before the server ever sees it. The resend loop is a + plain `for`, so that exception also skips seqnos 2 and 3, which are never retried and + never resolve. + + The comment on _recover_partition assumes the server dedups this retry; the client-side + guard fires first, so it never gets the chance. + """ + _FAKE_LAST_SEQNO.clear() + driver = _MultiFakeDescribeDriver([[_multi_partition(0), _multi_partition(1)]]) + chooser = _KeyMapChooser({"a": 0}) + settings = MultiWriterSettings( + topic="/local/topic", producer_id_prefix="pfx", partition_chooser=chooser, auto_seqno=auto_seqno + ) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _SeqnoGuardSubWriter), mock.patch( + "ydb._topic_writer.topic_writer_multi_asyncio._REPARTITION_DISCOVER_DELAY", 0 + ), mock.patch("ydb._topic_writer.topic_writer_multi_asyncio._REPARTITION_DISCOVER_ATTEMPTS", 2): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + futures = [ + await writer.write_with_ack_future( + PublicMessage(("m%d" % i).encode(), key="a", seqno=None if auto_seqno else i + 1) + ) + for i in range(3) + ] + assert set(writer._inflight[0]) == {1, 2, 3} + + # The server persisted seqno 1 and acked it, but the ack died with the stream. + _FAKE_LAST_SEQNO[0] = 1 + + try: + await writer._on_partition_overloaded(0) + + new_sub = writer._writers[0] + assert [m.seqno for m in new_sub.messages] == [2, 3], "messages after the persisted one were dropped" + assert futures[0].done() and futures[0].exception() is None, "persisted message must resolve as written" + + new_sub.resolve_all() + await asyncio.sleep(0) + assert all(f.done() for f in futures) + finally: + await writer.close(flush=False) + _retrieve_exceptions(futures) + + async def test_split_migration_does_not_block_on_an_uninitializable_child(self): + """Migration must not wedge the whole multi-writer when a child cannot be opened. + + Splits cascade (1->3->7 was observed live), so by the time we migrate, the child the + chooser picked may itself have split and gone inactive. _get_or_create_writer awaits + wait_init() on it while holding the orchestrator lock, and a writer against an inactive + partition never finishes init -- so every write, flush and further repartition blocks + behind it. This is the same failure the maxSeqNo probe caused before it was removed; + the unbounded wait_init under the lock survived it. + """ + _FAKE_LAST_SEQNO.clear() + _FAKE_HANGING_INIT_PARTITIONS.clear() + mapping = {"a": 0, "b": 1} + chooser = _KeyMapChooser(mapping) + before = [_multi_partition(0), _multi_partition(1)] + after = [ + _split_parent(0, children=[2, 3]), + _multi_partition(1), + _multi_partition(2, parents=[0]), + _multi_partition(3, parents=[0]), + ] + driver = _MultiFakeDescribeDriver([before, after]) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx", partition_chooser=chooser) + + with mock.patch( + "ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _HangingInitSubWriter + ), mock.patch("ydb._topic_writer.topic_writer_multi_asyncio._WRITER_INIT_TIMEOUT", 0.1): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + future = await writer.write_with_ack_future(PublicMessage(b"a", key="a")) + + mapping["a"] = 2 + _FAKE_HANGING_INIT_PARTITIONS.add(2) # child 2 split again before we got to it + + try: + try: + await asyncio.wait_for(writer._on_partition_overloaded(0), timeout=1) + except asyncio.TimeoutError: + pytest.fail("repartition blocked on an uninitializable child while holding the lock") + + # And the lock must be free afterwards: an unrelated partition still accepts writes. + await asyncio.wait_for(writer.write_with_ack_future(PublicMessage(b"b", key="b")), timeout=1) + finally: + _FAKE_HANGING_INIT_PARTITIONS.clear() + await writer.close(flush=False) + _retrieve_exceptions([future]) + + async def test_split_waits_for_children_to_cover_the_parent_range(self): + """A partially visible split must not retire the parent. + + DescribeTopic can show one child before its sibling becomes active. Retiring the parent + on that view drops its key range down to the single child, leaving the rest of the key + space uncovered -- and since routing only compares from_bound, keys from the missing + range then land in the left sibling. That puts one key on two branches of the partition + graph, which is the invariant the whole design exists to protect. + + Per the C++ producer spec an incomplete graph is a retry-with-backoff state, not a + successful split: "producer не должен выбирать случайную partition". + """ + before = [_multi_partition(0, from_bound=b"", to_bound=b"")] + partial = [ + _split_parent(0, children=[1, 2]), + _multi_partition(1, parents=[0], from_bound=b"", to_bound=b"\x80"), + ] + complete = partial + [_multi_partition(2, parents=[0], from_bound=b"\x80", to_bound=b"")] + driver = _MultiFakeDescribeDriver([before, partial, complete]) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx") + + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter), mock.patch( + "ydb._topic_writer.topic_writer_multi_asyncio._REPARTITION_DISCOVER_DELAY", 0 + ): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + assert isinstance(writer._chooser, PublicPartitionByKeyBound) + + await writer._on_partition_overloaded(0) + + assert set(writer._partitions) == {1, 2}, "parent retired before both children were visible" + assert sorted(p[-1] for p in writer._chooser._partitions) == [1, 2] + await writer.close(flush=False) + + async def test_ack_racing_the_split_is_not_resent_to_the_child(self): + """An ack that lands during the quiesce must count towards the dedup cut. + + Closing a retired sub-writer fails the acks it still holds, and those failures must not + reach the user because the messages are about to be migrated. Suppressing them must not + also swallow a SUCCESS landing in the same window: that ack is real, and dropping it + leaves _max_acked -- the dedup cut -- too low, so a message the parent already persisted + is resent to the child. That is the duplicate the cut exists to prevent. + """ + _FAKE_LAST_SEQNO.clear() + mapping = {"a": 0} + chooser = _KeyMapChooser(mapping) + before = [_multi_partition(0), _multi_partition(1)] + after = [ + _split_parent(0, children=[2, 3]), + _multi_partition(1), + _multi_partition(2, parents=[0]), + _multi_partition(3, parents=[0]), + ] + driver = _MultiFakeDescribeDriver([before, after]) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx", partition_chooser=chooser) + + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _AckOnCloseSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + future = await writer.write_with_ack_future(PublicMessage(b"a", key="a")) + + mapping["a"] = 2 + try: + await writer._on_partition_overloaded(0) + await asyncio.sleep(0) + + resent = [m.key for m in writer._writers[2].messages] if 2 in writer._writers else [] + assert resent == [], "a message the parent persisted was resent to the child" + assert future.done() and future.exception() is None + finally: + await writer.close(flush=False) + _retrieve_exceptions([future]) + + @pytest.mark.parametrize("auto_seqno", [False, True]) + async def test_split_resend_preserves_the_original_seqno(self, auto_seqno): + """Changing the destination must preserve the number assigned at admission.""" + _FAKE_LAST_SEQNO.clear() + mapping = {"a": 0, "b": 0} + chooser = _KeyMapChooser(mapping) + before = [_multi_partition(0), _multi_partition(1)] + after = [ + _split_parent(0, children=[2, 3]), + _multi_partition(1), + _multi_partition(2, parents=[0]), + _multi_partition(3, parents=[0]), + ] + driver = _MultiFakeDescribeDriver([before, after]) + settings = MultiWriterSettings( + topic="/local/topic", producer_id_prefix="pfx", partition_chooser=chooser, auto_seqno=auto_seqno + ) + + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + f_a = await writer.write_with_ack_future(PublicMessage(b"a", key="a", seqno=None if auto_seqno else 1)) + f_b = await writer.write_with_ack_future(PublicMessage(b"b", key="b", seqno=None if auto_seqno else 2)) + assert set(writer._inflight[0]) == {1, 2} + + # The split sends the two keys to different children; each keeps its own number. + mapping.update({"a": 2, "b": 3}) + try: + await writer._on_partition_overloaded(0) + + assert [m.seqno for m in writer._writers[2].messages] == [1] + assert [m.seqno for m in writer._writers[3].messages] == [2] + assert set(writer._inflight[2]) == {1} and set(writer._inflight[3]) == {2} + finally: + await writer.close(flush=False) + _retrieve_exceptions([f_a, f_b]) + + +@pytest.mark.asyncio +class TestTopicWriterMultiAsyncIOLifecycle: + """Lifecycle and error-path behaviour of the orchestrator. + + Separate from the routing/split tests: these are about what happens around the happy path -- + shutdown, destructors, background tasks and the branches that only run when something fails. + """ + + def _driver(self, partitions=None): + return _MultiFakeDescribeDriver([partitions or [_multi_partition(0), _multi_partition(1)]]) + + def _settings(self, **kwargs): + kwargs.setdefault("topic", "/local/topic") + kwargs.setdefault("producer_id_prefix", "pfx") + kwargs.setdefault("partition_chooser", _KeyMapChooser({"a": 0, "b": 1})) + return MultiWriterSettings(**kwargs) + + async def test_context_manager_closes_and_keeps_body_errors(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + async with TopicWriterMultiAsyncIO(self._driver(), self._settings()) as writer: + await writer.write(PublicMessage(b"a", key="a")) + assert writer._closed + + class TestException(Exception): + pass + + # A failure inside the block must survive close(): losing it would report a real + # error as an unrelated teardown problem. + with pytest.raises(TestException): + async with TopicWriterMultiAsyncIO(self._driver(), self._settings()) as writer: + raise TestException() + assert writer._closed + + async def test_unclosed_writer_schedules_a_close_on_delete(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer.wait_init() + + writer.__del__() # forgotten writer: the streams still have to be released + await asyncio.sleep(0) + await asyncio.sleep(0) + assert writer._closed + + writer.__del__() # already closed -> nothing scheduled, still no raise + + async def test_writes_are_refused_after_close(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer.wait_init() + await writer.close(flush=False) + + with pytest.raises(TopicWriterClosedError): + await writer.write(PublicMessage(b"a", key="a")) + with pytest.raises(TopicWriterClosedError): + await writer.flush() + + async def test_close_cancels_an_init_that_never_finished(self): + class NeverDescribes(_MultiFakeDescribeDriver): + async def __call__(self, *args, **kwargs): + await asyncio.Event().wait() + + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(NeverDescribes([[]]), self._settings()) + await asyncio.wait_for(writer.close(), timeout=1) + + # An init left running would keep describing a topic nobody writes to any more. + with pytest.raises(asyncio.CancelledError): + await writer._init_task + + async def test_close_reports_a_failing_flush_after_cleanup(self): + class FlushRaises(_FakeSubWriter): + async def flush(self): + raise RuntimeError("flush failed") + + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", FlushRaises): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer.write(PublicMessage(b"a", key="a")) + subwriter = writer._writers[0] + with pytest.raises(RuntimeError, match="flush failed"): + await writer.close() + assert writer._closed + assert subwriter.closed + assert not writer._writers + + async def test_write_with_ack_returns_results_for_one_and_many(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + + single = await writer.write_with_ack(PublicMessage(b"a", key="a")) + assert isinstance(single, PublicWriteResult.Written) + + many = await writer.write_with_ack([PublicMessage(b"a", key="a"), PublicMessage(b"b", key="b")]) + assert isinstance(many, list) and len(many) == 2 + + await writer.close(flush=False) + + async def test_a_failed_ack_reaches_the_caller(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + future = await writer.write_with_ack_future(PublicMessage(b"a", key="a")) + + writer._writers[0].pending[0].set_exception(RuntimeError("write rejected")) + await asyncio.sleep(0) + + assert isinstance(future.exception(), RuntimeError) + assert not writer._inflight.get(0), "a settled message must not stay in flight" + await writer.close(flush=False) + + async def test_a_cancelled_ack_leaves_the_message_in_flight(self): + """Cancellation is not an outcome: the writer is being torn down, and the message is + still owned by the orchestrator until a repartition or recovery decides its fate.""" + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + future = await writer.write_with_ack_future(PublicMessage(b"a", key="a")) + + writer._writers[0].pending[0].cancel() + await asyncio.sleep(0) + + assert not future.done() + assert set(writer._inflight[0]) == {1} + await writer.close(flush=False) + _retrieve_exceptions([future]) + + async def test_ack_failures_are_suppressed_while_the_partition_is_being_retired(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + future = await writer.write_with_ack_future(PublicMessage(b"a", key="a")) + + writer._retiring.add(0) + writer._writers[0].pending[0].set_exception(RuntimeError("session closed by us")) + await asyncio.sleep(0) + + # Expected noise from our own teardown: the message waits to be resent instead. + assert not future.done() + assert set(writer._inflight[0]) == {1} + await writer.close(flush=False) + _retrieve_exceptions([future]) + + async def test_probe_result_is_cached_per_producer(self): + _FAKE_LAST_SEQNO.clear() + _FAKE_LAST_SEQNO[0] = 42 + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer.wait_init() + + assert await writer._probe_server_seqno(0) == 42 + # A retired producer receives no further writes, so the answer is final: changing + # what the server would say must not change the cached cut. + _FAKE_LAST_SEQNO[0] = 99 + assert await writer._probe_server_seqno(0) == 42 + + await writer.close(flush=False) + + async def test_repartition_task_deregisters_itself_when_done(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer.wait_init() + + async def noop(partition_id): + return + + writer._on_partition_overloaded = noop + writer._schedule_repartition(0) + task = writer._repartition_tasks[0] + await task + await asyncio.sleep(0) + + # Left registered, a finished task would block every later signal for this partition. + assert 0 not in writer._repartition_tasks + await writer.close(flush=False) + + async def test_repartition_of_an_unknown_partition_is_a_no_op(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + driver = self._driver() + writer = TopicWriterMultiAsyncIO(driver, self._settings()) + await writer.wait_init() + describes = driver.describe_calls + + # Already retired by a sibling's event: nothing left to do, and re-describing would + # only race the handler that did retire it. + await writer._handle_repartition(99) + + assert driver.describe_calls == describes + await writer.close(flush=False) + + async def test_failing_inflight_of_an_empty_partition_is_a_no_op(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer.wait_init() + writer._fail_partition_inflight(0, RuntimeError("boom")) # must not raise + await writer.close(flush=False) + + async def test_idle_reaper_evicts_and_survives_a_failing_pass(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings(writer_idle_timeout_sec=3)) + await writer.write(PublicMessage(b"a", key="a")) + assert 0 in writer._writers + + writer._last_write_at[0] = writer._loop.time() - 1000 + + # One failing pass must not kill the reaper: it has to keep collecting later. + failed = {"once": False} + real_evict = writer._evict_idle_writers + + async def flaky(): + if not failed["once"]: + failed["once"] = True + raise RuntimeError("eviction failed") + await real_evict() + + async def flaky_then_stop(): + await flaky() + if failed["once"] and 0 not in writer._writers: + writer._closed = True # end the reaper loop once it has done its job + + writer._evict_idle_writers = flaky_then_stop + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.asyncio.sleep", _no_sleep): + await asyncio.wait_for(TopicWriterMultiAsyncIO._idle_reaper(weakref.ref(writer), 3), timeout=2) + + assert failed["once"] + assert 0 not in writer._writers, "an idle sub-writer should have been closed" + await writer.close(flush=False) + + async def test_idle_reaper_stops_when_the_writer_is_gone(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.asyncio.sleep", _no_sleep): + # A dead weakref is how the reaper learns the writer was garbage collected; it must + # end rather than keep a task alive forever. + await asyncio.wait_for(TopicWriterMultiAsyncIO._idle_reaper(lambda: None, 3), timeout=2) + + async def test_a_busy_partition_is_never_evicted(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings(writer_idle_timeout_sec=3)) + future = await writer.write_with_ack_future(PublicMessage(b"a", key="a")) + + writer._last_write_at[0] = writer._loop.time() - 1000 + await writer._evict_idle_writers() + + # Closing it would strand the un-acked message on a dead session. + assert 0 in writer._writers + await writer.close(flush=False) + _retrieve_exceptions([future]) + + async def test_close_failure_on_exit_is_raised_only_on_a_clean_body(self): + class CloseRaises(_FakeSubWriter): + async def close(self, flush=True): + raise RuntimeError("close failed") + + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer.wait_init() + writer._flush_impl = mock.AsyncMock(side_effect=RuntimeError("close failed")) + writer.close = mock.AsyncMock(side_effect=RuntimeError("close failed")) + + with pytest.raises(RuntimeError, match="close failed"): + await writer.__aexit__(None, None, None) + + # With an exception already travelling, the close failure must not replace it. + await writer.__aexit__(TypeError, TypeError("original"), None) + + async def test_delete_survives_a_loop_that_cannot_schedule(self): + """__del__ can run at interpreter shutdown, when scheduling no longer works.""" + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer.wait_init() + writer._loop = mock.Mock(is_closed=lambda: False, create_task=mock.Mock(side_effect=RuntimeError)) + + writer.__del__() # must not raise + + writer._closed = True + + async def test_close_survives_a_failing_flush(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer.wait_init() + writer._flush_impl = mock.AsyncMock(side_effect=RuntimeError("flush failed")) + + # Refusing to close on a flush error would leak every stream the writer holds. + with pytest.raises(RuntimeError, match="flush failed"): + await writer.close() + assert writer._closed + + async def test_repartition_cancellation_is_not_swallowed(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer.wait_init() + + async def cancelled(partition_id): + raise asyncio.CancelledError() + + # Cancellation means "we are shutting down", not "this partition failed": turning it + # into a recovery attempt would restart work close() is trying to stop. + writer._handle_repartition = cancelled + with pytest.raises(asyncio.CancelledError): + await writer._on_partition_overloaded(0) + + writer._handle_repartition = mock.AsyncMock(side_effect=RuntimeError("boom")) + writer._recover_partition = cancelled + with pytest.raises(asyncio.CancelledError): + await writer._on_partition_overloaded(0) + + await writer.close(flush=False) + + async def test_recovery_after_a_failed_repartition_keeps_the_partition(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + future = await writer.write_with_ack_future(PublicMessage(b"a", key="a")) + + writer._handle_repartition = mock.AsyncMock(side_effect=RuntimeError("describe blew up")) + await writer._on_partition_overloaded(0) + + # Repartition failed, but the partition is still ours, so the message is resent + # rather than failed. + assert 0 in writer._partitions + assert not future.done() + writer._writers[0].resolve_all() + await asyncio.sleep(0) + assert future.done() + + await writer.close(flush=False) + + async def test_migration_refuses_a_seqno_the_child_already_persisted(self): + _FAKE_LAST_SEQNO.clear() + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer.wait_init() + writer._server_init_seqno[2] = 100 + + # Resending it anyway would be rejected by the child's writer as a duplicate seqno, + # taking down the migration of everything behind it. + conflict = writer._migration_conflict(2, 50, {}) + assert isinstance(conflict, TopicWriterError) + assert "already persisted" in str(conflict) + + await writer.close(flush=False) + + async def test_idle_eviction_leaves_recently_used_writers_alone(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings(writer_idle_timeout_sec=1000)) + await writer.write(PublicMessage(b"a", key="a")) + await asyncio.sleep(0) # let the ack settle so the writer counts as idle + assert not writer._inflight.get(0) + + await writer._evict_idle_writers() + + assert 0 in writer._writers, "a writer used a moment ago is not idle" + await writer.close(flush=False) + + +def test_children_must_cover_the_parent_range(): + """Coverage decides whether a split may be committed to, so each way it can fail matters.""" + cover = TopicWriterMultiAsyncIO._children_cover_parent + parent = _multi_partition(0, from_bound=b"", to_bound=b"") + + # A child with no range at all cannot be reconciled with a bounded parent. + assert cover(parent, [_multi_partition(1, parents=[0])]) is False + + # A hole between the children: keys in it would fall back to the left sibling. + assert ( + cover( + parent, + [ + _multi_partition(1, parents=[0], from_bound=b"", to_bound=b"\x40"), + _multi_partition(2, parents=[0], from_bound=b"\x80", to_bound=b""), + ], + ) + is False + ) + + # A bounded parent fully tiled by its children. + bounded = _multi_partition(0, from_bound=b"a", to_bound=b"m") + assert ( + cover( + bounded, + [ + _multi_partition(1, parents=[0], from_bound=b"a", to_bound=b"f"), + _multi_partition(2, parents=[0], from_bound=b"f", to_bound=b"m"), + ], + ) + is True + ) + + # The same parent left short: the tail of its range has no owner. + assert cover(bounded, [_multi_partition(1, parents=[0], from_bound=b"a", to_bound=b"f")]) is False + + +class _AckOnChildInitSubWriter(_ControllableSubWriter): + """Child sub-writer whose init settles the parent's outstanding ack. + + Models the real race: opening the child's session yields, and the parent's ack can land in + that window -- after the migration decided to resend the message, but before it does. + """ + + parent_writer = None + child_partition_id = None + + async def wait_init(self): + # Only when the CHILD's own session is opened. The dedup-cut probe also opens a session + # (unpinned, so no partition id); acking there would settle the message through the cut + # instead of the race this is meant to reproduce. + parent = type(self).parent_writer + if parent is not None and self.partition_id == type(self).child_partition_id: + type(self).parent_writer = None + parent.resolve_all() + await asyncio.sleep(0) + return await super().wait_init() + + +@pytest.mark.asyncio +async def test_message_acked_while_opening_the_child_is_not_resent(): + """An ack that arrives mid-migration means the message is already written. + + Resending it would duplicate it, and the ack has already completed the caller's future, so + the migration has to notice it lost ownership rather than push it again. + """ + _FAKE_LAST_SEQNO.clear() + mapping = {"a": 0} + chooser = _KeyMapChooser(mapping) + before = [_multi_partition(0), _multi_partition(1)] + after = [ + _split_parent(0, children=[2, 3]), + _multi_partition(1), + _multi_partition(2, parents=[0]), + _multi_partition(3, parents=[0]), + ] + driver = _MultiFakeDescribeDriver([before, after]) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx", partition_chooser=chooser) + + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _AckOnChildInitSubWriter): + writer = TopicWriterMultiAsyncIO(driver, settings) + await writer.wait_init() + future = await writer.write_with_ack_future(PublicMessage(b"a", key="a")) + + _AckOnChildInitSubWriter.parent_writer = writer._writers[0] + _AckOnChildInitSubWriter.child_partition_id = 2 + mapping["a"] = 2 + await writer._on_partition_overloaded(0) + + assert future.done() and future.exception() is None + resent = [m.seqno for m in writer._writers[2].messages] if 2 in writer._writers else [] + assert resent == [], "a message acked mid-migration was resent anyway" + + await writer.close(flush=False) + + +@pytest.mark.asyncio +class TestTopicWriterMultiAsyncIOBranches: + """The remaining decision points: the side of each branch that only a specific state reaches. + + Most of these are guards against resolving a caller's future twice. A future is settled once + and raises if settled again, and that second attempt happens inside an ack callback where the + exception would be swallowed and the message silently stuck, so the guards are load-bearing. + """ + + def _driver(self): + return _MultiFakeDescribeDriver([[_multi_partition(0), _multi_partition(1)]]) + + def _settings(self, **kwargs): + kwargs.setdefault("topic", "/local/topic") + kwargs.setdefault("producer_id_prefix", "pfx") + kwargs.setdefault("partition_chooser", _KeyMapChooser({"a": 0, "b": 1})) + return MultiWriterSettings(**kwargs) + + def _settled_entry(self, writer, partition_id, seqno): + """Put an already-resolved message into the in-flight map.""" + future = writer._loop.create_future() + future.set_result(PublicWriteResult.Written(offset=1)) + entry = topic_writer_multi_asyncio._InflightMessage( + message=PublicMessage(b"x", key="a"), + user_future=future, + seqno=seqno, + partition_id=partition_id, + ) + writer._inflight.setdefault(partition_id, {})[seqno] = entry + return entry + + async def test_idle_eviction_can_be_switched_off(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings(writer_idle_timeout_sec=0)) + await writer.wait_init() + + # No reaper task at all, so nothing to cancel on close either. + assert writer._reaper_task is None + await writer.close(flush=False) + + async def test_describe_accepts_a_synchronous_driver(self): + """The sync facade hands the sync driver to the same code, which returns a result + directly instead of a coroutine.""" + + class SyncDriver(_MultiFakeDescribeDriver): + def __call__(self, request, stub, method, wrapper=None, *args, **kwargs): + description = _PublicDescription([_multi_partition(0), _multi_partition(1)]) + + class _Result: + def to_public(self): + return description + + return _Result() + + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(SyncDriver([[]]), self._settings()) + await writer.wait_init() + + assert set(writer._partitions) == {0, 1} + await writer.close(flush=False) + + async def test_already_settled_futures_are_left_alone(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer.wait_init() + + # close(): a message whose ack arrived while close was collecting the pending set. + self._settled_entry(writer, 0, 1) + await writer.close(flush=False) + + writer2 = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer2.wait_init() + entry = self._settled_entry(writer2, 0, 1) + # An unusable partition must not try to fail a message that already succeeded. + writer2._fail_partition_inflight(0, RuntimeError("boom")) + assert entry.user_future.exception() is None + await writer2.close(flush=False) + + async def test_settled_futures_survive_recovery_and_migration(self): + _FAKE_LAST_SEQNO.clear() + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer.wait_init() + + # Below the cut on the recovery path: reported as written, but it already is. + _FAKE_LAST_SEQNO[0] = 5 + entry = self._settled_entry(writer, 0, 3) + await writer._recover_partition(0) + assert entry.user_future.result().offset == 1, "the real ack must not be overwritten" + + # Same on the migration path. + entry2 = self._settled_entry(writer, 1, 3) + await writer._migrate_messages({1: 5}) + assert entry2.user_future.result().offset == 1 + + await writer.close(flush=False) + + async def test_settled_futures_survive_a_failed_migration(self): + _FAKE_LAST_SEQNO.clear() + chooser = _KeyMapChooser({"a": 0}) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings(partition_chooser=chooser)) + await writer.wait_init() + + entry = self._settled_entry(writer, 0, 7) + # Nothing owns the key any more: the tail is failed, but a settled message keeps its + # successful result rather than being turned into an error after the fact. + writer._chooser = _KeyMapChooser({}) + await writer._migrate_messages({0: 0}) + + assert entry.user_future.exception() is None + await writer.close(flush=False) + + async def test_conflicting_migration_leaves_a_settled_future_alone(self): + _FAKE_LAST_SEQNO.clear() + chooser = _KeyMapChooser({"a": 2}) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings(partition_chooser=chooser)) + await writer.wait_init() + writer._partitions[2] = _multi_partition(2) + + entry = self._settled_entry(writer, 0, 3) + # Set it where the fake reports it from: opening the child's writer would otherwise + # overwrite _server_init_seqno with what its init returns. + _FAKE_LAST_SEQNO[2] = 100 # the child already persisted past this seqno + + await writer._migrate_messages({0: 0}) + + assert entry.user_future.exception() is None + await writer.close(flush=False) + + async def test_recovery_is_skipped_for_an_already_retired_partition(self): + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer.wait_init() + + writer._handle_repartition = mock.AsyncMock(side_effect=RuntimeError("boom")) + writer._partitions.pop(0) # retired by a sibling's event while we were failing + + # Recreating a sub-writer for it would resurrect a partition we already gave up. + await writer._on_partition_overloaded(0) + + assert 0 not in writer._writers + await writer.close(flush=False) + + async def test_repartition_ignores_unknown_parents_and_known_children(self): + before = [_multi_partition(0), _multi_partition(1)] + # The child lists a parent we never held, and both children are already in our view. + after = [ + _split_parent(0, children=[2, 3]), + _multi_partition(1), + _multi_partition(2, parents=[0, 99]), + _multi_partition(3, parents=[0]), + ] + driver = _MultiFakeDescribeDriver([before, after]) + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _FakeSubWriter): + writer = TopicWriterMultiAsyncIO(driver, self._settings()) + await writer.wait_init() + writer._partitions[2] = _multi_partition(2) + writer._partitions[3] = _multi_partition(3) + + await writer._on_partition_overloaded(0) + + assert set(writer._partitions) == {1, 2, 3} + await writer.close(flush=False) + + async def test_a_late_ack_does_not_resolve_a_settled_message_twice(self): + """Both ack paths must tolerate a future that is already finished. + + A message can be completed by a dedup cut or by close() while its sub-writer ack is still + on its way. Setting a result twice raises, and this runs inside a done-callback where the + exception would be swallowed -- leaving a message that looks in flight forever. + """ + with mock.patch("ydb._topic_writer.topic_writer_multi_asyncio.WriterAsyncIO", _ControllableSubWriter): + writer = TopicWriterMultiAsyncIO(self._driver(), self._settings()) + await writer.wait_init() + + for seqno, outcome in ((1, "success"), (2, "failure")): + entry = self._settled_entry(writer, 0, seqno) + sub_future = writer._loop.create_future() + entry.sub_future = sub_future + writer._attach_ack(entry) + + if outcome == "success": + sub_future.set_result(PublicWriteResult.Written(offset=9)) + else: + sub_future.set_exception(RuntimeError("too late")) + await asyncio.sleep(0) + + # The original outcome stands, and the message is no longer in flight. + assert entry.user_future.result().offset == 1 + assert seqno not in writer._inflight.get(0, {}) + + await writer.close(flush=False) + + +def test_overlapping_children_do_not_extend_the_covered_range(): + """A child contained in what is already covered adds nothing. + + Letting it move the cursor would make an incomplete set look complete and retire a parent + whose range still has an unowned tail. + """ + cover = TopicWriterMultiAsyncIO._children_cover_parent + parent = _multi_partition(0, from_bound=b"", to_bound=b"") + + assert ( + cover( + parent, + [ + _multi_partition(1, parents=[0], from_bound=b"", to_bound=b"\x80"), + _multi_partition(2, parents=[0], from_bound=b"\x40", to_bound=b"\x60"), + ], + ) + is False + ) + + +def test_bound_chooser_keeps_existing_message_metadata(): + chooser = PublicPartitionByKeyBound() + chooser.add_partitions([PublicDescribeTopicResult.PartitionInfo(0, True, [], [], None, None)]) + message = PublicMessage(b"x", key="user-42", metadata_items={"trace": b"abc"}) + + chooser.choose_partition(message) + + assert message.metadata_items["trace"] == b"abc", "routing must not drop the caller's metadata" + assert PARTITION_KEY_METADATA_KEY in message.metadata_items + + +@pytest.mark.asyncio +class TestTopicWriterMultiAsyncIORegressions: + @pytest.fixture(autouse=True) + def reset_subwriters(self, monkeypatch): + _FAKE_LAST_SEQNO.clear() + _FAKE_HANGING_INIT_PARTITIONS.clear() + monkeypatch.setattr(topic_writer_multi_asyncio, "_REPARTITION_DISCOVER_DELAY", 0) + monkeypatch.setattr(topic_writer_multi_asyncio, "_WRITER_INIT_TIMEOUT", 0.01) + yield + _FAKE_LAST_SEQNO.clear() + _FAKE_HANGING_INIT_PARTITIONS.clear() + + def settings(self, **kwargs): + return MultiWriterSettings( + topic="/local/topic", + producer_id_prefix="pfx", + partition_chooser=PublicPartitionByKeyBound(key_hasher=lambda key: key.encode()), + writer_idle_timeout_sec=0, + **kwargs, + ) + + def partitions(self): + parent = _multi_partition(0, from_bound=b"", to_bound=b"") + left = _multi_partition(1, parents=[0], from_bound=b"", to_bound=b"m") + right = _multi_partition(2, parents=[0], from_bound=b"m", to_bound=b"") + return parent, left, right + + def merge_partitions(self): + before = [ + _multi_partition(0, from_bound=b"", to_bound=b"m"), + _multi_partition(1, from_bound=b"m", to_bound=b""), + ] + return before, [_multi_partition(2, parents=[0, 1], from_bound=b"", to_bound=b"")] + + @pytest.fixture + def opened_streams(self, monkeypatch): + opened = asyncio.Queue() + monkeypatch.setattr(topic_writer_multi_asyncio, "_WRITER_INIT_TIMEOUT", 5) + + async def create(*args, **kwargs): + stream = TestWriterAsyncIOReconnector.StreamWriterMock() + opened.put_nowait(stream) + return stream + + monkeypatch.setattr(WriterAsyncIOStream, "create", create) + return opened + + @pytest.mark.parametrize("flush", [True, False]) + async def test_close_rejects_new_and_waiting_writes(self, opened_streams, flush): + parent, _, _ = self.partitions() + writer = TopicWriterMultiAsyncIO( + _MultiFakeDescribeDriver([[parent]]), self.settings(max_buffer_messages=1, codec=PublicCodec.RAW) + ) + accepted = await writer.write_with_ack_future(b"first") + stream = await opened_streams.get() + waiting = asyncio.create_task(writer.write(b"waiting")) + await asyncio.sleep(0) + assert not waiting.done() + closing = asyncio.create_task(writer.close(flush=flush)) + try: + await asyncio.sleep(0) + with pytest.raises(TopicWriterClosedError): + await asyncio.wait_for(writer.write(b"late"), 1) + if flush: + assert not closing.done() + stream.from_server.put_nowait(TestWriterAsyncIOReconnector().make_default_ack_message(1)) + await asyncio.wait_for(accepted, 1) + await asyncio.wait_for(closing, 1) + with pytest.raises(TopicWriterClosedError): + await asyncio.wait_for(waiting, 1) + if not flush: + assert isinstance(accepted.exception(), TopicWriterStopped) + assert stream._closed + finally: + waiting.cancel() + await asyncio.gather(waiting, return_exceptions=True) + await writer.close(flush=False) + + async def test_batch_stops_at_full_buffer_and_delivers_accepted_prefix(self, opened_streams): + _, left, right = self.partitions() + settings = self.settings(max_buffer_messages=1, buffer_wait_timeout_sec=0, codec=PublicCodec.RAW) + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([[left, right]]), settings) + try: + batch = [ + PublicMessage(b"accepted", key="a"), + PublicMessage(b"buffer full", key="b"), + PublicMessage(b"another partition", key="z"), + ] + with pytest.raises(TopicWriterBufferFullError): + await asyncio.wait_for(writer.write(batch), 1) + + stream = await asyncio.wait_for(opened_streams.get(), 1) + sent = await asyncio.wait_for(stream.from_client.get(), 1) + assert [message.data for message in sent] == [b"accepted"] + stream.from_server.put_nowait(TestWriterAsyncIOReconnector().make_default_ack_message(sent[0].seq_no)) + await asyncio.wait_for(writer.flush(), 1) + assert stream.from_client.empty() + assert opened_streams.empty() + finally: + await writer.close(flush=False) + + async def test_retry_manual_seqno_after_full_buffer(self, opened_streams): + parent, _, _ = self.partitions() + settings = self.settings( + auto_seqno=False, max_buffer_messages=1, buffer_wait_timeout_sec=0, codec=PublicCodec.RAW + ) + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([[parent]]), settings) + try: + first = await writer.write_with_ack_future(PublicMessage(b"a", seqno=1)) + stream = await opened_streams.get() + with pytest.raises(TopicWriterError, match="must increase"): + await writer.write(PublicMessage(b"duplicate", seqno=1)) + second = PublicMessage(b"b", seqno=2) + with pytest.raises(TopicWriterBufferFullError): + await writer.write(second) + stream.from_server.put_nowait(TestWriterAsyncIOReconnector().make_default_ack_message(1)) + await asyncio.wait_for(first, 1) + retry = await writer.write_with_ack_future(second) + stream.from_server.put_nowait(TestWriterAsyncIOReconnector().make_default_ack_message(2)) + await asyncio.wait_for(retry, 1) + with pytest.raises(TopicWriterError, match="must increase"): + await writer.write(second) + finally: + await writer.close(flush=False) + + async def test_split_while_waiting_for_buffer_space(self, opened_streams): + parent, left, right = self.partitions() + driver = _MultiFakeDescribeDriver([[parent], [left, right]]) + writer = TopicWriterMultiAsyncIO(driver, self.settings(max_buffer_messages=1, codec=PublicCodec.RAW)) + try: + first = await writer.write_with_ack_future(PublicMessage(b"one", key="a")) + parent_stream = await opened_streams.get() + waiting = asyncio.create_task(writer.write_with_ack(PublicMessage(b"two", key="a"))) + await asyncio.sleep(0) + assert not waiting.done() + parent_stream.from_server.put_nowait(issues.Overloaded("inactive partition")) + await asyncio.wait_for(opened_streams.get(), 1) # retired producer's seqno probe + child = await asyncio.wait_for(opened_streams.get(), 1) + for seqno, data in [(1, b"one"), (2, b"two")]: + messages = await asyncio.wait_for(child.from_client.get(), 1) + assert [(m.seq_no, m.data) for m in messages] == [(seqno, data)] + child.from_server.put_nowait(TestWriterAsyncIOReconnector().make_default_ack_message(seqno)) + await asyncio.wait_for(asyncio.gather(first, waiting), 1) + finally: + await writer.close(flush=False) + + async def test_cancelled_write_leaves_merge_running(self, monkeypatch): + probe_started = asyncio.Event() + release_probe = asyncio.Event() + + class MergingWriter(_FakeSubWriter): + async def wait_init(self): + if self.partition_id == 1: + raise asyncio.TimeoutError() + if self.partition_id is None: + probe_started.set() + await release_probe.wait() + return await super().wait_init() + + async def write_with_ack_future(self, message): + if self.partition_id == 0: + return asyncio.get_running_loop().create_future() + return await super().write_with_ack_future(message) + + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", MergingWriter) + monkeypatch.setattr(topic_writer_multi_asyncio, "_WRITER_INIT_TIMEOUT", 5) + before, after = self.merge_partitions() + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([before, after]), self.settings()) + try: + accepted = await writer.write_with_ack_future(PublicMessage(b"accepted", key="a")) + discovering = asyncio.create_task(writer.write(PublicMessage(b"cancelled", key="z"))) + await asyncio.wait_for(probe_started.wait(), 1) + discovering.cancel() + with pytest.raises(asyncio.CancelledError): + await discovering + release_probe.set() + await asyncio.wait_for(accepted, 1) + await asyncio.wait_for(writer.flush(), 1) + assert [m.data for m in writer._writers[2].messages] == [b"accepted"] + finally: + release_probe.set() + await writer.close(flush=False) + + @pytest.mark.parametrize("server_seqno,invalid_seqno", [(0, -1), (0, 9), (0, 10), (30, 29), (30, 30)]) + async def test_manual_seqnos_increase_across_partitions_and_known_history( + self, monkeypatch, server_seqno, invalid_seqno + ): + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", _SeqnoGuardSubWriter) + before, _ = self.merge_partitions() + _FAKE_LAST_SEQNO[1] = server_seqno + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([before]), self.settings(auto_seqno=False)) + futures = [] + try: + first = PublicMessage(b"first", key="a", seqno=10) + futures.append(await writer.write_with_ack_future(first)) + writer._writers[0].resolve_all() + await futures[0] + + invalid = PublicMessage(b"invalid", key="z", seqno=invalid_seqno) + with pytest.raises(TopicWriterError, match="must increase across the whole multi-writer"): + await writer.write_with_ack_future(invalid) + assert not writer._inflight.get(1) + assert writer._writers[1].messages == [] + assert invalid.seqno == invalid_seqno + + next_seqno = max(10, server_seqno) + 10 + following = PublicMessage(b"next", key="z", seqno=next_seqno) + futures.append(await writer.write_with_ack_future(following)) + assert [message.seqno for message in writer._writers[1].messages] == [next_seqno] + assert first.seqno == 10 and following.seqno == next_seqno + writer._writers[1].resolve_all() + await asyncio.gather(*futures) + await writer.flush() + finally: + await writer.close(flush=False) + _retrieve_exceptions(futures) + + @pytest.mark.parametrize("seqno", [None, 0]) + async def test_auto_seqno_accepts_an_unset_number_like_the_single_writer(self, monkeypatch, seqno): + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", _FakeSubWriter) + parent, _, _ = self.partitions() + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([[parent]]), self.settings()) + message = PublicMessage(b"data", key="a", seqno=seqno) + try: + await writer.write_with_ack(message) + assert writer._writers[0].messages[0].seqno == 1 + assert message.seqno == seqno + finally: + await writer.close(flush=False) + + @pytest.mark.parametrize("trigger", [0, 1]) + @pytest.mark.parametrize("auto_seqno", [False, True]) + @pytest.mark.parametrize("cuts,expected", [({}, [1, 2, 3, 4]), ({0: 2}, [1, 3, 4]), ({1: 3}, [2, 4])]) + async def test_merge_orders_all_parent_tails_after_individual_cuts( + self, monkeypatch, trigger, auto_seqno, cuts, expected + ): + stopped = set() + probed = set() + + class GuardedWriter(_SeqnoGuardSubWriter): + async def close(self, flush=True): + if self.partition_id in (0, 1): + stopped.add(self.partition_id) + await super().close(flush) + + async def wait_init(self): + if self.partition_id is None: + assert stopped == {0, 1} + probed.add(self.producer_partition_id) + return await super().wait_init() + + async def write_with_ack_future(self, message): + if self.partition_id == 2: + assert probed == {0, 1} + return await super().write_with_ack_future(message) + + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", GuardedWriter) + before, after = self.merge_partitions() + writer = TopicWriterMultiAsyncIO( + _MultiFakeDescribeDriver([before, after]), self.settings(auto_seqno=auto_seqno) + ) + futures = [] + try: + for seqno, key in enumerate(["z", "a", "z", "a"], 1): + message = PublicMessage(key, key=key, seqno=None if auto_seqno else seqno) + futures.append(await writer.write_with_ack_future(message)) + _FAKE_LAST_SEQNO.update(cuts) + await writer._on_partition_overloaded(trigger) + assert set(writer._partitions) == {2} + assert [message.seqno for message in writer._writers[2].messages] == expected + for seqno, future in enumerate(futures, 1): + if seqno not in expected: + assert future.result().offset == -1 + writer._writers[2].resolve_all() + assert all(isinstance(result, PublicWriteResult.Written) for result in await asyncio.gather(*futures)) + finally: + await writer.close(flush=False) + _retrieve_exceptions(futures) + + async def test_merge_serializes_new_writes_and_sibling_recovery(self, monkeypatch): + opening = asyncio.Event() + release = asyncio.Event() + + class DelayedChild(_SeqnoGuardSubWriter): + async def wait_init(self): + if self.partition_id == 2: + opening.set() + await release.wait() + return await super().wait_init() + + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", DelayedChild) + monkeypatch.setattr(topic_writer_multi_asyncio, "_WRITER_INIT_TIMEOUT", 1) + before, after = self.merge_partitions() + driver = _MultiFakeDescribeDriver([before, after]) + writer = TopicWriterMultiAsyncIO(driver, self.settings()) + futures = [await writer.write_with_ack_future(PublicMessage(key, key=key)) for key in ["z", "a"]] + recovery = asyncio.create_task(writer._on_partition_overloaded(0)) + tasks = [recovery] + try: + await asyncio.wait_for(opening.wait(), 1) + new_write = asyncio.create_task(writer.write_with_ack_future(PublicMessage(b"new", key="a"))) + sibling = asyncio.create_task(writer._on_partition_overloaded(1)) + tasks.extend([new_write, sibling]) + await asyncio.sleep(0) + assert not new_write.done() + release.set() + await asyncio.wait_for(asyncio.gather(recovery, sibling), 1) + futures.append(await new_write) + assert driver.describe_calls == 2 + assert [message.seqno for message in writer._writers[2].messages] == [1, 2, 3] + writer._writers[2].resolve_all() + await asyncio.wait_for(asyncio.gather(*futures), 1) + finally: + release.set() + await asyncio.gather(*tasks, return_exceptions=True) + await writer.close(flush=False) + _retrieve_exceptions(futures) + + @pytest.mark.parametrize("fail_at", [1, 3]) + async def test_merge_admission_failure_settles_all_remaining_parent_tails(self, monkeypatch, fail_at): + class RejectingChild(_SeqnoGuardSubWriter): + async def write_with_ack_future(self, message): + if self.partition_id == 2 and message.seqno == fail_at: + raise TopicWriterError("child admission failed") + return await super().write_with_ack_future(message) + + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", RejectingChild) + before, after = self.merge_partitions() + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([before, after]), self.settings()) + futures = [await writer.write_with_ack_future(PublicMessage(key, key=key)) for key in ["z", "a", "z", "a"]] + try: + await writer._on_partition_overloaded(0) + child = writer._writers[2] + assert [message.seqno for message in child.messages] == list(range(1, fail_at)) + assert not writer._inflight.get(0) and not writer._inflight.get(1) + child.resolve_all() + results = await asyncio.wait_for(asyncio.gather(*futures, return_exceptions=True), 1) + assert all(isinstance(result, PublicWriteResult.Written) for result in results[: fail_at - 1]) + assert all(isinstance(result, TopicWriterError) for result in results[fail_at - 1 :]) + with pytest.raises(TopicWriterError, match="child admission failed"): + await writer.flush() + finally: + await writer.close(flush=False) + _retrieve_exceptions(futures) + + @pytest.mark.parametrize("failed_parent", [0, 1]) + async def test_merge_probe_failure_does_not_start_partial_transfer(self, monkeypatch, failed_parent): + class FailingProbe(_SeqnoGuardSubWriter): + async def wait_init(self): + if self.partition_id is None and self.producer_partition_id == failed_parent: + raise TopicWriterError("probe failed") + return await super().wait_init() + + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", FailingProbe) + before, after = self.merge_partitions() + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([before, after]), self.settings()) + futures = [await writer.write_with_ack_future(PublicMessage(key, key=key)) for key in ["z", "a"]] + try: + await writer._on_partition_overloaded(0) + assert not writer._writers + results = await asyncio.wait_for(asyncio.gather(*futures, return_exceptions=True), 1) + assert all(isinstance(result, TopicWriterError) for result in results) + assert not writer._inflight + finally: + await writer.close(flush=False) + _retrieve_exceptions(futures) + + @pytest.mark.parametrize("split_during_init", [False, True]) + async def test_merge_followed_by_split_preserves_both_parent_tails(self, monkeypatch, split_during_init): + class SplittingChild(_SeqnoGuardSubWriter): + async def wait_init(self): + if self.partition_id == 2: + raise TopicWriterPartitionSplitError() + return await super().wait_init() + + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", SplittingChild) + before, merged = self.merge_partitions() + after = [ + _split_parent(0, [2]), + _split_parent(1, [2]), + _split_parent(2, [3, 4], parents=[0, 1]), + _multi_partition(3, parents=[2], from_bound=b"", to_bound=b"m"), + _multi_partition(4, parents=[2], from_bound=b"m", to_bound=b""), + ] + descriptions = [before] + ([merged] if split_during_init else []) + [after] + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver(descriptions), self.settings()) + futures = [await writer.write_with_ack_future(PublicMessage(key, key=key)) for key in ["z", "a", "z", "a"]] + try: + await writer._on_partition_overloaded(0) + assert set(writer._partitions) == {3, 4} + assert [message.seqno for message in writer._writers[3].messages] == [2, 4] + assert [message.seqno for message in writer._writers[4].messages] == [1, 3] + for subwriter in writer._writers.values(): + subwriter.resolve_all() + await asyncio.wait_for(asyncio.gather(*futures), 1) + finally: + await writer.close(flush=False) + _retrieve_exceptions(futures) + + async def test_merge_discovers_all_connected_parents_and_their_other_children(self, monkeypatch): + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", _SeqnoGuardSubWriter) + before = [ + _multi_partition(0, from_bound=b"", to_bound=b"g"), + _multi_partition(1, from_bound=b"g", to_bound=b"t"), + _multi_partition(2, from_bound=b"t", to_bound=b""), + ] + after = [ + _split_parent(0, [6]), + _split_parent(1, [4, 5]), + _split_parent(2, [7]), + _split_parent(4, [6], parents=[1]), + _split_parent(5, [7], parents=[1]), + _multi_partition(6, parents=[0, 4], from_bound=b"", to_bound=b"m"), + _multi_partition(7, parents=[5, 2], from_bound=b"m", to_bound=b""), + ] + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([before, after]), self.settings()) + futures = [await writer.write_with_ack_future(PublicMessage(key, key=key)) for key in ["z", "a", "h", "q", "i"]] + try: + await writer._on_partition_overloaded(0) + assert set(writer._partitions) == {6, 7} + assert [message.seqno for message in writer._writers[6].messages] == [2, 3, 5] + assert [message.seqno for message in writer._writers[7].messages] == [1, 4] + for subwriter in writer._writers.values(): + subwriter.resolve_all() + await asyncio.wait_for(asyncio.gather(*futures), 1) + finally: + await writer.close(flush=False) + _retrieve_exceptions(futures) + + @pytest.mark.parametrize("operation", ["wait_init", "write", "flush"]) + @pytest.mark.parametrize("cancel", [False, True], ids=["timeout", "cancel"]) + async def test_interrupted_init_waiter_does_not_stop_writer(self, monkeypatch, operation, cancel): + started = asyncio.Event() + release = asyncio.Event() + + class DelayedDriver(_MultiFakeDescribeDriver): + async def __call__(self, *args, **kwargs): + started.set() + await release.wait() + return await super().__call__(*args, **kwargs) + + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", _FakeSubWriter) + parent, _, _ = self.partitions() + driver = DelayedDriver([[parent]]) + writer = TopicWriterMultiAsyncIO(driver, self.settings()) + args = (PublicMessage(b"interrupted", key="a"),) if operation == "write" else () + waiter = asyncio.create_task(getattr(writer, operation)(*args)) + other_waiter = asyncio.create_task(writer.wait_init()) + try: + await asyncio.wait_for(started.wait(), 1) + if cancel: + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + else: + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(waiter, 0.01) + + assert not writer._init_task.done() + assert not other_waiter.done() + release.set() + await asyncio.wait_for(other_waiter, 1) + await writer.wait_init() + result = await asyncio.wait_for(writer.write_with_ack(PublicMessage(b"accepted", key="a")), 1) + assert isinstance(result, PublicWriteResult.Written) + assert [message.data for message in writer._writers[0].messages] == [b"accepted"] + assert driver.describe_calls == 1 + finally: + release.set() + waiter.cancel() + other_waiter.cancel() + await asyncio.gather(waiter, other_waiter, return_exceptions=True) + await writer.close(flush=False) + + async def test_close_waits_for_shared_init_cancellation(self): + started = asyncio.Event() + cleaned_up = asyncio.Event() + + class DelayedDriver(_MultiFakeDescribeDriver): + async def __call__(self, *args, **kwargs): + started.set() + try: + await asyncio.Event().wait() + finally: + await asyncio.sleep(0) + cleaned_up.set() + + writer = TopicWriterMultiAsyncIO(DelayedDriver([[]]), self.settings()) + waiter = asyncio.create_task(writer.wait_init()) + try: + await asyncio.wait_for(started.wait(), 1) + await writer.close(flush=False) + assert cleaned_up.is_set() + assert writer._init_task.cancelled() + with pytest.raises(asyncio.CancelledError): + await waiter + assert not writer._partitions + finally: + await writer.close(flush=False) + await asyncio.gather(writer._init_task, waiter, return_exceptions=True) + + async def test_shared_init_error_reaches_all_waiters(self): + started = asyncio.Event() + release = asyncio.Event() + + class FailingDriver(_MultiFakeDescribeDriver): + async def __call__(self, *args, **kwargs): + started.set() + await release.wait() + raise issues.Unavailable("describe failed") + + writer = TopicWriterMultiAsyncIO(FailingDriver([[]]), self.settings()) + waiters = [asyncio.create_task(writer.wait_init()) for _ in range(2)] + try: + await asyncio.wait_for(started.wait(), 1) + release.set() + for waiter in waiters: + with pytest.raises(issues.Unavailable, match="describe failed"): + await waiter + with pytest.raises(issues.Unavailable, match="describe failed"): + await writer.wait_init() + finally: + release.set() + await writer.close(flush=False) + await asyncio.gather(*waiters, return_exceptions=True) + + @pytest.mark.parametrize("before_flush", [False, True]) + async def test_write_error_survives_removal_from_inflight(self, monkeypatch, before_flush): + error = TopicWriterError("write rejected") + + class FailingWriter(_ControllableSubWriter): + async def flush(self): + for future in self.pending: + if not future.done(): + future.set_exception(error) + + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", FailingWriter) + parent, _, _ = self.partitions() + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([[parent]]), self.settings()) + await writer.write(PublicMessage(b"payload", key="a")) + subwriter = writer._writers[0] + if before_flush: + await subwriter.flush() + await asyncio.sleep(0) + assert not writer._inflight[0] + + with pytest.raises(TopicWriterError, match="write rejected"): + await asyncio.wait_for(writer.flush(), 1) + with pytest.raises(TopicWriterError, match="write rejected"): + await asyncio.wait_for(writer.close(), 1) + assert writer._closed and subwriter.closed + assert not writer._writers and not writer._inflight + + async def test_context_manager_reports_write_errors_without_masking_body_errors(self, monkeypatch): + class FailingWriter(_ControllableSubWriter): + async def flush(self): + for future in self.pending: + if not future.done(): + future.set_exception(TopicWriterError("write rejected")) + + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", FailingWriter) + parent, _, _ = self.partitions() + with pytest.raises(TopicWriterError, match="write rejected"): + async with TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([[parent]]), self.settings()) as writer: + await writer.write(PublicMessage(b"payload", key="a")) + assert writer._closed + + with pytest.raises(ValueError, match="body failed"): + async with TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([[parent]]), self.settings()) as writer: + await writer.write(PublicMessage(b"payload", key="a")) + raise ValueError("body failed") + assert writer._closed + + @pytest.mark.parametrize("failure_index", [0, 1]) + @pytest.mark.parametrize("error_type", [TopicWriterBufferFullError, TopicWriterStopped]) + async def test_resend_admission_failure_settles_the_remaining_messages( + self, monkeypatch, failure_index, error_type + ): + error = error_type() if error_type is TopicWriterStopped else error_type("buffer full") + + class FailingChild(_ControllableSubWriter): + async def write_with_ack_future(self, message): + if self.partition_id == 1 and len(self.messages) == failure_index: + raise error + return await super().write_with_ack_future(message) + + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", FailingChild) + parent, left, right = self.partitions() + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([[parent], [left, right]]), self.settings()) + futures = [await writer.write_with_ack_future(PublicMessage(str(i), key="a")) for i in range(3)] + await writer._on_partition_overloaded(0) + writer._writers[1].resolve_all() + results = await asyncio.wait_for(asyncio.gather(*futures, return_exceptions=True), 1) + + assert all(isinstance(result, PublicWriteResult.Written) for result in results[:failure_index]) + assert results[failure_index:] == [error] * (3 - failure_index) + assert not any(writer._inflight.values()) + with pytest.raises(error_type): + await asyncio.wait_for(writer.flush(), 1) + with pytest.raises(error_type): + await asyncio.wait_for(writer.close(), 1) + + @pytest.mark.parametrize("split", [False, True]) + async def test_resend_uses_an_owned_snapshot(self, monkeypatch, split): + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", _ControllableSubWriter) + parent, left, right = self.partitions() + after = [left, right] if split else [parent] + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([[parent], after]), self.settings()) + metadata = {"trace": b"original"} + message = PublicMessage(b"original", key="a", metadata_items=metadata) + future = await writer.write_with_ack_future(message) + accepted = copy.deepcopy(writer._writers[0].messages[0]) + assert accepted.created_at is not None + assert message.seqno is None and message.created_at is None + assert PARTITION_KEY_METADATA_KEY not in metadata + message.data = b"changed" + message.key = "z" + message.seqno = 999 + metadata["trace"] = b"changed" + + await writer._on_partition_overloaded(0) + subwriter = writer._writers[1 if split else 0] + resent = subwriter.messages[0] + assert (resent.data, resent.key, resent.seqno, resent.created_at, resent.metadata_items) == ( + accepted.data, + accepted.key, + accepted.seqno, + accepted.created_at, + accepted.metadata_items, + ) + subwriter.resolve_all() + await asyncio.wait_for(future, 1) + await writer.close() + + async def test_created_at_validation_preserves_writer_settings(self, monkeypatch): + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", _FakeSubWriter) + parent, _, _ = self.partitions() + timestamp = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) + message = PublicMessage(b"payload", key="a", created_at=timestamp) + async with TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([[parent]]), self.settings()) as writer: + with pytest.raises(TopicWriterError, match="auto_created_at"): + await writer.write(message) + assert not writer._writers + async with TopicWriterMultiAsyncIO( + _MultiFakeDescribeDriver([[parent]]), self.settings(auto_created_at=False) + ) as writer: + await writer.write_with_ack(message) + assert writer._writers[0].messages[0].created_at == timestamp + + async def test_flush_reports_failure_while_another_partition_is_unavailable(self, monkeypatch): + class FailingWriter(_ControllableSubWriter): + async def flush(self): + if self.partition_id == 1: + self.pending[0].set_exception(TopicWriterError("write rejected")) + else: + await asyncio.Event().wait() + + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", FailingWriter) + _, left, right = self.partitions() + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([[left, right]]), self.settings()) + await writer.write(PublicMessage(b"rejected", key="a")) + pending = await writer.write_with_ack_future(PublicMessage(b"unavailable", key="z")) + with pytest.raises(TopicWriterError, match="write rejected"): + await asyncio.wait_for(writer.flush(), 1) + assert not pending.done() + with pytest.raises(TopicWriterError, match="write rejected"): + await asyncio.wait_for(writer.close(), 1) + assert isinstance(pending.exception(), TopicWriterStopped) + + @pytest.mark.parametrize("child_fails", [False, True]) + async def test_close_waits_for_child_ack_after_caller_times_out(self, monkeypatch, child_fails): + flushing_parent = asyncio.Event() + release_parent_flush = asyncio.Event() + + class SplitWriter(_ControllableSubWriter): + async def flush(self): + flushing_parent.set() + await release_parent_flush.wait() + + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", SplitWriter) + parent, left, right = self.partitions() + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([[parent], [left, right]]), self.settings()) + ack = await writer.write_with_ack_future(PublicMessage(b"payload", key="a")) + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(ack, timeout=0) + + closing = asyncio.create_task(writer.close()) + try: + await asyncio.wait_for(flushing_parent.wait(), timeout=1) + await writer._on_partition_overloaded(0) + child = writer._writers[1] + assert [message.data for message in child.messages] == [b"payload"] + release_parent_flush.set() + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(asyncio.shield(closing), timeout=0.02) + if child_fails: + child.pending[0].set_exception(TopicWriterError("child write failed")) + with pytest.raises(TopicWriterError, match="child write failed"): + await asyncio.wait_for(closing, timeout=1) + else: + child.resolve_all() + await asyncio.wait_for(closing, timeout=1) + finally: + release_parent_flush.set() + closing.cancel() + await asyncio.gather(closing, return_exceptions=True) + await writer.close(flush=False) + + async def test_cancelling_flush_does_not_cancel_accepted_messages(self, monkeypatch): + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", _ControllableSubWriter) + parent, _, _ = self.partitions() + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver([[parent]]), self.settings()) + future = await writer.write_with_ack_future(PublicMessage(b"payload", key="a")) + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(writer.flush(), 0.01) + assert not future.done() + writer._writers[0].resolve_all() + assert isinstance(await asyncio.wait_for(future, 1), PublicWriteResult.Written) + await writer.close() + + @pytest.mark.parametrize("idle_eviction", [False, True]) + async def test_split_before_opening_a_subwriter_refreshes_routing(self, monkeypatch, idle_eviction): + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", _HangingInitSubWriter) + parent, left, right = self.partitions() + driver = _MultiFakeDescribeDriver([[parent], [left, right]]) + writer = TopicWriterMultiAsyncIO(driver, self.settings()) + await writer.wait_init() + if idle_eviction: + previous = await writer.write_with_ack_future(PublicMessage(b"previous", key="a")) + writer._writers[0].resolve_all() + await previous + await writer._evict_idle_writers() + assert not writer._writers + _FAKE_HANGING_INIT_PARTITIONS.add(0) + + future = await asyncio.wait_for(writer.write_with_ack_future(PublicMessage(b"next", key="z")), 1) + assert driver.describe_calls == 2 + assert set(writer._partitions) == {1, 2} + writer._writers[2].resolve_all() + await asyncio.wait_for(future, 1) + await writer.close() + + async def test_single_auto_partition_without_bounds_waits_for_both_children(self, monkeypatch): + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", _FakeSubWriter) + _, left, right = self.partitions() + partial = [_split_parent(0, [1, 2]), left] + complete = partial + [right] + driver = _MultiFakeDescribeDriver([[_multi_partition(0)], partial, complete]) + writer = TopicWriterMultiAsyncIO(driver, self.settings()) + await writer.wait_init() + await writer._on_partition_overloaded(0) + assert driver.describe_calls == 3 + assert set(writer._partitions) == {1, 2} + await writer.write_with_ack(PublicMessage(b"right", key="z")) + assert writer._writers[2].messages[0].data == b"right" + await writer.close() + + async def test_init_waits_for_a_complete_partition_snapshot(self, monkeypatch): + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", _FakeSubWriter) + _, left, right = self.partitions() + driver = _MultiFakeDescribeDriver([[left], [left, right]]) + async with TopicWriterMultiAsyncIO(driver, self.settings()) as writer: + await writer.wait_init() + assert driver.describe_calls == 2 + await writer.write_with_ack(PublicMessage(b"right", key="z")) + + @pytest.mark.parametrize("split_during_init", [False, True]) + async def test_cascaded_splits_route_pending_messages_to_active_leaves(self, monkeypatch, split_during_init): + monkeypatch.setattr(topic_writer_multi_asyncio, "WriterAsyncIO", _HangingInitSubWriter) + parent, left, right = self.partitions() + grandleft = _multi_partition(3, parents=[1], from_bound=b"", to_bound=b"g") + grandright = _multi_partition(4, parents=[1], from_bound=b"g", to_bound=b"m") + after = [_split_parent(0, [1, 2]), _split_parent(1, [3, 4], parents=[0]), right, grandleft, grandright] + descriptions = [[parent]] + if split_during_init: + descriptions.append([left, right]) + descriptions.append(after) + writer = TopicWriterMultiAsyncIO(_MultiFakeDescribeDriver(descriptions), self.settings()) + futures = [await writer.write_with_ack_future(PublicMessage(key, key=key)) for key in ["a", "h", "z"]] + _FAKE_HANGING_INIT_PARTITIONS.update({0, 1}) + await asyncio.wait_for(writer._on_partition_overloaded(0), 1) + assert set(writer._partitions) == {2, 3, 4} + assert writer._writers[3].messages[0].key == "a" + assert writer._writers[4].messages[0].key == "h" + assert writer._writers[2].messages[0].key == "z" + for subwriter in writer._writers.values(): + subwriter.resolve_all() + await asyncio.wait_for(asyncio.gather(*futures), 1) + await writer.close() diff --git a/ydb/_topic_writer/topic_writer_multi_asyncio.py b/ydb/_topic_writer/topic_writer_multi_asyncio.py new file mode 100644 index 00000000..bee3348e --- /dev/null +++ b/ydb/_topic_writer/topic_writer_multi_asyncio.py @@ -0,0 +1,1018 @@ +from __future__ import annotations + +import asyncio +import concurrent.futures +import copy +import datetime +import inspect +import uuid +import logging +import weakref +from dataclasses import dataclass, field +from typing import Callable, Dict, Iterator, List, Mapping, Optional, Tuple, Union + +from .topic_writer import ( + Message, + PublicMessage, + PublicWriterSettings, + PublicWriteResult, + PublicWriteResultTypes, + TopicWriterClosedError, + TopicWriterError, + TopicWriterPartitionSplitError, + TopicWriterStopped, +) +from .topic_writer_asyncio import WriterAsyncIO +from .topic_writer_partition_chooser import ( + PublicPartitionByKeyBound, + PublicPartitionByKeyKafka, + PublicPartitionChooser, +) +from .. import _apis, issues +from ..settings import BaseRequestSettings +from .._topic_common.common import create_result_wrapper +from .._grpc.grpcwrapper.common_utils import to_thread +from .._grpc.grpcwrapper import ydb_topic as _ydb_topic +from .._grpc.grpcwrapper import ydb_topic_public_types as _ydb_topic_public_types +from .._grpc.grpcwrapper.ydb_topic_public_types import PublicAutoPartitioningStrategy, PublicCodec + +_PartitionInfo = _ydb_topic_public_types.PublicDescribeTopicResult.PartitionInfo + +logger = logging.getLogger(__name__) + +# An OVERLOADED response may be an in-progress split/merge (children not yet visible in +# DescribeTopic) or ordinary transient overload. Re-describe a few times before deciding. +_REPARTITION_DISCOVER_ATTEMPTS = 4 +_REPARTITION_DISCOVER_DELAY = 0.25 + +# Bound the RPC itself: cancelling an executor waiter cannot stop a running sync call. +_DESCRIBE_TIMEOUT = 30.0 + +# A per-partition sub-writer with no in-flight messages and no writes for this long is closed; +# it is recreated on demand. Set <= 0 to disable idle eviction. +_DEFAULT_WRITER_IDLE_TIMEOUT = 60.0 + +# A writer opened against a partition that is no longer active never finishes its init handshake. +# Since sub-writers are created while the orchestrator lock is held, an unbounded wait there stalls +# every write, flush and repartition, so the wait is capped. +_WRITER_INIT_TIMEOUT = 30.0 + + +class _IncompletePartitionTopology(TopicWriterError): + pass + + +@dataclass +class MultiWriterSettings: + """Settings for the multi-partition (write-by-key) topic writer. + + order of fields IS NOT stable, use keywords only + """ + + topic: str + producer_id_prefix: Optional[str] = None + partition_chooser: Optional[PublicPartitionChooser] = None + auto_seqno: bool = True + auto_created_at: bool = True + codec: Optional[PublicCodec] = None + encoders: Optional[Mapping[PublicCodec, Callable[[bytes], bytes]]] = None + encoder_executor: Optional[concurrent.futures.Executor] = None + max_buffer_size_bytes: Optional[int] = None + max_buffer_messages: Optional[int] = None + buffer_wait_timeout_sec: Optional[float] = None + # Idle per-partition sub-writers are closed after this many seconds and recreated on demand. + # None -> default; <= 0 disables eviction. + writer_idle_timeout_sec: Optional[float] = None + + def __post_init__(self): + if self.producer_id_prefix is None: + self.producer_id_prefix = uuid.uuid4().hex + # partition_chooser is left as-is; when None the writer picks one adaptively + # after describing the topic (Bound if the topic reports key ranges, else Kafka). + + +@dataclass +class _InflightMessage: + message: PublicMessage + user_future: asyncio.Future + seqno: int + partition_id: int + sub_future: Optional[asyncio.Future] = field(default=None) + _flush_waiter: Optional[asyncio.Future] = field(default=None, init=False) + + def flush_waiter(self) -> asyncio.Future: + # The caller can cancel user_future while this message is still being delivered. + if self._flush_waiter is None: + self._flush_waiter = self.user_future.get_loop().create_future() + return self._flush_waiter + + def finish(self, outcome: Union[PublicWriteResultTypes, BaseException]) -> None: + for future in (self.user_future, self._flush_waiter): + if future is None or future.done(): + continue + if isinstance(outcome, BaseException): + future.set_exception(outcome) + future.exception() # write() does not expose an acknowledgement to the caller. + else: + future.set_result(outcome) + + +def _is_overloaded(err: BaseException) -> bool: + return isinstance(err, issues.Overloaded) + + +class TopicWriterMultiAsyncIO: + """One logical writer that routes messages to per-partition sub-writers by key. + + Each partition is served by an ordinary :class:`WriterAsyncIO` (buffering, + encoding, reconnection and token refresh are reused as-is). On top of that this + class: + + * routes each message to a partition via the partition chooser; + * owns the in-flight messages and assigns their sequence numbers, so that on an + auto-partition split it can transparently resend the un-acked messages of the + split partition to its children — without duplicating messages that were + already persisted (the ``maxSeqNo`` cut). + """ + + def __init__(self, driver, settings: MultiWriterSettings, _parent=None): + self._loop = asyncio.get_running_loop() + self._driver = driver + self._parent = _parent # keep parent client alive against GC + self._settings = settings + # producer_id_prefix is guaranteed set in __post_init__ + prefix = settings.producer_id_prefix + assert prefix is not None + self._prefix: str = prefix + # Resolved in _init(): the configured chooser, or an adaptive default. + self._chooser: Optional[PublicPartitionChooser] = settings.partition_chooser + self._closing = False + self._closed = False + self._write_error: asyncio.Future[BaseException] = self._loop.create_future() + self._write_lock = asyncio.Lock() + self._lock = asyncio.Lock() + self._writers: Dict[int, WriterAsyncIO] = {} + self._partitions: Dict[int, object] = {} + # partition_id -> {seqno -> in-flight message}, un-acked messages we may resend. + self._inflight: Dict[int, Dict[int, _InflightMessage]] = {} + # One sequence for the whole writer, seeded above the last_seqno of every producer we + # open, so a message keeps its number when a split moves it to another partition. + self._seqno: int = 0 + # partition_id -> last persisted seqno reported by the current sub-writer's init. + self._server_init_seqno: Dict[int, int] = {} + # partition_id -> server's last persisted seqno for a retired producer. Final once read: + # nothing writes under that producer id again. + self._retired_max_seqno: Dict[int, int] = {} + # partition_id -> highest acked seqno. + self._max_acked: Dict[int, int] = {} + # Partitions whose sub-writer is being torn down for a repartition or a recovery. Ack + # failures from such a writer are expected and must not reach the user. + self._retiring: set = set() + # partition_id -> in-progress repartition task, so repeated OVERLOADED for one partition + # coalesce and close() can cancel and await them. + self._repartition_tasks: Dict[int, asyncio.Future] = {} + # partition_id -> monotonic time of the last write/creation, for idle eviction. + self._last_write_at: Dict[int, float] = {} + self._idle_timeout = ( + settings.writer_idle_timeout_sec + if settings.writer_idle_timeout_sec is not None + else _DEFAULT_WRITER_IDLE_TIMEOUT + ) + self._init_task = asyncio.ensure_future(self._init()) + self._reaper_task: Optional[asyncio.Future] = None + if self._idle_timeout > 0: + # Hold only a weakref so the reaper does not keep the writer alive against GC. + self._reaper_task = asyncio.ensure_future(self._idle_reaper(weakref.ref(self), self._idle_timeout)) + + async def __aenter__(self) -> "TopicWriterMultiAsyncIO": + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + try: + await self.close() + except BaseException: + if exc_val is None: + raise + + def __del__(self): + if self._closed or self._loop.is_closed(): + return + try: + logger.debug("Topic multi-writer was not closed properly. Consider using method close().") + task = self._loop.create_task(self.close(flush=False)) + task.set_name("close multiwriter") + except BaseException: + logger.warning("Something went wrong during multi-writer close in __del__") + + async def _describe(self): + req = _ydb_topic_public_types.DescribeTopicRequestParams(path=self._settings.topic, include_stats=False) + args = ( + req.to_proto(), + _apis.TopicService.Stub, + _apis.TopicService.DescribeTopic, + create_result_wrapper(_ydb_topic.DescribeTopicResult), + ) + settings = BaseRequestSettings().with_timeout(_DESCRIBE_TIMEOUT) + if inspect.iscoroutinefunction(self._driver.__call__): + res = self._driver(*args, settings=settings) + else: + res = await to_thread(self._driver, *args, executor=None, settings=settings) + if inspect.isawaitable(res): + res = await res + return res.to_public() + + async def _init(self): + for attempt in range(_REPARTITION_DISCOVER_ATTEMPTS): + description = await self._describe() + leaves = [p for p in description.partitions if p.active and not p.child_partition_ids] + chooser = self._chooser or self._default_chooser(leaves, description) + if leaves and ( + not isinstance(chooser, PublicPartitionByKeyBound) or self._children_cover_parent(None, leaves) + ): + chooser.add_partitions(leaves) + self._chooser = chooser + self._partitions = {p.partition_id: p for p in leaves} + return + if attempt + 1 < _REPARTITION_DISCOVER_ATTEMPTS: + await asyncio.sleep(_REPARTITION_DISCOVER_DELAY) + raise _IncompletePartitionTopology("active partitions do not cover the topic key range") + + @staticmethod + def _default_chooser(partitions, description=None) -> PublicPartitionChooser: + # Route by key range on auto-partitioned topics, else by Kafka hash. Prefer the + # topic's auto-partitioning strategy as the signal: a single auto partition can + # report no key_range at all (open b""..b""), yet its split-children are bounded, + # which only the Bound chooser can accept. + aps = getattr(description, "auto_partitioning_settings", None) + auto_enabled = aps is not None and aps.strategy not in ( + None, + PublicAutoPartitioningStrategy.UNSPECIFIED, + PublicAutoPartitioningStrategy.DISABLED, + ) + has_key_ranges = any(p.key_range is not None for p in partitions) + if auto_enabled or has_key_ranges: + return PublicPartitionByKeyBound() + return PublicPartitionByKeyKafka() + + async def wait_init(self): + await asyncio.shield(self._init_task) + + def _check_closed(self): + if self._closing or self._closed: + raise TopicWriterClosedError() + + def _build_writer_settings( + self, + partition_id: int, + with_split_hook: bool, + pin_partition: bool = True, + ) -> PublicWriterSettings: + return PublicWriterSettings( + topic=self._settings.topic, + producer_id="%s-%d" % (self._prefix, partition_id), + # Unpinned (pin_partition=False) leaves the session routed by message group instead, + # which is the only way to reach a partition that is no longer active. + partition_id=partition_id if pin_partition else None, + # The multi-writer assigns sequence numbers itself so it can resend + # messages to child partitions after a split, keeping them monotonic. + auto_seqno=False, + auto_created_at=False, + codec=self._settings.codec, + encoders=self._settings.encoders, + encoder_executor=self._settings.encoder_executor, + max_buffer_size_bytes=self._settings.max_buffer_size_bytes, + max_buffer_messages=self._settings.max_buffer_messages, + buffer_wait_timeout_sec=self._settings.buffer_wait_timeout_sec, + _on_check_retriable_error=self._make_overloaded_hook(partition_id) if with_split_hook else None, + ) + + async def _get_or_create_writer(self, partition_id: int) -> WriterAsyncIO: + writer = self._writers.get(partition_id) + if writer is None: + writer = WriterAsyncIO(self._driver, self._build_writer_settings(partition_id, with_split_hook=True)) + # Seed the seqno cursor from the producer's last persisted seqno so a + # stable producer_id_prefix resumes numbering instead of colliding. The wait is + # bounded: a partition that went inactive between the routing decision and this call + # never completes init, and we hold the orchestrator lock here. + try: + init_info = await asyncio.wait_for(writer.wait_init(), timeout=_WRITER_INIT_TIMEOUT) + except BaseException: + # Do not register a writer that never became usable, and do not leak its stream. + await self._safe_close(writer) + raise + self._writers[partition_id] = writer + self._last_write_at[partition_id] = self._loop.time() + last_seqno = init_info.last_seqno or 0 + self._server_init_seqno[partition_id] = last_seqno + # Lift the shared cursor above every producer we have opened: each partition has its + # own producer_id and therefore its own persisted history, and a seqno we hand out + # must be new in whichever partition the message ends up in. + self._seqno = max(self._seqno, last_seqno) + return writer + + async def _writer_for_migration(self, message: PublicMessage) -> Tuple[int, WriterAsyncIO]: + assert self._chooser is not None + for attempt in range(_REPARTITION_DISCOVER_ATTEMPTS): + partition_id = self._chooser.choose_partition(message) + try: + return partition_id, await self._get_or_create_writer(partition_id) + except (asyncio.TimeoutError, TopicWriterPartitionSplitError): + if attempt + 1 == _REPARTITION_DISCOVER_ATTEMPTS: + raise + await self._handle_repartition_locked(partition_id, recover=False) + if partition_id in self._partitions: + raise + raise AssertionError("unreachable") + + def _schedule_repartition(self, partition_id: int, *, refresh_only: bool = False) -> Optional[asyncio.Future]: + """Start (or join) the repartition of one partition. + + A burst of OVERLOADED for the same partition must not start several concurrent + recoveries of it, and the writer must own the task so close() can cancel and await it + instead of leaving it to describe topics and open sub-writers after shutdown. + """ + if self._closed: + return None + running = self._repartition_tasks.get(partition_id) + if running is not None and not running.done(): + return running + + recovery = ( + self._handle_repartition(partition_id, recover=False) + if refresh_only + else self._on_partition_overloaded(partition_id) + ) + task = self._loop.create_task(recovery) + task.set_name("multiwriter repartition %d" % partition_id) + self._repartition_tasks[partition_id] = task + + def _forget(finished: asyncio.Future) -> None: + if not finished.cancelled(): + finished.exception() # The write waiting for this refresh may have been cancelled. + if self._repartition_tasks.get(partition_id) is finished: + del self._repartition_tasks[partition_id] + + task.add_done_callback(_forget) + return task + + def _make_overloaded_hook(self, partition_id: int): + def hook(err: BaseException) -> bool: + if _is_overloaded(err): + logger.debug("multi-writer: partition %d overloaded, re-describing (split/merge)", partition_id) + self._schedule_repartition(partition_id) + return True + return False + + return hook + + def _attach_ack(self, entry: _InflightMessage) -> None: + sub_future = entry.sub_future + assert sub_future is not None + sub_future.add_done_callback(lambda f: self._on_sub_result(entry.partition_id, entry.seqno, f)) + + def _fail_message(self, entry: _InflightMessage, err: BaseException) -> None: + if not self._write_error.done(): + self._write_error.set_result(err) + entry.finish(err) + + def _on_sub_result(self, partition_id: int, seqno: int, sub_future: asyncio.Future) -> None: + entry = self._inflight.get(partition_id, {}).get(seqno) + if entry is None or entry.sub_future is not sub_future: + return # stale: the message was already resolved or moved to a child + + if sub_future.cancelled(): + return + + exc = sub_future.exception() + if exc is not None: + if isinstance(exc, TopicWriterPartitionSplitError) or partition_id in self._retiring: + # Expected while the partition is torn down: leave the message in flight so the + # repartition (or the in-place recovery) resends it. + return + self._inflight.get(partition_id, {}).pop(seqno, None) + self._fail_message(entry, exc) + return + + # A success is always honoured, including one that lands while the sub-writer is being + # closed: the server persisted the message, and ignoring the ack would leave the dedup + # cut too low and resend an already-written message to the child. + self._inflight.get(partition_id, {}).pop(seqno, None) + self._max_acked[partition_id] = max(self._max_acked.get(partition_id, 0), seqno) + entry.finish(sub_future.result()) + + async def _quiesce_writer(self, partition_id: int) -> None: + """Close the partition's sub-writer and settle the acks it was still holding. + + Closing fails every pending ack, which must not reach the user because those messages + are about to be resent. ``_retiring`` suppresses exactly those failures while letting + successes through, and the yield below gives already-scheduled ack callbacks a chance to + run before the caller reads the dedup cut. + """ + self._retiring.add(partition_id) + writer = self._writers.pop(partition_id, None) + if writer is not None: + await self._safe_close(writer) + await asyncio.sleep(0) + + @staticmethod + async def _safe_close(writer: WriterAsyncIO) -> None: + # A sub-writer stopped by the split hook re-raises TopicWriterPartitionSplitError from + # close(); closing a writer we are discarding must never abort a repartition. + try: + await writer.close(flush=False) + except Exception: # noqa: BLE001 + logger.debug("multi-writer: ignoring error while closing a discarded sub-writer", exc_info=True) + + async def _probe_server_seqno(self, partition_id: int) -> int: + """Ask the server how far this partition's producer actually got. + + The answer must come from the server, not from the acks we happened to receive: a message + can be persisted and its ack lost when the session dies, and treating it as unwritten is + exactly what produces a duplicate on resend. + + The session is opened WITHOUT a partition id. By the time we ask, the partition is + usually inactive, and a session pinned to an inactive partition never completes its init + handshake -- it would hang here holding the orchestrator lock. Unpinned, the session is + routed by message group and still reports this producer's persisted seqno. Both reference + implementations read the cut the same way (C++ `CreateWriteSession(..., false)` sets only + ProducerId/MessageGroupId, Go `createNonDirectWriter` only WithProducerID). + + Results are cached: a retired producer receives no further writes, so its value is final. + """ + cached = self._retired_max_seqno.get(partition_id) + if cached is not None: + return cached + + probe = WriterAsyncIO( + self._driver, + self._build_writer_settings(partition_id, with_split_hook=False, pin_partition=False), + ) + try: + init_info = await asyncio.wait_for(probe.wait_init(), timeout=_WRITER_INIT_TIMEOUT) + finally: + await self._safe_close(probe) + + last_seqno = init_info.last_seqno or 0 + self._retired_max_seqno[partition_id] = last_seqno + logger.debug("multi-writer: partition %d persisted up to seqno %d (server)", partition_id, last_seqno) + return last_seqno + + async def _max_seqno_cut(self, partition_id: int) -> int: + """Dedup cut for a repartition: messages at or below it were persisted to the retiring + partition and must not be resent to the child. + + Only this partition's own producer is asked. A message sitting here cannot have been + persisted under any producer it used earlier in its life: every move is gated by a cut of + at least that producer's server seqno, so anything already stored there was resolved as + written on the spot and never travelled. Its number is therefore strictly above that + value, and a retired producer never grows -- asking again would only repeat an answer that + is already known to be too low to matter. + + Walking further would not merely be redundant. A merge child has two parents whose + branches numbered independently, so the maximum over them pulls in the sibling branch's + history, which says nothing about messages that came down this branch and can be + arbitrarily higher than theirs. + + Raises if the server cannot be asked. Falling back to the highest ack we saw would look + like it worked while quietly reopening the duplicate window this exists to close; the + caller turns the failure into terminal errors on the affected messages instead. + """ + acked = self._max_acked.get(partition_id, 0) + cut = max(acked, await self._probe_server_seqno(partition_id)) + logger.debug( + "multi-writer: dedup cut for partition %d is %d (highest ack seen %d)", + partition_id, + cut, + acked, + ) + return cut + + @staticmethod + def _children_cover_parent(parent, children: List[_PartitionInfo]) -> bool: + """Validate coverage before retiring a parent, so a gap cannot misroute its keys.""" + parent_range = getattr(parent, "key_range", None) + if parent_range is None and all(child.key_range is None for child in children): + return bool(children) # fixed-partition topics do not report key ranges + + ranges = [] + for child in children: + child_range = getattr(child, "key_range", None) + if child_range is None: + return False # inconsistent with a bounded parent -> treat the view as incomplete + ranges.append((child_range.from_bound, child_range.to_bound)) + ranges.sort(key=lambda r: r[0]) + + # Coverage is "at least the parent's range", not "exactly": a merge child owns the ranges + # of both its parents, so it legitimately covers more than the parent we started from. + # A single auto partition may omit key_range before its first split. + parent_end = parent_range.to_bound if parent_range is not None else b"" + cursor = parent_range.from_bound if parent_range is not None else b"" + for from_bound, to_bound in ranges: + if from_bound > cursor: + return False # gap between the covered prefix and this child + if not to_bound: + return True # this child runs to the end of the key space + if to_bound > cursor: + cursor = to_bound + if parent_end and cursor >= parent_end: + return True + return False + + @staticmethod + def _active_descendants(partition_id: int, partitions: List[_PartitionInfo]) -> List[_PartitionInfo]: + by_id = {p.partition_id: p for p in partitions} + children = {p.partition_id: set(p.child_partition_ids) for p in partitions} + for partition in partitions: + for parent in partition.parent_partition_ids: + children.setdefault(parent, set()).add(partition.partition_id) + pending = list(children.get(partition_id, ())) + visited = {partition_id} + leaves = [] + while pending: + current_id = pending.pop() + if current_id in visited: + continue + visited.add(current_id) + current = by_id.get(current_id) + descendants = children.get(current_id, set()) + if current is not None and current.active and not descendants: + leaves.append(current) + else: + pending.extend(descendants) + return leaves + + async def _discover_children(self, partition_id: int) -> Tuple[List[int], List[_PartitionInfo]]: + """Find all connected retiring parents and their active descendants. + + A merge can connect parents through intermediate, already retired partitions. + Every added parent contributes all its descendants, which may reach further parents. + """ + for attempt in range(_REPARTITION_DISCOVER_ATTEMPTS): + description = await self._describe() + parents = {p.partition_id: set(p.parent_partition_ids) for p in description.partitions} + for partition in description.partitions: + for child_id in partition.child_partition_ids: + parents.setdefault(child_id, set()).add(partition.partition_id) + + children: Dict[int, _PartitionInfo] = {} + retired = set() + seen_ancestors = set() + pending = [partition_id] + complete = True + while pending: + old = pending.pop() + if old in retired: + continue + retired.add(old) + descendants = self._active_descendants(old, description.partitions) + complete = complete and self._children_cover_parent(self._partitions.get(old), descendants) + ancestors: List[int] = [] + for child in descendants: + children[child.partition_id] = child + ancestors.extend(parents.get(child.partition_id, ())) + while ancestors: + ancestor = ancestors.pop() + if ancestor in seen_ancestors: + continue + seen_ancestors.add(ancestor) + if ancestor in self._partitions: + if ancestor not in retired: + pending.append(ancestor) + else: + ancestors.extend(parents.get(ancestor, ())) + + if children and complete: + return sorted(retired), [children[child_id] for child_id in sorted(children)] + if attempt + 1 < _REPARTITION_DISCOVER_ATTEMPTS: + await asyncio.sleep(_REPARTITION_DISCOVER_DELAY) + if not children and any( + p.partition_id == partition_id and p.active and not p.child_partition_ids for p in description.partitions + ): + return [], [] + raise _IncompletePartitionTopology("active descendants do not cover partition %d" % partition_id) + + def _fail_partition_inflight(self, partition_id: int, err: BaseException) -> None: + """Give every in-flight message of an unusable partition a terminal outcome. + + Once repartition and recovery have both failed there is no sub-writer left to ack these + messages, so their futures would never resolve and flush()/close(flush=True) would wait + on them forever. An accepted message must always end in success or failure. + """ + entries = self._inflight.pop(partition_id, {}) + if not entries: + return + logger.error( + "multi-writer: partition %d is unusable, failing %d in-flight messages: %s", + partition_id, + len(entries), + err, + ) + for entry in entries.values(): + entry.sub_future = None + self._fail_message(entry, err) + + async def _on_partition_overloaded(self, partition_id: int): + """Entry point for the OVERLOADED hook: handle a repartition, or recover on failure. + + The hook force-stops the sub-writer, so if handling fails we must not leave the + partition's messages stranded — recreate the writer and resend them. If that fails too, + the messages are failed explicitly rather than left without an owner. + """ + try: + await self._handle_repartition(partition_id) + return + except asyncio.CancelledError: + raise + except Exception as err: + logger.exception("multi-writer: repartition of partition %d failed; recovering", partition_id) + failure: BaseException = err + + try: + async with self._lock: + if partition_id in self._partitions and not isinstance(failure, _IncompletePartitionTopology): + await self._recover_partition(partition_id) + return + except asyncio.CancelledError: + raise + except Exception as err: + logger.exception("multi-writer: recovery of partition %d failed", partition_id) + failure = err + + async with self._lock: + await self._quiesce_writer(partition_id) + self._retiring.discard(partition_id) + self._fail_partition_inflight(partition_id, failure) + + async def _handle_repartition(self, partition_id: int, *, recover: bool = True): + """Resolve an OVERLOADED partition under the routing lock.""" + async with self._lock: + await self._handle_repartition_locked(partition_id, recover=recover) + + async def _handle_repartition_locked(self, partition_id: int, *, recover: bool = True): + if partition_id not in self._partitions: + return # already handled by a sibling parent's event + assert self._chooser is not None + + retired, children = await self._discover_children(partition_id) + if not children: + if recover: + await self._recover_partition(partition_id) + return + + new_children = [c for c in children if c.partition_id not in self._partitions] + if new_children: + self._chooser.add_partitions(new_children) + for child in new_children: + self._partitions[child.partition_id] = child + for old in retired: + self._chooser.remove_partition(old) + self._partitions.pop(old, None) + + # Stop all retired writers before probing or migrating any of their messages. + try: + self._retiring.update(retired) + for old in retired: + await self._quiesce_writer(old) + + cuts = {} + for old in retired: + if self._inflight.get(old): + cuts[old] = await self._max_seqno_cut(old) + else: + self._inflight.pop(old, None) + await self._migrate_messages(cuts) + except Exception as err: + for old in retired: + self._fail_partition_inflight(old, err) + raise + finally: + self._retiring.difference_update(retired) + + async def _recover_partition(self, partition_id: int): + # The hook stopped the sub-writer; drop it and resend the partition's in-flight + # messages to a fresh writer for the SAME partition, keeping their seqnos. + await self._quiesce_writer(partition_id) + try: + writer = await self._get_or_create_writer(partition_id) + + # The partition is still active, so this writer's init carries the server's real last + # persisted seqno -- an exact dedup cut, unlike the split case. Messages at or below it + # were written and their ack was lost with the stream; resending one is not idempotent + # here, because the writer rejects a seqno it has already seen ("Message seqno is + # duplicated") before the server ever gets a chance to deduplicate it, and that error + # would abort the resend of every message after it. + cut = max(self._server_init_seqno.get(partition_id, 0), self._max_acked.get(partition_id, 0)) + for seqno, entry in sorted(self._inflight.get(partition_id, {}).items()): + if seqno <= cut: + self._inflight.get(partition_id, {}).pop(seqno, None) + self._max_acked[partition_id] = max(self._max_acked.get(partition_id, 0), seqno) + entry.finish(PublicWriteResult.Written(offset=-1)) + continue + entry.message.seqno = seqno + sub_future = await writer.write_with_ack_future(entry.message) + assert not isinstance(sub_future, list) # single message -> single future + entry.sub_future = sub_future + self._attach_ack(entry) + finally: + self._retiring.discard(partition_id) + + def _migration_conflict( + self, + child_id: int, + seqno: int, + target: Dict[int, _InflightMessage], + ) -> Optional[BaseException]: + """Reject a collision with the target's in-flight messages or persisted history.""" + if seqno in target: + return TopicWriterError("seqno %d is already in flight on partition %d" % (seqno, child_id)) + server_seqno = self._server_init_seqno.get(child_id, 0) + if seqno <= server_seqno: + return TopicWriterError( + "seqno %d cannot be resent to partition %d: its producer has already persisted" + " up to %d" % (seqno, child_id, server_seqno) + ) + return None + + async def _migrate_messages(self, cuts: Dict[int, int]): + entries = [] + for partition_id, cut in cuts.items(): + for seqno, entry in list(self._inflight.get(partition_id, {}).items()): + if seqno <= cut: + self._inflight.get(partition_id, {}).pop(seqno, None) + entry.finish(PublicWriteResult.Written(offset=-1)) + else: + entries.append((seqno, partition_id, entry)) + + # Sorting the combined tail also orders every target's subsequence, including merges. + for seqno, partition_id, entry in sorted(entries, key=lambda item: (item[0], item[1])): + if self._inflight.get(partition_id, {}).get(seqno) is not entry: + continue + assert self._chooser is not None + try: + child_id, child_writer = await self._writer_for_migration(entry.message) + except Exception as err: # noqa: BLE001 + # The message cannot be placed: no ready leaf owns its key, or the child itself + # went inactive (splits cascade). Fail it and everything after it -- dropping + # would lose the message silently, and skipping ahead would reorder the key. + logger.warning( + "multi-writer: cannot migrate messages of partition %d, failing seqno >= %d: %s", + partition_id, + seqno, + err, + ) + for old in cuts: + self._fail_partition_inflight(old, err) + break + + if self._inflight.get(partition_id, {}).get(seqno) is not entry: + continue # acked while the child writer was being opened -> nothing to resend + + target = self._inflight.setdefault(child_id, {}) + conflict = self._migration_conflict(child_id, seqno, target) + if conflict is not None: + logger.error("multi-writer: %s", conflict) + self._inflight.get(partition_id, {}).pop(seqno, None) + self._fail_message(entry, conflict) + continue + + entry.message.seqno = seqno + try: + sub_future = await child_writer.write_with_ack_future(entry.message) + except Exception as err: + for old in cuts: + self._fail_partition_inflight(old, err) + break + assert not isinstance(sub_future, list) + self._inflight.get(partition_id, {}).pop(seqno, None) + entry.partition_id = child_id + target[seqno] = entry + entry.sub_future = sub_future + self._attach_ack(entry) + + for old in cuts: + self._inflight.pop(old, None) + + async def write_with_ack_future( + self, + messages: Union[Message, List[Message]], + ) -> Union[asyncio.Future, List[asyncio.Future]]: + self._check_closed() + + input_single_message = not isinstance(messages, list) + raw = messages if isinstance(messages, list) else [messages] + converted = [] + for source in raw: + message = copy.copy(PublicMessage._create_message(source)) + if self._settings.auto_seqno: + if message.seqno not in (None, 0): + raise TopicWriterError("Explicit seqno and auto_seq setting is mutual exclusive") + elif message.seqno in (None, 0): + raise TopicWriterError("Empty seqno and auto_seq setting is disabled") + if message.metadata_items is not None: + message.metadata_items = message.metadata_items.copy() + if self._settings.auto_created_at: + if message.created_at is not None: + raise TopicWriterError( + "Explicit set auto_created_at and setting auto_created_at is mutual exclusive" + ) + message.created_at = datetime.datetime.now(datetime.timezone.utc) + converted.append(message) + + await self.wait_init() + futures: List[asyncio.Future] = [] + async with self._write_lock: + for message in converted: + futures.append(await self._admit_message(message)) + + return futures[0] if input_single_message else futures + + async def _admit_message(self, message: PublicMessage) -> asyncio.Future: + assert self._chooser is not None + attempts = 0 + while True: + async with self._lock: + self._check_closed() + partition_id = self._chooser.choose_partition(message) + try: + writer = await self._get_or_create_writer(partition_id) + self._check_closed() + seqno = self._seqno + 1 if self._settings.auto_seqno else message.seqno + assert seqno is not None + if seqno <= self._seqno: + raise TopicWriterError( + "Message seqno must increase across the whole multi-writer: %d <= %d" % (seqno, self._seqno) + ) + if not writer._buffer_is_full(): + return await self._enqueue_message(writer, partition_id, message, seqno) + except (asyncio.TimeoutError, TopicWriterPartitionSplitError) as err: + failure: Optional[BaseException] = err + else: + failure = None + + if failure is None: + # Shutdown and repartition must be able to take the routing lock while the buffer is full. + try: + await writer._wait_for_buffer_space() + except TopicWriterPartitionSplitError as err: + failure = err + except TopicWriterStopped: + self._check_closed() + if self._writers.get(partition_id) is writer: + raise + if failure is None or self._writers.get(partition_id) is not writer: + continue + + self._check_closed() + attempts += 1 + if attempts >= _REPARTITION_DISCOVER_ATTEMPTS: + raise failure + timed_out = isinstance(failure, asyncio.TimeoutError) + recovery = self._schedule_repartition(partition_id, refresh_only=timed_out) + assert recovery is not None + await asyncio.shield(recovery) + if timed_out and partition_id in self._partitions: + raise failure + + async def _enqueue_message( + self, writer: WriterAsyncIO, partition_id: int, message: PublicMessage, seqno: int + ) -> asyncio.Future: + message.seqno = seqno + sub_future = await writer.write_with_ack_future(message) + assert not isinstance(sub_future, list) + self._seqno = seqno + self._last_write_at[partition_id] = self._loop.time() + entry = _InflightMessage(message, self._loop.create_future(), seqno, partition_id, sub_future) + self._inflight.setdefault(partition_id, {})[seqno] = entry + self._attach_ack(entry) + return entry.user_future + + async def write(self, messages: Union[Message, List[Message]]): + await self.write_with_ack_future(messages) + + async def write_with_ack( + self, + messages: Union[Message, List[Message]], + ) -> Union[PublicWriteResultTypes, List[PublicWriteResultTypes]]: + futures = await self.write_with_ack_future(messages) + future_list = futures if isinstance(futures, list) else [futures] + await asyncio.wait(future_list) + results = [f.result() for f in future_list] + return results if isinstance(futures, list) else results[0] + + @staticmethod + async def _idle_reaper(mw_ref: "weakref.ref", idle_timeout: float): + interval = max(1.0, idle_timeout / 3) + try: + while True: + await asyncio.sleep(interval) + mw = mw_ref() + if mw is None or mw._closed: + return + try: + await mw._evict_idle_writers() + except Exception: # noqa: BLE001 + logger.debug("multi-writer: idle eviction pass failed", exc_info=True) + del mw # drop the strong ref so the writer can be GC'd while we sleep + except asyncio.CancelledError: + pass + + async def _evict_idle_writers(self): + # Close sub-writers with no in-flight messages that have not been written to for the idle + # timeout; they are recreated on demand. Done under the lock so a concurrent write cannot + # open a second session for the same producer while we close the old one. + now = self._loop.time() + async with self._lock: + for partition_id in list(self._writers.keys()): + if self._inflight.get(partition_id): + continue # has un-acked messages -> not idle + if now - self._last_write_at.get(partition_id, now) < self._idle_timeout: + continue + writer = self._writers.pop(partition_id) + self._last_write_at.pop(partition_id, None) + logger.debug("multi-writer: evicting idle sub-writer for partition %d", partition_id) + await self._safe_close(writer) + + def _pending_entries(self) -> Iterator[_InflightMessage]: + for entries in self._inflight.values(): + yield from entries.values() + + async def _flush_impl(self): + await self.wait_init() + if self._write_error.done(): + raise self._write_error.result() + async with self._lock: + writers = list(self._writers.values()) + pending = [entry.flush_waiter() for entry in self._pending_entries()] + + async def wait_for_acks(): + results = await asyncio.gather(*(w.flush() for w in writers), return_exceptions=True) + for result in results: + if isinstance(result, BaseException) and not isinstance(result, TopicWriterPartitionSplitError): + raise result + if pending: + # Cancelling this waiter must not cancel accepted messages. + await asyncio.wait(pending) + for future in pending: + future.result() + + waiter = self._loop.create_task(wait_for_acks()) + try: + await asyncio.wait([waiter, self._write_error], return_when=asyncio.FIRST_COMPLETED) + if self._write_error.done(): + raise self._write_error.result() + await waiter + finally: + if not waiter.done(): + waiter.cancel() + await asyncio.gather(waiter, return_exceptions=True) + + async def flush(self): + self._check_closed() + await self._flush_impl() + + async def close(self, *, flush: bool = True): + if self._closed: + return + + error: Optional[BaseException] = None + # Stop new admissions before taking the flush snapshot. Recovery still serves accepted writes. + self._closing = True + init_done = self._init_task.done() and not self._init_task.cancelled() and self._init_task.exception() is None + if flush and init_done: + try: + await self._flush_impl() + except BaseException as err: + error = err + logger.debug("multi-writer: flush during close failed", exc_info=True) + + self._closed = True + if not self._init_task.done(): + self._init_task.cancel() + if self._reaper_task is not None and not self._reaper_task.done(): + self._reaper_task.cancel() + await asyncio.gather(self._init_task, return_exceptions=True) + + # Stop repartitions before touching the writers: an in-flight one holds the lock, opens + # sub-writers and mutates the topology, none of which may outlive the multi-writer. + repartitions = list(self._repartition_tasks.values()) + self._repartition_tasks.clear() + for task in repartitions: + if not task.done(): + task.cancel() + if repartitions: + await asyncio.gather(*repartitions, return_exceptions=True) + + async with self._lock: + writers = list(self._writers.values()) + self._writers.clear() + pending = list(self._pending_entries()) + self._inflight.clear() + results = await asyncio.gather(*(w.close(flush=False) for w in writers), return_exceptions=True) + for entry in pending: + entry.finish(TopicWriterStopped()) + if flush: + for result in results: + if isinstance(result, BaseException) and not isinstance(result, TopicWriterPartitionSplitError): + error = error or result + if error is not None: + raise error diff --git a/ydb/_topic_writer/topic_writer_multi_sync.py b/ydb/_topic_writer/topic_writer_multi_sync.py new file mode 100644 index 00000000..5e599a74 --- /dev/null +++ b/ydb/_topic_writer/topic_writer_multi_sync.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import asyncio +import logging +import typing +from concurrent.futures import Future +from typing import List, Optional, Union + +from .._grpc.grpcwrapper.common_utils import SupportedDriverType +from .topic_writer import ( + Message, + PublicWriteResult, + TopicWriterClosedError, +) +from .topic_writer_multi_asyncio import MultiWriterSettings, TopicWriterMultiAsyncIO +from .._topic_common.common import ( + _get_shared_event_loop, + TimeoutType, + CallFromSyncToAsync, +) + +logger = logging.getLogger(__name__) + + +class TopicWriterMultiSync: + _caller: CallFromSyncToAsync + _async_writer: TopicWriterMultiAsyncIO + _closed: bool + _parent: typing.Any # need for prevent close parent client by GC + + def __init__( + self, + driver: SupportedDriverType, + settings: MultiWriterSettings, + *, + eventloop: Optional[asyncio.AbstractEventLoop] = None, + _parent=None, + ): + self._closed = False + + if eventloop: + loop = eventloop + else: + loop = _get_shared_event_loop() + + self._caller = CallFromSyncToAsync(loop) + + async def create_async_writer(): + return TopicWriterMultiAsyncIO(driver, settings) + + self._async_writer = self._caller.safe_call_with_result(create_async_writer(), None) + self._parent = _parent + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + try: + self.close() + except BaseException: + if exc_val is None: + raise + + def __del__(self): + if not self._closed: + try: + logger.debug("Topic multi-writer was not closed properly. Consider using method close().") + self.close(flush=False) + except BaseException: + logger.warning("Something went wrong during multi-writer close in __del__") + + def close(self, *, flush: bool = True, timeout: TimeoutType = None): + if self._closed: + return + + logger.debug("Close topic multi-writer") + self._closed = True + + self._caller.safe_call_with_result(self._async_writer.close(flush=flush), timeout) + + def _check_closed(self): + if self._closed: + raise TopicWriterClosedError() + + def async_flush(self) -> Future: + self._check_closed() + return self._caller.unsafe_call_with_future(self._async_writer.flush()) + + def flush(self, *, timeout: TimeoutType = None): + self._check_closed() + logger.debug("flush multi-writer") + return self._caller.unsafe_call_with_result(self._async_writer.flush(), timeout) + + def async_wait_init(self) -> Future: + self._check_closed() + return self._caller.unsafe_call_with_future(self._async_writer.wait_init()) + + def wait_init(self, *, timeout: TimeoutType = None): + self._check_closed() + logger.debug("wait multi-writer init") + return self._caller.unsafe_call_with_result(self._async_writer.wait_init(), timeout) + + def write( + self, + messages: Union[Message, List[Message]], + timeout: TimeoutType = None, + ): + self._check_closed() + logger.debug( + "write %s messages", + len(messages) if isinstance(messages, list) else 1, + ) + self._caller.safe_call_with_result(self._async_writer.write(messages), timeout) + + def async_write_with_ack( + self, + messages: Union[Message, List[Message]], + ) -> Future[Union[PublicWriteResult, List[PublicWriteResult]]]: + self._check_closed() + return self._caller.unsafe_call_with_future(self._async_writer.write_with_ack(messages)) + + def write_with_ack( + self, + messages: Union[Message, List[Message]], + timeout: Union[float, None] = None, + ) -> Union[PublicWriteResult, List[PublicWriteResult]]: + self._check_closed() + logger.debug( + "write_with_ack %s messages", + len(messages) if isinstance(messages, list) else 1, + ) + return self._caller.unsafe_call_with_result(self._async_writer.write_with_ack(messages), timeout=timeout) diff --git a/ydb/_topic_writer/topic_writer_partition_chooser.py b/ydb/_topic_writer/topic_writer_partition_chooser.py new file mode 100644 index 00000000..2cbb3504 --- /dev/null +++ b/ydb/_topic_writer/topic_writer_partition_chooser.py @@ -0,0 +1,195 @@ +import abc +import bisect +from typing import Callable, List, Tuple + +from .topic_writer import PublicMessage +from .._grpc.grpcwrapper.ydb_topic_public_types import PublicDescribeTopicResult + +_MASK32 = 0xFFFFFFFF +_MASK64 = 0xFFFFFFFFFFFFFFFF + +# Metadata key the server uses to validate bound-based partition selection. +PARTITION_KEY_METADATA_KEY = "__partition_key" + +PartitionInfo = PublicDescribeTopicResult.PartitionInfo + + +def murmur2_32(data: bytes, seed: int = 0) -> int: + """MurmurHash2, 32-bit, little-endian.""" + m = 0x5BD1E995 + r = 24 + n = len(data) + h = (seed ^ n) & _MASK32 + + body = n - (n % 4) + for i in range(0, body, 4): + k = int.from_bytes(data[i : i + 4], "little") + k = (k * m) & _MASK32 + k ^= k >> r + k = (k * m) & _MASK32 + h = (h * m) & _MASK32 + h ^= k + + rem = n % 4 + if rem: + tail = data[body:] + if rem >= 3: + h ^= tail[2] << 16 + if rem >= 2: + h ^= tail[1] << 8 + h ^= tail[0] + h = (h * m) & _MASK32 + + h ^= h >> 13 + h = (h * m) & _MASK32 + h ^= h >> 15 + return h & _MASK32 + + +def murmur64a(data: bytes, seed: int = 0) -> int: + """MurmurHash64A, 64-bit, little-endian. Matches the YDB Go SDK + ``xhash.Murmur2Hash64A`` and the C++ default partitioning key hasher.""" + m = 0xC6A4A7935BD1E995 + r = 47 + n = len(data) + h = (seed ^ ((n * m) & _MASK64)) & _MASK64 + + body = n - (n % 8) + for i in range(0, body, 8): + k = int.from_bytes(data[i : i + 8], "little") + k = (k * m) & _MASK64 + k ^= k >> r + k = (k * m) & _MASK64 + h ^= k + h = (h * m) & _MASK64 + + rem = n % 8 + if rem: + tail = data[body:] + for j in range(rem - 1, -1, -1): + h ^= tail[j] << (8 * j) + h = (h * m) & _MASK64 + + h ^= h >> r + h = (h * m) & _MASK64 + h ^= h >> r + return h & _MASK64 + + +def default_bound_key_hasher(key: str) -> bytes: + """Hash a routing key the way the YDB server expects for bound-based + partition selection: MurmurHash64A(seed=0) as 8 big-endian bytes.""" + return murmur64a(key.encode("utf-8"), 0).to_bytes(8, "big") + + +class PublicPartitionChooser(abc.ABC): + """Maps a message to a target partition id and tracks the live partition set.""" + + @abc.abstractmethod + def choose_partition(self, message: PublicMessage) -> int: ... + + @abc.abstractmethod + def add_partitions(self, partitions: List[PartitionInfo]) -> None: ... + + @abc.abstractmethod + def remove_partition(self, partition_id: int) -> None: ... + + +class PublicPartitionByKeyKafka(PublicPartitionChooser): + """Kafka-compatible routing for topics with a fixed partition count. + + Hashes UTF-8 keys with 32-bit MurmurHash2 (seed ``0x9747b28c``), clears the + sign bit and indexes partition IDs sorted in ascending order. + """ + + def __init__(self): + self._partitions: List[int] = [] + + def add_partitions(self, partitions: List[PartitionInfo]) -> None: + for p in partitions: + # Any key range at all means the topic is auto-partitioned, including the fully open + # one a single partition reports before its first split. Accepting that would work + # until the split, then break: the children come back bounded, and modulo routing + # cannot place a key by bounds. + if p.key_range is not None: + raise ValueError("PublicPartitionByKeyKafka does not support partition key ranges") + self._partitions.append(p.partition_id) + self._partitions.sort() + + def remove_partition(self, partition_id: int) -> None: + self._partitions = [p for p in self._partitions if p != partition_id] + + def choose_partition(self, message: PublicMessage) -> int: + if not self._partitions: + raise ValueError("no partitions configured for partition chooser") + # Apache Kafka's DefaultPartitioner applies toPositive() (mask the sign bit) to the + # murmur2 hash before the modulo; match it so the same key lands on the same partition. + h = murmur2_32((message.key or "").encode("utf-8"), 0x9747B28C) & 0x7FFFFFFF + return self._partitions[h % len(self._partitions)] + + +class PublicPartitionByKeyBound(PublicPartitionChooser): + """Server-accurate routing: hashes the key and selects the partition whose + ``[from_bound, to_bound)`` key range owns it. Mirrors YDB auto-partitioning, + so the same key lands where the server expects it. + """ + + def __init__(self, key_hasher: Callable[[str], bytes] = default_bound_key_hasher): + self._key_hasher = key_hasher + # (from_bound, to_bound, partition_id) sorted by from_bound. An empty from_bound is the + # start of the key space, an empty to_bound is its end. + self._partitions: List[Tuple[bytes, bytes, int]] = [] + + def add_partitions(self, partitions: List[PartitionInfo]) -> None: + # Normalise and sort before validating: the caller may pass partitions in any order + # (DescribeTopic does not promise one), and only the partition that sorts leftmost may + # carry an open lower bound. Validating in argument order rejects a valid set that + # happens to arrive reversed. + added: List[Tuple[bytes, bytes, int]] = [] + for p in partitions: + from_bound = p.key_range.from_bound if p.key_range is not None else b"" + to_bound = p.key_range.to_bound if p.key_range is not None else b"" + added.append((from_bound, to_bound, p.partition_id)) + added.sort(key=lambda x: x[0]) + + for i, (from_bound, _to_bound, partition_id) in enumerate(added): + if i > 0 and not from_bound: + raise ValueError( + "partition %d has no from_bound key range, but only the leftmost partition may" + " have an open lower bound" % partition_id + ) + + self._partitions.extend(added) + self._partitions.sort(key=lambda x: x[0]) + + def remove_partition(self, partition_id: int) -> None: + self._partitions = [p for p in self._partitions if p[-1] != partition_id] + + def choose_partition(self, message: PublicMessage) -> int: + if not self._partitions: + raise ValueError("no partitions configured for partition chooser") + hashed = self._key_hasher(message.key or "") + + if message.metadata_items is None: + message.metadata_items = {} + message.metadata_items[PARTITION_KEY_METADATA_KEY] = hashed + + # First partition whose from_bound is strictly greater than the hashed key; the owning + # partition is the previous one. Searched in place with a key rather than by building a + # list of bounds, which would allocate per message on the hot path. + idx = bisect.bisect_right(self._partitions, hashed, key=lambda entry: entry[0]) + if idx == 0: + raise RuntimeError("inconsistent partition bounds: lower-bound search returned 0") + + from_bound, to_bound, partition_id = self._partitions[idx - 1] + # A key range is [from_bound, to_bound); an empty to_bound means the range runs to the + # end of the key space. Landing past to_bound means the known partitions have a hole -- + # e.g. only one child of a split is visible yet -- and the greatest-lower-bound search + # silently points at the *sibling* that owns the range to the left. Refuse instead: + # routing a key into a sibling branch is exactly what breaks one-key-one-lineage. + if to_bound and hashed >= to_bound: + raise RuntimeError( + "key is not covered by any known partition: it is above partition %d to_bound;" + " the partition set is incomplete" % partition_id + ) + return partition_id diff --git a/ydb/_topic_writer/topic_writer_test.py b/ydb/_topic_writer/topic_writer_test.py index 018823f3..08fcf5c8 100644 --- a/ydb/_topic_writer/topic_writer_test.py +++ b/ydb/_topic_writer/topic_writer_test.py @@ -13,10 +13,35 @@ PublicMessage, PublicWriterSettings, TopicWriterBufferFullError, + TopicWriterClosedError, messages_to_proto_requests, ) -from .topic_writer_asyncio import WriterAsyncIOReconnector from .topic_writer_sync import WriterSync +from .topic_writer_multi_asyncio import MultiWriterSettings +from .topic_writer_multi_sync import TopicWriterMultiSync +from .topic_writer_partition_chooser import ( + murmur2_32, + murmur64a, + default_bound_key_hasher, + PublicPartitionByKeyKafka, + PublicPartitionByKeyBound, + PARTITION_KEY_METADATA_KEY, +) +from .._grpc.grpcwrapper.ydb_topic_public_types import PublicDescribeTopicResult + + +def _partition_info(partition_id: int, from_bound: bytes = None, to_bound: bytes = None): + key_range = None + if from_bound is not None or to_bound is not None: + key_range = PublicDescribeTopicResult.PartitionKeyRange(from_bound=from_bound or b"", to_bound=to_bound or b"") + return PublicDescribeTopicResult.PartitionInfo( + partition_id=partition_id, + active=True, + child_partition_ids=[], + parent_partition_ids=[], + partition_stats=None, + key_range=key_range, + ) def _make_msg(data: bytes, codec: PublicCodec = PublicCodec.RAW, seqno: int = 1) -> InternalMessage: @@ -164,7 +189,10 @@ def run(): @pytest.fixture def mock_reconnector(monkeypatch): def factory(reconnector_instance): - monkeypatch.setattr(WriterAsyncIOReconnector, "__new__", lambda cls, *a, **kw: reconnector_instance) + monkeypatch.setattr( + "ydb._topic_writer.topic_writer_asyncio.WriterAsyncIOReconnector", + mock.Mock(return_value=reconnector_instance), + ) return reconnector_instance return factory @@ -236,3 +264,397 @@ def do_write(): assert not write_errors, f"unexpected error: {write_errors}" writer.close(flush=False) + + +# Golden vectors generated from the YDB Go SDK pkg/xhash reference implementation +# (Murmur2Hash32 / Murmur2Hash64A, seed=0). A mismatch means keys would route to +# the wrong partition, so these are byte-exact assertions. +_MURMUR_GOLDEN = [ + ("", 0, 0), + ("a", 2456313694, 510903276987443985), + ("hello", 3848350155, 2191231550387646743), + ("hello world, murmur2 hash", 1305234166, 15193844207144850389), + ("мурмур2-хэш", 1364064206, 2094682108092698226), + ("user-42", 3766944517, 17854748655353381905), + ("0", 1111412596, 5533571732986600803), + ("key-with-длинный-unicode-🚀", 901185517, 8765931500921732560), +] + + +@pytest.mark.parametrize("text,h32,h64", _MURMUR_GOLDEN) +def test_murmur_hashes_match_go_golden_vectors(text, h32, h64): + data = text.encode("utf-8") + assert murmur2_32(data, 0) == h32 + assert murmur64a(data, 0) == h64 + + +def test_default_bound_key_hasher_is_big_endian_murmur64a(): + for text, _h32, h64 in _MURMUR_GOLDEN: + assert default_bound_key_hasher(text) == h64.to_bytes(8, "big") + + +# Generated with Apache Kafka 4.2 Utils.murmur2 and Utils.toPositive in Java. +# Each row contains the unsigned hash and partition indices for N=3, 7, 16. +_KAFKA_GOLDEN = [ + ("", 0x106E08D9, (0, 2, 9)), + ("a", 0xA2D0B27C, (1, 5, 12)), + ("b", 0x918C0FF4, (2, 6, 4)), + ("abc", 0x1C94221B, (0, 4, 11)), + ("abcd", 0xB11AB5F4, (2, 5, 4)), + ("abcde", 0x1B897EDD, (1, 4, 13)), + ("abcdef", 0x6F7FDAFC, (0, 5, 12)), + ("abcdefg", 0xEB595499, (1, 4, 9)), + ("abcdefgh", 0xC70D55DD, (0, 0, 13)), + ("hello", 0x7F1DDBBD, (0, 4, 13)), + ("user-42", 0x5700682C, (1, 1, 12)), + ("мурмур2-хэш", 0x071E5697, (1, 2, 7)), + ("0", 0x39E0B3C4, (2, 3, 4)), + ("zzz", 0x4712C76A, (1, 1, 10)), + ("a\x00b", 0xFED14E49, (0, 5, 9)), + ("🚀", 0x2D67D1C1, (1, 6, 1)), + ("\u00e9", 0x0B24F487, (0, 4, 7)), + ("e\u0301", 0x9D9EB11B, (1, 2, 11)), + ("key-with-длинный-unicode-🚀", 0xEA98B0BC, (1, 0, 12)), +] + + +@pytest.mark.parametrize("key,hash32,indices", _KAFKA_GOLDEN) +def test_kafka_chooser_matches_apache_kafka_partition(key, hash32, indices): + assert murmur2_32(key.encode("utf-8"), 0x9747B28C) == hash32 + for count, expected in zip((3, 7, 16), indices): + chooser = PublicPartitionByKeyKafka() + chooser.add_partitions([_partition_info(i) for i in reversed(range(count))]) + assert chooser.choose_partition(PublicMessage(b"x", key=key)) == expected + + +def test_kafka_chooser_maps_index_to_sorted_partition_ids(): + chooser = PublicPartitionByKeyKafka() + chooser.add_partitions([_partition_info(9), _partition_info(2)]) + chooser.add_partitions([_partition_info(5)]) + assert chooser.choose_partition(PublicMessage(b"x", key="a")) == 5 + + +def test_kafka_chooser_rejects_key_ranges(): + chooser = PublicPartitionByKeyKafka() + with pytest.raises(ValueError): + chooser.add_partitions([_partition_info(0, from_bound=b"\x10")]) + + +def test_kafka_chooser_raises_without_partitions(): + with pytest.raises(ValueError): + PublicPartitionByKeyKafka().choose_partition(PublicMessage(b"x", key="k")) + + +def test_bound_chooser_routes_into_owning_key_range(): + lo = b"\x55" * 8 + hi = b"\xaa" * 8 + chooser = PublicPartitionByKeyBound() + chooser.add_partitions([_partition_info(0), _partition_info(1, from_bound=lo), _partition_info(2, from_bound=hi)]) + + for key in ["a", "b", "user-42", "hello", "zzz", "мурмур2-хэш", "0"]: + hashed = default_bound_key_hasher(key) + if hashed < lo: + expected = 0 + elif hashed < hi: + expected = 1 + else: + expected = 2 + assert chooser.choose_partition(PublicMessage(b"x", key=key)) == expected + + +def test_bound_chooser_stamps_partition_key_metadata(): + chooser = PublicPartitionByKeyBound() + chooser.add_partitions([_partition_info(0)]) + message = PublicMessage(b"x", key="user-42") + chooser.choose_partition(message) + assert message.metadata_items[PARTITION_KEY_METADATA_KEY] == default_bound_key_hasher("user-42") + + +def test_bound_chooser_requires_from_bound_on_non_first_partition(): + chooser = PublicPartitionByKeyBound() + with pytest.raises(ValueError): + chooser.add_partitions([_partition_info(0), _partition_info(1)]) + + +def test_bound_chooser_accepts_partitions_in_any_order(): + """A valid range set must not be rejected just because it arrived out of order. + + DescribeTopic does not promise an ordering, so validation has to run on the sorted set: + the partition with the open lower bound is the leftmost one, not necessarily the first + argument. + """ + lo = b"\x55" * 8 + reversed_order = [_partition_info(1, from_bound=lo), _partition_info(0)] + + chooser = PublicPartitionByKeyBound() + chooser.add_partitions(reversed_order) + + assert sorted(p[-1] for p in chooser._partitions) == [0, 1] + + +def test_bound_chooser_rejects_key_above_the_last_to_bound(): + """A key outside every known range must fail routing instead of landing on a neighbour. + + The chooser only indexes `from_bound` and takes the greatest one <= the hashed key, so a + key above the last partition's `to_bound` silently lands in that partition. That happens + whenever the known partition set has a hole -- e.g. a split whose second child is not + visible in DescribeTopic yet -- and sends the key into a *sibling* branch of the partition + graph, breaking "one key -> one lineage". + + Per the C++ producer spec the lookup is two steps, not one: + partition = greatest from_bound <= key + check key < partition.to_bound + and a key that maps to no ready leaf must be retried against a refreshed graph. + + The exact exception type is a fix-time decision; the contract asserted here is only that + routing refuses to guess. + """ + chooser = PublicPartitionByKeyBound(key_hasher=lambda key: key.encode("utf-8")) + chooser.add_partitions([_partition_info(0, from_bound=b"", to_bound=b"m")]) + + assert chooser.choose_partition(PublicMessage(b"x", key="apple")) == 0 + with pytest.raises((ValueError, RuntimeError)): + chooser.choose_partition(PublicMessage(b"x", key="zebra")) + + +def test_bound_chooser_raises_without_partitions(): + with pytest.raises(ValueError): + PublicPartitionByKeyBound().choose_partition(PublicMessage(b"x", key="k")) + + +def test_bound_chooser_rejects_key_below_every_from_bound(): + """A key under the leftmost bound means the partition set lost its first partition. + + Nothing owns that key, and picking the nearest partition anyway would put it in a range the + server does not associate with it, so routing has to refuse instead of guessing. + """ + chooser = PublicPartitionByKeyBound(key_hasher=lambda key: key.encode("utf-8")) + chooser.add_partitions([_partition_info(1, from_bound=b"m", to_bound=b"")]) + + with pytest.raises(RuntimeError): + chooser.choose_partition(PublicMessage(b"x", key="apple")) + + +def test_kafka_chooser_rejects_a_fully_open_key_range(): + """An open range still means the topic is auto-partitioned. + + A single auto-partitioned partition reports ["", "") before its first split. Accepting it + here would work right up until that split, and then break: the children come back bounded + and modulo routing cannot place a key by bounds. + """ + chooser = PublicPartitionByKeyKafka() + with pytest.raises(ValueError): + chooser.add_partitions([_partition_info(0, from_bound=b"", to_bound=b"")]) + + +class _FakeMultiAsyncWriter: + """Stand-in for the async multi-writer that the sync facade drives.""" + + def __init__(self, driver=None, settings=None, _parent=None): + self.calls = [] + self.closed_with = None + + async def wait_init(self): + self.calls.append("wait_init") + + async def write(self, messages): + self.calls.append(("write", messages)) + + async def write_with_ack(self, messages): + self.calls.append(("write_with_ack", messages)) + return "ack" + + async def flush(self): + self.calls.append("flush") + + async def close(self, *, flush=True): + self.closed_with = flush + self.calls.append(("close", flush)) + + +class TestTopicWriterMultiSync: + """The sync facade owns no logic of its own: it must hand every call to the async writer + on the shared loop and report closure honestly. These check that nothing is dropped or + silently reordered on the way across the thread boundary.""" + + @pytest.mark.parametrize("close_during_describe", [False, True]) + def test_describe_does_not_block_loop_or_inherit_waiter_timeout(self, background_loop, close_during_describe): + started = threading.Event() + release = threading.Event() + request_settings = [] + + class DelayedDriver: + def __call__(self, request, stub, method, wrapper, settings=None): + request_settings.append(settings) + started.set() + if not release.wait(timeout=2): + raise RuntimeError("DescribeTopic blocked the event loop") + return mock.Mock( + to_public=mock.Mock( + return_value=mock.Mock( + partitions=[_partition_info(0)], + auto_partitioning_settings=None, + ) + ) + ) + + settings = MultiWriterSettings(topic="/local/topic", writer_idle_timeout_sec=0) + writer = TopicWriterMultiSync(DelayedDriver(), settings, eventloop=background_loop) + try: + assert started.wait(timeout=1) + asyncio.run_coroutine_threadsafe(asyncio.sleep(0), background_loop).result(timeout=1) + with pytest.raises(TimeoutError): + writer.wait_init(timeout=0.01) + + if close_during_describe: + writer.close(flush=False, timeout=1) + release.set() + asyncio.run_coroutine_threadsafe(background_loop.shutdown_default_executor(), background_loop).result(1) + if close_during_describe: + assert writer._async_writer._init_task.cancelled() + assert not writer._async_writer._partitions + else: + writer.wait_init(timeout=1) + writer.flush(timeout=1) + assert len(request_settings) == 1 + assert request_settings[0].timeout == 30 + finally: + release.set() + writer.close(flush=False, timeout=1) + + @pytest.fixture + def writer(self, background_loop, monkeypatch): + monkeypatch.setattr( + "ydb._topic_writer.topic_writer_multi_sync.TopicWriterMultiAsyncIO", + _FakeMultiAsyncWriter, + ) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx") + writer = TopicWriterMultiSync(mock.Mock(), settings, eventloop=background_loop) + yield writer + if not writer._closed: + writer.close(flush=False) + + def test_calls_are_forwarded_to_the_async_writer(self, writer): + message = PublicMessage(data=b"hello", key="k") + + writer.wait_init() + writer.write(message) + assert writer.write_with_ack(message) == "ack" + writer.flush() + + kinds = [c if isinstance(c, str) else c[0] for c in writer._async_writer.calls] + assert kinds == ["wait_init", "write", "write_with_ack", "flush"] + + def test_async_variants_return_futures(self, writer): + assert writer.async_wait_init().result(timeout=5) is None + assert writer.async_write_with_ack(PublicMessage(data=b"x", key="k")).result(timeout=5) == "ack" + assert writer.async_flush().result(timeout=5) is None + + def test_write_accepts_a_batch(self, writer): + messages = [PublicMessage(data=b"a", key="k"), PublicMessage(data=b"b", key="k")] + writer.write(messages) + assert writer._async_writer.calls[-1][0] == "write" + + def test_close_is_idempotent_and_flushes_by_default(self, writer): + writer.close() + assert writer._async_writer.closed_with is True + + # A second close must not reach the async writer again. + calls_before = len(writer._async_writer.calls) + writer.close() + assert len(writer._async_writer.calls) == calls_before + + def test_every_entry_point_refuses_to_work_after_close(self, writer): + writer.close(flush=False) + + message = PublicMessage(data=b"x", key="k") + for call in ( + lambda: writer.write(message), + lambda: writer.write_with_ack(message), + lambda: writer.flush(), + lambda: writer.wait_init(), + lambda: writer.async_flush(), + lambda: writer.async_wait_init(), + lambda: writer.async_write_with_ack(message), + ): + with pytest.raises(TopicWriterClosedError): + call() + + def test_context_manager_closes_on_exit(self, background_loop, monkeypatch): + monkeypatch.setattr( + "ydb._topic_writer.topic_writer_multi_sync.TopicWriterMultiAsyncIO", + _FakeMultiAsyncWriter, + ) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx") + with TopicWriterMultiSync(mock.Mock(), settings, eventloop=background_loop) as writer: + writer.write(PublicMessage(data=b"x", key="k")) + assert writer._closed + assert writer._async_writer.closed_with is True + + def test_unclosed_writer_is_closed_on_delete(self, background_loop, monkeypatch): + """__del__ is the last chance to release the streams a forgotten writer still holds.""" + monkeypatch.setattr( + "ydb._topic_writer.topic_writer_multi_sync.TopicWriterMultiAsyncIO", + _FakeMultiAsyncWriter, + ) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx") + writer = TopicWriterMultiSync(mock.Mock(), settings, eventloop=background_loop) + inner = writer._async_writer + + writer.__del__() + + assert writer._closed + assert inner.closed_with is False # never flush from a destructor + + def test_close_failure_surfaces_unless_the_body_already_failed(self, background_loop, monkeypatch): + """A failing close must not mask the caller's own exception. + + Losing the body's exception to a teardown error is how a real failure gets reported as + something unrelated, so close errors are only raised when the block exited cleanly. + """ + + class FailingClose(_FakeMultiAsyncWriter): + async def close(self, *, flush=True): + raise RuntimeError("close failed") + + monkeypatch.setattr("ydb._topic_writer.topic_writer_multi_sync.TopicWriterMultiAsyncIO", FailingClose) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx") + + with pytest.raises(RuntimeError, match="close failed"): + with TopicWriterMultiSync(mock.Mock(), settings, eventloop=background_loop): + pass + + class TestException(Exception): + pass + + with pytest.raises(TestException): + with TopicWriterMultiSync(mock.Mock(), settings, eventloop=background_loop): + raise TestException() + + def test_delete_swallows_close_failure(self, background_loop, monkeypatch): + """__del__ runs during garbage collection: raising there only produces noise.""" + + class FailingClose(_FakeMultiAsyncWriter): + async def close(self, *, flush=True): + raise RuntimeError("close failed") + + monkeypatch.setattr("ydb._topic_writer.topic_writer_multi_sync.TopicWriterMultiAsyncIO", FailingClose) + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx") + writer = TopicWriterMultiSync(mock.Mock(), settings, eventloop=background_loop) + + writer.__del__() # must not raise + + def test_falls_back_to_the_shared_event_loop(self, monkeypatch): + """Without an explicit loop the writer joins the client's shared one.""" + monkeypatch.setattr("ydb._topic_writer.topic_writer_multi_sync.TopicWriterMultiAsyncIO", _FakeMultiAsyncWriter) + loop = mock.Mock() + shared = mock.Mock(return_value=loop) + monkeypatch.setattr("ydb._topic_writer.topic_writer_multi_sync._get_shared_event_loop", shared) + caller = mock.Mock() + monkeypatch.setattr("ydb._topic_writer.topic_writer_multi_sync.CallFromSyncToAsync", caller) + + settings = MultiWriterSettings(topic="/local/topic", producer_id_prefix="pfx") + TopicWriterMultiSync(mock.Mock(), settings) + + shared.assert_called_once() + caller.assert_called_once_with(loop) diff --git a/ydb/topic.py b/ydb/topic.py index e76956ab..56fc930a 100644 --- a/ydb/topic.py +++ b/ydb/topic.py @@ -27,6 +27,12 @@ "TopicWriteResult", "TopicWriter", "TopicWriterAsyncIO", + "TopicWriterMulti", + "TopicWriterMultiAsyncIO", + "TopicWriterMultiSettings", + "TopicWriterPartitionChooser", + "TopicWriterPartitionByKeyKafka", + "TopicWriterPartitionByKeyBound", "TopicTxWriter", "TopicTxWriterAsyncIO", "TopicWriterInitInfo", @@ -81,6 +87,17 @@ from ._topic_writer.topic_writer_sync import WriterSync as TopicWriter from ._topic_writer.topic_writer_sync import TxWriterSync as TopicTxWriter +from ._topic_writer.topic_writer_multi_asyncio import ( # noqa: F401 + MultiWriterSettings as TopicWriterMultiSettings, + TopicWriterMultiAsyncIO, +) +from ._topic_writer.topic_writer_multi_sync import TopicWriterMultiSync as TopicWriterMulti +from ._topic_writer.topic_writer_partition_chooser import ( # noqa: F401 + PublicPartitionChooser as TopicWriterPartitionChooser, + PublicPartitionByKeyKafka as TopicWriterPartitionByKeyKafka, + PublicPartitionByKeyBound as TopicWriterPartitionByKeyBound, +) + from ._topic_common.common import ( wrap_operation as _wrap_operation, create_result_wrapper as _create_result_wrapper, @@ -335,7 +352,7 @@ def writer( partition_id: Union[int, None] = None, auto_seqno: bool = True, auto_created_at: bool = True, - codec: Optional[TopicCodec] = None, # default mean auto-select + codec: Optional[TopicCodec] = None, # default means auto-select # encoders: map[codec_code] func(encoded_bytes)->decoded_bytes # the func will be called from multiply threads in parallel. encoders: Optional[Mapping[_ydb_topic_public_types.PublicCodec, Callable[[bytes], bytes]]] = None, @@ -367,7 +384,7 @@ def tx_writer( partition_id: Union[int, None] = None, auto_seqno: bool = True, auto_created_at: bool = True, - codec: Optional[TopicCodec] = None, # default mean auto-select + codec: Optional[TopicCodec] = None, # default means auto-select # encoders: map[codec_code] func(encoded_bytes)->decoded_bytes # the func will be called from multiply threads in parallel. encoders: Optional[Mapping[_ydb_topic_public_types.PublicCodec, Callable[[bytes], bytes]]] = None, @@ -390,6 +407,36 @@ def tx_writer( return TopicTxWriterAsyncIO(tx=tx, driver=self._driver, settings=settings, _client=self) + def multiwriter( + self, + topic, + *, + producer_id_prefix: Optional[str] = None, # default - random + partition_chooser: Optional[TopicWriterPartitionChooser] = None, # default - route by key (Kafka hash) + auto_seqno: bool = True, + auto_created_at: bool = True, + codec: Optional[TopicCodec] = None, # default means auto-select + encoders: Optional[Mapping[_ydb_topic_public_types.PublicCodec, Callable[[bytes], bytes]]] = None, + encoder_executor: Optional[concurrent.futures.Executor] = None, # default shared client executor pool + max_buffer_size_bytes: Optional[int] = None, + max_buffer_messages: Optional[int] = None, + buffer_wait_timeout_sec: Optional[float] = None, + # Close idle per-partition sub-writers after this many seconds (recreated on demand). + # None = default; <= 0 disables eviction. + writer_idle_timeout_sec: Optional[float] = None, + ) -> TopicWriterMultiAsyncIO: + logger.debug("Create multi-writer for topic=%s", topic) + args = locals().copy() + del args["self"] + self._check_closed() + + settings = TopicWriterMultiSettings(**args) + + if not settings.encoder_executor: + settings.encoder_executor = self._executor + + return TopicWriterMultiAsyncIO(self._driver, settings, _parent=self) + @ydb_retry(retry_cancelled=True, idempotent=True) async def commit_offset( self, path: str, consumer: str, partition_id: int, offset: int, read_session_id: Optional[str] = None @@ -453,7 +500,7 @@ def __init__(self, driver: driver.Driver, settings: Optional[TopicClientSettings def __del__(self): if not self._closed: try: - logger.warning("Topic client was not closed properly. Consider using method close().") + logger.debug("Topic client was not closed properly. Consider using method close().") self.close() except BaseException: logger.warning("Something went wrong during topic client close in __del__") @@ -669,7 +716,7 @@ def writer( partition_id: Union[int, None] = None, auto_seqno: bool = True, auto_created_at: bool = True, - codec: Optional[TopicCodec] = None, # default mean auto-select + codec: Optional[TopicCodec] = None, # default means auto-select # encoders: map[codec_code] func(encoded_bytes)->decoded_bytes # the func will be called from multiply threads in parallel. encoders: Optional[Mapping[_ydb_topic_public_types.PublicCodec, Callable[[bytes], bytes]]] = None, @@ -702,7 +749,7 @@ def tx_writer( partition_id: Union[int, None] = None, auto_seqno: bool = True, auto_created_at: bool = True, - codec: Optional[TopicCodec] = None, # default mean auto-select + codec: Optional[TopicCodec] = None, # default means auto-select # encoders: map[codec_code] func(encoded_bytes)->decoded_bytes # the func will be called from multiply threads in parallel. encoders: Optional[Mapping[_ydb_topic_public_types.PublicCodec, Callable[[bytes], bytes]]] = None, @@ -726,6 +773,36 @@ def tx_writer( return TopicTxWriter(tx, self._driver, settings, _parent=self) + def multiwriter( + self, + topic, + *, + producer_id_prefix: Optional[str] = None, # default - random + partition_chooser: Optional[TopicWriterPartitionChooser] = None, # default - route by key (Kafka hash) + auto_seqno: bool = True, + auto_created_at: bool = True, + codec: Optional[TopicCodec] = None, # default means auto-select + encoders: Optional[Mapping[_ydb_topic_public_types.PublicCodec, Callable[[bytes], bytes]]] = None, + encoder_executor: Optional[concurrent.futures.Executor] = None, # default shared client executor pool + max_buffer_size_bytes: Optional[int] = None, + max_buffer_messages: Optional[int] = None, + buffer_wait_timeout_sec: Optional[float] = None, + # Close idle per-partition sub-writers after this many seconds (recreated on demand). + # None = default; <= 0 disables eviction. + writer_idle_timeout_sec: Optional[float] = None, + ) -> TopicWriterMulti: + logger.debug("Create multi-writer for topic=%s", topic) + args = locals().copy() + del args["self"] + self._check_closed() + + settings = TopicWriterMultiSettings(**args) + + if not settings.encoder_executor: + settings.encoder_executor = self._executor + + return TopicWriterMulti(self._driver, settings, _parent=self) + @ydb_retry(retry_cancelled=True, idempotent=True) def commit_offset( self, path: str, consumer: str, partition_id: int, offset: int, read_session_id: Optional[str] = None diff --git a/ydb/topic_test.py b/ydb/topic_test.py new file mode 100644 index 00000000..a1533fa5 --- /dev/null +++ b/ydb/topic_test.py @@ -0,0 +1,115 @@ +from unittest import mock + +import pytest + +from . import issues, topic + + +class _CapturedWriter: + """Records what the client factory built, so the test can check the wiring only.""" + + last = None + + def __init__(self, driver, settings, _parent=None): + self.driver = driver + self.settings = settings + self.parent = _parent + type(self).last = self + + +@pytest.fixture +def async_client(): + client = topic.TopicClientAsyncIO(mock.Mock()) + yield client + client._closed = True # skip the real close: nothing was actually opened + + +@pytest.fixture +def sync_client(): + # The sync client takes settings positionally, unlike the async one. + client = topic.TopicClient(mock.Mock(), None) + yield client + client._closed = True + + +@pytest.mark.parametrize( + "client_fixture, patched", [("async_client", "TopicWriterMultiAsyncIO"), ("sync_client", "TopicWriterMulti")] +) +def test_multiwriter_factory_passes_settings_through(client_fixture, patched, request, monkeypatch): + """The factory is the only place the caller's arguments turn into writer settings. + + A silently dropped argument here (a codec, a chooser, a buffer limit) would leave the writer + running on defaults while the caller believes otherwise, so every one of them is checked. + """ + client = request.getfixturevalue(client_fixture) + monkeypatch.setattr(topic, patched, _CapturedWriter) + + chooser = topic.TopicWriterPartitionByKeyKafka() + writer = client.multiwriter( + "/local/topic", + producer_id_prefix="pfx", + partition_chooser=chooser, + auto_seqno=False, + auto_created_at=False, + codec=topic.TopicCodec.RAW, + max_buffer_size_bytes=1024, + max_buffer_messages=10, + buffer_wait_timeout_sec=1.5, + writer_idle_timeout_sec=30, + ) + + assert writer is _CapturedWriter.last + assert writer.parent is client, "the writer must keep the client alive" + + settings = writer.settings + assert settings.topic == "/local/topic" + assert settings.producer_id_prefix == "pfx" + assert settings.partition_chooser is chooser + assert settings.auto_seqno is False + assert settings.auto_created_at is False + assert settings.codec == topic.TopicCodec.RAW + assert settings.max_buffer_size_bytes == 1024 + assert settings.max_buffer_messages == 10 + assert settings.buffer_wait_timeout_sec == 1.5 + assert settings.writer_idle_timeout_sec == 30 + # Encoding runs on the client's shared pool unless the caller brought its own. + assert settings.encoder_executor is client._executor + + +@pytest.mark.parametrize( + "client_fixture, patched", [("async_client", "TopicWriterMultiAsyncIO"), ("sync_client", "TopicWriterMulti")] +) +def test_multiwriter_keeps_a_caller_supplied_executor(client_fixture, patched, request, monkeypatch): + client = request.getfixturevalue(client_fixture) + monkeypatch.setattr(topic, patched, _CapturedWriter) + executor = mock.Mock() + + writer = client.multiwriter("/local/topic", encoder_executor=executor) + + assert writer.settings.encoder_executor is executor + + +@pytest.mark.parametrize("client_fixture", ["async_client", "sync_client"]) +def test_multiwriter_refuses_a_closed_client(client_fixture, request): + client = request.getfixturevalue(client_fixture) + client._closed = True + + with pytest.raises(issues.Error): + client.multiwriter("/local/topic") + + +def test_unclosed_sync_client_is_closed_on_delete(): + """__del__ is the last chance to release the executor a forgotten client still holds.""" + client = topic.TopicClient(mock.Mock(), None) + with mock.patch.object(topic.TopicClient, "close") as close: + client.__del__() + close.assert_called_once() + client._closed = True + + +def test_delete_of_a_closed_sync_client_does_nothing(): + client = topic.TopicClient(mock.Mock(), None) + client._closed = True + with mock.patch.object(topic.TopicClient, "close") as close: + client.__del__() + close.assert_not_called()