From f3b8458b323f6f7d346ea6231f5b9d1f55af7637 Mon Sep 17 00:00:00 2001 From: Alexandr Gorshenin Date: Mon, 5 Oct 2026 17:57:29 +0100 Subject: [PATCH 1/9] Enable test for splitted read --- .../ydb/topic/read/impl/AsyncReaderImpl.java | 9 +- .../ydb/topic/read/impl/ReadPartition.java | 9 ++ .../ydb/topic/FailableReaderInterceptor.java | 11 ++ .../topic/TopicReadersIntegrationTest.java | 121 +++++++++--------- 4 files changed, 78 insertions(+), 72 deletions(-) diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/AsyncReaderImpl.java b/topic/src/main/java/tech/ydb/topic/read/impl/AsyncReaderImpl.java index d5497f2f3..125ee9e7c 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/AsyncReaderImpl.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/AsyncReaderImpl.java @@ -148,15 +148,8 @@ private class AsyncHandler implements ReadSession.Handler { @Override public void onData(ReadSession.PartitionControl control, DataReceivedEvent event) { try { - int messagesCount = event.getMessages().size(); - long offsetStart = event.getMessages().get(0).getOffset(); - long offsetEnd = event.getMessages().get(event.getMessages().size() - 1).getOffset(); - logger.debug("[{}] DataReceivedEvent callback with {} message(s) (offsets {}-{}) is about " - + "to be called...", debugId, messagesCount, offsetStart, offsetEnd); - config.getMetrics().reportDelivered(messagesCount, event.getPartitionSession().getPath()); + config.getMetrics().reportDelivered(event.getMessages().size(), event.getPartitionSession().getPath()); eventHandler.onMessages(event); - logger.debug("[{}] DataReceivedEvent callback with {} message(s) (offsets {}-{}) " - + "successfully finished", debugId, messagesCount, offsetStart, offsetEnd); } catch (Throwable th) { failSession(th, "onMessages"); } finally { diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java index adf2be2f8..378e868cf 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java @@ -130,7 +130,16 @@ private void sendDataToReaders() { } DataReceivedEventImpl event = new DataReceivedEventImpl(partition, committer, list); + + int messagesCount = event.getMessages().size(); + long offsetStart = event.getMessages().get(0).getOffset(); + long offsetEnd = event.getMessages().get(event.getMessages().size() - 1).getOffset(); + logger.debug("[{}] onData with {} message(s) (offsets {}-{}) is about to be called...", + traceID, messagesCount, offsetStart, offsetEnd); session.getHandler().onData(this, event); + logger.debug("[{}] onData with {} message(s) (offsets {}-{}) successfully finished", + traceID, messagesCount, offsetStart, offsetEnd); + sendCommitOffsetAck(); } }); diff --git a/topic/src/test/java/tech/ydb/topic/FailableReaderInterceptor.java b/topic/src/test/java/tech/ydb/topic/FailableReaderInterceptor.java index 831f62345..7a1f3f8f1 100644 --- a/topic/src/test/java/tech/ydb/topic/FailableReaderInterceptor.java +++ b/topic/src/test/java/tech/ydb/topic/FailableReaderInterceptor.java @@ -18,6 +18,7 @@ import io.grpc.Status; import tech.ydb.proto.StatusCodesProtos; +import tech.ydb.proto.topic.YdbTopic; import tech.ydb.proto.topic.YdbTopic.StreamReadMessage.CommitOffsetRequest.PartitionCommitOffset; import tech.ydb.proto.topic.YdbTopic.StreamReadMessage.CommitOffsetResponse.PartitionCommittedOffset; import tech.ydb.proto.topic.YdbTopic.StreamReadMessage.FromClient; @@ -36,6 +37,7 @@ public class FailableReaderInterceptor implements Consumer readErrors = new HashMap<>(); private final Map> ackErrors = new ConcurrentHashMap<>(); private final Map> sendErrors = new ConcurrentHashMap<>(); + private volatile Consumer dataConsumer = null; public void reset() { initErrors.clear(); @@ -44,6 +46,7 @@ public void reset() { sendErrors.clear(); initCounter.set(0); readCounter.set(0); + dataConsumer = null; } @Override @@ -87,6 +90,10 @@ public void badSessionOnCommitWithOffset(long partitionID, long offset) { .put(offset, sendError(StatusCodesProtos.StatusIds.StatusCode.BAD_SESSION)); } + public void listenPartitionData(Consumer listener) { + this.dataConsumer = listener; + } + @Override public ClientCall interceptCall(MethodDescriptor method, CallOptions callOptions, Channel next) { @@ -207,6 +214,10 @@ public void onMessage(R message) { error = initErrors.get(initCounter.incrementAndGet()); } if (msg.hasReadResponse()) { + Consumer local = dataConsumer; + if (local != null) { + msg.getReadResponse().getPartitionDataList().forEach(local::accept); + } error = readErrors.get(readCounter.incrementAndGet()); } if (msg.hasCommitOffsetResponse()) { diff --git a/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java b/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java index 876d0e768..5bd045356 100644 --- a/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java +++ b/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java @@ -3,11 +3,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Map; + import java.util.Queue; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletableFuture; + import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -23,7 +25,6 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.slf4j.Logger; @@ -91,9 +92,17 @@ public class TopicReadersIntegrationTest { @BeforeClass public static void initClient() { client = TopicClient.newClient(ydbTransport).build(); + initTopips(); + } + + @AfterClass + public static void closeClient() { + dropTopics(); + client.close(); + } + + private static void initTopips() { logger.info("Create test topic {} ...", TEST_TOPIC); - client.dropTopic(TEST_TOPIC).join(); - client.dropTopic(SPLITTED_TOPIC).join(); client.createTopic(TEST_TOPIC, CreateTopicSettings.newBuilder() .addConsumer(Consumer.newBuilder().setName(TEST_CONSUMER1).build()) .setPartitioningSettings(PartitioningSettings.newBuilder() @@ -136,13 +145,11 @@ public static void initClient() { CompletableFuture.allOf(f6, f7).join(); } - @AfterClass - public static void closeClient() { + private static void dropTopics() { logger.info("Drop test topic {} ...", TEST_TOPIC); client.dropTopic(TEST_TOPIC).join(); logger.info("Drop test topic {} ...", SPLITTED_TOPIC); client.dropTopic(SPLITTED_TOPIC).join(); - client.close(); } @Before @@ -192,14 +199,19 @@ private static void writeToTopic(String topicPath, String producerId, int startF .build()); } + private static byte[] writedMsg(String producerID, int idx) { + byte[] msg = ("p" + producerID + "_msg" + idx).getBytes(); + byte[] data = new byte[100]; + Arrays.fill(data, (byte) 0x20); // fill spaces + System.arraycopy(msg, 0, data, 0, msg.length); + return data; + } + private static void writeToTopic(int startFrom, int count, WriterSettings settings) { SyncWriter writer = client.createSyncWriter(settings); writer.initAndWait(); for (int idx = 1; idx <= count; idx++) { - byte[] msg = ("p" + settings.getProducerId() + "_msg" + (startFrom + idx)).getBytes(); - byte[] data = new byte[100]; - System.arraycopy(msg, 0, data, 0, msg.length); - writer.send(tech.ydb.topic.write.Message.of(data)); + writer.send(tech.ydb.topic.write.Message.of(writedMsg(settings.getProducerId(), startFrom + idx))); } try { @@ -305,32 +317,45 @@ public void readAllSplittedTest() throws InterruptedException { ReaderSettings readerSettings = ReaderSettings.newBuilder() .addTopic(TopicReadSettings.newBuilder().setPath(SPLITTED_TOPIC).build()) .setConsumerName(TEST_CONSUMER1) + .setDecompressionExecutor(Runnable::run) + .setMaxBatchSize(100) .build(); - Map partitions = new ConcurrentHashMap<>(); - partitions.put("p0", 0L); - partitions.put("p1", 0L); - - CountDownLatch read = new CountDownLatch(1500); + BlockingQueue queue = new ArrayBlockingQueue<>(1); AsyncReader reader = client.createAsyncReader(readerSettings, ReadEventHandlersSettings.newBuilder() .setEventHandler((DataReceivedEvent event) -> { - for (Message msg : event.getMessages()) { - String producer = msg.getProducerId(); - Assert.assertTrue(partitions.containsKey(producer)); - partitions.put(msg.getProducerId(), 1L + partitions.get(msg.getProducerId())); - Assert.assertEquals(100, msg.getData().length); - String expected = "p" + producer + "_msg" + partitions.get(producer); - Assert.assertEquals(expected, new String(Arrays.copyOf(msg.getData(), expected.length()))); - read.countDown(); + try { + for (Message msg : event.getMessages()) { + Assert.assertTrue(queue.offer(msg, 60, TimeUnit.SECONDS)); + } + } catch (InterruptedException ex) { + throw new AssertionError("cannot process event", ex); } - event.commit(); - }).build()); + }).build()); - reader.init().join(); + CountDownLatch recieved = new CountDownLatch(1500); + PROXY.listenPartitionData(partitionData -> { + partitionData.getBatchesList().forEach(batch -> { + batch.getMessageDataList().forEach(msg -> recieved.countDown()); + }); + }); + + reader.init(); try { - Assert.assertTrue(read.await(30, TimeUnit.SECONDS)); - Assert.assertEquals(Long.valueOf(1000), partitions.get("p0")); - Assert.assertEquals(Long.valueOf(500), partitions.get("p1")); + // wait to recieve all messages + Assert.assertTrue(recieved.await(5, TimeUnit.SECONDS)); + + // validate all messages + int p0_idx = 0; + int p1_idx = 0; + while (p0_idx < 1000 || p1_idx < 500) { + Message msg = queue.poll(1, TimeUnit.SECONDS); + Assert.assertNotNull("cannot get msg " + (p0_idx + p1_idx), msg); + + int msg_idx = "p0".equals(msg.getProducerId()) ? ++p0_idx : ++p1_idx; + byte[] expected = writedMsg(msg.getProducerId(), msg_idx); + Assert.assertEquals(new String(expected), new String(msg.getData())); + } } finally { reader.shutdown().join(); } @@ -513,38 +538,6 @@ public void onMessages(DataReceivedEvent event) { ); } - @Test - @Ignore // requires auto-partitioning supporr - public void readAllSplittedWithoutCommitTest() throws InterruptedException { - ReaderSettings readerSettings = ReaderSettings.newBuilder() - .addTopic(TopicReadSettings.newBuilder().setPath(SPLITTED_TOPIC).build()) - .setConsumerName(TEST_CONSUMER1) - .build(); - - Map partitions = new ConcurrentHashMap<>(); - partitions.put("p0", 0L); - partitions.put("p1", 0L); - - CountDownLatch read = new CountDownLatch(1500); - AsyncReader reader = client.createAsyncReader(readerSettings, ReadEventHandlersSettings.newBuilder() - .setEventHandler((DataReceivedEvent event) -> { - for (Message msg : event.getMessages()) { - Assert.assertTrue(partitions.containsKey(msg.getProducerId())); - partitions.put(msg.getProducerId(), 1L + partitions.get(msg.getProducerId())); - read.countDown(); - } - }).build()); - - reader.init().join(); - try { - Assert.assertTrue(read.await(30, TimeUnit.SECONDS)); - Assert.assertEquals(Long.valueOf(1000), partitions.get("p0")); - Assert.assertEquals(Long.valueOf(500), partitions.get("p1")); - } finally { - reader.shutdown().join(); - } - } - @Test public void readAllByPartitionIdTest() throws InterruptedException { ReaderSettings readerSettings = ReaderSettings.newBuilder() From 1562fd70ba38c92efcf03341b6c02cedd00deb50 Mon Sep 17 00:00:00 2001 From: Alexandr Gorshenin Date: Thu, 8 Oct 2026 13:58:46 +0100 Subject: [PATCH 2/9] Added single thread reading test --- .../ydb/topic/read/impl/ReadPartition.java | 25 ++++++++-------- .../topic/TopicReadersIntegrationTest.java | 30 +++++++++++++++++++ 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java index 378e868cf..3a0be9111 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java @@ -218,21 +218,22 @@ public void confirm(StartPartitionSessionSettings settings) { Long readFrom = settings != null ? settings.getReadOffset() : null; Long commitTo = settings != null ? settings.getCommitOffset() : null; - String consumer = session.getConfig().getConsumerName(); - if (state.compareAndSet(State.INITED, State.STARTED)) { - if (commitTo != null) { - committer.updateCommittedOffset(commitTo); - queue.updateLastReadOffset(commitTo); // avoid lags of commits - } - - logger.info("[{}] Sending StartPartitionSessionResponse for {} and consumer \"{}\" with readOffset {} " - + "and commitOffset {}", traceID, partition, consumer, readFrom, commitTo); - session.sendStartPartition(partition, readFrom, commitTo); - sendDataToReaders(); - } else { + if (!state.compareAndSet(State.INITED, State.STARTED)) { logger.warn("[{}] Need to send StartPartitionSessionResponse, but the partition session is already {}", traceID, state.get()); + return; } + + if (commitTo != null) { + committer.updateCommittedOffset(commitTo); + queue.updateLastReadOffset(commitTo); // avoid lags of commits + } + + String consumer = session.getConfig().getConsumerName(); + logger.info("[{}] Sending StartPartitionSessionResponse for {} and consumer \"{}\" with readOffset {} " + + "and commitOffset {}", traceID, partition, consumer, readFrom, commitTo); + session.sendStartPartition(partition, readFrom, commitTo); + sendDataToReaders(); } } diff --git a/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java b/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java index 5bd045356..cffcd5d9e 100644 --- a/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java +++ b/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java @@ -258,6 +258,36 @@ public void singleThreadExecutorTest() throws Exception { executor.awaitTermination(5, TimeUnit.SECONDS); } + @Test + public void singleThreadReadAllTest() throws Exception { + ReaderSettings readerSettings = ReaderSettings.newBuilder() + .addTopic(TEST_TOPIC) + .setConsumerName(TEST_CONSUMER1) + .build(); + + CountDownLatch read = new CountDownLatch(3600); + + ExecutorService executor = Executors.newSingleThreadExecutor((r) -> new Thread(r, "test-executor")); + AsyncReader reader = client.createAsyncReader(readerSettings, ReadEventHandlersSettings.newBuilder() + .setExecutor(executor) + .setEventHandler((event) -> { + event.commit().join(); + event.getMessages().forEach(m -> read.countDown()); + }).build() + ); + + reader.init(); + + // wait for message committing + Assert.assertTrue(read.await(5, TimeUnit.SECONDS)); + + // stop reader + reader.shutdown(); + + executor.shutdown(); + executor.awaitTermination(5, TimeUnit.SECONDS); + } + @Test public void readAllTest() throws InterruptedException { ReaderSettings readerSettings = ReaderSettings.newBuilder() From a2250d44db97d7ea19ceeb98229cb3eb4225b9d2 Mon Sep 17 00:00:00 2001 From: Alexandr Gorshenin Date: Thu, 8 Oct 2026 15:16:26 +0100 Subject: [PATCH 3/9] Added test for all stream messages processing --- .../topic/TopicReadersIntegrationTest.java | 4 +- .../ydb/topic/read/impl/ReadStreamMock.java | 17 ++++++++ .../ydb/topic/read/impl/ReaderImplTest.java | 40 +++++++++++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java b/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java index cffcd5d9e..161c76a6e 100644 --- a/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java +++ b/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java @@ -3,13 +3,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; - import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; - +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReadStreamMock.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReadStreamMock.java index 52b320c48..66b367559 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReadStreamMock.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReadStreamMock.java @@ -123,6 +123,23 @@ public void responseStopPartition(long psid, boolean graceful) { observer.onNext(msg); } + public void responsePartitionSessionStatus(long psid) { + FromServer msg = FromServer.newBuilder() + .setStatus(StatusCodesProtos.StatusIds.StatusCode.SUCCESS) + .setPartitionSessionStatusResponse(YdbTopic.StreamReadMessage.PartitionSessionStatusResponse.newBuilder() + .setPartitionSessionId(psid) + .build()) + .build(); + observer.onNext(msg); + } + + public void responseEmpty() { + FromServer msg = FromServer.newBuilder() + .setStatus(StatusCodesProtos.StatusIds.StatusCode.SUCCESS) + .build(); + observer.onNext(msg); + } + public CommitAckResponse responseCommitAck() { return new CommitAckResponse(); } diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderImplTest.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderImplTest.java index cc25eb33f..246d3604f 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderImplTest.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderImplTest.java @@ -300,6 +300,46 @@ public void duplicateForcedStopPartitionRequestTest() { mock.assertIsClosed(); } + @Test + @HideLoggers(ReaderImpl.class) + public void onPartitionSessionStatusTest() { + ReaderSettings settings = ReaderSettings.newBuilder() + .addTopic(TOPIC1) + .setMaxMemoryUsageBytes(10200) + .setConsumerName("test") + .build(); + + ReadStreamMock mock = new ReadStreamMock(); + ReadConfig config = new ReadConfig(REGISTRY, Runnable::run, Runnable::run, Runnable::run, settings); + TestImpl reader = new TestImpl(mockRpc(mock), "test-reader", settings, config); + + // init reader + reader.start(); + mock.assertSentMessagesCount(1); + mock.assertLastMessage().isInitRequest("test", TOPIC1); + mock.responseInit("read-session-1"); + + mock.assertSentMessagesCount(2); + mock.assertLastMessage().isReadRequest(10200); + + // start partition + ArgumentCaptor started = ArgumentCaptor.forClass(StartPartitionSessionEvent.class); + mock.responseStartPartition(TOPIC1, 333, 100); + reader.verifyHandler().onPartitionStarted(started.capture()); + started.getValue().confirm(); + mock.assertSentMessagesCount(3); + mock.assertLastMessage().isStartPartition(1); + + // on partition status + mock.responsePartitionSessionStatus(1); // nothing + mock.responsePartitionSessionStatus(2); // nothing + mock.responseEmpty(); // nothing + + reader.close(); + reader.assertReaderIsStopped(Status.SUCCESS); + mock.assertIsClosed(); + } + @Test public void updateTokenTest() { ReaderSettings settings = ReaderSettings.newBuilder() From 1b08ab8a62bf11bd6a4ead69fde986c92ce258f5 Mon Sep 17 00:00:00 2001 From: Alexandr Gorshenin Date: Thu, 8 Oct 2026 18:10:53 +0100 Subject: [PATCH 4/9] Added support for child parititons pausing --- .../events/PartitionSessionEndedEvent.java | 10 ++ .../events/PartitionSessionStatusEvent.java | 7 -- .../topic/read/events/ReadEventHandler.java | 2 + .../ydb/topic/read/impl/AsyncReaderImpl.java | 10 ++ .../ydb/topic/read/impl/BufferManager.java | 8 +- .../ydb/topic/read/impl/ReadPartition.java | 95 +++++++++++++------ .../tech/ydb/topic/read/impl/ReadSession.java | 40 +++++++- .../tech/ydb/topic/read/impl/ReaderImpl.java | 4 + .../ydb/topic/read/impl/SyncReaderImpl.java | 6 ++ .../PartitionSessionEndedEventImpl.java | 28 ++++++ .../ydb/topic/read/impl/ReadStreamMock.java | 2 +- 11 files changed, 172 insertions(+), 40 deletions(-) create mode 100644 topic/src/main/java/tech/ydb/topic/read/events/PartitionSessionEndedEvent.java delete mode 100644 topic/src/main/java/tech/ydb/topic/read/events/PartitionSessionStatusEvent.java create mode 100644 topic/src/main/java/tech/ydb/topic/read/impl/events/PartitionSessionEndedEventImpl.java diff --git a/topic/src/main/java/tech/ydb/topic/read/events/PartitionSessionEndedEvent.java b/topic/src/main/java/tech/ydb/topic/read/events/PartitionSessionEndedEvent.java new file mode 100644 index 000000000..d5fb0f21f --- /dev/null +++ b/topic/src/main/java/tech/ydb/topic/read/events/PartitionSessionEndedEvent.java @@ -0,0 +1,10 @@ +package tech.ydb.topic.read.events; + +import tech.ydb.topic.read.PartitionSession; + +/** + * @author Nikolay Perfilov + */ +public interface PartitionSessionEndedEvent { + PartitionSession getPartitionSession(); +} diff --git a/topic/src/main/java/tech/ydb/topic/read/events/PartitionSessionStatusEvent.java b/topic/src/main/java/tech/ydb/topic/read/events/PartitionSessionStatusEvent.java deleted file mode 100644 index 7eead4274..000000000 --- a/topic/src/main/java/tech/ydb/topic/read/events/PartitionSessionStatusEvent.java +++ /dev/null @@ -1,7 +0,0 @@ -package tech.ydb.topic.read.events; - -/** - * @author Nikolay Perfilov - */ -public class PartitionSessionStatusEvent { -} diff --git a/topic/src/main/java/tech/ydb/topic/read/events/ReadEventHandler.java b/topic/src/main/java/tech/ydb/topic/read/events/ReadEventHandler.java index 0397e3f7b..bfa1334bd 100644 --- a/topic/src/main/java/tech/ydb/topic/read/events/ReadEventHandler.java +++ b/topic/src/main/java/tech/ydb/topic/read/events/ReadEventHandler.java @@ -23,6 +23,8 @@ default void onStopPartitionSession(StopPartitionSessionEvent event) { default void onPartitionSessionClosed(PartitionSessionClosedEvent event) { } + default void onPartitionSessionEnded(PartitionSessionEndedEvent event) { } + default void onReaderClosed(ReaderClosedEvent event) { } default void onSessionStarted(SessionStartedEvent event) { } diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/AsyncReaderImpl.java b/topic/src/main/java/tech/ydb/topic/read/impl/AsyncReaderImpl.java index 125ee9e7c..352847cfa 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/AsyncReaderImpl.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/AsyncReaderImpl.java @@ -23,6 +23,7 @@ import tech.ydb.topic.read.events.CommitOffsetAcknowledgementEvent; import tech.ydb.topic.read.events.DataReceivedEvent; import tech.ydb.topic.read.events.PartitionSessionClosedEvent; +import tech.ydb.topic.read.events.PartitionSessionEndedEvent; import tech.ydb.topic.read.events.ReadEventHandler; import tech.ydb.topic.read.events.ReaderClosedEvent; import tech.ydb.topic.read.events.StartPartitionSessionEvent; @@ -192,5 +193,14 @@ public void onPartitionClosed(PartitionSessionClosedEvent event) { failSession(th, "onPartitionSessionClosed"); } } + + @Override + public void onPartitionEnded(PartitionSessionEndedEvent event) { + try { + eventHandler.onPartitionSessionEnded(event); + } catch (Throwable th) { + failSession(th, "onPartitionSessionEnded"); + } + } } } diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java b/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java index 70bc9febe..fe9c122a4 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java @@ -123,11 +123,13 @@ public void releasePartition(Long id) { } // Thread safe - public void releaseRange(Long id, OffsetsRange range) { + public boolean releaseRange(Long id, OffsetsRange range) { PartitionBuffer part = partitions.get(id); if (part != null) { release(part.release(range)); + return part.isActive(); } + return false; } private void release(long total) { @@ -180,6 +182,10 @@ public void add(BatchBuffer range) { batches.add(range); } + public boolean isActive() { + return !batches.isEmpty(); + } + public long release(OffsetsRange range) { long released = 0; diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java index 3a0be9111..11ba78feb 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java @@ -17,6 +17,7 @@ import tech.ydb.topic.read.impl.events.CommitOffsetAcknowledgementEventImpl; import tech.ydb.topic.read.impl.events.DataReceivedEventImpl; import tech.ydb.topic.read.impl.events.PartitionSessionClosedEventImpl; +import tech.ydb.topic.read.impl.events.PartitionSessionEndedEventImpl; import tech.ydb.topic.read.impl.events.StartPartitionSessionEventImpl; import tech.ydb.topic.read.impl.events.StopPartitionSessionEventImpl; import tech.ydb.topic.settings.StartPartitionSessionSettings; @@ -58,6 +59,9 @@ private enum State { private final AtomicReference state = new AtomicReference<>(State.CREATED); private final AtomicReference commitOffsetAck = new AtomicReference<>(null); + private final AtomicReference partitonEnd = new AtomicReference<>(null); + private volatile boolean hasUnprocessedMessages = true; + private volatile boolean isPaused = true; ReadPartition(String traceID, ReadSession session, PartitionSession partition, long lastCommittedOffset) { this.traceID = traceID; @@ -74,6 +78,8 @@ private enum State { this.committer = new ReadPartitionCommitter(traceID, session, partition, lastCommittedOffset); this.decoder = new ReadPartitionDecoder(traceID, sessionDecoder, partition, committer, this::sendDataToReaders); this.queue = new ReadPartitionQueue(traceID, decoder, config.getMaxBatchSize(), lastCommittedOffset); + + logger.info("[{}] started for {}", traceID, partition); } @Override @@ -81,15 +87,22 @@ public boolean isActive() { return State.IS_ACTIVE.contains(state.get()); } + public PartitionSession getPartition() { + return partition; + } + @Override public void confirmRangeProcessed(OffsetsRange range) { - session.getBufferManager().releaseRange(partition.getId(), range); decoder.releaseRange(range); + hasUnprocessedMessages = session.getBufferManager().releaseRange(partition.getId(), range); + trySendPartitionEnded(); sendDataToReaders(); } - public PartitionSession getPartition() { - return partition; + public void confirmPartitionEnded(List childs) { + logger.info("[{}] got EndPartitionSession with child partitions {}", traceID, childs); + partitonEnd.set(new PartitionSessionEndedEventImpl(partition, childs)); + trySendPartitionEnded(); } public void confirmCommittedOffset(long committedOffset) { @@ -99,30 +112,59 @@ public void confirmCommittedOffset(long committedOffset) { sendDataToReaders(); } - private void sendCommitOffsetAck() { + public void release() { + isPaused = false; + if (state.get() != State.CREATED) { + logger.info("[{}] was unpaused and ready to send data", traceID); + sendDataToReaders(); + } + } + + private void trySendCommitOffsetAck() { CommitOffsetAcknowledgementEvent event = commitOffsetAck.getAndSet(null); if (event != null) { session.getHandler().onCommitAck(event); } } + private void trySendPartitionEnded() { + if (hasUnprocessedMessages) { + return; + } + PartitionSessionEndedEventImpl event = partitonEnd.getAndSet(null); + if (event != null) { + dataExecutor.execute(() -> session.getHandler().onPartitionEnded(event)); + logger.info("[{}] has finished processing and unpaused child partitions {}", + traceID, event.getChildPartitionIds()); + session.releasePartitions(event.getChildPartitionIds()); + } + } + public boolean addBatches(List batchList) { if (!isActive()) { return false; } - queue.addBatches(batchList); - long messagesCount = 0; - for (YdbTopic.StreamReadMessage.ReadResponse.Batch batch : batchList) { - messagesCount += batch.getMessageDataCount(); - } + + long messagesCount = batchList.stream() + .mapToInt(YdbTopic.StreamReadMessage.ReadResponse.Batch::getMessageDataCount) + .sum(); metrics.reportReceivedMessages(messagesCount, partition.getPath()); - sendDataToReaders(); + + if (messagesCount > 0) { + hasUnprocessedMessages = true; + queue.addBatches(batchList); + sendDataToReaders(); + } + return isActive(); } private void sendDataToReaders() { dataExecutor.execute(() -> { - sendCommitOffsetAck(); // ack may be sent even state is not active + trySendCommitOffsetAck(); // ack may be sent even state is not active + if (isPaused) { + return; + } while (state.get() == State.STARTED || state.get() == State.PRE_STOPPED) { List list = queue.getNextBatch(); if (list == null) { @@ -140,7 +182,7 @@ private void sendDataToReaders() { logger.debug("[{}] onData with {} message(s) (offsets {}-{}) successfully finished", traceID, messagesCount, offsetStart, offsetEnd); - sendCommitOffsetAck(); + trySendCommitOffsetAck(); } }); } @@ -154,7 +196,7 @@ public void close() { session.getBufferManager().releasePartition(partition.getId()); decoder.close(); committer.close(); - logger.info("[{}] with state {} was stopped", traceID, old); + logger.info("[{}] with state {} was closed", traceID, old); if (old != State.STOPPED && old != State.CREATED) { PartitionSessionClosedEventImpl event = new PartitionSessionClosedEventImpl(partition); @@ -163,10 +205,11 @@ public void close() { } } - public void start(long committed, OffsetsRange offsets) { - logger.info("[{}] Received StartPartitionSessionRequest for {} and consumer \"{}\" with committedOffset {} and " - + "partitionOffsets {}", traceID, partition, session.getConfig().getConsumerName(), committed, offsets); + public void start(long committed, OffsetsRange offsets, boolean paused) { + logger.info("[{}] got StartPartitionSessionRequest with committed offset {} and partition offsets {} {}", + traceID, committed, offsets, paused ? "[paused]" : ""); + isPaused = paused; committer.updateCommittedOffset(committed); queue.updateLastReadOffset(committed); // partition start event doesn't use partition's executors @@ -175,16 +218,15 @@ public void start(long committed, OffsetsRange offsets) { session.getHandler().onPartitionStarted(new StartEvent(committed, offsets)); return; } - logger.info("[{}] skipped onStart event because the partition session is already {}", traceID, state.get()); + logger.info("[{}] skipped start confirmation event because is already {}", traceID, state.get()); }); } public void stop(long committed) { - logger.info("[{}] Received graceful StopPartitionSessionRequest for {} with committedOffset {}", - traceID, partition, committed); + logger.info("[{}] got graceful StopPartitionSessionRequest with committedOffset {}", traceID, committed); if (state.compareAndSet(State.CREATED, State.STOPPED)) { - logger.info("[{}] was auto stopped because the partition is not initialized yet", traceID); + logger.info("[{}] stop was auto confirmed the partition is not initialized yet", traceID); session.sendStopPartition(partition); return; } @@ -204,7 +246,7 @@ public void stop(long committed) { session.getHandler().onPartitionStopped(new StopEvent(committed)); return; } - logger.warn("[{}] skipped onStop event because the partition session is already {}", traceID, state.get()); + logger.warn("[{}] skipped onStop because the partition session is already {}", traceID, state.get()); }); } @@ -219,7 +261,7 @@ public void confirm(StartPartitionSessionSettings settings) { Long commitTo = settings != null ? settings.getCommitOffset() : null; if (!state.compareAndSet(State.INITED, State.STARTED)) { - logger.warn("[{}] Need to send StartPartitionSessionResponse, but the partition session is already {}", + logger.warn("[{}] needs to send StartPartitionSessionResponse, but the partition session is already {}", traceID, state.get()); return; } @@ -229,9 +271,8 @@ public void confirm(StartPartitionSessionSettings settings) { queue.updateLastReadOffset(commitTo); // avoid lags of commits } - String consumer = session.getConfig().getConsumerName(); - logger.info("[{}] Sending StartPartitionSessionResponse for {} and consumer \"{}\" with readOffset {} " - + "and commitOffset {}", traceID, partition, consumer, readFrom, commitTo); + logger.info("[{}] sent StartPartitionSessionResponse with readOffset {} and commitOffset {}", traceID, + readFrom, commitTo); session.sendStartPartition(partition, readFrom, commitTo); sendDataToReaders(); } @@ -245,10 +286,10 @@ private class StopEvent extends StopPartitionSessionEventImpl { @Override public void confirm() { if (state.compareAndSet(State.PRE_STOPPED, State.STOPPED)) { - logger.info("[{}] Sending StopPartitionSessionResponse for {}", traceID, partition); + logger.info("[{}] sent StopPartitionSessionResponse", traceID); session.sendStopPartition(partition); } else { - logger.warn("[{}] Need to send StopPartitionSessionResponse, but the partition session is already {}", + logger.warn("[{}] needs to send StopPartitionSessionResponse, but the partition session is already {}", traceID, partition); } } diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java index bd9c30800..d4216511a 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java @@ -24,6 +24,7 @@ import tech.ydb.topic.read.events.CommitOffsetAcknowledgementEvent; import tech.ydb.topic.read.events.DataReceivedEvent; import tech.ydb.topic.read.events.PartitionSessionClosedEvent; +import tech.ydb.topic.read.events.PartitionSessionEndedEvent; import tech.ydb.topic.read.events.StartPartitionSessionEvent; import tech.ydb.topic.read.events.StopPartitionSessionEvent; @@ -34,6 +35,8 @@ public class ReadSession extends TopicStreamBase { private static final Logger logger = LoggerFactory.getLogger(ReadSession.class); + private static final Runnable EMPTY_RELEASER = () -> {}; + public interface PartitionControl { boolean isActive(); void confirmRangeProcessed(OffsetsRange range); @@ -43,6 +46,7 @@ public interface Handler { void onPartitionStarted(StartPartitionSessionEvent event); void onPartitionStopped(StopPartitionSessionEvent event); void onPartitionClosed(PartitionSessionClosedEvent event); + void onPartitionEnded(PartitionSessionEndedEvent event); void onData(PartitionControl control, DataReceivedEvent event); @@ -56,6 +60,7 @@ public interface Handler { private final Handler handler; private final Map partitions = new ConcurrentHashMap<>(); + private final Map partitionReleasers = new ConcurrentHashMap<>(); private volatile boolean isClosed = false; public ReadSession(String id, GrpcReadWriteStream stream, FromClient initReq, @@ -98,6 +103,7 @@ Handler getHandler() { public void closeAll() { isClosed = true; decoder.stop(); + partitionReleasers.clear(); partitions.values().forEach(ReadPartition::close); partitions.clear(); config.getMetrics().unregister(); @@ -161,6 +167,15 @@ public void sendStopPartition(PartitionSession ps) { } } + public void releasePartitions(List partitionIds) { + for (Long pid: partitionIds) { + Runnable releaser = partitionReleasers.remove(pid); + if (releaser != null) { + releaser.run(); + } + } + } + public void onInit(YdbTopic.StreamReadMessage.InitResponse response) { bufferManager.init(response.getSessionId()); config.getMetrics().register(partitions::size, bufferManager::getCreditBalanceBytes); @@ -187,14 +202,31 @@ public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionReq return; } - partition.start(committed, offsets); + boolean isPaused = (EMPTY_RELEASER == partitionReleasers.replace(pid, partition::release)); + partition.start(committed, offsets, isPaused); + } + + public void onEndPartition(YdbTopic.StreamReadMessage.EndPartitionSession request) { + long psid = request.getPartitionSessionId(); + ReadPartition partition = partitions.get(psid); + if (partition == null) { + logger.warn("[{}] Received EndPartitionSession for partition session {}, but have no such " + + "partition session running", debugId, psid); + return; + } + + for (Long pid: request.getChildPartitionIdsList()) { // register all child partitions + partitionReleasers.put(pid, EMPTY_RELEASER); + } + + partition.confirmPartitionEnded(request.getChildPartitionIdsList()); } - public void onClosePartition(long partitionSessionId) { - ReadPartition partition = partitions.remove(partitionSessionId); + public void onClosePartition(long psid) { + ReadPartition partition = partitions.remove(psid); if (partition == null) { logger.warn("[{}] Received force StopPartitionSessionRequest for partition session {}, " + - "but have no such partition session running", debugId, partitionSessionId); + "but have no such partition session running", debugId, psid); return; } diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReaderImpl.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReaderImpl.java index 13c27d643..e3548c093 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReaderImpl.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReaderImpl.java @@ -124,6 +124,8 @@ protected void onNext(ReadSession stream, FromServer message) { stream.onRead(message.getReadResponse()); } else if (message.hasCommitOffsetResponse()) { stream.onCommitOffset(message.getCommitOffsetResponse()); + } else if (message.hasEndPartitionSession()) { + stream.onEndPartition(message.getEndPartitionSession()); } else if (message.hasPartitionSessionStatusResponse()) { stream.onPartitionSessionStatus(message.getPartitionSessionStatusResponse()); } else if (message.hasUpdateTokenResponse()) { @@ -278,6 +280,8 @@ private static YdbTopic.StreamReadMessage.InitRequest buildInitRequest(ReaderSet builder.addTopicsReadSettings(buildTopicSettings(trs)); } + builder.setAutoPartitioningSupport(true); + return builder.build(); } } diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java b/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java index a555a3b29..0efd13af4 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java @@ -32,6 +32,7 @@ import tech.ydb.topic.read.events.CommitOffsetAcknowledgementEvent; import tech.ydb.topic.read.events.DataReceivedEvent; import tech.ydb.topic.read.events.PartitionSessionClosedEvent; +import tech.ydb.topic.read.events.PartitionSessionEndedEvent; import tech.ydb.topic.read.events.StartPartitionSessionEvent; import tech.ydb.topic.read.events.StopPartitionSessionEvent; import tech.ydb.topic.settings.ReaderSettings; @@ -274,6 +275,11 @@ public void onPartitionStopped(StopPartitionSessionEvent event) { public void onPartitionClosed(PartitionSessionClosedEvent event) { // Nothing } + + @Override + public void onPartitionEnded(PartitionSessionEndedEvent event) { + // Nothing + } } private static class MessageWrapper { diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/events/PartitionSessionEndedEventImpl.java b/topic/src/main/java/tech/ydb/topic/read/impl/events/PartitionSessionEndedEventImpl.java new file mode 100644 index 000000000..6231b8e05 --- /dev/null +++ b/topic/src/main/java/tech/ydb/topic/read/impl/events/PartitionSessionEndedEventImpl.java @@ -0,0 +1,28 @@ +package tech.ydb.topic.read.impl.events; + +import java.util.List; + +import tech.ydb.topic.read.PartitionSession; +import tech.ydb.topic.read.events.*; + +/** + * @author Nikolay Perfilov + */ +public class PartitionSessionEndedEventImpl implements PartitionSessionEndedEvent { + private final PartitionSession partition; + private final List childPartitionIds; + + public PartitionSessionEndedEventImpl(PartitionSession partition, List childPartitionIds) { + this.partition = partition; + this.childPartitionIds = childPartitionIds; + } + + @Override + public PartitionSession getPartitionSession() { + return partition; + } + + public List getChildPartitionIds() { + return childPartitionIds; + } +} diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReadStreamMock.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReadStreamMock.java index 66b367559..b609d8c5c 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReadStreamMock.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReadStreamMock.java @@ -264,7 +264,7 @@ public MessageAssert(FromClient msg) { public MessageAssert isInitRequest(String consumerName, String... topicPaths) { Assert.assertTrue("Msg is not init request", msg.hasInitRequest()); - Assert.assertFalse("Auto partition is disabled", msg.getInitRequest().getAutoPartitioningSupport()); + Assert.assertTrue("Auto partition is enabled", msg.getInitRequest().getAutoPartitioningSupport()); if (consumerName != null) { Assert.assertEquals("Wrong consumer in init request", consumerName, msg.getInitRequest().getConsumer()); } else { From 6a729fcd24b0844c7116ef7e4b5ef13916ac0634 Mon Sep 17 00:00:00 2001 From: Alexandr Gorshenin Date: Thu, 8 Oct 2026 18:42:52 +0100 Subject: [PATCH 5/9] Small fixes --- .../ydb/topic/read/impl/ReadPartition.java | 24 ++++++------- .../tech/ydb/topic/read/impl/ReadSession.java | 25 ++++++++----- .../ydb/topic/read/impl/TopicPartition.java | 36 +++++++++++++++++++ .../PartitionSessionEndedEventImpl.java | 24 ++++++++++--- .../topic/TopicReadersIntegrationTest.java | 17 ++++----- 5 files changed, 93 insertions(+), 33 deletions(-) create mode 100644 topic/src/main/java/tech/ydb/topic/read/impl/TopicPartition.java diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java index 11ba78feb..30a88bd85 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java @@ -60,7 +60,7 @@ private enum State { private final AtomicReference state = new AtomicReference<>(State.CREATED); private final AtomicReference commitOffsetAck = new AtomicReference<>(null); private final AtomicReference partitonEnd = new AtomicReference<>(null); - private volatile boolean hasUnprocessedMessages = true; + private volatile boolean hasUnprocessedMessages = false; private volatile boolean isPaused = true; ReadPartition(String traceID, ReadSession session, PartitionSession partition, long lastCommittedOffset) { @@ -99,9 +99,10 @@ public void confirmRangeProcessed(OffsetsRange range) { sendDataToReaders(); } - public void confirmPartitionEnded(List childs) { - logger.info("[{}] got EndPartitionSession with child partitions {}", traceID, childs); - partitonEnd.set(new PartitionSessionEndedEventImpl(partition, childs)); + public void confirmPartitionEnded(List childs) { + PartitionSessionEndedEventImpl event = new PartitionSessionEndedEventImpl(partition, childs); + logger.info("[{}] got EndPartitionSession with child partitions {}", traceID, event.getChildsString()); + partitonEnd.set(event); trySendPartitionEnded(); } @@ -112,7 +113,7 @@ public void confirmCommittedOffset(long committedOffset) { sendDataToReaders(); } - public void release() { + public void unpause() { isPaused = false; if (state.get() != State.CREATED) { logger.info("[{}] was unpaused and ready to send data", traceID); @@ -134,9 +135,9 @@ private void trySendPartitionEnded() { PartitionSessionEndedEventImpl event = partitonEnd.getAndSet(null); if (event != null) { dataExecutor.execute(() -> session.getHandler().onPartitionEnded(event)); - logger.info("[{}] has finished processing and unpaused child partitions {}", - traceID, event.getChildPartitionIds()); - session.releasePartitions(event.getChildPartitionIds()); + logger.info("[{}] has finished processing and unpaused child partitions {}", traceID, + event.getChildsString()); + session.releasePartitions(event.getChilds()); } } @@ -205,11 +206,10 @@ public void close() { } } - public void start(long committed, OffsetsRange offsets, boolean paused) { - logger.info("[{}] got StartPartitionSessionRequest with committed offset {} and partition offsets {} {}", - traceID, committed, offsets, paused ? "[paused]" : ""); + public void start(long committed, OffsetsRange offsets) { + logger.info("[{}] got StartPartitionSessionRequest with committed offset {} and partition offsets {}", + traceID, committed, offsets); - isPaused = paused; committer.updateCommittedOffset(committed); queue.updateLastReadOffset(committed); // partition start event doesn't use partition's executors diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java index d4216511a..c7323e93b 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java @@ -1,5 +1,6 @@ package tech.ydb.topic.read.impl; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -60,7 +61,7 @@ public interface Handler { private final Handler handler; private final Map partitions = new ConcurrentHashMap<>(); - private final Map partitionReleasers = new ConcurrentHashMap<>(); + private final Map partitionReleasers = new ConcurrentHashMap<>(); private volatile boolean isClosed = false; public ReadSession(String id, GrpcReadWriteStream stream, FromClient initReq, @@ -167,9 +168,9 @@ public void sendStopPartition(PartitionSession ps) { } } - public void releasePartitions(List partitionIds) { - for (Long pid: partitionIds) { - Runnable releaser = partitionReleasers.remove(pid); + public void releasePartitions(List partitions) { + for (TopicPartition partition: partitions) { + Runnable releaser = partitionReleasers.remove(partition); if (releaser != null) { releaser.run(); } @@ -202,8 +203,13 @@ public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionReq return; } - boolean isPaused = (EMPTY_RELEASER == partitionReleasers.replace(pid, partition::release)); - partition.start(committed, offsets, isPaused); + partition.start(committed, offsets); + TopicPartition tp = new TopicPartition(ps.getPath(), ps.getPartitionId()); + if (EMPTY_RELEASER == partitionReleasers.replace(tp, partition::unpause)) { + logger.info("[{}] was paused", tid); + } else { + partition.unpause(); + } } public void onEndPartition(YdbTopic.StreamReadMessage.EndPartitionSession request) { @@ -215,11 +221,14 @@ public void onEndPartition(YdbTopic.StreamReadMessage.EndPartitionSession reques return; } + List childs = new ArrayList<>(); for (Long pid: request.getChildPartitionIdsList()) { // register all child partitions - partitionReleasers.put(pid, EMPTY_RELEASER); + TopicPartition child = new TopicPartition(partition.getPartition().getPath(), pid); + partitionReleasers.put(child, EMPTY_RELEASER); + childs.add(child); } - partition.confirmPartitionEnded(request.getChildPartitionIdsList()); + partition.confirmPartitionEnded(childs); } public void onClosePartition(long psid) { diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/TopicPartition.java b/topic/src/main/java/tech/ydb/topic/read/impl/TopicPartition.java new file mode 100644 index 000000000..e154dbfb0 --- /dev/null +++ b/topic/src/main/java/tech/ydb/topic/read/impl/TopicPartition.java @@ -0,0 +1,36 @@ +package tech.ydb.topic.read.impl; + +/** + * + * @author Aleksandr Gorshenin {@literal } + */ +public class TopicPartition { + private final String topicPath; + private final long partitionId; + + public TopicPartition(String topicPath, long partitionId) { + this.topicPath = topicPath; + this.partitionId = partitionId; + } + + public long getPartitionId() { + return partitionId; + } + + @Override + public int hashCode() { + return 31 * Long.hashCode(partitionId) + topicPath.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof TopicPartition)) { + return false; + } + TopicPartition o = (TopicPartition) obj; + return partitionId == o.partitionId && topicPath.equals(o.topicPath); + } +} diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/events/PartitionSessionEndedEventImpl.java b/topic/src/main/java/tech/ydb/topic/read/impl/events/PartitionSessionEndedEventImpl.java index 6231b8e05..a5b4c7a31 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/events/PartitionSessionEndedEventImpl.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/events/PartitionSessionEndedEventImpl.java @@ -4,17 +4,18 @@ import tech.ydb.topic.read.PartitionSession; import tech.ydb.topic.read.events.*; +import tech.ydb.topic.read.impl.TopicPartition; /** * @author Nikolay Perfilov */ public class PartitionSessionEndedEventImpl implements PartitionSessionEndedEvent { private final PartitionSession partition; - private final List childPartitionIds; + private final List childs; - public PartitionSessionEndedEventImpl(PartitionSession partition, List childPartitionIds) { + public PartitionSessionEndedEventImpl(PartitionSession partition, List childs) { this.partition = partition; - this.childPartitionIds = childPartitionIds; + this.childs = childs; } @Override @@ -22,7 +23,20 @@ public PartitionSession getPartitionSession() { return partition; } - public List getChildPartitionIds() { - return childPartitionIds; + public List getChilds() { + return childs; + } + + public String getChildsString() { + StringBuilder sb = new StringBuilder("["); + boolean first = true; + for (TopicPartition child: childs) { + if (!first) { + sb.append(", "); + } + sb.append(child.getPartitionId()); + first = false; + } + return sb.append("]").toString(); } } diff --git a/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java b/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java index 161c76a6e..c4b4dba6f 100644 --- a/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java +++ b/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java @@ -275,15 +275,16 @@ public void singleThreadReadAllTest() throws Exception { ); reader.init(); + try { + // wait for message committing + Assert.assertTrue(read.await(5, TimeUnit.SECONDS)); + } finally { + // stop reader + reader.shutdown(); - // wait for message committing - Assert.assertTrue(read.await(5, TimeUnit.SECONDS)); - - // stop reader - reader.shutdown(); - - executor.shutdown(); - executor.awaitTermination(5, TimeUnit.SECONDS); + executor.shutdown(); + executor.awaitTermination(5, TimeUnit.SECONDS); + } } @Test From 8611b0f80a8a15a4f096799e944a39f74383c138 Mon Sep 17 00:00:00 2001 From: Alexandr Gorshenin Date: Thu, 8 Oct 2026 20:56:07 +0100 Subject: [PATCH 6/9] Added support for partitions with multiply parents --- .../tech/ydb/topic/read/impl/ReadSession.java | 53 ++++++++++++++----- .../ydb/topic/read/impl/TopicPartition.java | 7 +++ .../PartitionSessionEndedEventImpl.java | 2 +- .../topic/TopicReadersIntegrationTest.java | 4 +- .../ydb/topic/read/impl/ReaderImplTest.java | 1 - 5 files changed, 50 insertions(+), 17 deletions(-) diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java index c7323e93b..f9bf3a0b8 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -36,8 +37,6 @@ public class ReadSession extends TopicStreamBase { private static final Logger logger = LoggerFactory.getLogger(ReadSession.class); - private static final Runnable EMPTY_RELEASER = () -> {}; - public interface PartitionControl { boolean isActive(); void confirmRangeProcessed(OffsetsRange range); @@ -61,7 +60,7 @@ public interface Handler { private final Handler handler; private final Map partitions = new ConcurrentHashMap<>(); - private final Map partitionReleasers = new ConcurrentHashMap<>(); + private final Map locks = new ConcurrentHashMap<>(); private volatile boolean isClosed = false; public ReadSession(String id, GrpcReadWriteStream stream, FromClient initReq, @@ -104,7 +103,7 @@ Handler getHandler() { public void closeAll() { isClosed = true; decoder.stop(); - partitionReleasers.clear(); + locks.clear(); partitions.values().forEach(ReadPartition::close); partitions.clear(); config.getMetrics().unregister(); @@ -169,12 +168,11 @@ public void sendStopPartition(PartitionSession ps) { } public void releasePartitions(List partitions) { + List released = new ArrayList<>(); for (TopicPartition partition: partitions) { - Runnable releaser = partitionReleasers.remove(partition); - if (releaser != null) { - releaser.run(); - } + locks.computeIfPresent(partition, (key, lock) -> lock.release(released) ? null : lock); } + released.forEach(ReadPartition::unpause); } public void onInit(YdbTopic.StreamReadMessage.InitResponse response) { @@ -204,11 +202,12 @@ public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionReq } partition.start(committed, offsets); - TopicPartition tp = new TopicPartition(ps.getPath(), ps.getPartitionId()); - if (EMPTY_RELEASER == partitionReleasers.replace(tp, partition::unpause)) { - logger.info("[{}] was paused", tid); - } else { + PartitionLock lock = locks.get(new TopicPartition(ps)); + if (lock == null) { partition.unpause(); + } else { + logger.info("[{}] was paused", tid); + lock.bind(partition); } } @@ -224,7 +223,7 @@ public void onEndPartition(YdbTopic.StreamReadMessage.EndPartitionSession reques List childs = new ArrayList<>(); for (Long pid: request.getChildPartitionIdsList()) { // register all child partitions TopicPartition child = new TopicPartition(partition.getPartition().getPath(), pid); - partitionReleasers.put(child, EMPTY_RELEASER); + locks.compute(child, (key, existing) -> (existing != null ? existing : new PartitionLock()).lock()); childs.add(child); } @@ -314,4 +313,32 @@ public void accept(Long sizeToRequest) { .build()); } } + + private class PartitionLock { + private final AtomicInteger locks = new AtomicInteger(0); + private volatile ReadPartition partition; + + public PartitionLock lock() { + locks.incrementAndGet(); + return this; + } + + public boolean release(List released) { + if (locks.decrementAndGet() > 0) { + return false; + } + + if (partition != null) { + released.add(partition); + } + return true; + } + + public void bind(ReadPartition local) { + partition = local; + if (locks.get() <= 0) { + partition.unpause(); + } + } + } } diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/TopicPartition.java b/topic/src/main/java/tech/ydb/topic/read/impl/TopicPartition.java index e154dbfb0..6715039b1 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/TopicPartition.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/TopicPartition.java @@ -1,5 +1,7 @@ package tech.ydb.topic.read.impl; +import tech.ydb.topic.read.PartitionSession; + /** * * @author Aleksandr Gorshenin {@literal } @@ -8,6 +10,11 @@ public class TopicPartition { private final String topicPath; private final long partitionId; + public TopicPartition(PartitionSession ps) { + this.topicPath = ps.getPath(); + this.partitionId = ps.getPartitionId(); + } + public TopicPartition(String topicPath, long partitionId) { this.topicPath = topicPath; this.partitionId = partitionId; diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/events/PartitionSessionEndedEventImpl.java b/topic/src/main/java/tech/ydb/topic/read/impl/events/PartitionSessionEndedEventImpl.java index a5b4c7a31..6ed884b3a 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/events/PartitionSessionEndedEventImpl.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/events/PartitionSessionEndedEventImpl.java @@ -3,7 +3,7 @@ import java.util.List; import tech.ydb.topic.read.PartitionSession; -import tech.ydb.topic.read.events.*; +import tech.ydb.topic.read.events.PartitionSessionEndedEvent; import tech.ydb.topic.read.impl.TopicPartition; /** diff --git a/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java b/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java index c4b4dba6f..cc0c64efe 100644 --- a/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java +++ b/topic/src/test/java/tech/ydb/topic/TopicReadersIntegrationTest.java @@ -90,7 +90,7 @@ public class TopicReadersIntegrationTest { @BeforeClass public static void initClient() { client = TopicClient.newClient(ydbTransport).build(); - initTopips(); + initTopics(); } @AfterClass @@ -99,7 +99,7 @@ public static void closeClient() { client.close(); } - private static void initTopips() { + private static void initTopics() { logger.info("Create test topic {} ...", TEST_TOPIC); client.createTopic(TEST_TOPIC, CreateTopicSettings.newBuilder() .addConsumer(Consumer.newBuilder().setName(TEST_CONSUMER1).build()) diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderImplTest.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderImplTest.java index 246d3604f..f87006a45 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderImplTest.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderImplTest.java @@ -410,7 +410,6 @@ public void updateOffsetsInTxValidationTest() { ); reader.updateOffsetsInTransaction(active, Collections.singletonMap(TOPIC1, offsets), updateSettings); - txStatus.complete(Status.SUCCESS); mock.assertIsActive(); reader.close(); From 0b6dcd4d32b1ab5973effa6f049e99ab66e3fe52 Mon Sep 17 00:00:00 2001 From: Alexandr Gorshenin Date: Fri, 9 Oct 2026 11:05:38 +0100 Subject: [PATCH 7/9] Keep partition lockes by keys --- .../ydb/topic/read/impl/ReadPartition.java | 25 ++++-- .../tech/ydb/topic/read/impl/ReadSession.java | 85 +++++++++++++------ 2 files changed, 75 insertions(+), 35 deletions(-) diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java index 30a88bd85..c73906e73 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java @@ -59,7 +59,7 @@ private enum State { private final AtomicReference state = new AtomicReference<>(State.CREATED); private final AtomicReference commitOffsetAck = new AtomicReference<>(null); - private final AtomicReference partitonEnd = new AtomicReference<>(null); + private final AtomicReference partitionEnd = new AtomicReference<>(null); private volatile boolean hasUnprocessedMessages = false; private volatile boolean isPaused = true; @@ -102,7 +102,7 @@ public void confirmRangeProcessed(OffsetsRange range) { public void confirmPartitionEnded(List childs) { PartitionSessionEndedEventImpl event = new PartitionSessionEndedEventImpl(partition, childs); logger.info("[{}] got EndPartitionSession with child partitions {}", traceID, event.getChildsString()); - partitonEnd.set(event); + partitionEnd.set(event); trySendPartitionEnded(); } @@ -132,12 +132,14 @@ private void trySendPartitionEnded() { if (hasUnprocessedMessages) { return; } - PartitionSessionEndedEventImpl event = partitonEnd.getAndSet(null); + PartitionSessionEndedEventImpl event = partitionEnd.getAndSet(null); if (event != null) { - dataExecutor.execute(() -> session.getHandler().onPartitionEnded(event)); - logger.info("[{}] has finished processing and unpaused child partitions {}", traceID, - event.getChildsString()); - session.releasePartitions(event.getChilds()); + dataExecutor.execute(() -> { + logger.info("[{}] has finished processing and unpaused child partitions {}", traceID, + event.getChildsString()); + session.getHandler().onPartitionEnded(event); + session.releaseLocks(partition, event.getChilds()); + }); } } @@ -199,10 +201,15 @@ public void close() { committer.close(); logger.info("[{}] with state {} was closed", traceID, old); + PartitionSessionEndedEventImpl endEvent = partitionEnd.getAndSet(null); + if (endEvent != null) { + session.removeLocks(partition, endEvent.getChilds()); + } + if (old != State.STOPPED && old != State.CREATED) { - PartitionSessionClosedEventImpl event = new PartitionSessionClosedEventImpl(partition); + PartitionSessionClosedEventImpl closedEvent = new PartitionSessionClosedEventImpl(partition); // partition close event doesn't use partition's executors - controlExecutor.execute(() -> session.getHandler().onPartitionClosed(event)); + controlExecutor.execute(() -> session.getHandler().onPartitionClosed(closedEvent)); } } diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java index f9bf3a0b8..6b26f3aa1 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java @@ -1,10 +1,11 @@ package tech.ydb.topic.read.impl; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -167,14 +168,25 @@ public void sendStopPartition(PartitionSession ps) { } } - public void releasePartitions(List partitions) { + public void releaseLocks(PartitionSession ps, List childs) { List released = new ArrayList<>(); - for (TopicPartition partition: partitions) { - locks.computeIfPresent(partition, (key, lock) -> lock.release(released) ? null : lock); + for (TopicPartition partition: childs) { + locks.computeIfPresent(partition, (key, lock) -> lock.release(ps, released)); } released.forEach(ReadPartition::unpause); } + public void removeLocks(PartitionSession ps, List childs) { + List frozen = new ArrayList<>(); + for (TopicPartition partition: childs) { + locks.computeIfPresent(partition, (key, lock) -> lock.remove(ps, frozen)); + } + for (ReadPartition partition: frozen) { + logger.error("[{}] {} will be paused forever because the parent {} was closed", debugId, + partition.getPartition(), ps); + } + } + public void onInit(YdbTopic.StreamReadMessage.InitResponse response) { bufferManager.init(response.getSessionId()); config.getMetrics().register(partitions::size, bufferManager::getCreditBalanceBytes); @@ -202,12 +214,11 @@ public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionReq } partition.start(committed, offsets); - PartitionLock lock = locks.get(new TopicPartition(ps)); - if (lock == null) { - partition.unpause(); - } else { + TopicPartition lockKey = new TopicPartition(ps); + if (locks.computeIfPresent(lockKey, (key, lock) -> lock.bind(partition)) != null) { logger.info("[{}] was paused", tid); - lock.bind(partition); + } else { + partition.unpause(); } } @@ -220,10 +231,11 @@ public void onEndPartition(YdbTopic.StreamReadMessage.EndPartitionSession reques return; } + PartitionSession ps = partition.getPartition(); List childs = new ArrayList<>(); for (Long pid: request.getChildPartitionIdsList()) { // register all child partitions - TopicPartition child = new TopicPartition(partition.getPartition().getPath(), pid); - locks.compute(child, (key, existing) -> (existing != null ? existing : new PartitionLock()).lock()); + TopicPartition child = new TopicPartition(ps.getPath(), pid); + locks.compute(child, (key, lock) -> PartitionLock.lock(lock, ps)); childs.add(child); } @@ -314,31 +326,52 @@ public void accept(Long sizeToRequest) { } } - private class PartitionLock { - private final AtomicInteger locks = new AtomicInteger(0); - private volatile ReadPartition partition; + private static class PartitionLock { + private final Set lockedBy; + private final ReadPartition partition; - public PartitionLock lock() { - locks.incrementAndGet(); - return this; + private PartitionLock(Set lockedBy, ReadPartition partition) { + this.lockedBy = lockedBy; + this.partition = partition; } - public boolean release(List released) { - if (locks.decrementAndGet() > 0) { - return false; + public PartitionLock release(PartitionSession locker, List released) { + Set newSet = new HashSet<>(lockedBy); + newSet.remove(locker); + if (newSet.isEmpty()) { + if (partition != null) { + released.add(partition); + } + return null; + } + + return new PartitionLock(newSet, partition); + } + + public PartitionLock remove(PartitionSession locker, List hanged) { + if (!lockedBy.contains(locker)) { + return this; } if (partition != null) { - released.add(partition); + hanged.add(partition); } - return true; + return null; + } + + public PartitionLock bind(ReadPartition binded) { + return new PartitionLock(lockedBy, binded); } - public void bind(ReadPartition local) { - partition = local; - if (locks.get() <= 0) { - partition.unpause(); + public static PartitionLock lock(PartitionLock currentLock, PartitionSession locker) { + Set locks = new HashSet<>(); + ReadPartition partition = null; + if (currentLock != null) { + locks.addAll(currentLock.lockedBy); + partition = currentLock.partition; } + locks.add(locker); + return new PartitionLock(locks, partition); } } } From 5af2877ca8efbd1153ef9492069c55d9426031e2 Mon Sep 17 00:00:00 2001 From: Alexandr Gorshenin Date: Fri, 9 Oct 2026 12:04:18 +0100 Subject: [PATCH 8/9] Added unit tests for merged partitions --- .../ydb/topic/read/impl/ReadStreamMock.java | 17 +- .../ydb/topic/read/impl/ReaderImplTest.java | 158 +++++++++++++++++- 2 files changed, 170 insertions(+), 5 deletions(-) diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReadStreamMock.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReadStreamMock.java index b609d8c5c..99f7d8109 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReadStreamMock.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReadStreamMock.java @@ -123,12 +123,23 @@ public void responseStopPartition(long psid, boolean graceful) { observer.onNext(msg); } + public void responseEndPartition(long psid, long... childPartitionIds) { + YdbTopic.StreamReadMessage.EndPartitionSession.Builder end = YdbTopic.StreamReadMessage.EndPartitionSession + .newBuilder().setPartitionSessionId(psid); + for (long id: childPartitionIds) { + end.addChildPartitionIds(id); + } + observer.onNext(FromServer.newBuilder() + .setStatus(StatusCodesProtos.StatusIds.StatusCode.SUCCESS) + .setEndPartitionSession(end) + .build()); + } + public void responsePartitionSessionStatus(long psid) { FromServer msg = FromServer.newBuilder() .setStatus(StatusCodesProtos.StatusIds.StatusCode.SUCCESS) - .setPartitionSessionStatusResponse(YdbTopic.StreamReadMessage.PartitionSessionStatusResponse.newBuilder() - .setPartitionSessionId(psid) - .build()) + .setPartitionSessionStatusResponse(YdbTopic.StreamReadMessage.PartitionSessionStatusResponse + .newBuilder().setPartitionSessionId(psid).build()) .build(); observer.onNext(msg); } diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderImplTest.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderImplTest.java index f87006a45..f152f49eb 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderImplTest.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderImplTest.java @@ -25,11 +25,14 @@ import tech.ydb.core.Status; import tech.ydb.core.StatusCode; import tech.ydb.topic.TopicRpc; +import tech.ydb.topic.description.Codec; import tech.ydb.topic.description.CodecRegistry; import tech.ydb.topic.description.OffsetsRange; import tech.ydb.topic.read.PartitionOffsets; import tech.ydb.topic.read.PartitionSession; +import tech.ydb.topic.read.events.DataReceivedEvent; import tech.ydb.topic.read.events.PartitionSessionClosedEvent; +import tech.ydb.topic.read.events.PartitionSessionEndedEvent; import tech.ydb.topic.read.events.StartPartitionSessionEvent; import tech.ydb.topic.read.events.StopPartitionSessionEvent; import tech.ydb.topic.settings.ReaderSettings; @@ -45,6 +48,10 @@ public class ReaderImplTest { private static final String TOPIC1 = "/test/topic"; private static final String TOPIC2 = "/test/topic2"; + private static final byte[] MSG1 = new byte[] { 0x00 }; + private static final byte[] MSG2 = new byte[] { }; + private static final byte[] MSG3 = new byte[] { 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, 0x02 }; + @Rule public final HideLoggersRule hideLogger = new HideLoggersRule(); @@ -63,6 +70,15 @@ private static void assertIllegalArgument(String msg, ThrowingRunnable runnable) Assert.assertEquals(msg, ex.getMessage()); } + private static void assertMessages(long pid, long offset, List expected, DataReceivedEvent event) { + Assert.assertEquals(pid, event.getPartitionSession().getPartitionId()); + Assert.assertEquals(expected.size(), event.getMessages().size()); + for (int idx = 0; idx < event.getMessages().size(); idx++) { + Assert.assertEquals(offset + idx, event.getMessages().get(idx).getOffset()); + Assert.assertArrayEquals(expected.get(idx), event.getMessages().get(idx).getData()); + } + } + private static TestImpl startReader(ReadStreamMock mock) { return startReader(mock, Runnable::run, Runnable::run); } @@ -340,6 +356,140 @@ public void onPartitionSessionStatusTest() { mock.assertIsClosed(); } + @Test + public void mergedPartitionReadTest() { + ReadStreamMock mock = new ReadStreamMock(); + TestImpl reader = startReader(mock); + + PartitionSession ps1 = new PartitionSession(1, 100, TOPIC1); + PartitionSession ps2 = new PartitionSession(2, 200, TOPIC1); + PartitionSession ps3 = new PartitionSession(3, 300, TOPIC1); + PartitionSession ps4 = new PartitionSession(4, 400, TOPIC1); + + // start two parent partitions + mock.responseStartPartition(ps1, 0); + mock.responseStartPartition(ps2, 10); + + ArgumentCaptor start = ArgumentCaptor.forClass(StartPartitionSessionEvent.class); + ArgumentCaptor ended = ArgumentCaptor.forClass(PartitionSessionEndedEvent.class); + ArgumentCaptor ctrl = ArgumentCaptor.forClass(ReadSession.PartitionControl.class); + ArgumentCaptor dataEvent = ArgumentCaptor.forClass(DataReceivedEvent.class); + + reader.verifyHandler(Mockito.times(2)).onPartitionStarted(start.capture()); + start.getAllValues().forEach(StartPartitionSessionEvent::confirm); + + mock.responseData(100) + .partition(1, 0).batch(Codec.RAW, MSG1, MSG2, MSG3).and() + .partition(2, 10).batch(Codec.RAW, MSG3, MSG1, MSG2).and() + .send(); + + reader.verifyHandler(Mockito.times(2)).onData(ctrl.capture(), dataEvent.capture()); + ReadSession.PartitionControl ctrl1 = ctrl.getAllValues().get(0); + ReadSession.PartitionControl ctrl2 = ctrl.getAllValues().get(1); + assertMessages(100, 0, Arrays.asList(MSG1, MSG2, MSG3), dataEvent.getAllValues().get(0)); + assertMessages(200, 10, Arrays.asList(MSG3, MSG1, MSG2), dataEvent.getAllValues().get(1)); + reader.resetHandler(); + + mock.responseEndPartition(1, 300); + mock.responseEndPartition(2, 300); + + reader.verifyHandler(Mockito.never()).onPartitionEnded(ended.capture()); + + mock.responseStartPartition(ps3, 0); + mock.responseStartPartition(ps4, 0); + + reader.verifyHandler(Mockito.times(2)).onPartitionStarted(start.capture()); + start.getAllValues().forEach(StartPartitionSessionEvent::confirm); + + mock.responseData(100) + .partition(3, 30).batch(Codec.RAW, MSG3, MSG2, MSG1).and() + .partition(4, 40).batch(Codec.RAW, MSG2, MSG1, MSG3).and() + .send(); + + // Got only ps4 cause ps3 is paused + reader.verifyHandler(Mockito.times(1)).onData(ctrl.capture(), dataEvent.capture()); + assertMessages(400, 40, Arrays.asList(MSG2, MSG1, MSG3), dataEvent.getAllValues().get(2)); + reader.resetHandler(); + + ctrl2.confirmRangeProcessed(OffsetsRange.of(10, 13)); // release one of parents + + reader.verifyHandler(Mockito.times(1)).onPartitionEnded(ended.capture()); + reader.verifyHandler(Mockito.never()).onData(ctrl.capture(), dataEvent.capture()); + + ctrl1.confirmRangeProcessed(OffsetsRange.of(0, 3)); // release second of parents + + reader.verifyHandler(Mockito.times(2)).onPartitionEnded(ended.capture()); + reader.verifyHandler(Mockito.times(1)).onData(ctrl.capture(), dataEvent.capture()); + + assertMessages(300, 30, Arrays.asList(MSG3, MSG2, MSG1), dataEvent.getAllValues().get(3)); + + reader.close(); + mock.assertIsClosed(); + } + + @Test + @HideLoggers(ReadSession.class) + public void mergedPartitionWithClosedParentTest() { + ReadStreamMock mock = new ReadStreamMock(); + TestImpl reader = startReader(mock); + + PartitionSession ps1 = new PartitionSession(1, 100, TOPIC1); + PartitionSession ps2 = new PartitionSession(2, 200, TOPIC1); + PartitionSession ps3 = new PartitionSession(3, 300, TOPIC1); + + // start two parent partitions + mock.responseStartPartition(ps1, 30); + mock.responseStartPartition(ps2, 10); + + ArgumentCaptor start = ArgumentCaptor.forClass(StartPartitionSessionEvent.class); + ArgumentCaptor ended = ArgumentCaptor.forClass(PartitionSessionEndedEvent.class); + ArgumentCaptor ctrl = ArgumentCaptor.forClass(ReadSession.PartitionControl.class); + ArgumentCaptor dataEvent = ArgumentCaptor.forClass(DataReceivedEvent.class); + + reader.verifyHandler(Mockito.times(2)).onPartitionStarted(start.capture()); + start.getAllValues().forEach(StartPartitionSessionEvent::confirm); + + mock.responseData(100) + .partition(1, 40).batch(Codec.RAW, MSG1, MSG2, MSG3).and() + .partition(2, 10).batch(Codec.RAW, MSG3, MSG1, MSG2).and() + .send(); + + reader.verifyHandler(Mockito.times(2)).onData(ctrl.capture(), dataEvent.capture()); + ReadSession.PartitionControl ctrl1 = ctrl.getAllValues().get(0); + ReadSession.PartitionControl ctrl2 = ctrl.getAllValues().get(1); + assertMessages(100, 40, Arrays.asList(MSG1, MSG2, MSG3), dataEvent.getAllValues().get(0)); + assertMessages(200, 10, Arrays.asList(MSG3, MSG1, MSG2), dataEvent.getAllValues().get(1)); + reader.resetHandler(); + + mock.responseEndPartition(1, 300); + mock.responseEndPartition(2, 300); + + reader.verifyHandler(Mockito.never()).onPartitionEnded(ended.capture()); + + mock.responseStartPartition(ps3, 0); + + reader.verifyHandler(Mockito.times(1)).onPartitionStarted(start.capture()); + start.getAllValues().forEach(StartPartitionSessionEvent::confirm); + + mock.responseData(100) + .partition(3, 30).batch(Codec.RAW, MSG3, MSG2, MSG1).and() + .send(); + + // ps3 is paused + reader.verifyHandler(Mockito.never()).onData(ctrl.capture(), dataEvent.capture()); + reader.resetHandler(); + + ctrl1.confirmRangeProcessed(OffsetsRange.of(40, 43)); // release first parent + mock.responseStopPartition(2, false); // non graceful stopp of second parent + ctrl2.confirmRangeProcessed(OffsetsRange.of(10, 13)); // release has no effect + + // ps3 is paused forever + reader.verifyHandler(Mockito.never()).onData(ctrl.capture(), dataEvent.capture()); + + reader.close(); + mock.assertIsClosed(); + } + @Test public void updateTokenTest() { ReaderSettings settings = ReaderSettings.newBuilder() @@ -471,7 +621,7 @@ private TestImpl(TopicRpc rpc, String id, ReaderSettings settings, ReadConfig co this.handler = h; } - public TestImpl(TopicRpc rpc, String id, ReaderSettings settings, ReadConfig config) { + TestImpl(TopicRpc rpc, String id, ReaderSettings settings, ReadConfig config) { this(rpc, id, settings, config, Mockito.mock(ReadSession.Handler.class)); } @@ -493,6 +643,10 @@ public ReadSession.Handler verifyHandler(VerificationMode mode) { return Mockito.verify(handler, mode); } + public void resetHandler() { + Mockito.reset(handler); + } + public void assertReaderIsStopped(Status expected) { Assert.assertEquals(1, stops.size()); Assert.assertEquals(expected, stops.get(0)); @@ -523,7 +677,7 @@ public TestExecutor executeNext() { private static class TxMock implements YdbTransaction { private final CompletableFuture status; - public TxMock(CompletableFuture status) { + TxMock(CompletableFuture status) { this.status = status; } From 0f8f52ce9daceb1516cb04f50101b65162f2e66c Mon Sep 17 00:00:00 2001 From: Alexandr Gorshenin Date: Fri, 9 Oct 2026 14:54:43 +0100 Subject: [PATCH 9/9] Small fixes --- .../ydb/topic/read/impl/BufferManager.java | 13 +++-- .../ydb/topic/read/impl/ReadPartition.java | 57 ++++++++----------- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java b/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java index fe9c122a4..76d73d96e 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java @@ -122,14 +122,17 @@ public void releasePartition(Long id) { } } + public boolean hasPartition(Long id) { + PartitionBuffer part = partitions.get(id); + return part != null && !part.isEmpty(); + } + // Thread safe - public boolean releaseRange(Long id, OffsetsRange range) { + public void releaseRange(Long id, OffsetsRange range) { PartitionBuffer part = partitions.get(id); if (part != null) { release(part.release(range)); - return part.isActive(); } - return false; } private void release(long total) { @@ -182,8 +185,8 @@ public void add(BatchBuffer range) { batches.add(range); } - public boolean isActive() { - return !batches.isEmpty(); + public boolean isEmpty() { + return batches.isEmpty(); } public long release(OffsetsRange range) { diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java index c73906e73..88909462c 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java @@ -60,7 +60,6 @@ private enum State { private final AtomicReference state = new AtomicReference<>(State.CREATED); private final AtomicReference commitOffsetAck = new AtomicReference<>(null); private final AtomicReference partitionEnd = new AtomicReference<>(null); - private volatile boolean hasUnprocessedMessages = false; private volatile boolean isPaused = true; ReadPartition(String traceID, ReadSession session, PartitionSession partition, long lastCommittedOffset) { @@ -94,8 +93,7 @@ public PartitionSession getPartition() { @Override public void confirmRangeProcessed(OffsetsRange range) { decoder.releaseRange(range); - hasUnprocessedMessages = session.getBufferManager().releaseRange(partition.getId(), range); - trySendPartitionEnded(); + session.getBufferManager().releaseRange(partition.getId(), range); sendDataToReaders(); } @@ -103,7 +101,7 @@ public void confirmPartitionEnded(List childs) { PartitionSessionEndedEventImpl event = new PartitionSessionEndedEventImpl(partition, childs); logger.info("[{}] got EndPartitionSession with child partitions {}", traceID, event.getChildsString()); partitionEnd.set(event); - trySendPartitionEnded(); + sendDataToReaders(); } public void confirmCommittedOffset(long committedOffset) { @@ -114,33 +112,11 @@ public void confirmCommittedOffset(long committedOffset) { } public void unpause() { - isPaused = false; if (state.get() != State.CREATED) { logger.info("[{}] was unpaused and ready to send data", traceID); - sendDataToReaders(); - } - } - - private void trySendCommitOffsetAck() { - CommitOffsetAcknowledgementEvent event = commitOffsetAck.getAndSet(null); - if (event != null) { - session.getHandler().onCommitAck(event); - } - } - - private void trySendPartitionEnded() { - if (hasUnprocessedMessages) { - return; - } - PartitionSessionEndedEventImpl event = partitionEnd.getAndSet(null); - if (event != null) { - dataExecutor.execute(() -> { - logger.info("[{}] has finished processing and unpaused child partitions {}", traceID, - event.getChildsString()); - session.getHandler().onPartitionEnded(event); - session.releaseLocks(partition, event.getChilds()); - }); } + isPaused = false; + sendDataToReaders(); } public boolean addBatches(List batchList) { @@ -154,7 +130,6 @@ public boolean addBatches(List ba metrics.reportReceivedMessages(messagesCount, partition.getPath()); if (messagesCount > 0) { - hasUnprocessedMessages = true; queue.addBatches(batchList); sendDataToReaders(); } @@ -162,16 +137,24 @@ public boolean addBatches(List ba return isActive(); } + private void sendCommitOffsetAck() { + CommitOffsetAcknowledgementEvent event = commitOffsetAck.getAndSet(null); + if (event != null) { + session.getHandler().onCommitAck(event); + } + } + private void sendDataToReaders() { dataExecutor.execute(() -> { - trySendCommitOffsetAck(); // ack may be sent even state is not active + sendCommitOffsetAck(); // ack may be sent even state is not active if (isPaused) { return; } + while (state.get() == State.STARTED || state.get() == State.PRE_STOPPED) { List list = queue.getNextBatch(); if (list == null) { - return; + break; } DataReceivedEventImpl event = new DataReceivedEventImpl(partition, committer, list); @@ -185,7 +168,17 @@ private void sendDataToReaders() { logger.debug("[{}] onData with {} message(s) (offsets {}-{}) successfully finished", traceID, messagesCount, offsetStart, offsetEnd); - trySendCommitOffsetAck(); + sendCommitOffsetAck(); + } + + if (!session.getBufferManager().hasPartition(partition.getId())) { + PartitionSessionEndedEventImpl event = partitionEnd.getAndSet(null); + if (event != null) { + logger.info("[{}] has finished processing and unpaused child partitions {}", traceID, + event.getChildsString()); + session.getHandler().onPartitionEnded(event); + session.releaseLocks(partition, event.getChilds()); + } } }); }