From 47e18b905177c9ef276a7f422e8301ad01c7dd31 Mon Sep 17 00:00:00 2001 From: KirillKurdyukov Date: Thu, 8 Oct 2026 17:00:00 +0300 Subject: [PATCH 1/2] Add Topic Writer buffer byte gauges --- .../ydb/topic/write/impl/BufferManager.java | 6 +++ .../ydb/topic/write/impl/WriterMetrics.java | 25 +++++++++- .../ydb/topic/write/impl/WriterQueue.java | 2 + .../ydb/topic/write/impl/WriterImplTest.java | 47 +++++++++++++++++++ 4 files changed, 79 insertions(+), 1 deletion(-) diff --git a/topic/src/main/java/tech/ydb/topic/write/impl/BufferManager.java b/topic/src/main/java/tech/ydb/topic/write/impl/BufferManager.java index fe48f96b8..fe6bc46fb 100644 --- a/topic/src/main/java/tech/ydb/topic/write/impl/BufferManager.java +++ b/topic/src/main/java/tech/ydb/topic/write/impl/BufferManager.java @@ -45,6 +45,12 @@ public long getMaxSize() { return bufferMaxSize; } + long getUsedSize() { + long usedBlocks = (long) calculateBlocksCount(bufferMaxSize, blockBitsCount) + - blocksAvailable.availablePermits(); + return Math.max(0, usedBlocks << blockBitsCount); + } + public void close(Status status) { this.closed = status; // release all waiters diff --git a/topic/src/main/java/tech/ydb/topic/write/impl/WriterMetrics.java b/topic/src/main/java/tech/ydb/topic/write/impl/WriterMetrics.java index 0b2f99697..3cf097710 100644 --- a/topic/src/main/java/tech/ydb/topic/write/impl/WriterMetrics.java +++ b/topic/src/main/java/tech/ydb/topic/write/impl/WriterMetrics.java @@ -1,11 +1,14 @@ package tech.ydb.topic.write.impl; +import java.util.function.LongSupplier; + import tech.ydb.core.metrics.Attr; import tech.ydb.core.metrics.LongCounter; import tech.ydb.core.metrics.Meter; +import tech.ydb.core.metrics.MetricRegistration; /** - * Topic writer counters. + * Topic writer metrics. */ final class WriterMetrics { private static final String MESSAGE_UNIT = "{message}"; @@ -14,8 +17,12 @@ final class WriterMetrics { private final LongCounter sendingBytes; private final LongCounter writtenMessages; private final Attr[] commonAttributes; + private final Meter meter; + private MetricRegistration bufferUsedGauge = MetricRegistration.NOOP; + private MetricRegistration bufferLimitGauge = MetricRegistration.NOOP; WriterMetrics(Meter meter, String topic, String writerName) { + this.meter = meter; this.sendingMessages = meter.createCounter("ydb.topic.writer.sending.messages", MESSAGE_UNIT, "The number of messages accepted by the SDK for sending."); this.sendingBytes = meter.createCounter("ydb.topic.writer.sending.bytes", "By", @@ -27,6 +34,22 @@ final class WriterMetrics { : new Attr[]{Attr.of("topic", topic), Attr.of("writer.name", writerName)}; } + void register(LongSupplier bufferUsed, LongSupplier bufferLimit) { + bufferUsedGauge = meter.registerLongGauge("ydb.topic.writer.buffer.used.bytes", "By", + "The occupied budget of the writer buffer limiter.", + m -> m.record(bufferUsed.getAsLong(), commonAttributes)); + bufferLimitGauge = meter.registerLongGauge("ydb.topic.writer.buffer.limit.bytes", "By", + "The configured limit of the writer buffer limiter.", + m -> m.record(bufferLimit.getAsLong(), commonAttributes)); + } + + void unregister() { + bufferUsedGauge.close(); + bufferLimitGauge.close(); + bufferUsedGauge = MetricRegistration.NOOP; + bufferLimitGauge = MetricRegistration.NOOP; + } + void reportSending(long bytes) { sendingMessages.add(1, commonAttributes); sendingBytes.add(bytes, commonAttributes); diff --git a/topic/src/main/java/tech/ydb/topic/write/impl/WriterQueue.java b/topic/src/main/java/tech/ydb/topic/write/impl/WriterQueue.java index dcccb60bd..709725373 100644 --- a/topic/src/main/java/tech/ydb/topic/write/impl/WriterQueue.java +++ b/topic/src/main/java/tech/ydb/topic/write/impl/WriterQueue.java @@ -71,6 +71,7 @@ public WriterQueue(String debugId, WriterSettings settings, CodecRegistry codecR this.compressionExecutor = compressionExecutor; this.readyNotify = readyNotify; this.metrics = new WriterMetrics(settings.getMeter(), settings.getTopicPath(), settings.getWriterName()); + this.metrics.register(buffer::getUsedSize, buffer::getMaxSize); } CompletableFuture flush() { @@ -154,6 +155,7 @@ void confirmAck(WriteAck ack) { } void close(Status status) { + metrics.unregister(); buffer.close(status); while (!queue.isEmpty()) { diff --git a/topic/src/test/java/tech/ydb/topic/write/impl/WriterImplTest.java b/topic/src/test/java/tech/ydb/topic/write/impl/WriterImplTest.java index dd59e5999..c2e7db584 100644 --- a/topic/src/test/java/tech/ydb/topic/write/impl/WriterImplTest.java +++ b/topic/src/test/java/tech/ydb/topic/write/impl/WriterImplTest.java @@ -11,6 +11,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.LongSupplier; import org.junit.Assert; import org.junit.Test; @@ -23,7 +25,9 @@ import tech.ydb.core.grpc.GrpcReadStream; import tech.ydb.core.grpc.GrpcReadWriteStream; import tech.ydb.core.metrics.LongCounter; +import tech.ydb.core.metrics.LongMeasurement; import tech.ydb.core.metrics.Meter; +import tech.ydb.core.metrics.MetricRegistration; import tech.ydb.proto.StatusCodesProtos; import tech.ydb.proto.topic.YdbTopic; import tech.ydb.proto.topic.YdbTopic.StreamWriteMessage.FromClient; @@ -394,6 +398,49 @@ public LongCounter createCounter(String name, String unit, String description) { } } + @Test + public void writerBufferGaugesTrackBudgetAndLimit() throws QueueOverflowException { + Map gauges = new HashMap<>(); + Meter meter = new Meter() { + @Override + public MetricRegistration registerLongGauge(String name, String unit, String description, + Consumer callback) { + gauges.put(name, () -> { + long[] value = new long[1]; + callback.accept((observed, attrs) -> value[0] = observed); + return value[0]; + }); + return () -> gauges.remove(name); + } + }; + String used = "ydb.topic.writer.buffer.used.bytes"; + String limit = "ydb.topic.writer.buffer.limit.bytes"; + StreamMock stream = new StreamMock(); + WriterSettings settings = WriterSettings.newBuilder() + .setTopicPath("/test/topic").setCodec(Codec.RAW).setMaxSendBufferMemorySize(100) + .withMeter(meter, "writer").build(); + try (TopicClient client = TopicClientImpl.newClient(mockRpc(stream)) + .setCompressionExecutor(Runnable::run).build()) { + AsyncWriter writer = client.createAsyncWriter(settings); + try { + writer.init(); + stream.sendInitResponse(0); + Assert.assertEquals(0, gauges.get(used).getAsLong()); + Assert.assertEquals(100, gauges.get(limit).getAsLong()); + writer.send(Message.of(new byte[]{1, 2})); + writer.send(Message.of(new byte[]{3, 4, 5})); + Assert.assertEquals(5, gauges.get(used).getAsLong()); + Assert.assertEquals(100, gauges.get(limit).getAsLong()); + stream.sendAckResponse(2, 10); + Assert.assertEquals(0, gauges.get(used).getAsLong()); + } finally { + writer.shutdown(); + stream.close(Status.SUCCESS); + } + } + Assert.assertTrue(gauges.isEmpty()); + } + private static class StreamMock implements GrpcReadWriteStream { private final CompletableFuture future = new CompletableFuture<>(); private final List messages = new ArrayList<>(); From df2f243a05df12e67982aa62f6d41d5564b23bb0 Mon Sep 17 00:00:00 2001 From: KirillKurdyukov Date: Fri, 9 Oct 2026 00:12:13 +0300 Subject: [PATCH 2/2] Remove Writer buffer gauge clamp --- .../src/main/java/tech/ydb/topic/write/impl/BufferManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/topic/src/main/java/tech/ydb/topic/write/impl/BufferManager.java b/topic/src/main/java/tech/ydb/topic/write/impl/BufferManager.java index fe6bc46fb..28b0a2bac 100644 --- a/topic/src/main/java/tech/ydb/topic/write/impl/BufferManager.java +++ b/topic/src/main/java/tech/ydb/topic/write/impl/BufferManager.java @@ -48,7 +48,7 @@ public long getMaxSize() { long getUsedSize() { long usedBlocks = (long) calculateBlocksCount(bufferMaxSize, blockBitsCount) - blocksAvailable.availablePermits(); - return Math.max(0, usedBlocks << blockBitsCount); + return usedBlocks << blockBitsCount; } public void close(Status status) {