Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ public long getMaxSize() {
return bufferMaxSize;
}

long getUsedSize() {
long usedBlocks = (long) calculateBlocksCount(bufferMaxSize, blockBitsCount)
- blocksAvailable.availablePermits();
return usedBlocks << blockBitsCount;
}

public void close(Status status) {
this.closed = status;
// release all waiters
Expand Down
25 changes: 24 additions & 1 deletion topic/src/main/java/tech/ydb/topic/write/impl/WriterMetrics.java
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
package tech.ydb.topic.write.impl;

import java.util.function.LongSupplier;

import tech.ydb.core.metrics.Attr;
import tech.ydb.core.metrics.LongCounter;
import tech.ydb.core.metrics.Meter;
import tech.ydb.core.metrics.MetricRegistration;

/**
* Topic writer counters.
* Topic writer metrics.
*/
final class WriterMetrics {
private static final String MESSAGE_UNIT = "{message}";
Expand All @@ -14,8 +17,12 @@ final class WriterMetrics {
private final LongCounter sendingBytes;
private final LongCounter writtenMessages;
private final Attr[] commonAttributes;
private final Meter meter;
private MetricRegistration bufferUsedGauge = MetricRegistration.NOOP;
private MetricRegistration bufferLimitGauge = MetricRegistration.NOOP;

WriterMetrics(Meter meter, String topic, String writerName) {
this.meter = meter;
this.sendingMessages = meter.createCounter("ydb.topic.writer.sending.messages", MESSAGE_UNIT,
"The number of messages accepted by the SDK for sending.");
this.sendingBytes = meter.createCounter("ydb.topic.writer.sending.bytes", "By",
Expand All @@ -27,6 +34,22 @@ final class WriterMetrics {
: new Attr[]{Attr.of("topic", topic), Attr.of("writer.name", writerName)};
}

void register(LongSupplier bufferUsed, LongSupplier bufferLimit) {
bufferUsedGauge = meter.registerLongGauge("ydb.topic.writer.buffer.used.bytes", "By",
"The occupied budget of the writer buffer limiter.",
m -> m.record(bufferUsed.getAsLong(), commonAttributes));
bufferLimitGauge = meter.registerLongGauge("ydb.topic.writer.buffer.limit.bytes", "By",
"The configured limit of the writer buffer limiter.",
m -> m.record(bufferLimit.getAsLong(), commonAttributes));
}

void unregister() {
bufferUsedGauge.close();
bufferLimitGauge.close();
bufferUsedGauge = MetricRegistration.NOOP;
bufferLimitGauge = MetricRegistration.NOOP;
}

void reportSending(long bytes) {
sendingMessages.add(1, commonAttributes);
sendingBytes.add(bytes, commonAttributes);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ public WriterQueue(String debugId, WriterSettings settings, CodecRegistry codecR
this.compressionExecutor = compressionExecutor;
this.readyNotify = readyNotify;
this.metrics = new WriterMetrics(settings.getMeter(), settings.getTopicPath(), settings.getWriterName());
this.metrics.register(buffer::getUsedSize, buffer::getMaxSize);
}

CompletableFuture<Void> flush() {
Expand Down Expand Up @@ -154,6 +155,7 @@ void confirmAck(WriteAck ack) {
}

void close(Status status) {
metrics.unregister();
buffer.close(status);

while (!queue.isEmpty()) {
Expand Down
47 changes: 47 additions & 0 deletions topic/src/test/java/tech/ydb/topic/write/impl/WriterImplTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.function.LongSupplier;

import org.junit.Assert;
import org.junit.Test;
Expand All @@ -23,7 +25,9 @@
import tech.ydb.core.grpc.GrpcReadStream;
import tech.ydb.core.grpc.GrpcReadWriteStream;
import tech.ydb.core.metrics.LongCounter;
import tech.ydb.core.metrics.LongMeasurement;
import tech.ydb.core.metrics.Meter;
import tech.ydb.core.metrics.MetricRegistration;
import tech.ydb.proto.StatusCodesProtos;
import tech.ydb.proto.topic.YdbTopic;
import tech.ydb.proto.topic.YdbTopic.StreamWriteMessage.FromClient;
Expand Down Expand Up @@ -394,6 +398,49 @@ public LongCounter createCounter(String name, String unit, String description) {
}
}

@Test
public void writerBufferGaugesTrackBudgetAndLimit() throws QueueOverflowException {
Map<String, LongSupplier> gauges = new HashMap<>();
Meter meter = new Meter() {
@Override
public MetricRegistration registerLongGauge(String name, String unit, String description,
Consumer<LongMeasurement> callback) {
gauges.put(name, () -> {
long[] value = new long[1];
callback.accept((observed, attrs) -> value[0] = observed);
return value[0];
});
return () -> gauges.remove(name);
}
};
String used = "ydb.topic.writer.buffer.used.bytes";
String limit = "ydb.topic.writer.buffer.limit.bytes";
StreamMock stream = new StreamMock();
WriterSettings settings = WriterSettings.newBuilder()
.setTopicPath("/test/topic").setCodec(Codec.RAW).setMaxSendBufferMemorySize(100)
.withMeter(meter, "writer").build();
try (TopicClient client = TopicClientImpl.newClient(mockRpc(stream))
.setCompressionExecutor(Runnable::run).build()) {
AsyncWriter writer = client.createAsyncWriter(settings);
try {
writer.init();
stream.sendInitResponse(0);
Assert.assertEquals(0, gauges.get(used).getAsLong());
Assert.assertEquals(100, gauges.get(limit).getAsLong());
writer.send(Message.of(new byte[]{1, 2}));
writer.send(Message.of(new byte[]{3, 4, 5}));
Assert.assertEquals(5, gauges.get(used).getAsLong());
Assert.assertEquals(100, gauges.get(limit).getAsLong());
stream.sendAckResponse(2, 10);
Assert.assertEquals(0, gauges.get(used).getAsLong());
} finally {
writer.shutdown();
stream.close(Status.SUCCESS);
}
}
Assert.assertTrue(gauges.isEmpty());
}

private static class StreamMock implements GrpcReadWriteStream<FromServer, FromClient> {
private final CompletableFuture<Status> future = new CompletableFuture<>();
private final List<FromClient> messages = new ArrayList<>();
Expand Down
Loading