From c62a89150c7398d0d23978b9a2f373b115030cb7 Mon Sep 17 00:00:00 2001 From: KirillKurdyukov Date: Thu, 22 Oct 2026 16:34:27 +0300 Subject: [PATCH 1/4] Test Topic metric accounting --- .../topic/read/impl/ReaderMetricsTest.java | 356 ++++++++++++------ .../tech/ydb/topic/utils/RecordingMeter.java | 104 +++++ .../topic/write/impl/WriterMetricsTest.java | 226 +++++++---- 3 files changed, 497 insertions(+), 189 deletions(-) create mode 100644 topic/src/test/java/tech/ydb/topic/utils/RecordingMeter.java diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java index 6c656a8d3..1627a107c 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java @@ -1,14 +1,11 @@ package tech.ydb.topic.read.impl; -import java.util.ArrayList; +import java.io.IOException; import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Consumer; import org.junit.Assert; import org.junit.Test; @@ -16,153 +13,278 @@ import tech.ydb.core.Status; import tech.ydb.core.StatusCode; -import tech.ydb.core.metrics.Attr; -import tech.ydb.core.metrics.LongCounter; -import tech.ydb.core.metrics.LongMeasurement; -import tech.ydb.core.metrics.Meter; -import tech.ydb.core.metrics.MetricRegistration; import tech.ydb.topic.TopicClient; import tech.ydb.topic.TopicRpc; import tech.ydb.topic.description.Codec; import tech.ydb.topic.impl.TopicClientImpl; +import tech.ydb.topic.read.Message; import tech.ydb.topic.read.SyncReader; import tech.ydb.topic.settings.ReaderSettings; -import tech.ydb.topic.settings.TopicReadSettings; +import tech.ydb.topic.utils.RecordingMeter; public class ReaderMetricsTest { + private static final String PREFIX = "ydb.topic.reader."; private static final byte[] MSG1 = new byte[] { 1 }; private static final byte[] MSG2 = new byte[] { 2 }; private static final byte[] MSG3 = new byte[] { 3 }; + private final RecordingMeter meter = new RecordingMeter(); @Test public void readerCountersIncrementOnReceive() throws InterruptedException { - String delivered = "ydb.topic.reader.delivered.messages"; - String receivedMessages = "ydb.topic.reader.received.messages"; - String receivedBytes = "ydb.topic.reader.received.bytes"; - RecordingMeter meter = new RecordingMeter(); - ReadStreamMock stream = new ReadStreamMock(true); - TopicRpc rpc = Mockito.mock(TopicRpc.class); - Mockito.when(rpc.getScheduler()).thenReturn(Mockito.mock(ScheduledExecutorService.class)); - Mockito.when(rpc.readSession(Mockito.any(String.class))).thenReturn(stream); - - TopicClient client = TopicClientImpl.newClient(rpc).build(); - SyncReader reader = client.createSyncReader(ReaderSettings.newBuilder() - .addTopic(TopicReadSettings.newBuilder().setPath("/topic").build()) - .setConsumerName("consumer") - .withMeter(meter, "reader") - .build()); - - try { - reader.init(); - stream.assertLastMessage().isInitRequest("consumer", "/topic").hasReaderName("reader"); - stream.responseInit("read-session"); - stream.responseStartPartition("/topic", 42, 0); - stream.responseData(25).partition(1, 0) - .batch(Codec.RAW, MSG1, MSG2) - .and().send(); - Assert.assertEquals(2, meter.value(receivedMessages)); - Assert.assertEquals(25, meter.value(receivedBytes)); - - stream.responseData(30).partition(1, 2) - .batch(Codec.RAW, MSG1, MSG2, MSG3) - .and().send(); - Assert.assertEquals(5, meter.value(receivedMessages)); - Assert.assertEquals(55, meter.value(receivedBytes)); - - Assert.assertEquals(0, meter.value(delivered)); - meter.assertAttribute(receivedMessages, "topic", "/topic"); - meter.assertAttribute(receivedMessages, "reader.name", "reader"); - - reader.receive(); - reader.receive(); - Assert.assertEquals(2, meter.value(delivered)); - } finally { - reader.shutdown(); - client.close(); + try (ReaderFixture f = new ReaderFixture(meter)) { + f.start(); + f.stream.responseStartPartition("/topic", 42, 0); + f.stream.responseData(25).partition(1, 0).batch(Codec.RAW, MSG1, MSG2).and().send(); + Assert.assertEquals(2, meter.value(PREFIX + "received.messages")); + Assert.assertEquals(25, meter.value(PREFIX + "received.bytes")); + f.stream.responseData(30).partition(1, 2).batch(Codec.RAW, MSG1, MSG2, MSG3).and().send(); + Assert.assertEquals(5, meter.value(PREFIX + "received.messages")); + Assert.assertEquals(55, meter.value(PREFIX + "received.bytes")); + Assert.assertEquals(5, meter.collect(PREFIX + "local_buffer.messages")); + Assert.assertEquals(0, meter.value(PREFIX + "delivered.messages")); + meter.assertAttributes(PREFIX + "received.messages", + "consumer", "consumer", "reader.name", "reader", "topic", "/topic"); + Assert.assertNotNull(f.reader.receive(0, TimeUnit.SECONDS)); + Assert.assertNotNull(f.reader.receive(0, TimeUnit.SECONDS)); + Assert.assertEquals(2, meter.value(PREFIX + "delivered.messages")); + Assert.assertEquals(3, meter.collect(PREFIX + "local_buffer.messages")); + } + } + + @Test + public void commitMetricsFollowPublicAcknowledgement() throws InterruptedException { + try (ReaderFixture f = new ReaderFixture(meter)) { + f.start(); + f.stream.responseStartPartition("/topic", 42, 0); + f.stream.responseData(5).partition(1, 0).batch(Codec.RAW, MSG1).and().send(); + Message message = f.reader.receive(0, TimeUnit.SECONDS); + Assert.assertNotNull(message); + CompletableFuture committed = message.commit(); + f.stream.assertLastMessage().isCommit(1); + Assert.assertFalse(committed.isDone()); + Assert.assertEquals(1, meter.value(PREFIX + "commit.queued")); + Assert.assertEquals(1, meter.collect(PREFIX + "commit_offset.lag.max")); + f.stream.responseCommitAck().partition(1, 1).send(); + Assert.assertTrue(committed.isDone()); + Assert.assertFalse(committed.isCompletedExceptionally()); + Assert.assertEquals(1, meter.value(PREFIX + "commit.acknowledged")); + Assert.assertEquals(0, meter.collect(PREFIX + "commit_offset.lag.max")); } } + @Test + public void commitCountersUseAcceptedWidthsAndAcknowledgedFrontier() { + ReaderMetrics metrics = registeredMetrics(); + metrics.addPartition(1, 10); + metrics.reportCommitQueued(1, 15, 3, "/topic"); + metrics.reportCommitQueued(1, 12, 2, "/topic"); + Assert.assertEquals(5, meter.value(PREFIX + "commit.queued")); + Assert.assertEquals(5, meter.collect(PREFIX + "commit_offset.lag.max")); + metrics.reportCommitAcknowledged(1, 13, "/topic"); + metrics.reportCommitAcknowledged(1, 13, "/topic"); + metrics.reportCommitAcknowledged(1, 12, "/topic"); + Assert.assertEquals(3, meter.value(PREFIX + "commit.acknowledged")); + metrics.reportCommitAcknowledged(1, 15, "/topic"); + Assert.assertEquals(5, meter.value(PREFIX + "commit.acknowledged")); + Assert.assertEquals(0, meter.collect(PREFIX + "commit_offset.lag.max")); + for (String name : Arrays.asList("commit.queued", "commit.acknowledged")) { + meter.assertAttributes(PREFIX + name, "consumer", "consumer", "reader.name", "reader", "topic", "/topic"); + } + } + + @Test + public void commitLagUsesMaximumAndDropsClosedPartitions() { + ReaderMetrics metrics = registeredMetrics(); + metrics.addPartition(1, 0); + metrics.addPartition(2, 0); + metrics.reportCommitQueued(1, 20, 2, "/topic"); + metrics.reportCommitQueued(1, 10, 1, "/topic"); + metrics.reportCommitQueued(2, 7, 7, "/topic"); + Assert.assertEquals(20, meter.collect(PREFIX + "commit_offset.lag.max")); + metrics.updateCommittedOffset(1, 25); + Assert.assertEquals(7, meter.collect(PREFIX + "commit_offset.lag.max")); + metrics.removePartition(2); + Assert.assertEquals(0, meter.collect(PREFIX + "commit_offset.lag.max")); + Assert.assertEquals(1, meter.collect(PREFIX + "partition_session.count")); + } + + @Test + public void bufferCountsUseCurrentSourcesAndRemoveClosedOnes() { + ReaderMetrics metrics = registeredMetrics(); + ReaderMetrics.Source first = source(2, 0); + ReaderMetrics.Source second = source(3, 0); + metrics.addSource(first); + metrics.addSource(second); + Assert.assertEquals(5, meter.collect(PREFIX + "local_buffer.messages")); + Mockito.when(first.getMessagesCount()).thenReturn(4L); + Assert.assertEquals(7, meter.collect(PREFIX + "local_buffer.messages")); + metrics.removeSource(first); + Assert.assertEquals(3, meter.collect(PREFIX + "local_buffer.messages")); + metrics.removeSource(second); + Assert.assertEquals(0, meter.collect(PREFIX + "local_buffer.messages")); + meter.assertAttributes(PREFIX + "local_buffer.messages", "consumer", "consumer", "reader.name", "reader"); + } + + @Test + public void bufferAgeUsesOldestNonZeroSourceInSeconds() { + ReaderMetrics metrics = registeredMetrics(); + Assert.assertEquals(0, meter.collectDouble(PREFIX + "local_buffer.message_age.max"), 0); + long oldest = System.nanoTime() - TimeUnit.DAYS.toNanos(1); + long newer = oldest + TimeUnit.HOURS.toNanos(12); + ReaderMetrics.Source first = source(1, oldest); + metrics.addSource(first); + metrics.addSource(source(2, newer)); + metrics.addSource(source(3, 0)); + assertBufferAge(oldest); + metrics.removeSource(first); + assertBufferAge(newer); + metrics.unregister(); + Assert.assertTrue(meter.doubleGauges.isEmpty()); + } + @Test public void gaugesObservePartitionSessionsAndProtocolCredit() throws InterruptedException { - String partitions = "ydb.topic.reader.partition_session.count"; - String credit = "ydb.topic.reader.credit_balance_bytes"; - RecordingMeter meter = new RecordingMeter(); - ReadStreamMock stream = new ReadStreamMock(true); - TopicRpc rpc = Mockito.mock(TopicRpc.class); - Mockito.when(rpc.getScheduler()).thenReturn(Mockito.mock(ScheduledExecutorService.class)); - Mockito.when(rpc.readSession(Mockito.anyString())).thenReturn(stream); - TopicClient client = TopicClientImpl.newClient(rpc).build(); - SyncReader reader = client.createSyncReader(ReaderSettings.newBuilder() - .addTopic("/topic").setConsumerName("consumer") - .setMaxMemoryUsageBytes(100).withMeter(meter, "reader").build()); - try { + try (ReaderFixture f = new ReaderFixture(meter)) { Assert.assertTrue(meter.gauges.isEmpty()); - reader.init(); + f.reader.init(); Assert.assertTrue(meter.gauges.isEmpty()); - stream.responseInit("session"); - Assert.assertEquals(0, meter.collect(partitions)); - Assert.assertEquals(100, meter.collect(credit)); - stream.responseStartPartition("/topic", 42, 0); - Assert.assertEquals(1, meter.collect(partitions)); - Assert.assertEquals(100, meter.collect(credit)); - stream.responseData(20).partition(1, 0).batch(Codec.RAW, MSG1, MSG2).and().send(); - Assert.assertEquals(80, meter.collect(credit)); - Assert.assertNotNull(reader.receive(1, TimeUnit.SECONDS)); - Assert.assertEquals(80, meter.collect(credit)); - Assert.assertNotNull(reader.receive(1, TimeUnit.SECONDS)); - Assert.assertEquals(100, meter.collect(credit)); - stream.closeStream(Status.of(StatusCode.OVERLOADED)); + f.stream.responseInit("session"); + assertReaderGauges(0, 100); + f.stream.responseStartPartition("/topic", 42, 0); + assertReaderGauges(1, 100); + f.stream.responseData(20).partition(1, 0).batch(Codec.RAW, MSG1, MSG2).and().send(); + assertReaderGauges(1, 80); + Assert.assertNotNull(f.reader.receive(0, TimeUnit.SECONDS)); + assertReaderGauges(1, 80); + Assert.assertNotNull(f.reader.receive(0, TimeUnit.SECONDS)); + assertReaderGauges(1, 100); + f.stream.closeStream(Status.of(StatusCode.OVERLOADED)); Assert.assertTrue(meter.gauges.isEmpty()); - } finally { - reader.shutdown(); - client.close(); } Assert.assertTrue(meter.gauges.isEmpty()); } - private static class RecordingMeter implements Meter { - private final Map counters = new ConcurrentHashMap<>(); - private final Map attributes = new ConcurrentHashMap<>(); - private final Map>> gauges = new ConcurrentHashMap<>(); + @Test + public void creditCanBeNegativeWhenServerExceedsWindow() throws InterruptedException { + try (ReaderFixture f = new ReaderFixture(meter)) { + f.start(); + f.stream.responseStartPartition("/topic", 42, 0); + f.stream.responseData(120).partition(1, 0).batch(Codec.RAW, MSG1).and().send(); + assertReaderGauges(1, -20); + Assert.assertNotNull(f.reader.receive(0, TimeUnit.SECONDS)); + assertReaderGauges(1, 100); + } + } - @Override - public MetricRegistration registerLongGauge( - String name, String unit, String description, Consumer callback) { - List> callbacks = gauges.computeIfAbsent(name, key -> new ArrayList<>()); - callbacks.add(callback); - return () -> { - callbacks.remove(callback); - if (callbacks.isEmpty()) { - gauges.remove(name, callbacks); - } - }; + @Test + public void nonPositiveDeltasDoNotChangeCounters() { + ReaderMetrics metrics = new ReaderMetrics(meter, "consumer", "reader"); + metrics.reportDelivered(2, "/topic"); + metrics.reportReceivedMessages(3, "/topic"); + metrics.reportReceivedBytes(5); + for (long ignored : new long[]{0, -1}) { + metrics.reportDelivered(ignored, "/topic"); + metrics.reportReceivedMessages(ignored, "/topic"); + metrics.reportReceivedBytes(ignored); } + Assert.assertEquals(2, meter.value(PREFIX + "delivered.messages")); + Assert.assertEquals(3, meter.value(PREFIX + "received.messages")); + Assert.assertEquals(5, meter.value(PREFIX + "received.bytes")); + Assert.assertEquals("By", meter.units.get(PREFIX + "received.bytes")); + } - long collect(String name) { - Long[] value = new Long[1]; - Assert.assertEquals(1, gauges.get(name).size()); - gauges.get(name).get(0).accept((observed, attrs) -> value[0] = observed); - return value[0]; + @Test + public void decodedDurationsUseSeconds() { + ReaderMetrics metrics = new ReaderMetrics(meter, "consumer", "reader"); + metrics.reportDecoded(250_000_000, 1_500_000_000, Codec.GZIP, "/topic", null); + Assert.assertEquals(Collections.singletonList(0.25), meter.histograms.get(PREFIX + "compression.duration")); + Assert.assertEquals(Collections.singletonList(1.5), meter.histograms.get(PREFIX + "message.processing.duration")); + for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { + Assert.assertEquals("s", meter.units.get(PREFIX + name)); + meter.assertAttributes(PREFIX + name, "consumer", "consumer", "reader.name", "reader", + "topic", "/topic", "codec", "gzip", "outcome", "ok"); } + } - @Override - public LongCounter createCounter(String name, String unit, String description) { - AtomicLong counter = counters.computeIfAbsent(name, key -> new AtomicLong()); - return (value, attrs) -> { - counter.addAndGet(value); - attributes.put(name, attrs); - }; + @Test + public void decodedFailuresKeepCodecAndErrorWithoutConsumer() { + ReaderMetrics metrics = new ReaderMetrics(meter, null, "reader"); + metrics.reportDecoded(500_000_000, 2_000_000_000, 10001, "/topic", new IOException()); + Assert.assertEquals(Collections.singletonList(0.5), meter.histograms.get(PREFIX + "compression.duration")); + Assert.assertEquals(Collections.singletonList(2.0), meter.histograms.get(PREFIX + "message.processing.duration")); + for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { + meter.assertAttributes(PREFIX + name, "reader.name", "reader", "topic", "/topic", + "codec", "10001", "outcome", "error", "error.type", "java.io.IOException"); } + } - long value(String name) { - AtomicLong counter = counters.get(name); - return counter == null ? 0 : counter.get(); + @Test + public void sessionErrorsCountRetryAndStopOnce() { + ReaderMetrics metrics = new ReaderMetrics(meter, "consumer", "reader"); + metrics.reportSessionError(Status.of(StatusCode.TRANSPORT_UNAVAILABLE), true); + Assert.assertEquals(1, meter.value(PREFIX + "session.errors")); + meter.assertAttributes(PREFIX + "session.errors", "consumer", "consumer", "reader.name", "reader", + "retry_decision", "retry", "status_code", "TRANSPORT_UNAVAILABLE", "error.type", "transport_error"); + metrics.reportSessionError(Status.of(StatusCode.BAD_REQUEST), false); + Assert.assertEquals(2, meter.value(PREFIX + "session.errors")); + meter.assertAttributes(PREFIX + "session.errors", "consumer", "consumer", "reader.name", "reader", + "retry_decision", "stop", "status_code", "BAD_REQUEST", "error.type", "ydb_error"); + } + + private ReaderMetrics registeredMetrics() { + ReaderMetrics metrics = new ReaderMetrics(meter, "consumer", "reader"); + metrics.register(() -> 100); + return metrics; + } + + private static ReaderMetrics.Source source(long count, long timestamp) { + ReaderMetrics.Source source = Mockito.mock(ReaderMetrics.Source.class); + Mockito.when(source.getMessagesCount()).thenReturn(count); + Mockito.when(source.getOldestReceivedTimestamp()).thenReturn(timestamp); + return source; + } + + private void assertBufferAge(long timestamp) { + long before = System.nanoTime(); + double age = meter.collectDouble(PREFIX + "local_buffer.message_age.max"); + long after = System.nanoTime(); + Assert.assertEquals((before - timestamp) / 1_000_000_000d, age, (after - before) / 1_000_000_000d); + Assert.assertEquals("s", meter.units.get(PREFIX + "local_buffer.message_age.max")); + } + + private void assertReaderGauges(long partitions, long credit) { + Assert.assertEquals(partitions, meter.collect(PREFIX + "partition_session.count")); + Assert.assertEquals(credit, meter.collect(PREFIX + "credit_balance_bytes")); + } + + private static final class ReaderFixture implements AutoCloseable { + private final ReadStreamMock stream = new ReadStreamMock(true); + private final TopicClient client; + private final SyncReader reader; + + ReaderFixture(RecordingMeter meter) { + TopicRpc rpc = Mockito.mock(TopicRpc.class); + Mockito.when(rpc.getScheduler()).thenReturn(Mockito.mock(ScheduledExecutorService.class)); + Mockito.when(rpc.readSession(Mockito.anyString())).thenReturn(stream); + client = TopicClientImpl.newClient(rpc).build(); + reader = client.createSyncReader(ReaderSettings.newBuilder().addTopic("/topic") + .setConsumerName("consumer").setMaxMemoryUsageBytes(100).withMeter(meter, "reader").build()); } - void assertAttribute(String name, String key, String value) { - Assert.assertTrue(Arrays.stream(attributes.get(name)) - .anyMatch(attr -> attr.getKey().equals(key) && attr.getValue().equals(value))); + void start() { + reader.init(); + stream.assertLastMessage().isInitRequest("consumer", "/topic").hasReaderName("reader"); + stream.responseInit("read-session"); + } + + @Override + public void close() { + try { + reader.shutdown(); + } finally { + client.close(); + } } } } diff --git a/topic/src/test/java/tech/ydb/topic/utils/RecordingMeter.java b/topic/src/test/java/tech/ydb/topic/utils/RecordingMeter.java new file mode 100644 index 000000000..a26217934 --- /dev/null +++ b/topic/src/test/java/tech/ydb/topic/utils/RecordingMeter.java @@ -0,0 +1,104 @@ +package tech.ydb.topic.utils; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; +import java.util.function.DoubleSupplier; +import java.util.function.LongSupplier; +import java.util.stream.Collectors; + +import org.junit.Assert; + +import tech.ydb.core.metrics.Attr; +import tech.ydb.core.metrics.DoubleHistogram; +import tech.ydb.core.metrics.DoubleMeasurement; +import tech.ydb.core.metrics.LongCounter; +import tech.ydb.core.metrics.LongMeasurement; +import tech.ydb.core.metrics.Meter; +import tech.ydb.core.metrics.MetricRegistration; + +public final class RecordingMeter implements Meter { + public final Map> histograms = new ConcurrentHashMap<>(); + public final Map gauges = new ConcurrentHashMap<>(); + public final Map doubleGauges = new ConcurrentHashMap<>(); + public final Map units = new ConcurrentHashMap<>(); + private final Map counters = new ConcurrentHashMap<>(); + private final Map attributes = new ConcurrentHashMap<>(); + + @Override + public LongCounter createCounter(String name, String unit, String description) { + units.put(name, unit); + AtomicLong counter = counters.computeIfAbsent(name, key -> new AtomicLong()); + return (value, attrs) -> { + counter.addAndGet(value); + attributes.put(name, attrs); + }; + } + + @Override + public DoubleHistogram createHistogram(String name, String unit, String description) { + units.put(name, unit); + List samples = histograms.computeIfAbsent(name, key -> new CopyOnWriteArrayList<>()); + return (value, attrs) -> { + samples.add(value); + attributes.put(name, attrs); + }; + } + + @Override + public MetricRegistration registerLongGauge( + String name, String unit, String description, Consumer callback) { + units.put(name, unit); + Assert.assertNull(gauges.put(name, () -> { + Long[] value = new Long[1]; + callback.accept((observed, attrs) -> { + value[0] = observed; + attributes.put(name, attrs); + }); + return value[0]; + })); + return () -> gauges.remove(name); + } + + @Override + public MetricRegistration registerDoubleGauge( + String name, String unit, String description, Consumer callback) { + units.put(name, unit); + Assert.assertNull(doubleGauges.put(name, () -> { + Double[] value = new Double[1]; + callback.accept((observed, attrs) -> { + value[0] = observed; + attributes.put(name, attrs); + }); + return value[0]; + })); + return () -> doubleGauges.remove(name); + } + + public long value(String name) { + AtomicLong value = counters.get(name); + return value == null ? 0 : value.get(); + } + + public long collect(String name) { + return gauges.get(name).getAsLong(); + } + + public double collectDouble(String name) { + return doubleGauges.get(name).getAsDouble(); + } + + public void assertAttributes(String name, String... pairs) { + Map expected = new HashMap<>(); + for (int i = 0; i < pairs.length; i += 2) { + expected.put(pairs[i], pairs[i + 1]); + } + Assert.assertEquals(expected, Arrays.stream(attributes.get(name)) + .collect(Collectors.toMap(Attr::getKey, Attr::getValue))); + } +} diff --git a/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java b/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java index 1a4e7b03b..0f67e22d8 100644 --- a/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java +++ b/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java @@ -1,105 +1,164 @@ package tech.ydb.topic.write.impl; -import java.util.HashMap; -import java.util.Map; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.concurrent.ScheduledExecutorService; -import java.util.function.Consumer; -import java.util.function.LongSupplier; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; import tech.ydb.core.Status; -import tech.ydb.core.metrics.LongCounter; -import tech.ydb.core.metrics.LongMeasurement; -import tech.ydb.core.metrics.Meter; -import tech.ydb.core.metrics.MetricRegistration; +import tech.ydb.core.StatusCode; import tech.ydb.topic.TopicClient; import tech.ydb.topic.TopicRpc; import tech.ydb.topic.description.Codec; import tech.ydb.topic.impl.TopicClientImpl; import tech.ydb.topic.settings.WriterSettings; +import tech.ydb.topic.utils.RecordingMeter; import tech.ydb.topic.write.AsyncWriter; import tech.ydb.topic.write.Message; import tech.ydb.topic.write.QueueOverflowException; public class WriterMetricsTest { + private static final String PREFIX = "ydb.topic.writer."; + private final RecordingMeter meter = new RecordingMeter(); @Test public void writerCountersCountAcceptedMessagesAndAcknowledgements() throws QueueOverflowException { - Map counters = new HashMap<>(); - Meter meter = new Meter() { - @Override - public LongCounter createCounter(String name, String unit, String description) { - counters.put(name, 0L); - return (value, attrs) -> counters.put(name, counters.get(name) + value); - } - }; - WriteStreamMock stream = new WriteStreamMock(); - WriterSettings settings = WriterSettings.newBuilder() - .setTopicPath("/test/topic").setCodec(Codec.RAW) - .withMeter(meter, "writer").build(); - try (TopicClient client = TopicClientImpl.newClient(mockRpc(stream)) - .setCompressionExecutor(Runnable::run).build()) { - AsyncWriter writer = client.createAsyncWriter(settings); - try { - writer.init(); - stream.sendInitResponse(0); - writer.send(Message.of(new byte[]{1, 2})); - writer.send(Message.of(new byte[]{3, 4, 5})); - stream.sendAckResponse(2, 10); - Assert.assertEquals(Long.valueOf(2), counters.get("ydb.topic.writer.sending.messages")); - Assert.assertEquals(Long.valueOf(5), counters.get("ydb.topic.writer.sending.bytes")); - Assert.assertEquals(Long.valueOf(2), counters.get("ydb.topic.writer.written.messages")); - } finally { - writer.shutdown(); - stream.close(Status.SUCCESS); - } + try (WriterFixture f = new WriterFixture(meter)) { + f.writer.send(Message.of(new byte[]{1, 2})); + f.writer.send(Message.of(new byte[]{3, 4, 5})); + f.writer.send(Message.of(new byte[0])); + Assert.assertEquals(3, meter.value(PREFIX + "sending.messages")); + Assert.assertEquals(5, meter.value(PREFIX + "sending.bytes")); + Assert.assertEquals(0, meter.value(PREFIX + "written.messages")); + f.stream.sendAckResponse(2, 10); + Assert.assertEquals(2, meter.value(PREFIX + "written.messages")); + f.stream.sendAckResponse(2, 10); + Assert.assertEquals(2, meter.value(PREFIX + "written.messages")); + f.stream.sendAckResponse(3, 11); + Assert.assertEquals(3, meter.value(PREFIX + "written.messages")); + Assert.assertEquals(3, meter.histograms.get(PREFIX + "message.ack.duration").size()); } } @Test public void writerBufferGaugesTrackBudgetAndLimit() throws QueueOverflowException { - Map gauges = new HashMap<>(); - Meter meter = new Meter() { - @Override - public MetricRegistration registerLongGauge(String name, String unit, String description, - Consumer callback) { - gauges.put(name, () -> { - long[] value = new long[1]; - callback.accept((observed, attrs) -> value[0] = observed); - return value[0]; - }); - return () -> gauges.remove(name); - } - }; String used = "ydb.topic.writer.buffer.used.bytes"; String limit = "ydb.topic.writer.buffer.limit.bytes"; - WriteStreamMock stream = new WriteStreamMock(); - WriterSettings settings = WriterSettings.newBuilder() - .setTopicPath("/test/topic").setCodec(Codec.RAW).setMaxSendBufferMemorySize(100) - .withMeter(meter, "writer").build(); - try (TopicClient client = TopicClientImpl.newClient(mockRpc(stream)) - .setCompressionExecutor(Runnable::run).build()) { - AsyncWriter writer = client.createAsyncWriter(settings); - try { - writer.init(); - stream.sendInitResponse(0); - Assert.assertEquals(0, gauges.get(used).getAsLong()); - Assert.assertEquals(100, gauges.get(limit).getAsLong()); - writer.send(Message.of(new byte[]{1, 2})); - writer.send(Message.of(new byte[]{3, 4, 5})); - Assert.assertEquals(5, gauges.get(used).getAsLong()); - Assert.assertEquals(100, gauges.get(limit).getAsLong()); - stream.sendAckResponse(2, 10); - Assert.assertEquals(0, gauges.get(used).getAsLong()); - } finally { - writer.shutdown(); - stream.close(Status.SUCCESS); - } + try (WriterFixture f = new WriterFixture(meter)) { + Assert.assertEquals(0, meter.collect(used)); + Assert.assertEquals(100, meter.collect(limit)); + Assert.assertEquals(0, meter.collectDouble(PREFIX + "sending.oldest_age"), 0); + f.writer.send(Message.of(new byte[]{1, 2})); + f.writer.send(Message.of(new byte[]{3, 4, 5})); + Assert.assertEquals(5, meter.collect(used)); + Assert.assertEquals(100, meter.collect(limit)); + f.stream.sendAckResponse(2, 10); + Assert.assertEquals(0, meter.collect(used)); + Assert.assertEquals(0, meter.collectDouble(PREFIX + "sending.oldest_age"), 0); } - Assert.assertTrue(gauges.isEmpty()); + Assert.assertTrue(meter.gauges.isEmpty()); + Assert.assertTrue(meter.doubleGauges.isEmpty()); + } + + @Test + public void rejectedAdmissionDoesNotChangeCounters() throws QueueOverflowException { + try (WriterFixture f = new WriterFixture(meter)) { + f.writer.send(Message.of(new byte[100])); + Assert.assertThrows(QueueOverflowException.class, () -> f.writer.send(Message.of(new byte[]{1}))); + Assert.assertEquals(1, meter.value(PREFIX + "sending.messages")); + Assert.assertEquals(100, meter.value(PREFIX + "sending.bytes")); + Assert.assertEquals(0, meter.value(PREFIX + "written.messages")); + Assert.assertEquals(1, meter.histograms.get(PREFIX + "message.processing.duration").size()); + } + } + + @Test + public void emptyBodyStillCountsAsAnAcceptedMessage() { + WriterMetrics metrics = new WriterMetrics(meter, "/topic", "writer"); + metrics.reportSending(0); + metrics.reportSending(5); + metrics.reportWritten(); + Assert.assertEquals(2, meter.value(PREFIX + "sending.messages")); + Assert.assertEquals(5, meter.value(PREFIX + "sending.bytes")); + Assert.assertEquals(1, meter.value(PREFIX + "written.messages")); + Assert.assertEquals("By", meter.units.get(PREFIX + "sending.bytes")); + } + + @Test + public void preparationDurationsUseSeconds() { + WriterMetrics metrics = new WriterMetrics(meter, "/topic", "writer"); + metrics.reportCompression(250_000_000, Codec.GZIP, null); + metrics.reportProcessing(1_500_000_000, Codec.GZIP, null); + Assert.assertEquals(Collections.singletonList(0.25), meter.histograms.get(PREFIX + "compression.duration")); + Assert.assertEquals(Collections.singletonList(1.5), meter.histograms.get(PREFIX + "message.processing.duration")); + for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { + Assert.assertEquals("s", meter.units.get(PREFIX + name)); + meter.assertAttributes(PREFIX + name, + "topic", "/topic", "writer.name", "writer", "codec", "gzip", "outcome", "ok"); + } + } + + @Test + public void preparationFailuresKeepCodecAndError() { + WriterMetrics metrics = new WriterMetrics(meter, "/topic", "writer"); + metrics.reportCompression(500_000_000, 10001, new IOException()); + metrics.reportProcessing(2_000_000_000, 10001, new IOException()); + Assert.assertEquals(Collections.singletonList(0.5), meter.histograms.get(PREFIX + "compression.duration")); + Assert.assertEquals(Collections.singletonList(2.0), meter.histograms.get(PREFIX + "message.processing.duration")); + for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { + meter.assertAttributes(PREFIX + name, "topic", "/topic", "writer.name", "writer", + "codec", "10001", "outcome", "error", "error.type", "java.io.IOException"); + } + } + + @Test + public void ackAndBufferWaitUseMonotonicSeconds() { + WriterMetrics metrics = new WriterMetrics(meter, "/topic", "writer"); + long timestamp = System.nanoTime() - 2_000_000_000L; + metrics.reportMessageAckDuration(timestamp); + metrics.reportBufferWaitDuration(timestamp); + long end = System.nanoTime(); + for (String name : Arrays.asList("message.ack.duration", "buffer.wait.duration")) { + List samples = meter.histograms.get(PREFIX + name); + Assert.assertEquals(1, samples.size()); + assertElapsedSeconds(name, samples.get(0), timestamp, end); + } + } + + @Test + public void oldestAgeUsesMonotonicSeconds() { + WriterMetrics metrics = new WriterMetrics(meter, "/topic", "writer"); + long timestamp = System.nanoTime() - 2_000_000_000L; + metrics.register(() -> 5, () -> 10, () -> timestamp); + double age = meter.collectDouble(PREFIX + "sending.oldest_age"); + assertElapsedSeconds("sending.oldest_age", age, timestamp, System.nanoTime()); + metrics.unregister(); + Assert.assertTrue(meter.doubleGauges.isEmpty()); + } + + @Test + public void sessionErrorsCountRetryAndStopOnce() { + WriterMetrics metrics = new WriterMetrics(meter, "/topic", "writer"); + metrics.reportSessionError(Status.of(StatusCode.TRANSPORT_UNAVAILABLE), true); + Assert.assertEquals(1, meter.value(PREFIX + "session.errors")); + meter.assertAttributes(PREFIX + "session.errors", "topic", "/topic", "writer.name", "writer", + "retry_decision", "retry", "status_code", "TRANSPORT_UNAVAILABLE", "error.type", "transport_error"); + metrics.reportSessionError(Status.of(StatusCode.BAD_REQUEST), false); + Assert.assertEquals(2, meter.value(PREFIX + "session.errors")); + meter.assertAttributes(PREFIX + "session.errors", "topic", "/topic", "writer.name", "writer", + "retry_decision", "stop", "status_code", "BAD_REQUEST", "error.type", "ydb_error"); + } + + private void assertElapsedSeconds(String name, double value, long timestamp, long end) { + Assert.assertTrue(value >= 2); + Assert.assertTrue(value <= (end - timestamp) / 1_000_000_000d); + Assert.assertEquals("s", meter.units.get(PREFIX + name)); + meter.assertAttributes(PREFIX + name, "topic", "/topic", "writer.name", "writer"); } private static TopicRpc mockRpc(WriteStreamMock stream) { @@ -109,4 +168,27 @@ private static TopicRpc mockRpc(WriteStreamMock stream) { return rpc; } + private static final class WriterFixture implements AutoCloseable { + private final WriteStreamMock stream = new WriteStreamMock(); + private final TopicClient client = TopicClientImpl.newClient(mockRpc(stream)) + .setCompressionExecutor(Runnable::run).build(); + private final AsyncWriter writer; + + WriterFixture(RecordingMeter meter) { + writer = client.createAsyncWriter(WriterSettings.newBuilder().setTopicPath("/test/topic") + .setCodec(Codec.RAW).setMaxSendBufferMemorySize(100).withMeter(meter, "writer").build()); + writer.init(); + stream.sendInitResponse(0); + } + + @Override + public void close() { + try { + writer.shutdown(); + stream.close(Status.SUCCESS); + } finally { + client.close(); + } + } + } } From 565e04e440f43d42d4db747c9e230f92a844a0e5 Mon Sep 17 00:00:00 2001 From: KirillKurdyukov Date: Thu, 22 Oct 2026 17:47:29 +0300 Subject: [PATCH 2/4] Simplify Topic metric tests to happy paths --- .../topic/read/impl/ReaderMetricsTest.java | 264 +++--------------- .../topic/write/impl/WriterMetricsTest.java | 154 ++-------- 2 files changed, 66 insertions(+), 352 deletions(-) diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java index 1627a107c..e86d81041 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java @@ -1,8 +1,6 @@ package tech.ydb.topic.read.impl; -import java.io.IOException; import java.util.Arrays; -import java.util.Collections; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -11,8 +9,6 @@ import org.junit.Test; import org.mockito.Mockito; -import tech.ydb.core.Status; -import tech.ydb.core.StatusCode; import tech.ydb.topic.TopicClient; import tech.ydb.topic.TopicRpc; import tech.ydb.topic.description.Codec; @@ -24,238 +20,57 @@ public class ReaderMetricsTest { private static final String PREFIX = "ydb.topic.reader."; - - private static final byte[] MSG1 = new byte[] { 1 }; - private static final byte[] MSG2 = new byte[] { 2 }; - private static final byte[] MSG3 = new byte[] { 3 }; + private static final byte[] MSG1 = new byte[] { 1, 2 }; + private static final byte[] MSG2 = new byte[] { 3, 4, 5 }; private final RecordingMeter meter = new RecordingMeter(); @Test - public void readerCountersIncrementOnReceive() throws InterruptedException { + public void readAndCommitMetrics() throws Exception { try (ReaderFixture f = new ReaderFixture(meter)) { f.start(); f.stream.responseStartPartition("/topic", 42, 0); - f.stream.responseData(25).partition(1, 0).batch(Codec.RAW, MSG1, MSG2).and().send(); + Assert.assertEquals(1, meter.collect(PREFIX + "partition_session.count")); + Assert.assertEquals(100, meter.collect(PREFIX + "credit_balance_bytes")); + f.stream.responseData(20).partition(1, 0).batch(Codec.GZIP, MSG1, MSG2).and().send(); Assert.assertEquals(2, meter.value(PREFIX + "received.messages")); - Assert.assertEquals(25, meter.value(PREFIX + "received.bytes")); - f.stream.responseData(30).partition(1, 2).batch(Codec.RAW, MSG1, MSG2, MSG3).and().send(); - Assert.assertEquals(5, meter.value(PREFIX + "received.messages")); - Assert.assertEquals(55, meter.value(PREFIX + "received.bytes")); - Assert.assertEquals(5, meter.collect(PREFIX + "local_buffer.messages")); + Assert.assertEquals(20, meter.value(PREFIX + "received.bytes")); Assert.assertEquals(0, meter.value(PREFIX + "delivered.messages")); - meter.assertAttributes(PREFIX + "received.messages", - "consumer", "consumer", "reader.name", "reader", "topic", "/topic"); - Assert.assertNotNull(f.reader.receive(0, TimeUnit.SECONDS)); - Assert.assertNotNull(f.reader.receive(0, TimeUnit.SECONDS)); + Assert.assertEquals(2, meter.collect(PREFIX + "local_buffer.messages")); + Assert.assertTrue(meter.collectDouble(PREFIX + "local_buffer.message_age.max") > 0); + Message first = f.reader.receive(0, TimeUnit.SECONDS); + Message second = f.reader.receive(0, TimeUnit.SECONDS); + Assert.assertNotNull(first); + Assert.assertNotNull(second); + Assert.assertArrayEquals(MSG1, first.getData()); + Assert.assertArrayEquals(MSG2, second.getData()); Assert.assertEquals(2, meter.value(PREFIX + "delivered.messages")); - Assert.assertEquals(3, meter.collect(PREFIX + "local_buffer.messages")); - } - } - - @Test - public void commitMetricsFollowPublicAcknowledgement() throws InterruptedException { - try (ReaderFixture f = new ReaderFixture(meter)) { - f.start(); - f.stream.responseStartPartition("/topic", 42, 0); - f.stream.responseData(5).partition(1, 0).batch(Codec.RAW, MSG1).and().send(); - Message message = f.reader.receive(0, TimeUnit.SECONDS); - Assert.assertNotNull(message); - CompletableFuture committed = message.commit(); - f.stream.assertLastMessage().isCommit(1); + Assert.assertEquals(0, meter.collect(PREFIX + "local_buffer.messages")); + Assert.assertEquals(0, meter.collectDouble(PREFIX + "local_buffer.message_age.max"), 0); + Assert.assertEquals(100, meter.collect(PREFIX + "credit_balance_bytes")); + CompletableFuture committed = CompletableFuture.allOf(first.commit(), second.commit()); Assert.assertFalse(committed.isDone()); - Assert.assertEquals(1, meter.value(PREFIX + "commit.queued")); - Assert.assertEquals(1, meter.collect(PREFIX + "commit_offset.lag.max")); - f.stream.responseCommitAck().partition(1, 1).send(); - Assert.assertTrue(committed.isDone()); - Assert.assertFalse(committed.isCompletedExceptionally()); - Assert.assertEquals(1, meter.value(PREFIX + "commit.acknowledged")); + Assert.assertEquals(2, meter.value(PREFIX + "commit.queued")); + Assert.assertEquals(0, meter.value(PREFIX + "commit.acknowledged")); + Assert.assertEquals(2, meter.collect(PREFIX + "commit_offset.lag.max")); + f.stream.responseCommitAck().partition(1, 2).send(); + committed.get(1, TimeUnit.SECONDS); + Assert.assertEquals(2, meter.value(PREFIX + "commit.acknowledged")); Assert.assertEquals(0, meter.collect(PREFIX + "commit_offset.lag.max")); + for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { + Assert.assertEquals(2, meter.histograms.get(PREFIX + name).size()); + Assert.assertEquals("s", meter.units.get(PREFIX + name)); + meter.assertAttributes(PREFIX + name, "consumer", "consumer", "reader.name", "reader", + "topic", "/topic", "codec", "gzip", "outcome", "ok"); + } + meter.assertAttributes(PREFIX + "received.messages", + "consumer", "consumer", "reader.name", "reader", "topic", "/topic"); + Assert.assertEquals("By", meter.units.get(PREFIX + "received.bytes")); + f.stream.responseStopPartition(1, true); + Assert.assertEquals(0, meter.collect(PREFIX + "partition_session.count")); } - } - - @Test - public void commitCountersUseAcceptedWidthsAndAcknowledgedFrontier() { - ReaderMetrics metrics = registeredMetrics(); - metrics.addPartition(1, 10); - metrics.reportCommitQueued(1, 15, 3, "/topic"); - metrics.reportCommitQueued(1, 12, 2, "/topic"); - Assert.assertEquals(5, meter.value(PREFIX + "commit.queued")); - Assert.assertEquals(5, meter.collect(PREFIX + "commit_offset.lag.max")); - metrics.reportCommitAcknowledged(1, 13, "/topic"); - metrics.reportCommitAcknowledged(1, 13, "/topic"); - metrics.reportCommitAcknowledged(1, 12, "/topic"); - Assert.assertEquals(3, meter.value(PREFIX + "commit.acknowledged")); - metrics.reportCommitAcknowledged(1, 15, "/topic"); - Assert.assertEquals(5, meter.value(PREFIX + "commit.acknowledged")); - Assert.assertEquals(0, meter.collect(PREFIX + "commit_offset.lag.max")); - for (String name : Arrays.asList("commit.queued", "commit.acknowledged")) { - meter.assertAttributes(PREFIX + name, "consumer", "consumer", "reader.name", "reader", "topic", "/topic"); - } - } - - @Test - public void commitLagUsesMaximumAndDropsClosedPartitions() { - ReaderMetrics metrics = registeredMetrics(); - metrics.addPartition(1, 0); - metrics.addPartition(2, 0); - metrics.reportCommitQueued(1, 20, 2, "/topic"); - metrics.reportCommitQueued(1, 10, 1, "/topic"); - metrics.reportCommitQueued(2, 7, 7, "/topic"); - Assert.assertEquals(20, meter.collect(PREFIX + "commit_offset.lag.max")); - metrics.updateCommittedOffset(1, 25); - Assert.assertEquals(7, meter.collect(PREFIX + "commit_offset.lag.max")); - metrics.removePartition(2); - Assert.assertEquals(0, meter.collect(PREFIX + "commit_offset.lag.max")); - Assert.assertEquals(1, meter.collect(PREFIX + "partition_session.count")); - } - - @Test - public void bufferCountsUseCurrentSourcesAndRemoveClosedOnes() { - ReaderMetrics metrics = registeredMetrics(); - ReaderMetrics.Source first = source(2, 0); - ReaderMetrics.Source second = source(3, 0); - metrics.addSource(first); - metrics.addSource(second); - Assert.assertEquals(5, meter.collect(PREFIX + "local_buffer.messages")); - Mockito.when(first.getMessagesCount()).thenReturn(4L); - Assert.assertEquals(7, meter.collect(PREFIX + "local_buffer.messages")); - metrics.removeSource(first); - Assert.assertEquals(3, meter.collect(PREFIX + "local_buffer.messages")); - metrics.removeSource(second); - Assert.assertEquals(0, meter.collect(PREFIX + "local_buffer.messages")); - meter.assertAttributes(PREFIX + "local_buffer.messages", "consumer", "consumer", "reader.name", "reader"); - } - - @Test - public void bufferAgeUsesOldestNonZeroSourceInSeconds() { - ReaderMetrics metrics = registeredMetrics(); - Assert.assertEquals(0, meter.collectDouble(PREFIX + "local_buffer.message_age.max"), 0); - long oldest = System.nanoTime() - TimeUnit.DAYS.toNanos(1); - long newer = oldest + TimeUnit.HOURS.toNanos(12); - ReaderMetrics.Source first = source(1, oldest); - metrics.addSource(first); - metrics.addSource(source(2, newer)); - metrics.addSource(source(3, 0)); - assertBufferAge(oldest); - metrics.removeSource(first); - assertBufferAge(newer); - metrics.unregister(); - Assert.assertTrue(meter.doubleGauges.isEmpty()); - } - - @Test - public void gaugesObservePartitionSessionsAndProtocolCredit() throws InterruptedException { - try (ReaderFixture f = new ReaderFixture(meter)) { - Assert.assertTrue(meter.gauges.isEmpty()); - f.reader.init(); - Assert.assertTrue(meter.gauges.isEmpty()); - f.stream.responseInit("session"); - assertReaderGauges(0, 100); - f.stream.responseStartPartition("/topic", 42, 0); - assertReaderGauges(1, 100); - f.stream.responseData(20).partition(1, 0).batch(Codec.RAW, MSG1, MSG2).and().send(); - assertReaderGauges(1, 80); - Assert.assertNotNull(f.reader.receive(0, TimeUnit.SECONDS)); - assertReaderGauges(1, 80); - Assert.assertNotNull(f.reader.receive(0, TimeUnit.SECONDS)); - assertReaderGauges(1, 100); - f.stream.closeStream(Status.of(StatusCode.OVERLOADED)); - Assert.assertTrue(meter.gauges.isEmpty()); - } + Assert.assertEquals(0, meter.value(PREFIX + "session.errors")); Assert.assertTrue(meter.gauges.isEmpty()); - } - - @Test - public void creditCanBeNegativeWhenServerExceedsWindow() throws InterruptedException { - try (ReaderFixture f = new ReaderFixture(meter)) { - f.start(); - f.stream.responseStartPartition("/topic", 42, 0); - f.stream.responseData(120).partition(1, 0).batch(Codec.RAW, MSG1).and().send(); - assertReaderGauges(1, -20); - Assert.assertNotNull(f.reader.receive(0, TimeUnit.SECONDS)); - assertReaderGauges(1, 100); - } - } - - @Test - public void nonPositiveDeltasDoNotChangeCounters() { - ReaderMetrics metrics = new ReaderMetrics(meter, "consumer", "reader"); - metrics.reportDelivered(2, "/topic"); - metrics.reportReceivedMessages(3, "/topic"); - metrics.reportReceivedBytes(5); - for (long ignored : new long[]{0, -1}) { - metrics.reportDelivered(ignored, "/topic"); - metrics.reportReceivedMessages(ignored, "/topic"); - metrics.reportReceivedBytes(ignored); - } - Assert.assertEquals(2, meter.value(PREFIX + "delivered.messages")); - Assert.assertEquals(3, meter.value(PREFIX + "received.messages")); - Assert.assertEquals(5, meter.value(PREFIX + "received.bytes")); - Assert.assertEquals("By", meter.units.get(PREFIX + "received.bytes")); - } - - @Test - public void decodedDurationsUseSeconds() { - ReaderMetrics metrics = new ReaderMetrics(meter, "consumer", "reader"); - metrics.reportDecoded(250_000_000, 1_500_000_000, Codec.GZIP, "/topic", null); - Assert.assertEquals(Collections.singletonList(0.25), meter.histograms.get(PREFIX + "compression.duration")); - Assert.assertEquals(Collections.singletonList(1.5), meter.histograms.get(PREFIX + "message.processing.duration")); - for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { - Assert.assertEquals("s", meter.units.get(PREFIX + name)); - meter.assertAttributes(PREFIX + name, "consumer", "consumer", "reader.name", "reader", - "topic", "/topic", "codec", "gzip", "outcome", "ok"); - } - } - - @Test - public void decodedFailuresKeepCodecAndErrorWithoutConsumer() { - ReaderMetrics metrics = new ReaderMetrics(meter, null, "reader"); - metrics.reportDecoded(500_000_000, 2_000_000_000, 10001, "/topic", new IOException()); - Assert.assertEquals(Collections.singletonList(0.5), meter.histograms.get(PREFIX + "compression.duration")); - Assert.assertEquals(Collections.singletonList(2.0), meter.histograms.get(PREFIX + "message.processing.duration")); - for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { - meter.assertAttributes(PREFIX + name, "reader.name", "reader", "topic", "/topic", - "codec", "10001", "outcome", "error", "error.type", "java.io.IOException"); - } - } - - @Test - public void sessionErrorsCountRetryAndStopOnce() { - ReaderMetrics metrics = new ReaderMetrics(meter, "consumer", "reader"); - metrics.reportSessionError(Status.of(StatusCode.TRANSPORT_UNAVAILABLE), true); - Assert.assertEquals(1, meter.value(PREFIX + "session.errors")); - meter.assertAttributes(PREFIX + "session.errors", "consumer", "consumer", "reader.name", "reader", - "retry_decision", "retry", "status_code", "TRANSPORT_UNAVAILABLE", "error.type", "transport_error"); - metrics.reportSessionError(Status.of(StatusCode.BAD_REQUEST), false); - Assert.assertEquals(2, meter.value(PREFIX + "session.errors")); - meter.assertAttributes(PREFIX + "session.errors", "consumer", "consumer", "reader.name", "reader", - "retry_decision", "stop", "status_code", "BAD_REQUEST", "error.type", "ydb_error"); - } - - private ReaderMetrics registeredMetrics() { - ReaderMetrics metrics = new ReaderMetrics(meter, "consumer", "reader"); - metrics.register(() -> 100); - return metrics; - } - - private static ReaderMetrics.Source source(long count, long timestamp) { - ReaderMetrics.Source source = Mockito.mock(ReaderMetrics.Source.class); - Mockito.when(source.getMessagesCount()).thenReturn(count); - Mockito.when(source.getOldestReceivedTimestamp()).thenReturn(timestamp); - return source; - } - - private void assertBufferAge(long timestamp) { - long before = System.nanoTime(); - double age = meter.collectDouble(PREFIX + "local_buffer.message_age.max"); - long after = System.nanoTime(); - Assert.assertEquals((before - timestamp) / 1_000_000_000d, age, (after - before) / 1_000_000_000d); - Assert.assertEquals("s", meter.units.get(PREFIX + "local_buffer.message_age.max")); - } - - private void assertReaderGauges(long partitions, long credit) { - Assert.assertEquals(partitions, meter.collect(PREFIX + "partition_session.count")); - Assert.assertEquals(credit, meter.collect(PREFIX + "credit_balance_bytes")); + Assert.assertTrue(meter.doubleGauges.isEmpty()); } private static final class ReaderFixture implements AutoCloseable { @@ -269,7 +84,8 @@ private static final class ReaderFixture implements AutoCloseable { Mockito.when(rpc.readSession(Mockito.anyString())).thenReturn(stream); client = TopicClientImpl.newClient(rpc).build(); reader = client.createSyncReader(ReaderSettings.newBuilder().addTopic("/topic") - .setConsumerName("consumer").setMaxMemoryUsageBytes(100).withMeter(meter, "reader").build()); + .setConsumerName("consumer").setMaxMemoryUsageBytes(100) + .setDecompressionExecutor(Runnable::run).withMeter(meter, "reader").build()); } void start() { diff --git a/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java b/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java index 0f67e22d8..29fdba113 100644 --- a/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java +++ b/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java @@ -1,17 +1,15 @@ package tech.ydb.topic.write.impl; -import java.io.IOException; import java.util.Arrays; -import java.util.Collections; -import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; import tech.ydb.core.Status; -import tech.ydb.core.StatusCode; import tech.ydb.topic.TopicClient; import tech.ydb.topic.TopicRpc; import tech.ydb.topic.description.Codec; @@ -20,147 +18,47 @@ import tech.ydb.topic.utils.RecordingMeter; import tech.ydb.topic.write.AsyncWriter; import tech.ydb.topic.write.Message; -import tech.ydb.topic.write.QueueOverflowException; public class WriterMetricsTest { private static final String PREFIX = "ydb.topic.writer."; private final RecordingMeter meter = new RecordingMeter(); @Test - public void writerCountersCountAcceptedMessagesAndAcknowledgements() throws QueueOverflowException { + public void sendAndAcknowledgeMetrics() throws Exception { try (WriterFixture f = new WriterFixture(meter)) { - f.writer.send(Message.of(new byte[]{1, 2})); - f.writer.send(Message.of(new byte[]{3, 4, 5})); - f.writer.send(Message.of(new byte[0])); - Assert.assertEquals(3, meter.value(PREFIX + "sending.messages")); + Assert.assertEquals(0, meter.collect(PREFIX + "buffer.used.bytes")); + Assert.assertEquals(100, meter.collect(PREFIX + "buffer.limit.bytes")); + CompletableFuture written = CompletableFuture.allOf( + f.writer.send(Message.of(new byte[]{1, 2})), + f.writer.send(Message.of(new byte[]{3, 4, 5}))); + Assert.assertFalse(written.isDone()); + Assert.assertEquals(2, meter.value(PREFIX + "sending.messages")); Assert.assertEquals(5, meter.value(PREFIX + "sending.bytes")); Assert.assertEquals(0, meter.value(PREFIX + "written.messages")); + Assert.assertEquals(5, meter.collect(PREFIX + "buffer.used.bytes")); + Assert.assertTrue(meter.collectDouble(PREFIX + "sending.oldest_age") > 0); f.stream.sendAckResponse(2, 10); + written.get(1, TimeUnit.SECONDS); Assert.assertEquals(2, meter.value(PREFIX + "written.messages")); - f.stream.sendAckResponse(2, 10); - Assert.assertEquals(2, meter.value(PREFIX + "written.messages")); - f.stream.sendAckResponse(3, 11); - Assert.assertEquals(3, meter.value(PREFIX + "written.messages")); - Assert.assertEquals(3, meter.histograms.get(PREFIX + "message.ack.duration").size()); - } - } - - @Test - public void writerBufferGaugesTrackBudgetAndLimit() throws QueueOverflowException { - String used = "ydb.topic.writer.buffer.used.bytes"; - String limit = "ydb.topic.writer.buffer.limit.bytes"; - try (WriterFixture f = new WriterFixture(meter)) { - Assert.assertEquals(0, meter.collect(used)); - Assert.assertEquals(100, meter.collect(limit)); - Assert.assertEquals(0, meter.collectDouble(PREFIX + "sending.oldest_age"), 0); - f.writer.send(Message.of(new byte[]{1, 2})); - f.writer.send(Message.of(new byte[]{3, 4, 5})); - Assert.assertEquals(5, meter.collect(used)); - Assert.assertEquals(100, meter.collect(limit)); - f.stream.sendAckResponse(2, 10); - Assert.assertEquals(0, meter.collect(used)); + Assert.assertEquals(0, meter.collect(PREFIX + "buffer.used.bytes")); + Assert.assertEquals(100, meter.collect(PREFIX + "buffer.limit.bytes")); Assert.assertEquals(0, meter.collectDouble(PREFIX + "sending.oldest_age"), 0); + for (String name : Arrays.asList("message.ack.duration", "message.processing.duration")) { + Assert.assertEquals(2, meter.histograms.get(PREFIX + name).size()); + Assert.assertEquals("s", meter.units.get(PREFIX + name)); + } + Assert.assertTrue(meter.histograms.get(PREFIX + "buffer.wait.duration").isEmpty()); + Assert.assertTrue(meter.histograms.get(PREFIX + "compression.duration").isEmpty()); + meter.assertAttributes(PREFIX + "sending.messages", "topic", "/test/topic", "writer.name", "writer"); + meter.assertAttributes(PREFIX + "message.processing.duration", + "topic", "/test/topic", "writer.name", "writer", "codec", "raw", "outcome", "ok"); + Assert.assertEquals("By", meter.units.get(PREFIX + "sending.bytes")); } + Assert.assertEquals(0, meter.value(PREFIX + "session.errors")); Assert.assertTrue(meter.gauges.isEmpty()); Assert.assertTrue(meter.doubleGauges.isEmpty()); } - @Test - public void rejectedAdmissionDoesNotChangeCounters() throws QueueOverflowException { - try (WriterFixture f = new WriterFixture(meter)) { - f.writer.send(Message.of(new byte[100])); - Assert.assertThrows(QueueOverflowException.class, () -> f.writer.send(Message.of(new byte[]{1}))); - Assert.assertEquals(1, meter.value(PREFIX + "sending.messages")); - Assert.assertEquals(100, meter.value(PREFIX + "sending.bytes")); - Assert.assertEquals(0, meter.value(PREFIX + "written.messages")); - Assert.assertEquals(1, meter.histograms.get(PREFIX + "message.processing.duration").size()); - } - } - - @Test - public void emptyBodyStillCountsAsAnAcceptedMessage() { - WriterMetrics metrics = new WriterMetrics(meter, "/topic", "writer"); - metrics.reportSending(0); - metrics.reportSending(5); - metrics.reportWritten(); - Assert.assertEquals(2, meter.value(PREFIX + "sending.messages")); - Assert.assertEquals(5, meter.value(PREFIX + "sending.bytes")); - Assert.assertEquals(1, meter.value(PREFIX + "written.messages")); - Assert.assertEquals("By", meter.units.get(PREFIX + "sending.bytes")); - } - - @Test - public void preparationDurationsUseSeconds() { - WriterMetrics metrics = new WriterMetrics(meter, "/topic", "writer"); - metrics.reportCompression(250_000_000, Codec.GZIP, null); - metrics.reportProcessing(1_500_000_000, Codec.GZIP, null); - Assert.assertEquals(Collections.singletonList(0.25), meter.histograms.get(PREFIX + "compression.duration")); - Assert.assertEquals(Collections.singletonList(1.5), meter.histograms.get(PREFIX + "message.processing.duration")); - for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { - Assert.assertEquals("s", meter.units.get(PREFIX + name)); - meter.assertAttributes(PREFIX + name, - "topic", "/topic", "writer.name", "writer", "codec", "gzip", "outcome", "ok"); - } - } - - @Test - public void preparationFailuresKeepCodecAndError() { - WriterMetrics metrics = new WriterMetrics(meter, "/topic", "writer"); - metrics.reportCompression(500_000_000, 10001, new IOException()); - metrics.reportProcessing(2_000_000_000, 10001, new IOException()); - Assert.assertEquals(Collections.singletonList(0.5), meter.histograms.get(PREFIX + "compression.duration")); - Assert.assertEquals(Collections.singletonList(2.0), meter.histograms.get(PREFIX + "message.processing.duration")); - for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { - meter.assertAttributes(PREFIX + name, "topic", "/topic", "writer.name", "writer", - "codec", "10001", "outcome", "error", "error.type", "java.io.IOException"); - } - } - - @Test - public void ackAndBufferWaitUseMonotonicSeconds() { - WriterMetrics metrics = new WriterMetrics(meter, "/topic", "writer"); - long timestamp = System.nanoTime() - 2_000_000_000L; - metrics.reportMessageAckDuration(timestamp); - metrics.reportBufferWaitDuration(timestamp); - long end = System.nanoTime(); - for (String name : Arrays.asList("message.ack.duration", "buffer.wait.duration")) { - List samples = meter.histograms.get(PREFIX + name); - Assert.assertEquals(1, samples.size()); - assertElapsedSeconds(name, samples.get(0), timestamp, end); - } - } - - @Test - public void oldestAgeUsesMonotonicSeconds() { - WriterMetrics metrics = new WriterMetrics(meter, "/topic", "writer"); - long timestamp = System.nanoTime() - 2_000_000_000L; - metrics.register(() -> 5, () -> 10, () -> timestamp); - double age = meter.collectDouble(PREFIX + "sending.oldest_age"); - assertElapsedSeconds("sending.oldest_age", age, timestamp, System.nanoTime()); - metrics.unregister(); - Assert.assertTrue(meter.doubleGauges.isEmpty()); - } - - @Test - public void sessionErrorsCountRetryAndStopOnce() { - WriterMetrics metrics = new WriterMetrics(meter, "/topic", "writer"); - metrics.reportSessionError(Status.of(StatusCode.TRANSPORT_UNAVAILABLE), true); - Assert.assertEquals(1, meter.value(PREFIX + "session.errors")); - meter.assertAttributes(PREFIX + "session.errors", "topic", "/topic", "writer.name", "writer", - "retry_decision", "retry", "status_code", "TRANSPORT_UNAVAILABLE", "error.type", "transport_error"); - metrics.reportSessionError(Status.of(StatusCode.BAD_REQUEST), false); - Assert.assertEquals(2, meter.value(PREFIX + "session.errors")); - meter.assertAttributes(PREFIX + "session.errors", "topic", "/topic", "writer.name", "writer", - "retry_decision", "stop", "status_code", "BAD_REQUEST", "error.type", "ydb_error"); - } - - private void assertElapsedSeconds(String name, double value, long timestamp, long end) { - Assert.assertTrue(value >= 2); - Assert.assertTrue(value <= (end - timestamp) / 1_000_000_000d); - Assert.assertEquals("s", meter.units.get(PREFIX + name)); - meter.assertAttributes(PREFIX + name, "topic", "/topic", "writer.name", "writer"); - } - private static TopicRpc mockRpc(WriteStreamMock stream) { TopicRpc rpc = Mockito.mock(TopicRpc.class); Mockito.when(rpc.getScheduler()).thenReturn(Mockito.mock(ScheduledExecutorService.class)); From 349d3f06d5da345a1c5b4f1d13636f8faba8a94e Mon Sep 17 00:00:00 2001 From: KirillKurdyukov Date: Fri, 23 Oct 2026 15:31:10 +0300 Subject: [PATCH 3/4] Validate Topic metrics with exact attributes --- .../topic/read/impl/ReaderMetricsTest.java | 214 ++++++++++++------ .../tech/ydb/topic/utils/InMemoryMeter.java | 150 ++++++++++++ .../ydb/topic/utils/InMemoryMeterTest.java | 63 ++++++ .../tech/ydb/topic/utils/RecordingMeter.java | 104 --------- .../topic/write/impl/WriterMetricsTest.java | 175 +++++++++----- 5 files changed, 479 insertions(+), 227 deletions(-) create mode 100644 topic/src/test/java/tech/ydb/topic/utils/InMemoryMeter.java create mode 100644 topic/src/test/java/tech/ydb/topic/utils/InMemoryMeterTest.java delete mode 100644 topic/src/test/java/tech/ydb/topic/utils/RecordingMeter.java diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java index e86d81041..a789e7131 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java @@ -1,6 +1,7 @@ package tech.ydb.topic.read.impl; import java.util.Arrays; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -9,6 +10,9 @@ import org.junit.Test; import org.mockito.Mockito; +import tech.ydb.common.retry.RetryConfig; +import tech.ydb.core.Status; +import tech.ydb.core.StatusCode; import tech.ydb.topic.TopicClient; import tech.ydb.topic.TopicRpc; import tech.ydb.topic.description.Codec; @@ -16,91 +20,165 @@ import tech.ydb.topic.read.Message; import tech.ydb.topic.read.SyncReader; import tech.ydb.topic.settings.ReaderSettings; -import tech.ydb.topic.utils.RecordingMeter; +import tech.ydb.topic.utils.InMemoryMeter; + +import static tech.ydb.topic.utils.InMemoryMeter.attributes; public class ReaderMetricsTest { private static final String PREFIX = "ydb.topic.reader."; private static final byte[] MSG1 = new byte[] { 1, 2 }; private static final byte[] MSG2 = new byte[] { 3, 4, 5 }; - private final RecordingMeter meter = new RecordingMeter(); + private static final Map COMMON = attributes("consumer", "consumer", "reader.name", "reader"); + private static final Map TOPIC = attributes( + "consumer", "consumer", "reader.name", "reader", "topic", "/topic"); + private static final Map OTHER = attributes( + "consumer", "consumer", "reader.name", "reader", "topic", "/other-topic"); + private final InMemoryMeter meter = new InMemoryMeter(); @Test public void readAndCommitMetrics() throws Exception { - try (ReaderFixture f = new ReaderFixture(meter)) { - f.start(); - f.stream.responseStartPartition("/topic", 42, 0); - Assert.assertEquals(1, meter.collect(PREFIX + "partition_session.count")); - Assert.assertEquals(100, meter.collect(PREFIX + "credit_balance_bytes")); - f.stream.responseData(20).partition(1, 0).batch(Codec.GZIP, MSG1, MSG2).and().send(); - Assert.assertEquals(2, meter.value(PREFIX + "received.messages")); - Assert.assertEquals(20, meter.value(PREFIX + "received.bytes")); - Assert.assertEquals(0, meter.value(PREFIX + "delivered.messages")); - Assert.assertEquals(2, meter.collect(PREFIX + "local_buffer.messages")); - Assert.assertTrue(meter.collectDouble(PREFIX + "local_buffer.message_age.max") > 0); - Message first = f.reader.receive(0, TimeUnit.SECONDS); - Message second = f.reader.receive(0, TimeUnit.SECONDS); - Assert.assertNotNull(first); - Assert.assertNotNull(second); - Assert.assertArrayEquals(MSG1, first.getData()); - Assert.assertArrayEquals(MSG2, second.getData()); - Assert.assertEquals(2, meter.value(PREFIX + "delivered.messages")); - Assert.assertEquals(0, meter.collect(PREFIX + "local_buffer.messages")); - Assert.assertEquals(0, meter.collectDouble(PREFIX + "local_buffer.message_age.max"), 0); - Assert.assertEquals(100, meter.collect(PREFIX + "credit_balance_bytes")); - CompletableFuture committed = CompletableFuture.allOf(first.commit(), second.commit()); - Assert.assertFalse(committed.isDone()); - Assert.assertEquals(2, meter.value(PREFIX + "commit.queued")); - Assert.assertEquals(0, meter.value(PREFIX + "commit.acknowledged")); - Assert.assertEquals(2, meter.collect(PREFIX + "commit_offset.lag.max")); - f.stream.responseCommitAck().partition(1, 2).send(); - committed.get(1, TimeUnit.SECONDS); - Assert.assertEquals(2, meter.value(PREFIX + "commit.acknowledged")); - Assert.assertEquals(0, meter.collect(PREFIX + "commit_offset.lag.max")); - for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { - Assert.assertEquals(2, meter.histograms.get(PREFIX + name).size()); - Assert.assertEquals("s", meter.units.get(PREFIX + name)); - meter.assertAttributes(PREFIX + name, "consumer", "consumer", "reader.name", "reader", - "topic", "/topic", "codec", "gzip", "outcome", "ok"); + ReadStreamMock stream = new ReadStreamMock(true); + try (TopicClient client = TopicClientImpl.newClient(mockRpc(stream)).build()) { + SyncReader reader = client.createSyncReader(ReaderSettings.newBuilder() + .addTopic("/topic").addTopic("/other-topic").setConsumerName("consumer") + .setMaxMemoryUsageBytes(100).setDecompressionExecutor(Runnable::run) + .withMeter(meter, "reader").build()); + try { + reader.init(); + stream.assertLastMessage().isInitRequest("consumer", "/topic", "/other-topic").hasReaderName("reader"); + stream.responseInit("read-session"); + stream.responseStartPartition("/topic", 42, 0); + stream.responseStartPartition("/other-topic", 43, 0); + meter.assertLongGauge(PREFIX + "partition_session.count", 2, COMMON); + meter.assertLongGauge(PREFIX + "credit_balance_bytes", 100, COMMON); + stream.responseData(12).partition(1, 0).batch(Codec.GZIP, MSG1, MSG2).and().send(); + meter.assertCounter(PREFIX + "received.messages", 2, TOPIC); + meter.assertLongGauge(PREFIX + "credit_balance_bytes", 88, COMMON); + stream.responseData(8).partition(2, 0).batch(Codec.GZIP, MSG1).and().send(); + meter.assertCounter(PREFIX + "received.messages", 2, TOPIC); + meter.assertCounter(PREFIX + "received.messages", 1, OTHER); + meter.assertCounter(PREFIX + "received.bytes", 20, COMMON); + meter.assertLongGauge(PREFIX + "credit_balance_bytes", 80, COMMON); + meter.assertCounter(PREFIX + "delivered.messages", 0, TOPIC); + meter.assertCounter(PREFIX + "delivered.messages", 0, OTHER); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 3, COMMON); + meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", COMMON); + Message first = reader.receive(0, TimeUnit.SECONDS); + Message second = reader.receive(0, TimeUnit.SECONDS); + Message third = reader.receive(0, TimeUnit.SECONDS); + Assert.assertNotNull(first); + Assert.assertNotNull(second); + Assert.assertNotNull(third); + Assert.assertArrayEquals(MSG1, first.getData()); + Assert.assertArrayEquals(MSG2, second.getData()); + Assert.assertArrayEquals(MSG1, third.getData()); + meter.assertCounter(PREFIX + "delivered.messages", 2, TOPIC); + meter.assertCounter(PREFIX + "delivered.messages", 1, OTHER); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, COMMON); + meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, COMMON); + // The final 8 released bytes stay below the 10-byte credit-request threshold. + meter.assertLongGauge(PREFIX + "credit_balance_bytes", 92, COMMON); + CompletableFuture committed = CompletableFuture.allOf( + first.commit(), second.commit(), third.commit()); + Assert.assertFalse(committed.isDone()); + meter.assertCounter(PREFIX + "commit.queued", 2, TOPIC); + meter.assertCounter(PREFIX + "commit.queued", 1, OTHER); + meter.assertNoCounterSamples(PREFIX + "commit.acknowledged"); + meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 2, COMMON); + stream.responseCommitAck().partition(1, 2).partition(2, 1).send(); + committed.get(1, TimeUnit.SECONDS); + meter.assertCounter(PREFIX + "commit.acknowledged", 2, TOPIC); + meter.assertCounter(PREFIX + "commit.acknowledged", 1, OTHER); + meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 0, COMMON); + for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { + meter.assertHistogram(PREFIX + name, "s", 2, attributes("consumer", "consumer", + "reader.name", "reader", "topic", "/topic", "codec", "gzip", "outcome", "ok")); + meter.assertHistogram(PREFIX + name, "s", 1, attributes("consumer", "consumer", + "reader.name", "reader", "topic", "/other-topic", "codec", "gzip", "outcome", "ok")); + } + meter.assertUnit(PREFIX + "received.bytes", "By"); + stream.responseStopPartition(1, true); + stream.responseStopPartition(2, true); + meter.assertLongGauge(PREFIX + "partition_session.count", 0, COMMON); + } finally { + reader.shutdown(); } - meter.assertAttributes(PREFIX + "received.messages", - "consumer", "consumer", "reader.name", "reader", "topic", "/topic"); - Assert.assertEquals("By", meter.units.get(PREFIX + "received.bytes")); - f.stream.responseStopPartition(1, true); - Assert.assertEquals(0, meter.collect(PREFIX + "partition_session.count")); } - Assert.assertEquals(0, meter.value(PREFIX + "session.errors")); - Assert.assertTrue(meter.gauges.isEmpty()); - Assert.assertTrue(meter.doubleGauges.isEmpty()); + meter.assertNoCounterSamples(PREFIX + "session.errors"); + meter.assertNoGauges(); } - private static final class ReaderFixture implements AutoCloseable { - private final ReadStreamMock stream = new ReadStreamMock(true); - private final TopicClient client; - private final SyncReader reader; - - ReaderFixture(RecordingMeter meter) { - TopicRpc rpc = Mockito.mock(TopicRpc.class); - Mockito.when(rpc.getScheduler()).thenReturn(Mockito.mock(ScheduledExecutorService.class)); - Mockito.when(rpc.readSession(Mockito.anyString())).thenReturn(stream); - client = TopicClientImpl.newClient(rpc).build(); - reader = client.createSyncReader(ReaderSettings.newBuilder().addTopic("/topic") - .setConsumerName("consumer").setMaxMemoryUsageBytes(100) - .setDecompressionExecutor(Runnable::run).withMeter(meter, "reader").build()); - } - - void start() { - reader.init(); - stream.assertLastMessage().isInitRequest("consumer", "/topic").hasReaderName("reader"); - stream.responseInit("read-session"); + @Test + public void readWithoutConsumerMetrics() throws Exception { + ReadStreamMock stream = new ReadStreamMock(true); + Map common = attributes("reader.name", "without-consumer"); + Map topic = attributes("reader.name", "without-consumer", "topic", "/topic"); + try (TopicClient client = TopicClientImpl.newClient(mockRpc(stream)).build()) { + SyncReader reader = client.createSyncReader(ReaderSettings.newBuilder().addTopic("/topic") + .withoutConsumer().setDecompressionExecutor(Runnable::run) + .withMeter(meter, "without-consumer").build()); + try { + reader.init(); + stream.assertLastMessage().isInitRequest(null, "/topic").hasReaderName("without-consumer"); + stream.responseInit("without-consumer-session"); + stream.responseStartPartition("/topic", 42, 0); + stream.responseData(8).partition(1, 0).batch(Codec.GZIP, MSG1).and().send(); + meter.assertCounter(PREFIX + "received.messages", 1, topic); + meter.assertCounter(PREFIX + "received.bytes", 8, common); + Message message = reader.receive(0, TimeUnit.SECONDS); + Assert.assertNotNull(message); + Assert.assertArrayEquals(MSG1, message.getData()); + meter.assertCounter(PREFIX + "delivered.messages", 1, topic); + meter.assertLongGauge(PREFIX + "partition_session.count", 1, common); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, common); + for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { + meter.assertHistogram(PREFIX + name, "s", 1, attributes("reader.name", "without-consumer", + "topic", "/topic", "codec", "gzip", "outcome", "ok")); + } + } finally { + reader.shutdown(); + } } + meter.assertNoCounterSamples(PREFIX + "session.errors"); + meter.assertNoGauges(); + } - @Override - public void close() { + @Test + public void sessionErrorsFollowRetriesAndStop() throws Exception { + ReadStreamMock first = new ReadStreamMock(true); + ReadStreamMock second = new ReadStreamMock(true); + ReadStreamMock third = new ReadStreamMock(true); + RetryConfig retries = status -> (number, elapsed) -> number < 2 ? 0 : -1; + Map retry = attributes("consumer", "consumer", "reader.name", "reader", + "retry_decision", "retry", "status_code", "TRANSPORT_UNAVAILABLE", "error.type", "transport_error"); + Map stop = attributes("consumer", "consumer", "reader.name", "reader", + "retry_decision", "stop", "status_code", "TRANSPORT_UNAVAILABLE", "error.type", "transport_error"); + try (TopicClient client = TopicClientImpl.newClient(mockRpc(first, second, third)).build()) { + SyncReader reader = client.createSyncReader(ReaderSettings.newBuilder().addTopic("/topic") + .setConsumerName("consumer").setRetryConfig(retries).withMeter(meter, "reader").build()); try { - reader.shutdown(); + reader.init(); + first.responseInit("first-session"); + first.closeStream(Status.of(StatusCode.TRANSPORT_UNAVAILABLE)); + second.closeStream(Status.of(StatusCode.TRANSPORT_UNAVAILABLE)); + third.assertLastMessage().isInitRequest("consumer", "/topic"); + meter.assertCounter(PREFIX + "session.errors", 2, retry); + meter.assertCounter(PREFIX + "session.errors", 0, stop); + third.closeStream(Status.of(StatusCode.TRANSPORT_UNAVAILABLE)); } finally { - client.close(); + reader.shutdown(); } } + meter.assertCounter(PREFIX + "session.errors", 2, retry); + meter.assertCounter(PREFIX + "session.errors", 1, stop); + meter.assertCounterSeries(PREFIX + "session.errors", 2); + meter.assertNoGauges(); + } + + private static TopicRpc mockRpc(ReadStreamMock first, ReadStreamMock... rest) { + TopicRpc rpc = Mockito.mock(TopicRpc.class); + Mockito.when(rpc.getScheduler()).thenReturn(Mockito.mock(ScheduledExecutorService.class)); + Mockito.when(rpc.readSession(Mockito.anyString())).thenReturn(first, rest); + return rpc; } } diff --git a/topic/src/test/java/tech/ydb/topic/utils/InMemoryMeter.java b/topic/src/test/java/tech/ydb/topic/utils/InMemoryMeter.java new file mode 100644 index 000000000..8cbbfea14 --- /dev/null +++ b/topic/src/test/java/tech/ydb/topic/utils/InMemoryMeter.java @@ -0,0 +1,150 @@ +package tech.ydb.topic.utils; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +import org.junit.Assert; + +import tech.ydb.core.metrics.Attr; +import tech.ydb.core.metrics.DoubleHistogram; +import tech.ydb.core.metrics.DoubleMeasurement; +import tech.ydb.core.metrics.LongCounter; +import tech.ydb.core.metrics.LongMeasurement; +import tech.ydb.core.metrics.Meter; +import tech.ydb.core.metrics.MetricRegistration; + +public final class InMemoryMeter implements Meter { + private final Map, AtomicLong>> counters = new ConcurrentHashMap<>(); + private final Map, List>> histograms = new ConcurrentHashMap<>(); + private final Map>> longGauges = new ConcurrentHashMap<>(); + private final Map>> doubleGauges = new ConcurrentHashMap<>(); + private final Map units = new ConcurrentHashMap<>(); + + @Override + public LongCounter createCounter(String name, String unit, String description) { + units.put(name, unit); + Map, AtomicLong> values = counters.computeIfAbsent(name, key -> new ConcurrentHashMap<>()); + return (value, attrs) -> values.computeIfAbsent(key(attrs), key -> new AtomicLong()).addAndGet(value); + } + + @Override + public DoubleHistogram createHistogram(String name, String unit, String description) { + units.put(name, unit); + Map, List> values = histograms.computeIfAbsent(name, + key -> new ConcurrentHashMap<>()); + return (value, attrs) -> values.computeIfAbsent(key(attrs), key -> new CopyOnWriteArrayList<>()).add(value); + } + + @Override + public MetricRegistration registerLongGauge( + String name, String unit, String description, Consumer callback) { + units.put(name, unit); + List> callbacks = longGauges.computeIfAbsent(name, + key -> new CopyOnWriteArrayList<>()); + callbacks.add(callback); + return () -> callbacks.remove(callback); + } + + @Override + public MetricRegistration registerDoubleGauge( + String name, String unit, String description, Consumer callback) { + units.put(name, unit); + List> callbacks = doubleGauges.computeIfAbsent(name, + key -> new CopyOnWriteArrayList<>()); + callbacks.add(callback); + return () -> callbacks.remove(callback); + } + + public void assertCounter(String name, long expected, Map attributes) { + Map, AtomicLong> values = counters.get(name); + Assert.assertNotNull(name, values); + AtomicLong value = values.get(attributes); + Assert.assertEquals(name + " " + attributes, expected, value == null ? 0 : value.get()); + } + + public void assertCounterSeries(String name, int expected) { + Assert.assertNotNull(name, counters.get(name)); + Assert.assertEquals(name, expected, counters.get(name).size()); + } + + public void assertNoCounterSamples(String name) { + Assert.assertNotNull(name, counters.get(name)); + Assert.assertTrue(name, counters.get(name).isEmpty()); + } + + public void assertHistogram(String name, String unit, int expected, Map attributes) { + assertUnit(name, unit); + Map, List> values = histograms.get(name); + Assert.assertNotNull(name, values); + List samples = values.get(attributes); + Assert.assertEquals(name + " " + attributes, expected, samples == null ? 0 : samples.size()); + } + + public void assertNoHistogramSamples(String name) { + Assert.assertNotNull(name, histograms.get(name)); + Assert.assertTrue(name, histograms.get(name).isEmpty()); + } + + public void assertLongGauge(String name, long expected, Map attributes) { + List> callbacks = longGauges.get(name); + Assert.assertNotNull(name, callbacks); + Map, Long> values = new HashMap<>(); + for (Consumer callback : callbacks) { + callback.accept((value, attrs) -> Assert.assertNull(name + " duplicate series", + values.put(key(attrs), value))); + } + Assert.assertTrue(name + " " + attributes, values.containsKey(attributes)); + Assert.assertEquals(name + " " + attributes, expected, values.get(attributes).longValue()); + } + + public void assertDoubleGauge(String name, double expected, Map attributes) { + Assert.assertEquals(name + " " + attributes, expected, doubleGaugeValue(name, attributes), 0); + } + + public void assertPositiveDoubleGauge(String name, Map attributes) { + Assert.assertTrue(name + " " + attributes, doubleGaugeValue(name, attributes) > 0); + } + + public void assertUnit(String name, String expected) { + Assert.assertEquals(name, expected, units.get(name)); + } + + public void assertNoGauges() { + Assert.assertTrue(longGauges.toString(), longGauges.values().stream().allMatch(List::isEmpty)); + Assert.assertTrue(doubleGauges.toString(), doubleGauges.values().stream().allMatch(List::isEmpty)); + } + + private double doubleGaugeValue(String name, Map attributes) { + List> callbacks = doubleGauges.get(name); + Assert.assertNotNull(name, callbacks); + Map, Double> values = new HashMap<>(); + for (Consumer callback : callbacks) { + callback.accept((value, attrs) -> Assert.assertNull(name + " duplicate series", + values.put(key(attrs), value))); + } + Assert.assertTrue(name + " " + attributes, values.containsKey(attributes)); + return values.get(attributes); + } + + public static Map attributes(String... pairs) { + Map result = new HashMap<>(); + for (int i = 0; i < pairs.length; i += 2) { + result.put(pairs[i], pairs[i + 1]); + } + return Collections.unmodifiableMap(result); + } + + private static Map key(Attr[] attrs) { + Map result = new HashMap<>(); + for (Attr attr : attrs) { + result.put(attr.getKey(), attr.getValue()); + } + return Collections.unmodifiableMap(result); + } +} diff --git a/topic/src/test/java/tech/ydb/topic/utils/InMemoryMeterTest.java b/topic/src/test/java/tech/ydb/topic/utils/InMemoryMeterTest.java new file mode 100644 index 000000000..0ef9036de --- /dev/null +++ b/topic/src/test/java/tech/ydb/topic/utils/InMemoryMeterTest.java @@ -0,0 +1,63 @@ +package tech.ydb.topic.utils; + +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; + +import tech.ydb.core.metrics.Attr; +import tech.ydb.core.metrics.DoubleHistogram; +import tech.ydb.core.metrics.LongCounter; +import tech.ydb.core.metrics.MetricRegistration; + +import static tech.ydb.topic.utils.InMemoryMeter.attributes; + +public class InMemoryMeterTest { + @Test + public void assertionsBindInstrumentAndAttributes() { + InMemoryMeter meter = new InMemoryMeter(); + Map first = attributes("topic", "/first"); + Map second = attributes("topic", "/second"); + Attr a = Attr.of("topic", "/first"); + Attr b = Attr.of("topic", "/second"); + LongCounter counter = meter.createCounter("counter", "{message}", ""); + counter.add(2, a); + counter.add(3, b); + meter.assertCounter("counter", 2, first); + meter.assertCounter("counter", 3, second); + meter.assertCounterSeries("counter", 2); + Assert.assertThrows(AssertionError.class, () -> meter.assertCounter("counter", 3, first)); + Assert.assertThrows(AssertionError.class, () -> meter.assertCounter("missing", 2, first)); + Assert.assertThrows(AssertionError.class, () -> meter.assertNoCounterSamples("counter")); + DoubleHistogram histogram = meter.createHistogram("histogram", "s", ""); + histogram.record(0.25, a); + histogram.record(0.5, a); + histogram.record(0.75, b); + meter.assertHistogram("histogram", "s", 2, first); + meter.assertHistogram("histogram", "s", 1, second); + Assert.assertThrows(AssertionError.class, () -> meter.assertHistogram("histogram", "s", 1, first)); + Assert.assertThrows(AssertionError.class, () -> meter.assertNoHistogramSamples("histogram")); + MetricRegistration one = meter.registerLongGauge("gauge", "By", "", m -> m.record(2, a)); + MetricRegistration two = meter.registerLongGauge("gauge", "By", "", m -> m.record(3, b)); + MetricRegistration doubles = meter.registerDoubleGauge("double", "s", "", m -> { + m.record(2.5, a); + m.record(3.5, b); + }); + try { + meter.assertLongGauge("gauge", 2, first); + meter.assertLongGauge("gauge", 3, second); + Assert.assertThrows(AssertionError.class, () -> meter.assertLongGauge("gauge", 3, first)); + meter.assertDoubleGauge("double", 2.5, first); + meter.assertDoubleGauge("double", 3.5, second); + Assert.assertThrows(AssertionError.class, () -> meter.assertDoubleGauge("double", 3.5, first)); + one.close(); + meter.assertLongGauge("gauge", 3, second); + Assert.assertThrows(AssertionError.class, () -> meter.assertLongGauge("gauge", 2, first)); + } finally { + one.close(); + two.close(); + doubles.close(); + } + meter.assertNoGauges(); + } +} diff --git a/topic/src/test/java/tech/ydb/topic/utils/RecordingMeter.java b/topic/src/test/java/tech/ydb/topic/utils/RecordingMeter.java deleted file mode 100644 index a26217934..000000000 --- a/topic/src/test/java/tech/ydb/topic/utils/RecordingMeter.java +++ /dev/null @@ -1,104 +0,0 @@ -package tech.ydb.topic.utils; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Consumer; -import java.util.function.DoubleSupplier; -import java.util.function.LongSupplier; -import java.util.stream.Collectors; - -import org.junit.Assert; - -import tech.ydb.core.metrics.Attr; -import tech.ydb.core.metrics.DoubleHistogram; -import tech.ydb.core.metrics.DoubleMeasurement; -import tech.ydb.core.metrics.LongCounter; -import tech.ydb.core.metrics.LongMeasurement; -import tech.ydb.core.metrics.Meter; -import tech.ydb.core.metrics.MetricRegistration; - -public final class RecordingMeter implements Meter { - public final Map> histograms = new ConcurrentHashMap<>(); - public final Map gauges = new ConcurrentHashMap<>(); - public final Map doubleGauges = new ConcurrentHashMap<>(); - public final Map units = new ConcurrentHashMap<>(); - private final Map counters = new ConcurrentHashMap<>(); - private final Map attributes = new ConcurrentHashMap<>(); - - @Override - public LongCounter createCounter(String name, String unit, String description) { - units.put(name, unit); - AtomicLong counter = counters.computeIfAbsent(name, key -> new AtomicLong()); - return (value, attrs) -> { - counter.addAndGet(value); - attributes.put(name, attrs); - }; - } - - @Override - public DoubleHistogram createHistogram(String name, String unit, String description) { - units.put(name, unit); - List samples = histograms.computeIfAbsent(name, key -> new CopyOnWriteArrayList<>()); - return (value, attrs) -> { - samples.add(value); - attributes.put(name, attrs); - }; - } - - @Override - public MetricRegistration registerLongGauge( - String name, String unit, String description, Consumer callback) { - units.put(name, unit); - Assert.assertNull(gauges.put(name, () -> { - Long[] value = new Long[1]; - callback.accept((observed, attrs) -> { - value[0] = observed; - attributes.put(name, attrs); - }); - return value[0]; - })); - return () -> gauges.remove(name); - } - - @Override - public MetricRegistration registerDoubleGauge( - String name, String unit, String description, Consumer callback) { - units.put(name, unit); - Assert.assertNull(doubleGauges.put(name, () -> { - Double[] value = new Double[1]; - callback.accept((observed, attrs) -> { - value[0] = observed; - attributes.put(name, attrs); - }); - return value[0]; - })); - return () -> doubleGauges.remove(name); - } - - public long value(String name) { - AtomicLong value = counters.get(name); - return value == null ? 0 : value.get(); - } - - public long collect(String name) { - return gauges.get(name).getAsLong(); - } - - public double collectDouble(String name) { - return doubleGauges.get(name).getAsDouble(); - } - - public void assertAttributes(String name, String... pairs) { - Map expected = new HashMap<>(); - for (int i = 0; i < pairs.length; i += 2) { - expected.put(pairs[i], pairs[i + 1]); - } - Assert.assertEquals(expected, Arrays.stream(attributes.get(name)) - .collect(Collectors.toMap(Attr::getKey, Attr::getValue))); - } -} diff --git a/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java b/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java index 29fdba113..4128b8a40 100644 --- a/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java +++ b/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java @@ -1,6 +1,7 @@ package tech.ydb.topic.write.impl; import java.util.Arrays; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -9,84 +10,148 @@ import org.junit.Test; import org.mockito.Mockito; +import tech.ydb.common.retry.RetryConfig; import tech.ydb.core.Status; +import tech.ydb.core.StatusCode; import tech.ydb.topic.TopicClient; import tech.ydb.topic.TopicRpc; import tech.ydb.topic.description.Codec; import tech.ydb.topic.impl.TopicClientImpl; import tech.ydb.topic.settings.WriterSettings; -import tech.ydb.topic.utils.RecordingMeter; +import tech.ydb.topic.utils.InMemoryMeter; import tech.ydb.topic.write.AsyncWriter; import tech.ydb.topic.write.Message; +import tech.ydb.topic.write.SyncWriter; + +import static tech.ydb.topic.utils.InMemoryMeter.attributes; public class WriterMetricsTest { private static final String PREFIX = "ydb.topic.writer."; - private final RecordingMeter meter = new RecordingMeter(); + private static final Map COMMON = attributes("topic", "/test/topic", "writer.name", "writer"); + private final InMemoryMeter meter = new InMemoryMeter(); @Test public void sendAndAcknowledgeMetrics() throws Exception { - try (WriterFixture f = new WriterFixture(meter)) { - Assert.assertEquals(0, meter.collect(PREFIX + "buffer.used.bytes")); - Assert.assertEquals(100, meter.collect(PREFIX + "buffer.limit.bytes")); - CompletableFuture written = CompletableFuture.allOf( - f.writer.send(Message.of(new byte[]{1, 2})), - f.writer.send(Message.of(new byte[]{3, 4, 5}))); - Assert.assertFalse(written.isDone()); - Assert.assertEquals(2, meter.value(PREFIX + "sending.messages")); - Assert.assertEquals(5, meter.value(PREFIX + "sending.bytes")); - Assert.assertEquals(0, meter.value(PREFIX + "written.messages")); - Assert.assertEquals(5, meter.collect(PREFIX + "buffer.used.bytes")); - Assert.assertTrue(meter.collectDouble(PREFIX + "sending.oldest_age") > 0); - f.stream.sendAckResponse(2, 10); - written.get(1, TimeUnit.SECONDS); - Assert.assertEquals(2, meter.value(PREFIX + "written.messages")); - Assert.assertEquals(0, meter.collect(PREFIX + "buffer.used.bytes")); - Assert.assertEquals(100, meter.collect(PREFIX + "buffer.limit.bytes")); - Assert.assertEquals(0, meter.collectDouble(PREFIX + "sending.oldest_age"), 0); - for (String name : Arrays.asList("message.ack.duration", "message.processing.duration")) { - Assert.assertEquals(2, meter.histograms.get(PREFIX + name).size()); - Assert.assertEquals("s", meter.units.get(PREFIX + name)); + WriteStreamMock stream = new WriteStreamMock(); + try (TopicClient client = TopicClientImpl.newClient(mockRpc(stream)) + .setCompressionExecutor(Runnable::run).build()) { + AsyncWriter writer = client.createAsyncWriter(WriterSettings.newBuilder().setTopicPath("/test/topic") + .setCodec(Codec.RAW).setMaxSendBufferMemorySize(100).withMeter(meter, "writer").build()); + try { + writer.init(); + stream.sendInitResponse(0); + meter.assertLongGauge(PREFIX + "buffer.used.bytes", 0, COMMON); + meter.assertLongGauge(PREFIX + "buffer.limit.bytes", 100, COMMON); + CompletableFuture written = CompletableFuture.allOf( + writer.send(Message.of(new byte[]{1, 2})), writer.send(Message.of(new byte[]{3, 4, 5}))); + Assert.assertFalse(written.isDone()); + meter.assertCounter(PREFIX + "sending.messages", 2, COMMON); + meter.assertCounter(PREFIX + "sending.bytes", 5, COMMON); + meter.assertNoCounterSamples(PREFIX + "written.messages"); + meter.assertLongGauge(PREFIX + "buffer.used.bytes", 5, COMMON); + meter.assertPositiveDoubleGauge(PREFIX + "sending.oldest_age", COMMON); + stream.sendAckResponse(2, 10); + written.get(1, TimeUnit.SECONDS); + meter.assertCounter(PREFIX + "written.messages", 2, COMMON); + meter.assertLongGauge(PREFIX + "buffer.used.bytes", 0, COMMON); + meter.assertLongGauge(PREFIX + "buffer.limit.bytes", 100, COMMON); + meter.assertDoubleGauge(PREFIX + "sending.oldest_age", 0, COMMON); + meter.assertHistogram(PREFIX + "message.ack.duration", "s", 2, COMMON); + meter.assertHistogram(PREFIX + "message.processing.duration", "s", 2, attributes( + "topic", "/test/topic", "writer.name", "writer", "codec", "raw", "outcome", "ok")); + meter.assertNoHistogramSamples(PREFIX + "buffer.wait.duration"); + meter.assertNoHistogramSamples(PREFIX + "compression.duration"); + meter.assertUnit(PREFIX + "sending.bytes", "By"); + } finally { + CompletableFuture closed = writer.shutdown(); + stream.close(Status.SUCCESS); + closed.get(1, TimeUnit.SECONDS); } - Assert.assertTrue(meter.histograms.get(PREFIX + "buffer.wait.duration").isEmpty()); - Assert.assertTrue(meter.histograms.get(PREFIX + "compression.duration").isEmpty()); - meter.assertAttributes(PREFIX + "sending.messages", "topic", "/test/topic", "writer.name", "writer"); - meter.assertAttributes(PREFIX + "message.processing.duration", - "topic", "/test/topic", "writer.name", "writer", "codec", "raw", "outcome", "ok"); - Assert.assertEquals("By", meter.units.get(PREFIX + "sending.bytes")); } - Assert.assertEquals(0, meter.value(PREFIX + "session.errors")); - Assert.assertTrue(meter.gauges.isEmpty()); - Assert.assertTrue(meter.doubleGauges.isEmpty()); - } - - private static TopicRpc mockRpc(WriteStreamMock stream) { - TopicRpc rpc = Mockito.mock(TopicRpc.class); - Mockito.when(rpc.getScheduler()).thenReturn(Mockito.mock(ScheduledExecutorService.class)); - Mockito.when(rpc.writeSession(Mockito.any(String.class))).thenReturn(stream); - return rpc; + meter.assertNoCounterSamples(PREFIX + "session.errors"); + meter.assertNoGauges(); } - private static final class WriterFixture implements AutoCloseable { - private final WriteStreamMock stream = new WriteStreamMock(); - private final TopicClient client = TopicClientImpl.newClient(mockRpc(stream)) - .setCompressionExecutor(Runnable::run).build(); - private final AsyncWriter writer; - - WriterFixture(RecordingMeter meter) { - writer = client.createAsyncWriter(WriterSettings.newBuilder().setTopicPath("/test/topic") - .setCodec(Codec.RAW).setMaxSendBufferMemorySize(100).withMeter(meter, "writer").build()); - writer.init(); - stream.sendInitResponse(0); + @Test + public void compressedSyncSendMetrics() throws Exception { + WriteStreamMock stream = Mockito.spy(new WriteStreamMock()); + Mockito.doAnswer(invocation -> { + invocation.callRealMethod(); + stream.close(Status.SUCCESS); + return null; + }).when(stream).close(); + try (TopicClient client = TopicClientImpl.newClient(mockRpc(stream)) + .setCompressionExecutor(Runnable::run).build()) { + SyncWriter writer = client.createSyncWriter(WriterSettings.newBuilder().setTopicPath("/test/topic") + .setCodec(Codec.GZIP).setMaxSendBufferMemorySize(100).withMeter(meter, "writer").build()); + try { + writer.init(); + stream.sendInitResponse(0); + writer.send(Message.of(new byte[]{1, 2}), 1, TimeUnit.SECONDS); + writer.send(Message.of(new byte[]{3, 4, 5}), 1, TimeUnit.SECONDS); + meter.assertCounter(PREFIX + "sending.messages", 2, COMMON); + meter.assertCounter(PREFIX + "sending.bytes", 5, COMMON); + meter.assertNoCounterSamples(PREFIX + "written.messages"); + // The two GZIP bodies occupy 22 and 23 bytes. + meter.assertLongGauge(PREFIX + "buffer.used.bytes", 45, COMMON); + stream.sendAckResponse(2, 10); + meter.assertCounter(PREFIX + "written.messages", 2, COMMON); + meter.assertLongGauge(PREFIX + "buffer.used.bytes", 0, COMMON); + for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { + meter.assertHistogram(PREFIX + name, "s", 2, attributes( + "topic", "/test/topic", "writer.name", "writer", "codec", "gzip", "outcome", "ok")); + } + for (String name : Arrays.asList("buffer.wait.duration", "message.ack.duration")) { + meter.assertHistogram(PREFIX + name, "s", 2, COMMON); + } + } finally { + writer.shutdown(1, TimeUnit.SECONDS); + } } + meter.assertNoCounterSamples(PREFIX + "session.errors"); + meter.assertNoGauges(); + } - @Override - public void close() { + @Test + public void sessionErrorsFollowRetriesAndStop() throws Exception { + WriteStreamMock first = new WriteStreamMock(); + WriteStreamMock second = new WriteStreamMock(); + WriteStreamMock third = new WriteStreamMock(); + RetryConfig retries = status -> (number, elapsed) -> number < 2 ? 0 : -1; + Map retry = attributes("topic", "/test/topic", "writer.name", "writer", + "retry_decision", "retry", "status_code", "TRANSPORT_UNAVAILABLE", "error.type", "transport_error"); + Map stop = attributes("topic", "/test/topic", "writer.name", "writer", + "retry_decision", "stop", "status_code", "TRANSPORT_UNAVAILABLE", "error.type", "transport_error"); + try (TopicClient client = TopicClientImpl.newClient(mockRpc(first, second, third)) + .setCompressionExecutor(Runnable::run).build()) { + AsyncWriter writer = client.createAsyncWriter(WriterSettings.newBuilder().setTopicPath("/test/topic") + .setRetryConfig(retries).withMeter(meter, "writer").build()); try { - writer.shutdown(); - stream.close(Status.SUCCESS); + writer.init(); + first.sendInitResponse(0); + first.close(Status.of(StatusCode.TRANSPORT_UNAVAILABLE)); + second.close(Status.of(StatusCode.TRANSPORT_UNAVAILABLE)); + meter.assertCounter(PREFIX + "session.errors", 2, retry); + meter.assertCounter(PREFIX + "session.errors", 0, stop); + third.close(Status.of(StatusCode.TRANSPORT_UNAVAILABLE)); } finally { - client.close(); + CompletableFuture closed = writer.shutdown(); + first.close(Status.SUCCESS); + second.close(Status.SUCCESS); + third.close(Status.SUCCESS); + closed.get(1, TimeUnit.SECONDS); } } + meter.assertCounter(PREFIX + "session.errors", 2, retry); + meter.assertCounter(PREFIX + "session.errors", 1, stop); + meter.assertCounterSeries(PREFIX + "session.errors", 2); + meter.assertNoGauges(); + } + + private static TopicRpc mockRpc(WriteStreamMock first, WriteStreamMock... rest) { + TopicRpc rpc = Mockito.mock(TopicRpc.class); + Mockito.when(rpc.getScheduler()).thenReturn(Mockito.mock(ScheduledExecutorService.class)); + Mockito.when(rpc.writeSession(Mockito.anyString())).thenReturn(first, rest); + return rpc; } } From 70e345bacfdce178b079ba7e0f9c116b66b55853 Mon Sep 17 00:00:00 2001 From: KirillKurdyukov Date: Mon, 26 Oct 2026 10:04:33 +0300 Subject: [PATCH 4/4] Replace meter self-test with codec failure coverage --- .../topic/read/impl/ReaderMetricsTest.java | 43 +++++++++++++ .../ydb/topic/utils/InMemoryMeterTest.java | 63 ------------------- .../topic/write/impl/WriterMetricsTest.java | 46 ++++++++++++++ 3 files changed, 89 insertions(+), 63 deletions(-) delete mode 100644 topic/src/test/java/tech/ydb/topic/utils/InMemoryMeterTest.java diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java index a789e7131..a70f57ee8 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java @@ -1,5 +1,7 @@ package tech.ydb.topic.read.impl; +import java.io.IOException; +import java.io.InputStream; import java.util.Arrays; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -17,6 +19,7 @@ import tech.ydb.topic.TopicRpc; import tech.ydb.topic.description.Codec; import tech.ydb.topic.impl.TopicClientImpl; +import tech.ydb.topic.read.DecompressionException; import tech.ydb.topic.read.Message; import tech.ydb.topic.read.SyncReader; import tech.ydb.topic.settings.ReaderSettings; @@ -175,6 +178,46 @@ public void sessionErrorsFollowRetriesAndStop() throws Exception { meter.assertNoGauges(); } + @Test + public void decompressionFailureMetrics() throws Exception { + ReadStreamMock stream = new ReadStreamMock(true); + IOException problem = new IOException("decode failure"); + Codec codec = Mockito.mock(Codec.class, Mockito.CALLS_REAL_METHODS); + Mockito.when(codec.getId()).thenReturn(Codec.GZIP); + Mockito.doThrow(problem).when(codec).decode(Mockito.any(InputStream.class)); + try (TopicClient client = TopicClientImpl.newClient(mockRpc(stream)).registerCodec(codec).build()) { + SyncReader reader = client.createSyncReader(ReaderSettings.newBuilder().addTopic("/topic") + .setConsumerName("consumer").setDecompressionExecutor(Runnable::run) + .withMeter(meter, "reader").build()); + try { + reader.init(); + stream.responseInit("decode-failure-session"); + stream.responseStartPartition("/topic", 42, 0); + stream.responseData(20).partition(1, 0).batch(Codec.GZIP, MSG1, MSG2).and().send(); + for (int i = 0; i < 2; i++) { + Message message = reader.receive(0, TimeUnit.SECONDS); + Assert.assertNotNull(message); + DecompressionException error = Assert.assertThrows(DecompressionException.class, message::getData); + Assert.assertSame(problem, error.getCause()); + } + meter.assertCounter(PREFIX + "received.messages", 2, TOPIC); + meter.assertCounter(PREFIX + "delivered.messages", 2, TOPIC); + meter.assertNoCounterSamples(PREFIX + "commit.acknowledged"); + for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { + meter.assertHistogram(PREFIX + name, "s", 2, attributes("consumer", "consumer", + "reader.name", "reader", "topic", "/topic", "codec", "gzip", "outcome", "error", + "error.type", "java.io.IOException")); + meter.assertHistogram(PREFIX + name, "s", 0, attributes("consumer", "consumer", + "reader.name", "reader", "topic", "/topic", "codec", "gzip", "outcome", "ok")); + } + } finally { + reader.shutdown(); + } + } + meter.assertNoCounterSamples(PREFIX + "session.errors"); + meter.assertNoGauges(); + } + private static TopicRpc mockRpc(ReadStreamMock first, ReadStreamMock... rest) { TopicRpc rpc = Mockito.mock(TopicRpc.class); Mockito.when(rpc.getScheduler()).thenReturn(Mockito.mock(ScheduledExecutorService.class)); diff --git a/topic/src/test/java/tech/ydb/topic/utils/InMemoryMeterTest.java b/topic/src/test/java/tech/ydb/topic/utils/InMemoryMeterTest.java deleted file mode 100644 index 0ef9036de..000000000 --- a/topic/src/test/java/tech/ydb/topic/utils/InMemoryMeterTest.java +++ /dev/null @@ -1,63 +0,0 @@ -package tech.ydb.topic.utils; - -import java.util.Map; - -import org.junit.Assert; -import org.junit.Test; - -import tech.ydb.core.metrics.Attr; -import tech.ydb.core.metrics.DoubleHistogram; -import tech.ydb.core.metrics.LongCounter; -import tech.ydb.core.metrics.MetricRegistration; - -import static tech.ydb.topic.utils.InMemoryMeter.attributes; - -public class InMemoryMeterTest { - @Test - public void assertionsBindInstrumentAndAttributes() { - InMemoryMeter meter = new InMemoryMeter(); - Map first = attributes("topic", "/first"); - Map second = attributes("topic", "/second"); - Attr a = Attr.of("topic", "/first"); - Attr b = Attr.of("topic", "/second"); - LongCounter counter = meter.createCounter("counter", "{message}", ""); - counter.add(2, a); - counter.add(3, b); - meter.assertCounter("counter", 2, first); - meter.assertCounter("counter", 3, second); - meter.assertCounterSeries("counter", 2); - Assert.assertThrows(AssertionError.class, () -> meter.assertCounter("counter", 3, first)); - Assert.assertThrows(AssertionError.class, () -> meter.assertCounter("missing", 2, first)); - Assert.assertThrows(AssertionError.class, () -> meter.assertNoCounterSamples("counter")); - DoubleHistogram histogram = meter.createHistogram("histogram", "s", ""); - histogram.record(0.25, a); - histogram.record(0.5, a); - histogram.record(0.75, b); - meter.assertHistogram("histogram", "s", 2, first); - meter.assertHistogram("histogram", "s", 1, second); - Assert.assertThrows(AssertionError.class, () -> meter.assertHistogram("histogram", "s", 1, first)); - Assert.assertThrows(AssertionError.class, () -> meter.assertNoHistogramSamples("histogram")); - MetricRegistration one = meter.registerLongGauge("gauge", "By", "", m -> m.record(2, a)); - MetricRegistration two = meter.registerLongGauge("gauge", "By", "", m -> m.record(3, b)); - MetricRegistration doubles = meter.registerDoubleGauge("double", "s", "", m -> { - m.record(2.5, a); - m.record(3.5, b); - }); - try { - meter.assertLongGauge("gauge", 2, first); - meter.assertLongGauge("gauge", 3, second); - Assert.assertThrows(AssertionError.class, () -> meter.assertLongGauge("gauge", 3, first)); - meter.assertDoubleGauge("double", 2.5, first); - meter.assertDoubleGauge("double", 3.5, second); - Assert.assertThrows(AssertionError.class, () -> meter.assertDoubleGauge("double", 3.5, first)); - one.close(); - meter.assertLongGauge("gauge", 3, second); - Assert.assertThrows(AssertionError.class, () -> meter.assertLongGauge("gauge", 2, first)); - } finally { - one.close(); - two.close(); - doubles.close(); - } - meter.assertNoGauges(); - } -} diff --git a/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java b/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java index 4128b8a40..db38dfebc 100644 --- a/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java +++ b/topic/src/test/java/tech/ydb/topic/write/impl/WriterMetricsTest.java @@ -1,8 +1,11 @@ package tech.ydb.topic.write.impl; +import java.io.IOException; +import java.io.OutputStream; import java.util.Arrays; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -22,6 +25,7 @@ import tech.ydb.topic.write.AsyncWriter; import tech.ydb.topic.write.Message; import tech.ydb.topic.write.SyncWriter; +import tech.ydb.topic.write.WriteAck; import static tech.ydb.topic.utils.InMemoryMeter.attributes; @@ -148,6 +152,48 @@ public void sessionErrorsFollowRetriesAndStop() throws Exception { meter.assertNoGauges(); } + @Test + public void compressionFailureMetrics() throws Exception { + WriteStreamMock stream = new WriteStreamMock(); + IOException problem = new IOException("encode failure"); + Codec codec = Mockito.mock(Codec.class, Mockito.CALLS_REAL_METHODS); + Mockito.when(codec.getId()).thenReturn(Codec.GZIP); + Mockito.doThrow(problem).when(codec).encode(Mockito.any(OutputStream.class)); + try (TopicClient client = TopicClientImpl.newClient(mockRpc(stream)).registerCodec(codec) + .setCompressionExecutor(Runnable::run).build()) { + AsyncWriter writer = client.createAsyncWriter(WriterSettings.newBuilder().setTopicPath("/test/topic") + .setCodec(Codec.GZIP).withMeter(meter, "writer").build()); + try { + writer.init(); + stream.sendInitResponse(0); + CompletableFuture first = writer.send(Message.of(new byte[]{1, 2})); + CompletableFuture second = writer.send(Message.of(new byte[]{3, 4, 5})); + Assert.assertSame(problem, Assert.assertThrows(ExecutionException.class, + () -> first.get(1, TimeUnit.SECONDS)).getCause()); + Assert.assertSame(problem, Assert.assertThrows(ExecutionException.class, + () -> second.get(1, TimeUnit.SECONDS)).getCause()); + meter.assertCounter(PREFIX + "sending.messages", 2, COMMON); + meter.assertCounter(PREFIX + "sending.bytes", 5, COMMON); + meter.assertNoCounterSamples(PREFIX + "written.messages"); + meter.assertNoHistogramSamples(PREFIX + "message.ack.duration"); + meter.assertLongGauge(PREFIX + "buffer.used.bytes", 0, COMMON); + for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { + meter.assertHistogram(PREFIX + name, "s", 2, attributes("topic", "/test/topic", + "writer.name", "writer", "codec", "gzip", "outcome", "error", + "error.type", "java.io.IOException")); + meter.assertHistogram(PREFIX + name, "s", 0, attributes("topic", "/test/topic", + "writer.name", "writer", "codec", "gzip", "outcome", "ok")); + } + } finally { + CompletableFuture closed = writer.shutdown(); + stream.close(Status.SUCCESS); + closed.get(1, TimeUnit.SECONDS); + } + } + meter.assertNoCounterSamples(PREFIX + "session.errors"); + meter.assertNoGauges(); + } + private static TopicRpc mockRpc(WriteStreamMock first, WriteStreamMock... rest) { TopicRpc rpc = Mockito.mock(TopicRpc.class); Mockito.when(rpc.getScheduler()).thenReturn(Mockito.mock(ScheduledExecutorService.class));