diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java index e93007762..483bb8897 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java @@ -190,6 +190,7 @@ public void close() { return; } + metrics.removeSource(queue); session.getBufferManager().releasePartition(partition.getId()); decoder.close(); committer.close(); @@ -211,6 +212,7 @@ public void start(long committed, OffsetsRange offsets) { logger.info("[{}] got StartPartitionSessionRequest with committed offset {} and partition offsets {}", traceID, committed, offsets); + metrics.addSource(queue); committer.updateCommittedOffset(committed); queue.updateLastReadOffset(committed); // partition start event doesn't use partition's executors diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartitionQueue.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartitionQueue.java index 50ca6b33d..0b3755c64 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartitionQueue.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartitionQueue.java @@ -16,7 +16,7 @@ /** * @author Nikolay Perfilov */ -class ReadPartitionQueue { +class ReadPartitionQueue implements ReaderMetrics.Source { private static final Logger logger = LoggerFactory.getLogger(ReadPartition.class); private final String traceID; @@ -36,6 +36,17 @@ void updateLastReadOffset(long offset) { lastReadOffset = offset; } + @Override + public long getMessagesCount() { + return readingQueue.size(); + } + + @Override + public long getOldestReceivedTimestamp() { + MessageImpl first = readingQueue.peek(); + return first == null ? 0 : first.getReceivedAt(); + } + void addBatches(List batchList) { for (YdbTopic.StreamReadMessage.ReadResponse.Batch batch : batchList) { if (batch.getMessageDataCount() == 0) { diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java index a8d48ad51..4c53d2e94 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java @@ -128,6 +128,10 @@ public boolean commitOffsets(PartitionSession session, List ranges .build()) .build(); + for (OffsetsRange range : rangesToCommit) { + config.getMetrics().reportCommitQueued(session.getId(), range.getEnd(), + range.getEnd() - range.getStart(), session.getPath()); + } send(FromClient.newBuilder().setCommitOffsetRequest(req).build()); return true; } @@ -145,6 +149,7 @@ public void sendStartPartition(PartitionSession ps, Long readFrom, Long commitTo } if (commitTo != null) { resp.setCommitOffset(commitTo); + config.getMetrics().updateCommittedOffset(ps.getId(), commitTo); } send(YdbTopic.StreamReadMessage.FromClient.newBuilder().setStartPartitionSessionResponse(resp.build()).build()); @@ -164,6 +169,7 @@ public void sendStopPartition(PartitionSession ps) { ReadPartition partition = partitions.remove(ps.getId()); if (partition != null) { + config.getMetrics().removePartition(ps.getId()); partition.close(); } } @@ -189,7 +195,7 @@ public void removeLocks(PartitionSession ps, List childs) { public void onInit(YdbTopic.StreamReadMessage.InitResponse response) { bufferManager.init(response.getSessionId()); - config.getMetrics().register(partitions::size, bufferManager::getCreditBalanceBytes); + config.getMetrics().register(bufferManager::getCreditBalanceBytes); } public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionRequest req) { @@ -218,6 +224,7 @@ public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionReq return; } + config.getMetrics().addPartition(psid, committed); partition.start(committed, offsets); TopicPartition lockKey = new TopicPartition(ps); if (locks.computeIfPresent(lockKey, (key, lock) -> lock.bind(partition)) != null) { @@ -256,6 +263,7 @@ public void onClosePartition(long psid) { } logger.info("[{}] Received force StopPartitionSessionRequest for {} ", debugId, partition.getPartition()); + config.getMetrics().removePartition(psid); partition.close(); } @@ -264,6 +272,7 @@ public void onStopPartition(YdbTopic.StreamReadMessage.StopPartitionSessionReque long psid = request.getPartitionSessionId(); ReadPartition partition = partitions.get(psid); if (partition != null) { + config.getMetrics().updateCommittedOffset(psid, committedOffset); partition.stop(committedOffset); return; } @@ -304,6 +313,8 @@ public void onCommitOffset(YdbTopic.StreamReadMessage.CommitOffsetResponse respo continue; } + config.getMetrics().reportCommitAcknowledged(offset.getPartitionSessionId(), + offset.getCommittedOffset(), partition.getPartition().getPath()); partition.confirmCommittedOffset(offset.getCommittedOffset()); } } diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java index a20c3ea60..72e9ac192 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java @@ -1,6 +1,10 @@ package tech.ydb.topic.read.impl; import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.LongSupplier; import tech.ydb.core.Status; @@ -16,19 +20,31 @@ * Topic reader metrics. */ final class ReaderMetrics { + interface Source { + long getMessagesCount(); + long getOldestReceivedTimestamp(); + } + private static final String MESSAGE_UNIT = "{message}"; private final LongCounter deliveredMessages; private final LongCounter receivedMessages; private final LongCounter receivedBytes; + private final LongCounter commitQueued; + private final LongCounter commitAcknowledged; private final LongCounter sessionErrors; private final DoubleHistogram compressionDuration; private final DoubleHistogram processingDuration; private final Attr[] commonAttributes; private final boolean enabled; private final Meter meter; + private final Set sources = ConcurrentHashMap.newKeySet(); + private final Map partitions = new ConcurrentHashMap<>(); private MetricRegistration partitionsGauge = MetricRegistration.NOOP; private MetricRegistration creditGauge = MetricRegistration.NOOP; + private MetricRegistration commitLagGauge = MetricRegistration.NOOP; + private MetricRegistration bufferMessagesGauge = MetricRegistration.NOOP; + private MetricRegistration ageGauge = MetricRegistration.NOOP; ReaderMetrics(Meter meter, String consumer, String readerName) { this.meter = meter; @@ -41,6 +57,10 @@ final class ReaderMetrics { "The number of messages accepted by the SDK for an active partition session."); this.receivedBytes = meter.createCounter("ydb.topic.reader.received.bytes", "By", "The protocol bytes_size received in read responses."); + this.commitQueued = meter.createCounter("ydb.topic.reader.commit.queued", MESSAGE_UNIT, + "The number of messages in commit ranges accepted by the SDK."); + this.commitAcknowledged = meter.createCounter("ydb.topic.reader.commit.acknowledged", MESSAGE_UNIT, + "The number of messages acknowledged by advances of the committed offset."); this.sessionErrors = meter.createCounter("ydb.topic.reader.session.errors", "{error}", "The number of reader stream session errors by retry decision."); this.compressionDuration = meter.createHistogram("ydb.topic.reader.compression.duration", "s", @@ -50,21 +70,85 @@ final class ReaderMetrics { this.commonAttributes = createCommonAttributes(consumer, readerName); } - void register(LongSupplier partitionCount, LongSupplier bufferBudget) { + void addSource(Source source) { + sources.add(source); + } + + void removeSource(Source source) { + sources.remove(source); + } + + void addPartition(long id, long committedOffset) { + partitions.put(id, new CommitOffsets(committedOffset)); + } + + void removePartition(long id) { + partitions.remove(id); + } + + void updateCommittedOffset(long id, long offset) { + CommitOffsets offsets = partitions.get(id); + if (offsets != null) { + offsets.committed.accumulateAndGet(offset, Math::max); + } + } + + private long getCommitOffsetLagMax() { + return partitions.values().stream() + .mapToLong(p -> Math.max(0, p.requested.get() - p.committed.get())).max().orElse(0); + } + + private long getLocalBufferMessages() { + return sources.stream().mapToLong(Source::getMessagesCount).sum(); + } + + private long getOldestReceivedTimestamp() { + long oldest = 0; + for (Source source : sources) { + long timestamp = source.getOldestReceivedTimestamp(); + if (timestamp != 0 && (oldest == 0 || timestamp < oldest)) { + oldest = timestamp; + } + } + return oldest; + } + + void register(LongSupplier bufferBudget) { partitionsGauge = meter.registerLongGauge( "ydb.topic.reader.partition_session.count", "{session}", "The number of partition sessions currently in the reader session processing lifecycle.", - m -> m.record(partitionCount.getAsLong(), commonAttributes)); + m -> m.record(partitions.size(), commonAttributes)); creditGauge = meter.registerLongGauge("ydb.topic.reader.credit_balance_bytes", "By", "The protocol credit granted to the server and not yet consumed by read responses.", m -> m.record(bufferBudget.getAsLong(), commonAttributes)); + commitLagGauge = meter.registerLongGauge("ydb.topic.reader.commit_offset.lag.max", MESSAGE_UNIT, + "The maximum gap between requested and acknowledged commit offsets.", + m -> m.record(getCommitOffsetLagMax(), commonAttributes)); + bufferMessagesGauge = meter.registerLongGauge("ydb.topic.reader.local_buffer.messages", MESSAGE_UNIT, + "The number of messages currently buffered by the reader.", + m -> m.record(getLocalBufferMessages(), commonAttributes)); + ageGauge = meter.registerDoubleGauge("ydb.topic.reader.local_buffer.message_age.max", "s", + "The age of the oldest batch retained in the reader's local buffer.", + m -> { + long timestamp = getOldestReceivedTimestamp(); + m.record(timestamp == 0 ? 0 : (System.nanoTime() - timestamp) / 1_000_000_000d, + commonAttributes); + }); } void unregister() { partitionsGauge.close(); creditGauge.close(); + commitLagGauge.close(); + bufferMessagesGauge.close(); + ageGauge.close(); partitionsGauge = MetricRegistration.NOOP; creditGauge = MetricRegistration.NOOP; + commitLagGauge = MetricRegistration.NOOP; + bufferMessagesGauge = MetricRegistration.NOOP; + ageGauge = MetricRegistration.NOOP; + sources.clear(); + partitions.clear(); } void reportDelivered(long messages, String topic) { @@ -93,6 +177,22 @@ void reportSessionError(Status status, boolean retry) { sessionErrors.add(1, attributes); } + void reportCommitQueued(long partitionId, long offset, long messages, String topic) { + CommitOffsets offsets = partitions.get(partitionId); + if (offsets != null) { + offsets.requested.accumulateAndGet(offset, Math::max); + } + report(commitQueued, messages, topic); + } + + void reportCommitAcknowledged(long partitionId, long offset, String topic) { + CommitOffsets offsets = partitions.get(partitionId); + if (offsets != null) { + long old = offsets.committed.getAndAccumulate(offset, Math::max); + report(commitAcknowledged, offset > old ? offset - old : 0, topic); + } + } + void reportDecoded(long compressionNanos, long processingNanos, int codec, String topic, Throwable error) { recordDuration(compressionDuration, compressionNanos, codec, topic, error); recordDuration(processingDuration, processingNanos, codec, topic, error); @@ -138,4 +238,14 @@ private static Attr[] createCommonAttributes(String consumer, String readerName) } return new Attr[]{Attr.of("consumer", consumer), Attr.of("reader.name", readerName)}; } + + private static class CommitOffsets { + private final AtomicLong requested; + private final AtomicLong committed; + + CommitOffsets(long offset) { + this.requested = new AtomicLong(offset); + this.committed = new AtomicLong(offset); + } + } } diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java b/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java index 0efd13af4..984c997da 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java @@ -223,6 +223,18 @@ private class Impl extends ReaderImpl { @Override public void onSessionStarted(String sessionId) { SyncReaderImpl.this.sessionId = sessionId; + config.getMetrics().addSource(new ReaderMetrics.Source() { + @Override + public long getMessagesCount() { + return queue.size(); + } + + @Override + public long getOldestReceivedTimestamp() { + return queue.stream().filter(MessageWrapper::isActive) + .mapToLong(message -> ((MessageImpl) message.msg).getReceivedAt()).min().orElse(0); + } + }); SyncReaderImpl.this.initFuture.complete(null); }