Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions topic/src/main/java/tech/ydb/topic/read/impl/BatchMeta.java
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ public class BatchMeta {
private final Map<String, String> writeSessionMeta;
private final int codec;
private final Instant writtenAt;
private final long receivedAt = System.nanoTime();

public BatchMeta(YdbTopic.StreamReadMessage.ReadResponse.Batch batch) {
this.producerId = batch.getProducerId();
Expand All @@ -22,6 +23,10 @@ public BatchMeta(YdbTopic.StreamReadMessage.ReadResponse.Batch batch) {
this.writtenAt = ProtobufUtils.protoToInstant(batch.getWrittenAt());
}

long getReceivedAt() {
return receivedAt;
}

public String getProducerId() {
return producerId;
}
Expand Down
4 changes: 4 additions & 0 deletions topic/src/main/java/tech/ydb/topic/read/impl/MessageImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ public MessageImpl(PartitionSession session, MessageCommitter committer, BatchMe

public abstract boolean isReady();

long getReceivedAt() {
return batchMeta.getReceivedAt();
}

@Override
public long getOffset() {
return offset;
Expand Down
18 changes: 18 additions & 0 deletions topic/src/main/java/tech/ydb/topic/read/impl/ReadConfig.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package tech.ydb.topic.read.impl;

import java.util.concurrent.Executor;
import java.util.function.LongSupplier;

import tech.ydb.topic.description.CodecRegistry;
import tech.ydb.topic.settings.ReaderSettings;
Expand All @@ -18,9 +19,16 @@ public class ReadConfig {
private final long maxMemoryUsageBytes;
private final int maxBatchSize;
private final ReaderMetrics metrics;
private final LongSupplier readyMessages;
private final LongSupplier readyOldestTimestamp;

public ReadConfig(CodecRegistry codecRegistry, Executor control, Executor processor, Executor decompressor,
ReaderSettings settings) {
this(codecRegistry, control, processor, decompressor, settings, () -> 0, () -> 0);
}

public ReadConfig(CodecRegistry codecRegistry, Executor control, Executor processor, Executor decompressor,
ReaderSettings settings, LongSupplier readyMessages, LongSupplier readyOldestTimestamp) {
this.codecRegistry = codecRegistry;
this.control = control;
this.processor = processor;
Expand All @@ -29,6 +37,8 @@ public ReadConfig(CodecRegistry codecRegistry, Executor control, Executor proces
this.maxMemoryUsageBytes = settings.getMaxMemoryUsageBytes();
this.maxBatchSize = settings.getMaxBatchSize();
this.metrics = new ReaderMetrics(settings.getMeter(), settings.getConsumerName(), settings.getReaderName());
this.readyMessages = readyMessages;
this.readyOldestTimestamp = readyOldestTimestamp;
}

public CodecRegistry getCodecRegistry() {
Expand Down Expand Up @@ -59,6 +69,14 @@ public int getMaxBatchSize() {
return maxBatchSize;
}

public long getReadyMessages() {
return readyMessages.getAsLong();
}

public long getReadyOldestTimestamp() {
return readyOldestTimestamp.getAsLong();
}

ReaderMetrics getMetrics() {
return metrics;
}
Expand Down
24 changes: 21 additions & 3 deletions topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,8 @@ private enum State {
this.controlExecutor = config.getControlExecutor();

this.committer = new ReadPartitionCommitter(traceID, session, partition, lastCommittedOffset);
this.decoder = new ReadPartitionDecoder(traceID, sessionDecoder, partition, committer, this::sendDataToReaders);
this.decoder = new ReadPartitionDecoder(traceID, sessionDecoder, partition, committer,
this::sendDataToReaders, metrics);
this.queue = new ReadPartitionQueue(traceID, decoder, config.getMaxBatchSize(), lastCommittedOffset);

logger.info("[{}] started for {}", traceID, partition);
Expand All @@ -90,6 +91,14 @@ public PartitionSession getPartition() {
return partition;
}

public long getCommitOffsetLag() {
return committer.getCommitOffsetLag();
}

void updateCommitTarget(long offset) {
committer.updateCommitTarget(offset);
}

@Override
public void confirmRangeProcessed(OffsetsRange range) {
decoder.releaseRange(range);
Expand All @@ -104,11 +113,20 @@ public void confirmPartitionEnded(List<TopicPartition> childs) {
sendDataToReaders();
}

public void confirmCommittedOffset(long committedOffset) {
committer.updateCommittedOffset(committedOffset);
long getOldestReceivedTimestamp() {
return queue.getOldestReceivedTimestamp();
}

long getLocalBufferMessages() {
return queue.getLocalBufferMessages();
}

public long confirmCommittedOffset(long committedOffset) {
long acknowledged = committer.updateCommittedOffset(committedOffset);
committer.completePendingCommits();
commitOffsetAck.set(new CommitOffsetAcknowledgementEventImpl(partition, committedOffset));
sendDataToReaders();
return acknowledged;
}

public void unpause() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,24 +32,36 @@ class ReadPartitionCommitter implements MessageCommitter {
private final ReentrantLock commitFuturesLock = new ReentrantLock();

private final AtomicLong lastCommittedOffset;
private final AtomicLong commitTargetOffset;

ReadPartitionCommitter(String traceID, ReadSession session, PartitionSession partition, long lastCommittedOffset) {
this.traceID = traceID;
this.session = session;
this.partition = partition;
this.lastCommittedOffset = new AtomicLong(lastCommittedOffset);
this.commitTargetOffset = new AtomicLong(lastCommittedOffset);
}

private RuntimeException partitionIsClosedException() {
return new RuntimeException("" + partition + " is already stopped");
}

public void updateCommittedOffset(long offset) {
long old = lastCommittedOffset.get();
if (old != lastCommittedOffset.accumulateAndGet(offset, Math::max)) {
public long getCommitOffsetLag() {
return Math.max(0, commitTargetOffset.get() - lastCommittedOffset.get());
}

void updateCommitTarget(long offset) {
commitTargetOffset.accumulateAndGet(offset, Math::max);
}

public long updateCommittedOffset(long offset) {
long old = lastCommittedOffset.getAndAccumulate(offset, Math::max);
if (offset > old) {
logger.debug("[{}] Updated last committed offset: {}. Previous committed offset: {} "
+ "(diff is {} message(s)).", traceID, offset, old, offset - old);
return offset - old;
}
return 0;
}

public void completePendingCommits() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import tech.ydb.core.metrics.Meter;
import tech.ydb.proto.topic.YdbTopic;
import tech.ydb.topic.description.Codec;
import tech.ydb.topic.description.CodecRegistry;
Expand All @@ -31,6 +32,7 @@ class ReadPartitionDecoder {
private final PartitionSession partition;
private final MessageCommitter committer;
private final MessageDecoder decoder;
private final ReaderMetrics metrics;
private final Runnable readyHandler;

private final AtomicLong allocatedTotal = new AtomicLong(0);
Expand All @@ -39,8 +41,14 @@ class ReadPartitionDecoder {

ReadPartitionDecoder(String traceId, MessageDecoder decoder, PartitionSession partition,
MessageCommitter committer, Runnable readyHandler) {
this(traceId, decoder, partition, committer, readyHandler, new ReaderMetrics(Meter.NOOP, null, null));
}

ReadPartitionDecoder(String traceId, MessageDecoder decoder, PartitionSession partition,
MessageCommitter committer, Runnable readyHandler, ReaderMetrics metrics) {
this.traceID = traceId;
this.decoder = decoder;
this.metrics = metrics;
this.partition = partition;
this.committer = committer;
this.readyHandler = readyHandler;
Expand Down Expand Up @@ -181,6 +189,7 @@ public void decode(CodecRegistry registry) {
return;
}

long compressionStarted = System.nanoTime();
try {
Codec codec = registry.getCodec(codecCode);
if (codec == null) {
Expand All @@ -201,6 +210,8 @@ public void decode(CodecRegistry registry) {
problem = new IOException("Cannot decode message", ex);
}
} finally {
metrics.reportDecoded(System.nanoTime() - compressionStarted,
System.nanoTime() - getReceivedAt(), codecCode, partition.getPath(), problem);
if (problem == null) {
origin = null;
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,15 @@ void updateLastReadOffset(long offset) {
lastReadOffset = offset;
}

long getLocalBufferMessages() {
return readingQueue.size();
}

long getOldestReceivedTimestamp() {
MessageImpl first = readingQueue.peek();
return first == null ? 0 : first.getReceivedAt();
}

void addBatches(List<YdbTopic.StreamReadMessage.ReadResponse.Batch> batchList) {
for (YdbTopic.StreamReadMessage.ReadResponse.Batch batch : batchList) {
if (batch.getMessageDataCount() == 0) {
Expand Down
32 changes: 30 additions & 2 deletions topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,10 @@ public boolean commitOffsets(PartitionSession session, List<OffsetsRange> ranges
.build())
.build();

for (OffsetsRange range : rangesToCommit) {
partition.updateCommitTarget(range.getEnd());
config.getMetrics().reportCommitQueued(range.getEnd() - range.getStart(), session.getPath());
}
send(FromClient.newBuilder().setCommitOffsetRequest(req).build());
return true;
}
Expand Down Expand Up @@ -189,7 +193,30 @@ public void removeLocks(PartitionSession ps, List<TopicPartition> childs) {

public void onInit(YdbTopic.StreamReadMessage.InitResponse response) {
bufferManager.init(response.getSessionId());
config.getMetrics().register(partitions::size, bufferManager::getCreditBalanceBytes);
config.getMetrics().register(partitions::size, bufferManager::getCreditBalanceBytes,
this::getCommitOffsetLagMax, this::getLocalBufferMessages, this::getOldestReceivedTimestamp);
}

private long getCommitOffsetLagMax() {
return partitions.values().stream().mapToLong(ReadPartition::getCommitOffsetLag).max().orElse(0);
}

private long getLocalBufferMessages() {
return config.getReadyMessages() + partitions.values().stream()
.filter(ReadPartition::isActive).mapToLong(ReadPartition::getLocalBufferMessages).sum();
}

private long getOldestReceivedTimestamp() {
long oldestTimestamp = config.getReadyOldestTimestamp();
for (ReadPartition partition : partitions.values()) {
if (partition.isActive()) {
long timestamp = partition.getOldestReceivedTimestamp();
if (timestamp != 0 && (oldestTimestamp == 0 || timestamp < oldestTimestamp)) {
oldestTimestamp = timestamp;
}
}
}
return oldestTimestamp;
}

public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionRequest req) {
Expand Down Expand Up @@ -304,7 +331,8 @@ public void onCommitOffset(YdbTopic.StreamReadMessage.CommitOffsetResponse respo
continue;
}

partition.confirmCommittedOffset(offset.getCommittedOffset());
long acknowledged = partition.confirmCommittedOffset(offset.getCommittedOffset());
config.getMetrics().reportCommitAcknowledged(acknowledged, partition.getPartition().getPath());
}
}

Expand Down
Loading
Loading