diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java index bed9721f3aaa3..18e0fb296be93 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java @@ -54,6 +54,8 @@ import java.io.IOException; import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Properties; @@ -63,6 +65,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.LockSupport; import java.util.stream.Collectors; +import java.util.stream.LongStream; import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; import static org.junit.Assert.fail; @@ -430,6 +433,137 @@ public void testPollUnsubscribedTopics() throws Exception { } } + @Test + public void testInitialTopicRetainsProgressAfterUnsubscribe() throws Exception { + final String topicName = "topic_initial_retained_progress"; + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("create database root.retain_progress"); + for (int i = 0; i < 20; ++i) { + session.executeNonQueryStatement( + String.format("insert into root.retain_progress.d1(time, s1) values (%s, 1)", i)); + } + session.executeNonQueryStatement("flush"); + } + + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + final Properties properties = new Properties(); + properties.put(TopicConstant.MODE_KEY, TopicConstant.MODE_INITIAL_VALUE); + properties.put(TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, true); + session.createTopic(topicName, properties); + } + + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() + .host(host) + .port(port) + .consumerId("consumer_retained_progress") + .consumerGroupId("group_retained_progress") + .autoCommit(false) + .buildPullConsumer()) { + consumer.open(); + consumer.subscribe(topicName); + + Assert.assertEquals( + LongStream.range(0, 20).boxed().collect(Collectors.toList()), + pollTimestamps(consumer, 20, true)); + + try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement( + "insert into root.retain_progress.d1(time, s1) values (20, 1)"); + session.executeNonQueryStatement("flush"); + } + + // Leave this delivery uncommitted. The server must requeue it when the last consumer leaves. + Assert.assertEquals(Collections.singletonList(20L), pollTimestamps(consumer, 1, false)); + + consumer.unsubscribe(topicName); + try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement( + "insert into root.retain_progress.d1(time, s1) values (21, 1)"); + session.executeNonQueryStatement("flush"); + } + + consumer.subscribe(topicName); + Assert.assertEquals(Arrays.asList(20L, 21L), pollTimestamps(consumer, 2, true)); + consumer.unsubscribe(topicName); + } + } + + @Test + public void testInitialTopicDoesNotRetainProgressByDefault() throws Exception { + final String topicName = "topic_initial_unretained_progress"; + final String host = EnvFactory.getEnv().getIP(); + final int port = Integer.parseInt(EnvFactory.getEnv().getPort()); + try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("create database root.unretained_progress"); + for (int i = 0; i < 3; ++i) { + session.executeNonQueryStatement( + String.format("insert into root.unretained_progress.d1(time, s1) values (%s, 1)", i)); + } + session.executeNonQueryStatement("flush"); + } + + try (final SubscriptionTreeSession session = new SubscriptionTreeSession(host, port)) { + session.open(); + final Properties properties = new Properties(); + properties.put(TopicConstant.MODE_KEY, TopicConstant.MODE_INITIAL_VALUE); + session.createTopic(topicName, properties); + } + + try (final SubscriptionTreePullConsumer consumer = + new SubscriptionTreePullConsumer.Builder() + .host(host) + .port(port) + .consumerId("consumer_unretained_progress") + .consumerGroupId("group_unretained_progress") + .autoCommit(false) + .buildPullConsumer()) { + consumer.open(); + consumer.subscribe(topicName); + Assert.assertEquals(Arrays.asList(0L, 1L, 2L), pollTimestamps(consumer, 3, true)); + consumer.unsubscribe(topicName); + + try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement( + "insert into root.unretained_progress.d1(time, s1) values (3, 1)"); + session.executeNonQueryStatement("flush"); + } + + consumer.subscribe(topicName); + Assert.assertEquals(Arrays.asList(0L, 1L, 2L, 3L), pollTimestamps(consumer, 4, true)); + consumer.unsubscribe(topicName); + } + } + + private static List pollTimestamps( + final SubscriptionTreePullConsumer consumer, final int expectedRows, final boolean commit) { + final List timestamps = new ArrayList<>(); + AWAIT.untilAsserted( + () -> { + final List messages = consumer.poll(Duration.ofSeconds(1)); + for (final SubscriptionMessage message : messages) { + for (final ResultSet dataSet : message.getResultSets()) { + try (final ResultSet result = dataSet) { + final SubscriptionRecordHandler.SubscriptionResultSet records = + (SubscriptionRecordHandler.SubscriptionResultSet) result; + while (records.hasNext()) { + timestamps.add(records.nextRecord().getTimestamp()); + } + } + } + } + if (commit) { + consumer.commitSync(messages); + } + Assert.assertEquals(expectedRows, timestamps.size()); + }); + Collections.sort(timestamps); + return timestamps; + } + @Ignore @Test public void testTsFileDeduplication() { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java index a6b1e95ae3cc2..5edc2a1f785aa 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java @@ -32,6 +32,8 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTemporaryMetaInCoordinator; import org.apache.iotdb.commons.pipe.config.PipeConfig; +import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.manager.ConfigManager; @@ -188,9 +190,35 @@ private void parseHeartbeatAndSaveMetaChangeLocally( && PipeTaskAgent.isHistoryOnlyPipe(staticMeta.getSourceParameters()); final boolean hasReliableDataRegionReport = pipeHeartbeat.hasCompletedDataRegionReport(staticMeta); + final boolean isCompletedEmptySnapshot = + PipeStaticMeta.isSubscriptionPipe(staticMeta.getPipeName()) + && staticMeta + .getSourceParameters() + .getBooleanOrDefault( + SystemConstant.SUBSCRIPTION_EMPTY_SNAPSHOT_COMPLETED_KEY, false); if (isFiniteInternalPipe && hasReliableDataRegionReport - && temporaryMeta.getCompletedDataRegionIds().containsAll(requiredDataRegionIds)) { + && (requiredDataRegionIds.isEmpty() || isCompletedEmptySnapshot) + && isRetainedSubscriptionPipe(staticMeta)) { + if (!isCompletedEmptySnapshot) { + // Keep the original snapshot boundary across unsubscribe and recovery. There is no + // sink queue for an empty snapshot, so persist and push its completion explicitly. + staticMeta + .getSourceParameters() + .addAttribute( + SystemConstant.SUBSCRIPTION_EMPTY_SNAPSHOT_COMPLETED_KEY, + Boolean.TRUE.toString()); + needWriteConsensusOnConfigNodes.set(true); + needPushPipeMetaToDataNodes.set(true); + } + // The heartbeat is keyed by the pre-completion static meta. + continue; + } + if (isFiniteInternalPipe + && hasReliableDataRegionReport + && (isCompletedEmptySnapshot + || temporaryMeta.getCompletedDataRegionIds().containsAll(requiredDataRegionIds)) + && !isRetainedSubscriptionPipe(staticMeta)) { PipeLogger.log( LOGGER::info, ManagerMessages.ALL_DATANODES_REPORTED_HISTORICAL_PIPE_COMPLETED, @@ -298,6 +326,23 @@ private void parseHeartbeatAndSaveMetaChangeLocally( } } + private boolean isRetainedSubscriptionPipe(final PipeStaticMeta staticMeta) { + if (!PipeStaticMeta.isSubscriptionPipe(staticMeta.getPipeName())) { + return false; + } + final String topicName = + staticMeta.getSinkParameters().getStringOrDefault(PipeSinkConstant.SINK_TOPIC_KEY, ""); + final String consumerGroupId = + staticMeta + .getSinkParameters() + .getStringOrDefault(PipeSinkConstant.SINK_CONSUMER_GROUP_KEY, ""); + return configManager + .getSubscriptionManager() + .getSubscriptionCoordinator() + .getSubscriptionInfo() + .shouldRetainSubscriptionPipe(consumerGroupId, topicName, staticMeta.visibleUnder(true)); + } + /** * Collect DataRegion ids that this pipe must wait on before auto-drop. Schema / Config regions * are skipped; DataRegions that the source pattern will not listen to are skipped as well, using diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java index 79a735ded7ff3..bbc948bfe8aa9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfo.java @@ -411,6 +411,7 @@ private void validateTopicConfig(final TopicConfig topicConfig) throws Subscript validateColumnFilter(topicConfig); validateTagFilter(topicConfig); validateIncrementalTopicRetentionConfig(topicConfig); + validateRetainProgressAfterUnsubscribe(topicConfig); validateRecycleUncommittedEventInterval(topicConfig); final Long ownerLeaseDurationMs = @@ -654,9 +655,7 @@ private void validateRecycleUncommittedEventInterval(final TopicConfig topicConf private void validateIncrementalTopicRetentionConfig(final TopicConfig topicConfig) throws SubscriptionException { if (!hasTopicAttributeIgnoreCase(topicConfig, TopicConstant.RETENTION_BYTES_KEY) - && !hasTopicAttributeIgnoreCase(topicConfig, TopicConstant.RETENTION_MS_KEY) - && !hasTopicAttributeIgnoreCase( - topicConfig, TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY)) { + && !hasTopicAttributeIgnoreCase(topicConfig, TopicConstant.RETENTION_MS_KEY)) { return; } @@ -665,17 +664,13 @@ private void validateIncrementalTopicRetentionConfig(final TopicConfig topicConf String.format( ConfigNodeMessages .EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_ATTRIBUTES_ARG_ARE_ONLY_SUPPORTED_FOR_INCREMENTAL_TOPICS_55CB0BF2, - List.of( - TopicConstant.RETENTION_BYTES_KEY, - TopicConstant.RETENTION_MS_KEY, - TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY)); + List.of(TopicConstant.RETENTION_BYTES_KEY, TopicConstant.RETENTION_MS_KEY)); LOGGER.warn(exceptionMessage); throw new SubscriptionException(exceptionMessage); } validateRetentionValue(topicConfig, TopicConstant.RETENTION_BYTES_KEY); validateRetentionValue(topicConfig, TopicConstant.RETENTION_MS_KEY); - validateRetainProgressAfterUnsubscribe(topicConfig); } private void validateRetainProgressAfterUnsubscribe(final TopicConfig topicConfig) @@ -964,6 +959,12 @@ private TSStatus alterTopicInternal(final AlterTopicPlan plan) { if (consumerGroupMeta.visibleUnder(isTableModel) && !consumerGroupMeta.isTopicSubscribedByConsumerGroup( plan.getTopicMeta().getTopicName())) { + if (consumerGroupMeta + .getRetainedTopicNames() + .contains(plan.getTopicMeta().getTopicName())) { + // The AlterTopicProcedure drops the retained Pipe before clearing this marker. + continue; + } commitProgressKeeper.removeTopicProgress( consumerGroupMeta.getConsumerGroupId(), plan.getTopicMeta().getTopicName()); removeEmptyConsumerGroupWithoutProgress(consumerGroupMeta); @@ -1004,25 +1005,75 @@ public TSStatus alterMultipleTopics(final AlterMultipleTopicsPlan plan) { public TSStatus dropTopic(DropTopicPlan plan) { acquireWriteLock(); try { + final TopicMeta topicMeta = + plan.isTableModelSet() + ? topicMetaKeeper.getTopicMeta(plan.getTopicName(), plan.isTableModel()) + : topicMetaKeeper.getTopicMeta(plan.getTopicName()); + final boolean isTableModel = + plan.isTableModelSet() + ? plan.isTableModel() + : Objects.nonNull(topicMeta) && topicMeta.visibleUnderTableModel(); + final boolean isModelScoped = plan.isTableModelSet() || Objects.nonNull(topicMeta); for (final ConsumerGroupMeta consumerGroupMeta : consumerGroupMetaKeeper.getAllConsumerGroupMeta()) { - if (!plan.isTableModelSet() || consumerGroupMeta.visibleUnder(plan.isTableModel())) { + if (!isModelScoped || consumerGroupMeta.visibleUnder(isTableModel)) { + if (isModelScoped + && consumerGroupMeta.getRetainedTopicNames().contains(plan.getTopicName())) { + // The DropTopicProcedure drops the retained Pipe before clearing this marker. + continue; + } commitProgressKeeper.removeTopicProgress( consumerGroupMeta.getConsumerGroupId(), plan.getTopicName()); removeEmptyConsumerGroupWithoutProgress(consumerGroupMeta); } } - if (plan.isTableModelSet()) { - topicMetaKeeper.removeTopicMeta(plan.getTopicName(), plan.isTableModel()); - } else { - topicMetaKeeper.removeTopicMeta(plan.getTopicName()); - } + topicMetaKeeper.removeTopicMeta(plan.getTopicName(), isTableModel); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } finally { releaseWriteLock(); } } + public List getConsumerGroupsRetainingTopic( + final String topicName, final boolean isTableModel) { + acquireReadLock(); + try { + final List consumerGroupMetas = new ArrayList<>(); + for (final ConsumerGroupMeta meta : consumerGroupMetaKeeper.getAllConsumerGroupMeta()) { + if (meta.visibleUnder(isTableModel) && meta.getRetainedTopicNames().contains(topicName)) { + consumerGroupMetas.add(meta.deepCopy()); + } + } + return consumerGroupMetas; + } finally { + releaseReadLock(); + } + } + + public boolean shouldRetainSubscriptionPipe( + final String consumerGroupId, final String topicName, final boolean isTableModel) { + acquireReadLock(); + try { + final ConsumerGroupMeta consumerGroupMeta = + consumerGroupMetaKeeper.getConsumerGroupMeta(consumerGroupId); + if (Objects.isNull(consumerGroupMeta) || !consumerGroupMeta.visibleUnder(isTableModel)) { + return false; + } + if (consumerGroupMeta.getRetainedTopicNames().contains(topicName)) { + return true; + } + // A snapshot may complete before the last unsubscribe creates the retained marker. Keep + // its Pipe while subscribed as well, so resubscribing cannot recreate and replay it. + final TopicMeta topicMeta = topicMetaKeeper.getTopicMeta(topicName, isTableModel); + return consumerGroupMeta.isTopicSubscribedByConsumerGroup(topicName) + && Objects.nonNull(topicMeta) + && topicMeta.getConfig().isSnapshotMode() + && topicMeta.getConfig().isProgressRetainedAfterUnsubscribe(); + } finally { + releaseReadLock(); + } + } + public TSStatus handleTopicMetaChanges(TopicHandleMetaChangePlan plan) { acquireWriteLock(); try { @@ -1185,6 +1236,12 @@ public TSStatus alterConsumerGroup(AlterConsumerGroupPlan plan) { ? consumerGroupMetaKeeper.getConsumerGroupMeta(consumerGroupId) : null; if (Objects.nonNull(currentConsumerGroupMeta)) { + final Set removedRetainedTopics = + new HashSet<>(currentConsumerGroupMeta.getRetainedTopicNames()); + removedRetainedTopics.removeAll(consumerGroupMeta.getRetainedTopicNames()); + removedRetainedTopics.removeAll(consumerGroupMeta.getSubscribedTopicNames()); + removedRetainedTopics.forEach( + topicName -> commitProgressKeeper.removeTopicProgress(consumerGroupId, topicName)); ConsumerGroupMeta.getTopicsUnsubByGroup(currentConsumerGroupMeta, consumerGroupMeta) .forEach( topicName -> { @@ -1196,7 +1253,7 @@ public TSStatus alterConsumerGroup(AlterConsumerGroupPlan plan) { commitProgressKeeper.removeTopicProgress(consumerGroupId, topicName); } }); - if (consumerGroupMeta.isEmpty()) { + if (consumerGroupMeta.isEmpty() && !currentConsumerGroupMeta.isEmpty()) { final Set retainedTopicNames = new HashSet<>(); for (final TopicMeta topicMeta : topicMetaKeeper.getAllTopicMeta()) { if (topicMeta.visibleUnderTableModel() == currentConsumerGroupMeta.visibleUnder(true) @@ -1210,6 +1267,7 @@ public TSStatus alterConsumerGroup(AlterConsumerGroupPlan plan) { } consumerGroupMetaKeeper.removeConsumerGroupMeta(consumerGroupId); if (!consumerGroupMeta.isEmpty() + || !consumerGroupMeta.getRetainedTopicNames().isEmpty() || commitProgressKeeper.hasConsumerGroupProgress(consumerGroupId)) { consumerGroupMetaKeeper.addConsumerGroupMeta(consumerGroupId, consumerGroupMeta); } @@ -1222,11 +1280,35 @@ public TSStatus alterConsumerGroup(AlterConsumerGroupPlan plan) { private void removeEmptyConsumerGroupWithoutProgress(final ConsumerGroupMeta consumerGroupMeta) { if (consumerGroupMeta.isEmpty() + && consumerGroupMeta.getRetainedTopicNames().isEmpty() && !commitProgressKeeper.hasConsumerGroupProgress(consumerGroupMeta.getConsumerGroupId())) { consumerGroupMetaKeeper.removeConsumerGroupMeta(consumerGroupMeta.getConsumerGroupId()); } } + public void retainSubscriptionProgress( + final ConsumerGroupMeta existingMeta, final ConsumerGroupMeta updatedMeta) { + if (Objects.isNull(existingMeta)) { + return; + } + acquireReadLock(); + try { + for (final String topicName : + ConsumerGroupMeta.getTopicsUnsubByGroup(existingMeta, updatedMeta)) { + final TopicMeta topicMeta = + topicMetaKeeper.getTopicMeta(topicName, existingMeta.visibleUnder(true)); + if (Objects.nonNull(topicMeta) + && !topicMeta.getConfig().isIncrementalMode() + && topicMeta.getConfig().isProgressRetainedAfterUnsubscribe()) { + updatedMeta.retainTopic(topicName); + } + } + updatedMeta.getSubscribedTopicNames().forEach(updatedMeta::removeRetainedTopic); + } finally { + releaseReadLock(); + } + } + public TSStatus handleConsumerGroupMetaChanges(ConsumerGroupHandleMetaChangePlan plan) { acquireWriteLock(); try { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/consumer/AlterConsumerGroupProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/consumer/AlterConsumerGroupProcedure.java index faf9cce8a68ff..22a131f5bfb35 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/consumer/AlterConsumerGroupProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/consumer/AlterConsumerGroupProcedure.java @@ -81,6 +81,9 @@ public boolean executeFromValidate(ConfigNodeProcedureEnv env) throws Subscripti LOGGER.info(ProcedureMessages.ALTERCONSUMERGROUPPROCEDURE_EXECUTEFROMVALIDATE_TRY_TO_VALIDATE); validateAndGetOldAndNewMeta(env); + subscriptionInfo + .get() + .retainSubscriptionProgress(existingConsumerGroupMeta, updatedConsumerGroupMeta); return true; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java index 04e29edbb94c7..9ca82dbb05808 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java @@ -135,10 +135,15 @@ protected boolean executeFromValidate(final ConfigNodeProcedureEnv env) final String pipeName = PipeStaticMeta.generateSubscriptionPipeName(topicName, consumerGroupId); - if (!subscriptionInfo.get().isTopicSubscribedByConsumerGroup(topicName, consumerGroupId) - // even if there existed subscription meta, if there is no corresponding pipe meta, it - // will try to create the pipe - || !pipeTaskInfo.get().isPipeExisted(pipeName, topicMeta.visibleUnderTableModel())) { + final boolean pipeExists = + pipeTaskInfo.get().isPipeExisted(pipeName, topicMeta.visibleUnderTableModel()); + final boolean topicAlreadySubscribed = + consumerGroupMeta.getSubscribedTopicNames().contains(topicName); + final boolean topicProgressRetained = + consumerGroupMeta.getRetainedTopicNames().contains(topicName); + // A retained subscription keeps the original Pipe and its sink queue alive. Reuse it + // only while the group is subscribed or has an explicit retained-progress marker. + if (!pipeExists || (!topicAlreadySubscribed && !topicProgressRetained)) { createPipeProcedures.add( new CreatePipeProcedureV2( new TCreatePipeReq() @@ -227,12 +232,12 @@ protected void executeFromOperateOnDataNodes(final ConfigNodeProcedureEnv env) consensusTopicNames); } - // Push pipe meta to data nodes (only for non-consensus pipe-based topics) - if (!createPipeProcedures.isEmpty()) { - final List pipeStaticMetas = - createPipeProcedures.stream() - .map(CreatePipeProcedureV2::getPipeStaticMeta) - .collect(Collectors.toList()); + // Push pipe meta to data nodes (only for newly created non-consensus pipes). + final List pipeStaticMetas = + createPipeProcedures.stream() + .map(CreatePipeProcedureV2::getPipeStaticMeta) + .collect(Collectors.toList()); + if (!pipeStaticMetas.isEmpty()) { final String exceptionMessage = AbstractOperatePipeProcedureV2.parsePushPipeMetaExceptionForPipe( null, pushMultiPipeMetaToDataNodes(pipeStaticMetas, env)); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java index b4c16985aef6c..a2b2b44333146 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java @@ -121,6 +121,10 @@ protected boolean executeFromValidate(final ConfigNodeProcedureEnv env) continue; } + if (topicMeta.getConfig().isProgressRetainedAfterUnsubscribe()) { + continue; + } + // Topic will be subscribed by no consumers in this group dropPipeProcedures.add( new DropPipeProcedureV2( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedure.java index fb7ff2f6332b5..63052a951ac2d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedure.java @@ -20,14 +20,26 @@ package org.apache.iotdb.confignode.procedure.impl.subscription.topic; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.subscription.meta.consumer.CommitProgressKeeper; +import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.CreatePipePlanV2; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.DropPipePlanV2; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.OperateMultiplePipesPlanV2; +import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.AlterConsumerGroupPlan; +import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.runtime.CommitProgressHandleMetaChangePlan; import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.AlterTopicPlan; import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.apache.iotdb.confignode.i18n.ProcedureMessages; import org.apache.iotdb.confignode.persistence.subscription.SubscriptionInfo; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; -import org.apache.iotdb.confignode.procedure.impl.subscription.AbstractOperateSubscriptionProcedure; +import org.apache.iotdb.confignode.procedure.impl.pipe.AbstractOperatePipeProcedureV2; +import org.apache.iotdb.confignode.procedure.impl.pipe.task.DropPipeProcedureV2; import org.apache.iotdb.confignode.procedure.impl.subscription.SubscriptionOperation; +import org.apache.iotdb.confignode.procedure.impl.subscription.subscription.AbstractOperateSubscriptionAndPipeProcedure; import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.rpc.RpcUtils; @@ -41,13 +53,15 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; -public class AlterTopicProcedure extends AbstractOperateSubscriptionProcedure { +public class AlterTopicProcedure extends AbstractOperateSubscriptionAndPipeProcedure { private static final Logger LOGGER = LoggerFactory.getLogger(AlterTopicProcedure.class); @@ -60,6 +74,11 @@ public class AlterTopicProcedure extends AbstractOperateSubscriptionProcedure { // initial TopicMeta snapshot was built. private Map updatedTopicAttributes; + private final List retainedConsumerGroupMetas = new ArrayList<>(); + private final List retainedPipeProcedures = new ArrayList<>(); + private Map retainedProgress = new HashMap<>(); + private boolean serializeRetainedProgressState; + public AlterTopicProcedure() { super(); } @@ -71,6 +90,13 @@ public AlterTopicProcedure(final boolean shouldMergeUpdatedTopicAttributes) { } } + public AlterTopicProcedure( + final boolean shouldMergeUpdatedTopicAttributes, + final boolean serializeRetainedProgressState) { + this(shouldMergeUpdatedTopicAttributes); + this.serializeRetainedProgressState = serializeRetainedProgressState; + } + public AlterTopicProcedure(TopicMeta updatedTopicMeta) { super(); this.updatedTopicMeta = updatedTopicMeta; @@ -115,6 +141,12 @@ public boolean shouldMergeUpdatedTopicAttributes() { return Objects.nonNull(updatedTopicAttributes); } + public boolean shouldSerializeRetainedProgressState() { + return serializeRetainedProgressState + || !retainedConsumerGroupMetas.isEmpty() + || !retainedPipeProcedures.isEmpty(); + } + @Override protected SubscriptionOperation getOperation() { return SubscriptionOperation.ALTER_TOPIC; @@ -124,6 +156,10 @@ protected SubscriptionOperation getOperation() { public boolean executeFromValidate(ConfigNodeProcedureEnv env) throws SubscriptionException { LOGGER.info(ProcedureMessages.ALTERTOPICPROCEDURE_EXECUTEFROMVALIDATE); + retainedConsumerGroupMetas.clear(); + retainedPipeProcedures.clear(); + retainedProgress.clear(); + existedTopicMeta = subscriptionInfo .get() @@ -136,23 +172,74 @@ public boolean executeFromValidate(ConfigNodeProcedureEnv env) throws Subscripti subscriptionInfo.get().validateBeforeAlteringTopic(updatedTopicMeta); + if (Objects.nonNull(existedTopicMeta) + && existedTopicMeta.getConfig().isProgressRetainedAfterUnsubscribe() + && !updatedTopicMeta.getConfig().isProgressRetainedAfterUnsubscribe()) { + prepareRetainedProgressCleanup(env); + } + return true; } + private void prepareRetainedProgressCleanup(final ConfigNodeProcedureEnv env) { + final String topicName = updatedTopicMeta.getTopicName(); + final boolean isTableModel = updatedTopicMeta.visibleUnderTableModel(); + for (final ConsumerGroupMeta consumerGroupMeta : + subscriptionInfo.get().getConsumerGroupsRetainingTopic(topicName, isTableModel)) { + retainedConsumerGroupMetas.add(consumerGroupMeta); + retainedProgress.putAll( + subscriptionInfo + .get() + .getCommitProgressKeeper() + .getTopicProgress(consumerGroupMeta.getConsumerGroupId(), topicName)); + final String pipeName = + PipeStaticMeta.generateSubscriptionPipeName( + topicName, consumerGroupMeta.getConsumerGroupId()); + if (pipeTaskInfo.get().isPipeExisted(pipeName, isTableModel)) { + final DropPipeProcedureV2 dropPipeProcedure = + new DropPipeProcedureV2(pipeName, isTableModel, pipeTaskInfo); + dropPipeProcedure.executeFromValidateTask(env); + dropPipeProcedure.executeFromCalculateInfoForTask(env); + retainedPipeProcedures.add(dropPipeProcedure); + } + } + } + @Override public void executeFromOperateOnConfigNodes(ConfigNodeProcedureEnv env) throws SubscriptionException { LOGGER.info( ProcedureMessages.ALTERTOPICPROCEDURE_EXECUTEFROMOPERATEONCONFIGNODES_TRY_TO_ALTER_TOPIC); - TSStatus response; + if (!retainedPipeProcedures.isEmpty()) { + final List dropPipePlans = + retainedPipeProcedures.stream() + .map( + procedure -> + new DropPipePlanV2( + procedure.getPipeName(), updatedTopicMeta.visibleUnderTableModel())) + .collect(Collectors.toList()); + writeConsensusPlan(env, new OperateMultiplePipesPlanV2(dropPipePlans)); + } + for (final ConsumerGroupMeta consumerGroupMeta : getUpdatedRetainedConsumerGroupMetas()) { + writeConsensusPlan(env, new AlterConsumerGroupPlan(consumerGroupMeta)); + } + writeAlterTopicPlan(env, updatedTopicMeta); + } + + private void writeAlterTopicPlan(final ConfigNodeProcedureEnv env, final TopicMeta topicMeta) + throws SubscriptionException { + writeConsensusPlan(env, new AlterTopicPlan(topicMeta)); + } + + private void writeConsensusPlan(final ConfigNodeProcedureEnv env, final ConfigPhysicalPlan plan) + throws SubscriptionException { + final TSStatus response; try { - response = - env.getConfigManager().getConsensusManager().write(new AlterTopicPlan(updatedTopicMeta)); + response = env.getConfigManager().getConsensusManager().write(plan); } catch (ConsensusException e) { LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e); - response = - new TSStatus(TSStatusCode.ALTER_TOPIC_ERROR.getStatusCode()).setMessage(e.getMessage()); + throw new SubscriptionException(e.getMessage()); } if (response.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new SubscriptionException( @@ -164,6 +251,16 @@ public void executeFromOperateOnConfigNodes(ConfigNodeProcedureEnv env) } } + private List getUpdatedRetainedConsumerGroupMetas() { + final List updatedMetas = new ArrayList<>(); + for (final ConsumerGroupMeta existingMeta : retainedConsumerGroupMetas) { + final ConsumerGroupMeta updatedMeta = existingMeta.deepCopy(); + updatedMeta.removeRetainedTopic(updatedTopicMeta.getTopicName()); + updatedMetas.add(updatedMeta); + } + return updatedMetas; + } + @Override public void executeFromOperateOnDataNodes(ConfigNodeProcedureEnv env) throws SubscriptionException, IOException { @@ -171,6 +268,24 @@ public void executeFromOperateOnDataNodes(ConfigNodeProcedureEnv env) ProcedureMessages.ALTERTOPICPROCEDURE_EXECUTEFROMOPERATEONDATANODES, updatedTopicMeta.getTopicName()); + if (!retainedPipeProcedures.isEmpty()) { + final String exceptionMessage = + AbstractOperatePipeProcedureV2.parsePushPipeMetaExceptionForPipe( + null, dropMultiPipeOnDataNodes(retainedPipeProcedures, env)); + if (!exceptionMessage.isEmpty()) { + throw new SubscriptionException(exceptionMessage); + } + } + + for (final ConsumerGroupMeta consumerGroupMeta : getUpdatedRetainedConsumerGroupMetas()) { + final List consumerGroupStatuses = + env.pushSingleConsumerGroupOnDataNode(consumerGroupMeta.serialize()); + if (RpcUtils.squashResponseStatusList(consumerGroupStatuses).getCode() + != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new SubscriptionException(consumerGroupStatuses.toString()); + } + } + final List statuses = env.pushSingleTopicOnDataNode(updatedTopicMeta.serialize()); if (RpcUtils.squashResponseStatusList(statuses).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { @@ -198,23 +313,35 @@ public void rollbackFromOperateOnConfigNodes(ConfigNodeProcedureEnv env) ProcedureMessages.ALTERTOPICPROCEDURE_ROLLBACKFROMOPERATEONCONFIGNODES, updatedTopicMeta.getTopicName()); - TSStatus response; - try { - response = - env.getConfigManager().getConsensusManager().write(new AlterTopicPlan(existedTopicMeta)); - } catch (ConsensusException e) { - LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e); - response = - new TSStatus(TSStatusCode.ALTER_TOPIC_ERROR.getStatusCode()).setMessage(e.getMessage()); + writeAlterTopicPlan(env, existedTopicMeta); + if (!retainedPipeProcedures.isEmpty()) { + for (final DropPipeProcedureV2 dropPipeProcedure : retainedPipeProcedures) { + final PipeMeta pipeMeta = dropPipeProcedure.getPipeMetaToDrop(); + if (!pipeTaskInfo + .get() + .isPipeExisted( + pipeMeta.getStaticMeta().getPipeName(), + pipeMeta.getStaticMeta().visibleUnderTableModel())) { + writeConsensusPlan( + env, new CreatePipePlanV2(pipeMeta.getStaticMeta(), pipeMeta.getRuntimeMeta())); + } + } } - if (response.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - throw new SubscriptionException( - String.format( - ProcedureMessages.FAILED_TO_ROLLBACK_FROM_ALTERING_TOPIC_ON_CONFIG_NODES_BECAUSE, - updatedTopicMeta, - existedTopicMeta, - response)); + for (final ConsumerGroupMeta consumerGroupMeta : retainedConsumerGroupMetas) { + writeConsensusPlan(env, new AlterConsumerGroupPlan(consumerGroupMeta)); } + restoreRetainedProgress(env); + } + + private void restoreRetainedProgress(final ConfigNodeProcedureEnv env) + throws SubscriptionException { + if (retainedProgress.isEmpty()) { + return; + } + final Map progress = + subscriptionInfo.get().getCommitProgressKeeper().getAllRegionProgress(); + progress.putAll(retainedProgress); + writeConsensusPlan(env, new CommitProgressHandleMetaChangePlan(progress)); } @Override @@ -224,6 +351,15 @@ public void rollbackFromOperateOnDataNodes(ConfigNodeProcedureEnv env) ProcedureMessages.ALTERTOPICPROCEDURE_ROLLBACKFROMOPERATEONDATANODES, updatedTopicMeta.getTopicName()); + for (final ConsumerGroupMeta consumerGroupMeta : retainedConsumerGroupMetas) { + final List consumerGroupStatuses = + env.pushSingleConsumerGroupOnDataNode(consumerGroupMeta.serialize()); + if (RpcUtils.squashResponseStatusList(consumerGroupStatuses).getCode() + != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new SubscriptionException(consumerGroupStatuses.toString()); + } + } + final List statuses = env.pushSingleTopicOnDataNode(existedTopicMeta.serialize()); if (RpcUtils.squashResponseStatusList(statuses).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { @@ -235,15 +371,19 @@ public void rollbackFromOperateOnDataNodes(ConfigNodeProcedureEnv env) existedTopicMeta, statuses)); } + if (!retainedPipeProcedures.isEmpty()) { + final String exceptionMessage = + AbstractOperatePipeProcedureV2.parsePushPipeMetaExceptionForPipe( + null, AbstractOperatePipeProcedureV2.pushPipeMetaToDataNodes(env, pipeTaskInfo)); + if (!exceptionMessage.isEmpty()) { + throw new SubscriptionException(exceptionMessage); + } + } } @Override public void serialize(DataOutputStream stream) throws IOException { - stream.writeShort( - (shouldMergeUpdatedTopicAttributes() - ? ProcedureType.ALTER_TOPIC_WITH_ATTRIBUTES_PROCEDURE - : ProcedureType.ALTER_TOPIC_PROCEDURE) - .getTypeCode()); + stream.writeShort(getProcedureType().getTypeCode()); super.serialize(stream); ReadWriteIOUtils.write(updatedTopicMeta != null, stream); @@ -259,6 +399,28 @@ public void serialize(DataOutputStream stream) throws IOException { if (shouldMergeUpdatedTopicAttributes()) { ReadWriteIOUtils.write(updatedTopicAttributes, stream); } + if (shouldSerializeRetainedProgressState()) { + ReadWriteIOUtils.write(retainedConsumerGroupMetas.size(), stream); + for (final ConsumerGroupMeta consumerGroupMeta : retainedConsumerGroupMetas) { + consumerGroupMeta.serialize(stream); + } + ReadWriteIOUtils.write(retainedPipeProcedures.size(), stream); + for (final DropPipeProcedureV2 pipeProcedure : retainedPipeProcedures) { + pipeProcedure.serialize(stream); + } + CommitProgressKeeper.serializeRegionProgressMapToStream(retainedProgress, stream); + } + } + + private ProcedureType getProcedureType() { + if (shouldSerializeRetainedProgressState()) { + return shouldMergeUpdatedTopicAttributes() + ? ProcedureType.ALTER_TOPIC_WITH_ATTRIBUTES_AND_RETAINED_PROGRESS_PROCEDURE + : ProcedureType.ALTER_TOPIC_WITH_RETAINED_PROGRESS_PROCEDURE; + } + return shouldMergeUpdatedTopicAttributes() + ? ProcedureType.ALTER_TOPIC_WITH_ATTRIBUTES_PROCEDURE + : ProcedureType.ALTER_TOPIC_PROCEDURE; } @Override @@ -276,6 +438,20 @@ public void deserialize(ByteBuffer byteBuffer) { if (shouldMergeUpdatedTopicAttributes()) { updatedTopicAttributes = ReadWriteIOUtils.readMap(byteBuffer); } + if (shouldSerializeRetainedProgressState()) { + int size = ReadWriteIOUtils.readInt(byteBuffer); + for (int i = 0; i < size; i++) { + retainedConsumerGroupMetas.add(ConsumerGroupMeta.deserialize(byteBuffer)); + } + size = ReadWriteIOUtils.readInt(byteBuffer); + for (int i = 0; i < size; i++) { + ReadWriteIOUtils.readShort(byteBuffer); + final DropPipeProcedureV2 pipeProcedure = new DropPipeProcedureV2(); + pipeProcedure.deserialize(byteBuffer); + retainedPipeProcedures.add(pipeProcedure); + } + retainedProgress = CommitProgressKeeper.deserializeRegionProgressFromBuffer(byteBuffer); + } } @Override @@ -292,7 +468,10 @@ public boolean equals(Object o) { && getCycles() == that.getCycles() && Objects.equals(updatedTopicMeta, that.updatedTopicMeta) && Objects.equals(existedTopicMeta, that.existedTopicMeta) - && Objects.equals(updatedTopicAttributes, that.updatedTopicAttributes); + && Objects.equals(updatedTopicAttributes, that.updatedTopicAttributes) + && Objects.equals(retainedConsumerGroupMetas, that.retainedConsumerGroupMetas) + && Objects.equals(retainedPipeProcedures, that.retainedPipeProcedures) + && Objects.equals(retainedProgress, that.retainedProgress); } @Override @@ -303,6 +482,9 @@ public int hashCode() { getCycles(), updatedTopicMeta, existedTopicMeta, - updatedTopicAttributes); + updatedTopicAttributes, + retainedConsumerGroupMetas, + retainedPipeProcedures, + retainedProgress); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/DropTopicProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/DropTopicProcedure.java index 8a95811fce5c4..5d69675f76a1e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/DropTopicProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/DropTopicProcedure.java @@ -20,12 +20,26 @@ package org.apache.iotdb.confignode.procedure.impl.subscription.topic; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.subscription.meta.consumer.CommitProgressKeeper; +import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; +import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.CreatePipePlanV2; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.DropPipePlanV2; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.OperateMultiplePipesPlanV2; +import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.AlterConsumerGroupPlan; +import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.runtime.CommitProgressHandleMetaChangePlan; +import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.CreateTopicPlan; import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.DropTopicPlan; import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.apache.iotdb.confignode.i18n.ProcedureMessages; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; -import org.apache.iotdb.confignode.procedure.impl.subscription.AbstractOperateSubscriptionProcedure; +import org.apache.iotdb.confignode.procedure.impl.pipe.AbstractOperatePipeProcedureV2; +import org.apache.iotdb.confignode.procedure.impl.pipe.task.DropPipeProcedureV2; import org.apache.iotdb.confignode.procedure.impl.subscription.SubscriptionOperation; +import org.apache.iotdb.confignode.procedure.impl.subscription.subscription.AbstractOperateSubscriptionAndPipeProcedure; import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.rpc.RpcUtils; @@ -39,16 +53,24 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; -public class DropTopicProcedure extends AbstractOperateSubscriptionProcedure { +public class DropTopicProcedure extends AbstractOperateSubscriptionAndPipeProcedure { private static final Logger LOGGER = LoggerFactory.getLogger(DropTopicProcedure.class); private String topicName; private boolean isTableModel; private boolean isTableModelSet; + private TopicMeta existedTopicMeta; + private final List retainedConsumerGroupMetas = new ArrayList<>(); + private final List retainedPipeProcedures = new ArrayList<>(); + private Map retainedProgress = new HashMap<>(); + private boolean serializeRetainedProgressState; public DropTopicProcedure() { super(); @@ -65,6 +87,18 @@ public DropTopicProcedure(String topicName, boolean isTableModel) { this.isTableModelSet = true; } + public DropTopicProcedure(final boolean serializeRetainedProgressState) { + super(); + this.serializeRetainedProgressState = serializeRetainedProgressState; + } + + public boolean shouldSerializeRetainedProgressState() { + return serializeRetainedProgressState + || Objects.nonNull(existedTopicMeta) + || !retainedConsumerGroupMetas.isEmpty() + || !retainedPipeProcedures.isEmpty(); + } + @Override protected SubscriptionOperation getOperation() { return SubscriptionOperation.DROP_TOPIC; @@ -74,11 +108,43 @@ protected SubscriptionOperation getOperation() { protected boolean executeFromValidate(ConfigNodeProcedureEnv env) throws SubscriptionException { LOGGER.info(ProcedureMessages.DROPTOPICPROCEDURE_EXECUTEFROMVALIDATE, topicName); + retainedConsumerGroupMetas.clear(); + retainedPipeProcedures.clear(); + retainedProgress.clear(); + if (isTableModelSet) { subscriptionInfo.get().validateBeforeDroppingTopic(topicName, isTableModel); } else { subscriptionInfo.get().validateBeforeDroppingTopic(topicName); } + + existedTopicMeta = + isTableModelSet + ? subscriptionInfo.get().deepCopyTopicMeta(topicName, isTableModel) + : subscriptionInfo.get().deepCopyTopicMeta(topicName); + if (Objects.nonNull(existedTopicMeta) + && existedTopicMeta.getConfig().isProgressRetainedAfterUnsubscribe()) { + final boolean model = existedTopicMeta.visibleUnderTableModel(); + for (final ConsumerGroupMeta consumerGroupMeta : + subscriptionInfo.get().getConsumerGroupsRetainingTopic(topicName, model)) { + retainedConsumerGroupMetas.add(consumerGroupMeta); + retainedProgress.putAll( + subscriptionInfo + .get() + .getCommitProgressKeeper() + .getTopicProgress(consumerGroupMeta.getConsumerGroupId(), topicName)); + final String pipeName = + PipeStaticMeta.generateSubscriptionPipeName( + topicName, consumerGroupMeta.getConsumerGroupId()); + if (pipeTaskInfo.get().isPipeExisted(pipeName, model)) { + final DropPipeProcedureV2 dropPipeProcedure = + new DropPipeProcedureV2(pipeName, model, pipeTaskInfo); + dropPipeProcedure.executeFromValidateTask(env); + dropPipeProcedure.executeFromCalculateInfoForTask(env); + retainedPipeProcedures.add(dropPipeProcedure); + } + } + } return true; } @@ -87,15 +153,29 @@ protected void executeFromOperateOnConfigNodes(ConfigNodeProcedureEnv env) throws SubscriptionException { LOGGER.info(ProcedureMessages.DROPTOPICPROCEDURE_EXECUTEFROMOPERATEONCONFIGNODES, topicName); + final boolean model = getTopicModel(); + writeConsensusPlan( + env, + existedTopicMeta == null && !isTableModelSet + ? new DropTopicPlan(topicName) + : new DropTopicPlan(topicName, model)); + if (!retainedPipeProcedures.isEmpty()) { + final List dropPipePlans = + retainedPipeProcedures.stream() + .map(proc -> new DropPipePlanV2(proc.getPipeName(), model)) + .collect(java.util.stream.Collectors.toList()); + writeConsensusPlan(env, new OperateMultiplePipesPlanV2(dropPipePlans)); + } + for (final ConsumerGroupMeta consumerGroupMeta : getUpdatedRetainedConsumerGroupMetas()) { + writeConsensusPlan(env, new AlterConsumerGroupPlan(consumerGroupMeta)); + } + } + + private void writeConsensusPlan(final ConfigNodeProcedureEnv env, final ConfigPhysicalPlan plan) + throws SubscriptionException { TSStatus response; try { - response = - env.getConfigManager() - .getConsensusManager() - .write( - isTableModelSet - ? new DropTopicPlan(topicName, isTableModel) - : new DropTopicPlan(topicName)); + response = env.getConfigManager().getConsensusManager().write(plan); } catch (ConsensusException e) { LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e); response = @@ -108,14 +188,46 @@ protected void executeFromOperateOnConfigNodes(ConfigNodeProcedureEnv env) } } + private List getUpdatedRetainedConsumerGroupMetas() { + final List updatedMetas = new ArrayList<>(); + for (final ConsumerGroupMeta existingMeta : retainedConsumerGroupMetas) { + final ConsumerGroupMeta updatedMeta = existingMeta.deepCopy(); + updatedMeta.removeRetainedTopic(topicName); + updatedMetas.add(updatedMeta); + } + return updatedMetas; + } + + private boolean getTopicModel() { + return existedTopicMeta == null ? isTableModel : existedTopicMeta.visibleUnderTableModel(); + } + @Override - protected void executeFromOperateOnDataNodes(ConfigNodeProcedureEnv env) { + protected void executeFromOperateOnDataNodes(ConfigNodeProcedureEnv env) + throws SubscriptionException, IOException { LOGGER.info(ProcedureMessages.DROPTOPICPROCEDURE_EXECUTEFROMOPERATEONDATANODES, topicName); + if (!retainedPipeProcedures.isEmpty()) { + final String exceptionMessage = + AbstractOperatePipeProcedureV2.parsePushPipeMetaExceptionForPipe( + null, dropMultiPipeOnDataNodes(retainedPipeProcedures, env)); + if (!exceptionMessage.isEmpty()) { + throw new SubscriptionException(exceptionMessage); + } + } + for (final ConsumerGroupMeta consumerGroupMeta : getUpdatedRetainedConsumerGroupMetas()) { + final List consumerGroupStatuses = + env.pushSingleConsumerGroupOnDataNode(consumerGroupMeta.serialize()); + if (RpcUtils.squashResponseStatusList(consumerGroupStatuses).getCode() + != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new SubscriptionException(consumerGroupStatuses.toString()); + } + } + final List statuses = - isTableModelSet - ? env.dropSingleTopicOnDataNode(topicName, isTableModel) - : env.dropSingleTopicOnDataNode(topicName); + existedTopicMeta == null && !isTableModelSet + ? env.dropSingleTopicOnDataNode(topicName) + : env.dropSingleTopicOnDataNode(topicName, getTopicModel()); if (RpcUtils.squashResponseStatusList(statuses).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // throw exception instead of logging warn, do not rely on metadata synchronization @@ -131,22 +243,98 @@ protected void rollbackFromValidate(ConfigNodeProcedureEnv env) { } @Override - protected void rollbackFromOperateOnConfigNodes(ConfigNodeProcedureEnv env) { + protected void rollbackFromOperateOnConfigNodes(ConfigNodeProcedureEnv env) + throws SubscriptionException { LOGGER.info(ProcedureMessages.DROPTOPICPROCEDURE_ROLLBACKFROMCREATEONCONFIGNODES, topicName); + if (Objects.nonNull(existedTopicMeta)) { + writeConsensusPlan(env, new CreateTopicPlan(existedTopicMeta)); + for (final DropPipeProcedureV2 dropPipeProcedure : retainedPipeProcedures) { + final PipeMeta pipeMeta = dropPipeProcedure.getPipeMetaToDrop(); + if (!pipeTaskInfo + .get() + .isPipeExisted( + pipeMeta.getStaticMeta().getPipeName(), + pipeMeta.getStaticMeta().visibleUnderTableModel())) { + writeConsensusPlan( + env, new CreatePipePlanV2(pipeMeta.getStaticMeta(), pipeMeta.getRuntimeMeta())); + } + } + for (final ConsumerGroupMeta consumerGroupMeta : retainedConsumerGroupMetas) { + writeConsensusPlan(env, new AlterConsumerGroupPlan(consumerGroupMeta)); + } + restoreRetainedProgress(env); + } + } + + private void restoreRetainedProgress(final ConfigNodeProcedureEnv env) + throws SubscriptionException { + if (retainedProgress.isEmpty()) { + return; + } + final Map progress = + subscriptionInfo.get().getCommitProgressKeeper().getAllRegionProgress(); + progress.putAll(retainedProgress); + writeConsensusPlan(env, new CommitProgressHandleMetaChangePlan(progress)); } @Override - protected void rollbackFromOperateOnDataNodes(ConfigNodeProcedureEnv env) { + protected void rollbackFromOperateOnDataNodes(ConfigNodeProcedureEnv env) + throws SubscriptionException, IOException { LOGGER.info(ProcedureMessages.DROPTOPICPROCEDURE_ROLLBACKFROMCREATEONDATANODES, topicName); + if (Objects.nonNull(existedTopicMeta)) { + for (final ConsumerGroupMeta consumerGroupMeta : retainedConsumerGroupMetas) { + final List statuses = + env.pushSingleConsumerGroupOnDataNode(consumerGroupMeta.serialize()); + if (RpcUtils.squashResponseStatusList(statuses).getCode() + != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new SubscriptionException(statuses.toString()); + } + } + final List statuses = env.pushSingleTopicOnDataNode(existedTopicMeta.serialize()); + if (RpcUtils.squashResponseStatusList(statuses).getCode() + != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new SubscriptionException(statuses.toString()); + } + if (!retainedPipeProcedures.isEmpty()) { + final String exceptionMessage = + AbstractOperatePipeProcedureV2.parsePushPipeMetaExceptionForPipe( + null, AbstractOperatePipeProcedureV2.pushPipeMetaToDataNodes(env, pipeTaskInfo)); + if (!exceptionMessage.isEmpty()) { + throw new SubscriptionException(exceptionMessage); + } + } + } } @Override public void serialize(DataOutputStream stream) throws IOException { - stream.writeShort(ProcedureType.DROP_TOPIC_PROCEDURE.getTypeCode()); + stream.writeShort( + (shouldSerializeRetainedProgressState() + ? ProcedureType.DROP_TOPIC_WITH_RETAINED_PROGRESS_PROCEDURE + : ProcedureType.DROP_TOPIC_PROCEDURE) + .getTypeCode()); super.serialize(stream); ReadWriteIOUtils.write(topicName, stream); - if (isTableModelSet) { + if (shouldSerializeRetainedProgressState()) { + ReadWriteIOUtils.write(Objects.nonNull(existedTopicMeta), stream); + if (Objects.nonNull(existedTopicMeta)) { + existedTopicMeta.serialize(stream); + } + ReadWriteIOUtils.write(isTableModelSet, stream); + if (isTableModelSet) { + ReadWriteIOUtils.write(isTableModel, stream); + } + ReadWriteIOUtils.write(retainedConsumerGroupMetas.size(), stream); + for (final ConsumerGroupMeta consumerGroupMeta : retainedConsumerGroupMetas) { + consumerGroupMeta.serialize(stream); + } + ReadWriteIOUtils.write(retainedPipeProcedures.size(), stream); + for (final DropPipeProcedureV2 pipeProcedure : retainedPipeProcedures) { + pipeProcedure.serialize(stream); + } + CommitProgressKeeper.serializeRegionProgressMapToStream(retainedProgress, stream); + } else if (isTableModelSet) { ReadWriteIOUtils.write(isTableModel, stream); } } @@ -156,9 +344,31 @@ public void deserialize(ByteBuffer byteBuffer) { super.deserialize(byteBuffer); topicName = ReadWriteIOUtils.readString(byteBuffer); - isTableModelSet = byteBuffer.hasRemaining(); - if (isTableModelSet) { - isTableModel = ReadWriteIOUtils.readBool(byteBuffer); + if (shouldSerializeRetainedProgressState()) { + if (ReadWriteIOUtils.readBool(byteBuffer)) { + existedTopicMeta = TopicMeta.deserialize(byteBuffer); + } + isTableModelSet = ReadWriteIOUtils.readBool(byteBuffer); + if (isTableModelSet) { + isTableModel = ReadWriteIOUtils.readBool(byteBuffer); + } + int size = ReadWriteIOUtils.readInt(byteBuffer); + for (int i = 0; i < size; i++) { + retainedConsumerGroupMetas.add(ConsumerGroupMeta.deserialize(byteBuffer)); + } + size = ReadWriteIOUtils.readInt(byteBuffer); + for (int i = 0; i < size; i++) { + ReadWriteIOUtils.readShort(byteBuffer); + final DropPipeProcedureV2 pipeProcedure = new DropPipeProcedureV2(); + pipeProcedure.deserialize(byteBuffer); + retainedPipeProcedures.add(pipeProcedure); + } + retainedProgress = CommitProgressKeeper.deserializeRegionProgressFromBuffer(byteBuffer); + } else { + isTableModelSet = byteBuffer.hasRemaining(); + if (isTableModelSet) { + isTableModel = ReadWriteIOUtils.readBool(byteBuffer); + } } } @@ -176,12 +386,25 @@ public boolean equals(Object o) { && getCycles() == that.getCycles() && Objects.equals(topicName, that.topicName) && isTableModel == that.isTableModel - && isTableModelSet == that.isTableModelSet; + && isTableModelSet == that.isTableModelSet + && Objects.equals(existedTopicMeta, that.existedTopicMeta) + && Objects.equals(retainedConsumerGroupMetas, that.retainedConsumerGroupMetas) + && Objects.equals(retainedPipeProcedures, that.retainedPipeProcedures) + && Objects.equals(retainedProgress, that.retainedProgress); } @Override public int hashCode() { return Objects.hash( - getProcId(), getCurrentState(), getCycles(), topicName, isTableModel, isTableModelSet); + getProcId(), + getCurrentState(), + getCycles(), + topicName, + isTableModel, + isTableModelSet, + existedTopicMeta, + retainedConsumerGroupMetas, + retainedPipeProcedures, + retainedProgress); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java index 0f6b2fa75e1af..26d6ae0240815 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java @@ -382,12 +382,21 @@ public Procedure create(ByteBuffer buffer) throws IOException { case DROP_TOPIC_PROCEDURE: procedure = new DropTopicProcedure(); break; + case DROP_TOPIC_WITH_RETAINED_PROGRESS_PROCEDURE: + procedure = new DropTopicProcedure(true); + break; case ALTER_TOPIC_PROCEDURE: procedure = new AlterTopicProcedure(); break; case ALTER_TOPIC_WITH_ATTRIBUTES_PROCEDURE: procedure = new AlterTopicProcedure(true); break; + case ALTER_TOPIC_WITH_RETAINED_PROGRESS_PROCEDURE: + procedure = new AlterTopicProcedure(false, true); + break; + case ALTER_TOPIC_WITH_ATTRIBUTES_AND_RETAINED_PROGRESS_PROCEDURE: + procedure = new AlterTopicProcedure(true, true); + break; case TOPIC_META_SYNC_PROCEDURE: procedure = new TopicMetaSyncProcedure(); break; @@ -550,9 +559,17 @@ public static ProcedureType getProcedureType(final Procedure procedure) { } else if (procedure instanceof CreateTopicProcedure) { return ProcedureType.CREATE_TOPIC_PROCEDURE; } else if (procedure instanceof DropTopicProcedure) { - return ProcedureType.DROP_TOPIC_PROCEDURE; + return ((DropTopicProcedure) procedure).shouldSerializeRetainedProgressState() + ? ProcedureType.DROP_TOPIC_WITH_RETAINED_PROGRESS_PROCEDURE + : ProcedureType.DROP_TOPIC_PROCEDURE; } else if (procedure instanceof AlterTopicProcedure) { - return ((AlterTopicProcedure) procedure).shouldMergeUpdatedTopicAttributes() + final AlterTopicProcedure alterTopicProcedure = (AlterTopicProcedure) procedure; + if (alterTopicProcedure.shouldSerializeRetainedProgressState()) { + return alterTopicProcedure.shouldMergeUpdatedTopicAttributes() + ? ProcedureType.ALTER_TOPIC_WITH_ATTRIBUTES_AND_RETAINED_PROGRESS_PROCEDURE + : ProcedureType.ALTER_TOPIC_WITH_RETAINED_PROGRESS_PROCEDURE; + } + return alterTopicProcedure.shouldMergeUpdatedTopicAttributes() ? ProcedureType.ALTER_TOPIC_WITH_ATTRIBUTES_PROCEDURE : ProcedureType.ALTER_TOPIC_PROCEDURE; } else if (procedure instanceof TopicMetaSyncProcedure) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java index 7514d06bce48e..3656cddc714b2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java @@ -172,6 +172,9 @@ public enum ProcedureType { COMMIT_PROGRESS_SYNC_PROCEDURE((short) 1510), SUBSCRIPTION_HANDLE_LEADER_CHANGE_PROCEDURE((short) 1511), ALTER_TOPIC_WITH_ATTRIBUTES_PROCEDURE((short) 1512), + ALTER_TOPIC_WITH_RETAINED_PROGRESS_PROCEDURE((short) 1513), + ALTER_TOPIC_WITH_ATTRIBUTES_AND_RETAINED_PROGRESS_PROCEDURE((short) 1514), + DROP_TOPIC_WITH_RETAINED_PROGRESS_PROCEDURE((short) 1515), /** Other */ @TestOnly diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java index ef4c03549afc7..3fb4f7e50813a 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java @@ -34,6 +34,8 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTemporaryMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTemporaryMetaInCoordinator; +import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.CreatePipePlanV2; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.manager.ProcedureManager; @@ -42,7 +44,10 @@ import org.apache.iotdb.confignode.manager.pipe.coordinator.PipeManager; import org.apache.iotdb.confignode.manager.pipe.coordinator.runtime.PipeRuntimeCoordinator; import org.apache.iotdb.confignode.manager.pipe.coordinator.task.PipeTaskCoordinator; +import org.apache.iotdb.confignode.manager.subscription.SubscriptionCoordinator; +import org.apache.iotdb.confignode.manager.subscription.SubscriptionManager; import org.apache.iotdb.confignode.persistence.pipe.PipeTaskInfo; +import org.apache.iotdb.confignode.persistence.subscription.SubscriptionInfo; import org.junit.After; import org.junit.Assert; @@ -638,6 +643,132 @@ public void testParseHeartbeatKeepsCompletedDataRegionAfterLeaderChange() throws Assert.assertNull(pipeTaskInfo.getPipeMetaByPipeName("test_pipe")); } + @Test + public void testParseHeartbeatRetainsCompletedSnapshotSubscriptionPipe() throws Exception { + CommonDescriptor.getInstance().getConfig().setSeperatedPipeHeartbeatEnabled(false); + + for (final boolean retainProgress : new boolean[] {false, true}) { + final String topicName = "snapshot_topic_" + retainProgress; + final String groupId = "snapshot_group_" + retainProgress; + final Map sourceAttributes = new HashMap<>(); + sourceAttributes.put("source.mode", "snapshot"); + sourceAttributes.put("source.realtime.enable", Boolean.FALSE.toString()); + final Map sinkAttributes = new HashMap<>(); + sinkAttributes.put(PipeSinkConstant.SINK_TOPIC_KEY, topicName); + sinkAttributes.put(PipeSinkConstant.SINK_CONSUMER_GROUP_KEY, groupId); + final PipeMeta pipeMeta = + new PipeMeta( + new PipeStaticMeta( + PipeStaticMeta.generateSubscriptionPipeName(topicName, groupId), + 1L, + sourceAttributes, + Collections.emptyMap(), + sinkAttributes), + new PipeRuntimeMeta( + new ConcurrentHashMap<>( + Collections.singletonMap( + 1, new PipeTaskMeta(MinimumProgressIndex.INSTANCE, DATA_NODE_ID))))); + final PipeTaskInfo pipeTaskInfo = new PipeTaskInfo(); + pipeTaskInfo.createPipe( + new CreatePipePlanV2(pipeMeta.getStaticMeta(), pipeMeta.getRuntimeMeta())); + final ParserTestContext context = createParserTestContext(1, pipeTaskInfo); + when(context.subscriptionInfo.shouldRetainSubscriptionPipe(groupId, topicName, false)) + .thenReturn(retainProgress); + + context.parser.parseHeartbeat( + 1, + createPipeHeartbeatWithCompletedRegions(pipeMeta, false, Collections.singletonList(1))); + + if (retainProgress) { + Assert.assertNotNull( + pipeTaskInfo.getPipeMetaByPipeName(pipeMeta.getStaticMeta().getPipeName())); + verify(context.procedureManager, never()).pipeHandleMetaChange(anyBoolean(), anyBoolean()); + } else { + Assert.assertNull( + pipeTaskInfo.getPipeMetaByPipeName(pipeMeta.getStaticMeta().getPipeName())); + verify(context.procedureManager).pipeHandleMetaChange(true, true); + } + } + } + + @Test + public void testParseHeartbeatCompletesRetainedSnapshotSubscriptionPipeWithoutDataRegions() + throws Exception { + CommonDescriptor.getInstance().getConfig().setSeperatedPipeHeartbeatEnabled(false); + + for (final boolean tableModel : new boolean[] {false, true}) { + final String topicName = "snapshot_topic_without_data_regions"; + final String groupId = "snapshot_group_without_data_regions"; + final Map sourceAttributes = new HashMap<>(); + sourceAttributes.put(SystemConstant.SQL_DIALECT_KEY, tableModel ? "table" : "tree"); + sourceAttributes.put("source.mode", "snapshot"); + sourceAttributes.put("source.realtime.enable", Boolean.FALSE.toString()); + final Map sinkAttributes = new HashMap<>(); + sinkAttributes.put(PipeSinkConstant.SINK_TOPIC_KEY, topicName); + sinkAttributes.put(PipeSinkConstant.SINK_CONSUMER_GROUP_KEY, groupId); + final PipeMeta pipeMeta = + new PipeMeta( + new PipeStaticMeta( + PipeStaticMeta.generateSubscriptionPipeName(topicName, groupId), + 1L, + sourceAttributes, + Collections.emptyMap(), + sinkAttributes), + new PipeRuntimeMeta()); + final PipeTaskInfo pipeTaskInfo = new PipeTaskInfo(); + pipeTaskInfo.createPipe( + new CreatePipePlanV2(pipeMeta.getStaticMeta(), pipeMeta.getRuntimeMeta())); + final ParserTestContext context = createParserTestContext(1, pipeTaskInfo); + when(context.subscriptionInfo.shouldRetainSubscriptionPipe(groupId, topicName, tableModel)) + .thenReturn(true); + + // Legacy heartbeats do not establish that a DataNode has received the empty snapshot. + context.parser.parseHeartbeat(1, createPipeHeartbeat(pipeMeta, false)); + Assert.assertFalse( + sourceAttributes.containsKey(SystemConstant.SUBSCRIPTION_EMPTY_SNAPSHOT_COMPLETED_KEY)); + verify(context.procedureManager, never()).pipeHandleMetaChange(anyBoolean(), anyBoolean()); + + context.parser.parseHeartbeat( + 1, createPipeHeartbeatWithCompletedRegions(pipeMeta, false, Collections.emptyList())); + + final PipeMeta completedPipe = + pipeTaskInfo.getPipeMetaByPipeName(pipeMeta.getStaticMeta().getPipeName(), tableModel); + Assert.assertNotNull(completedPipe); + Assert.assertTrue( + completedPipe + .getStaticMeta() + .getSourceParameters() + .getBooleanOrDefault( + SystemConstant.SUBSCRIPTION_EMPTY_SNAPSHOT_COMPLETED_KEY, false)); + final PipeMeta recovered = PipeMeta.deserialize4Coordinator(completedPipe.serialize()); + Assert.assertEquals(completedPipe.getStaticMeta(), recovered.getStaticMeta()); + verify(context.procedureManager).pipeHandleMetaChange(true, true); + + // Region creation after completion must neither reopen the snapshot nor prevent cleanup. + for (final PipeMeta meta : new PipeMeta[] {completedPipe, recovered}) { + meta.getRuntimeMeta() + .getConsensusGroupId2TaskMetaMap() + .put(7, new PipeTaskMeta(MinimumProgressIndex.INSTANCE, DATA_NODE_ID)); + } + // Completion is durable and idempotent; a later metadata sync must keep the same snapshot. + context.parser.parseHeartbeat( + 1, createPipeHeartbeatWithCompletedRegions(recovered, false, Collections.emptyList())); + Assert.assertSame( + completedPipe, + pipeTaskInfo.getPipeMetaByPipeName(pipeMeta.getStaticMeta().getPipeName(), tableModel)); + verify(context.procedureManager, times(1)).pipeHandleMetaChange(true, true); + + // Turning retention off still releases the completed Pipe. + when(context.subscriptionInfo.shouldRetainSubscriptionPipe(groupId, topicName, tableModel)) + .thenReturn(false); + context.parser.parseHeartbeat( + 1, createPipeHeartbeatWithCompletedRegions(recovered, false, Collections.emptyList())); + Assert.assertNull( + pipeTaskInfo.getPipeMetaByPipeName(pipeMeta.getStaticMeta().getPipeName(), tableModel)); + verify(context.procedureManager, times(2)).pipeHandleMetaChange(true, true); + } + } + private ParserTestContext createParserTestContext(final int registeredDataNodeCount) { return createParserTestContext(registeredDataNodeCount, new PipeTaskInfo()); } @@ -653,15 +784,24 @@ private ParserTestContext createParserTestContext( final PipeTaskCoordinator pipeTaskCoordinator = Mockito.mock(PipeTaskCoordinator.class); final PartitionManager partitionManager = Mockito.mock(PartitionManager.class); final ExecutorService procedureSubmitter = Mockito.mock(ExecutorService.class); + final SubscriptionManager subscriptionManager = Mockito.mock(SubscriptionManager.class); + final SubscriptionCoordinator subscriptionCoordinator = + Mockito.mock(SubscriptionCoordinator.class); + final SubscriptionInfo subscriptionInfo = Mockito.mock(SubscriptionInfo.class); when(configManager.getNodeManager()).thenReturn(nodeManager); when(configManager.getProcedureManager()).thenReturn(procedureManager); when(configManager.getPipeManager()).thenReturn(pipeManager); + when(configManager.getSubscriptionManager()).thenReturn(subscriptionManager); when(nodeManager.getRegisteredDataNodeCount()).thenReturn(registeredDataNodeCount); when(pipeManager.getPipeRuntimeCoordinator()).thenReturn(pipeRuntimeCoordinator); when(pipeManager.getPipeTaskCoordinator()).thenReturn(pipeTaskCoordinator); when(pipeRuntimeCoordinator.getProcedureSubmitter()).thenReturn(procedureSubmitter); when(configManager.getPartitionManager()).thenReturn(partitionManager); + when(subscriptionManager.getSubscriptionCoordinator()).thenReturn(subscriptionCoordinator); + when(subscriptionCoordinator.getSubscriptionInfo()).thenReturn(subscriptionInfo); + when(subscriptionInfo.shouldRetainSubscriptionPipe(any(), any(), anyBoolean())) + .thenReturn(false); when(partitionManager.isRegionGroupExists(any(TConsensusGroupId.class))).thenReturn(true); when(pipeTaskCoordinator.tryLock()).thenReturn(new AtomicReference<>(pipeTaskInfo)); when(procedureManager.pipeHandleMetaChange(anyBoolean(), anyBoolean())).thenReturn(true); @@ -673,7 +813,8 @@ private ParserTestContext createParserTestContext( .when(procedureSubmitter) .submit(any(Runnable.class)); - return new ParserTestContext(new PipeHeartbeatParser(configManager), procedureManager); + return new ParserTestContext( + new PipeHeartbeatParser(configManager), procedureManager, subscriptionInfo); } private void createPipe( @@ -806,11 +947,15 @@ private PipeHeartbeat emptyHeartbeat() { private static class ParserTestContext { private final PipeHeartbeatParser parser; private final ProcedureManager procedureManager; + private final SubscriptionInfo subscriptionInfo; private ParserTestContext( - final PipeHeartbeatParser parser, final ProcedureManager procedureManager) { + final PipeHeartbeatParser parser, + final ProcedureManager procedureManager, + final SubscriptionInfo subscriptionInfo) { this.parser = parser; this.procedureManager = procedureManager; + this.subscriptionInfo = subscriptionInfo; } } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTest.java index 35cd29dc6178f..bda8048099980 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTest.java @@ -414,6 +414,58 @@ public void testAlterConsumerGroupRetainsUnsubscribedTopicProgressWhenConfigured Assert.assertNotNull(keeper.getRegionProgress(progressKey)); } + @Test + public void testUnsubscribingRetainedInitialAndSnapshotTopicsPersistsMarkers() { + for (final String mode : + new String[] {TopicConstant.MODE_INITIAL_VALUE, TopicConstant.MODE_SNAPSHOT_VALUE}) { + final String consumerGroupId = "consumer-group-" + mode; + final String consumerId = "consumer"; + final String topicName = "retained-topic-" + mode; + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map topicAttributes = new HashMap<>(); + topicAttributes.put(TopicConstant.MODE_KEY, mode); + topicAttributes.put(TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, "true"); + subscriptionInfo.createTopic( + new CreateTopicPlan(new TopicMeta(topicName, 1L, topicAttributes))); + + final ConsumerGroupMeta currentMeta = + new ConsumerGroupMeta( + consumerGroupId, 1L, new ConsumerMeta(consumerId, 1L, Collections.emptyMap())); + currentMeta.addSubscription(consumerId, Collections.singleton(topicName)); + subscriptionInfo.alterConsumerGroup(new AlterConsumerGroupPlan(currentMeta)); + + Assert.assertEquals( + TopicConstant.MODE_SNAPSHOT_VALUE.equals(mode), + subscriptionInfo.shouldRetainSubscriptionPipe(consumerGroupId, topicName, false)); + Assert.assertFalse( + subscriptionInfo.shouldRetainSubscriptionPipe(consumerGroupId, topicName, true)); + + final ConsumerGroupMeta updatedMeta = currentMeta.deepCopy(); + updatedMeta.removeSubscription(consumerId, Collections.singleton(topicName)); + subscriptionInfo.retainSubscriptionProgress(currentMeta, updatedMeta); + subscriptionInfo.alterConsumerGroup(new AlterConsumerGroupPlan(updatedMeta)); + + Assert.assertTrue(updatedMeta.getRetainedTopicNames().contains(topicName)); + Assert.assertTrue( + subscriptionInfo.shouldRetainSubscriptionPipe(consumerGroupId, topicName, false)); + + final ConsumerGroupMeta rejoinedMeta = updatedMeta.deepCopy(); + rejoinedMeta.addSubscription(consumerId, Collections.singleton(topicName)); + subscriptionInfo.retainSubscriptionProgress(updatedMeta, rejoinedMeta); + subscriptionInfo.alterConsumerGroup(new AlterConsumerGroupPlan(rejoinedMeta)); + Assert.assertTrue(rejoinedMeta.getRetainedTopicNames().isEmpty()); + Assert.assertEquals( + TopicConstant.MODE_SNAPSHOT_VALUE.equals(mode), + subscriptionInfo.shouldRetainSubscriptionPipe(consumerGroupId, topicName, false)); + + topicAttributes.put(TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, "false"); + subscriptionInfo.alterTopic( + new AlterTopicPlan(new TopicMeta(topicName, 2L, topicAttributes))); + Assert.assertFalse( + subscriptionInfo.shouldRetainSubscriptionPipe(consumerGroupId, topicName, false)); + } + } + @Test public void testAlterTopicDisablingRetentionRemovesUnsubscribedProgress() { final String consumerGroupId = "consumer-group"; diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java index fd97bebe74ffc..6ede9a970bdfb 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/subscription/SubscriptionInfoTopicValidationTest.java @@ -596,21 +596,54 @@ public void testAcceptRetainingProgressOnIncrementalTopic() throws Exception { } @Test - public void testRejectRetainingProgressOnInitialTopic() { + public void testAcceptRetainingProgressOnInitialTopic() throws Exception { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + for (final String value : new String[] {"true", " FALSE "}) { + final Map attributes = newInitialTableTopicAttributes(); + attributes.put(TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, value); + + Assert.assertTrue( + subscriptionInfo.validateBeforeCreatingTopic( + new TCreateTopicReq("initial_topic").setTopicAttributes(attributes))); + } + } + + @Test + public void testAcceptRetainingProgressOnSnapshotTopic() throws Exception { final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); final Map attributes = newInitialTableTopicAttributes(); + attributes.put(TopicConstant.MODE_KEY, TopicConstant.MODE_SNAPSHOT_VALUE); attributes.put(TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, "true"); + Assert.assertTrue( + subscriptionInfo.validateBeforeCreatingTopic( + new TCreateTopicReq("snapshot_topic").setTopicAttributes(attributes))); + } + + @Test + public void testInitialRetainProgressDoesNotAllowIncrementalRetentionAttributes() { + final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); + final Map attributes = newInitialTableTopicAttributes(); + attributes.put(TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, "true"); + attributes.put(TopicConstant.RETENTION_BYTES_KEY, "1024"); + assertCreateRejected(subscriptionInfo, attributes, "only supported for incremental topics"); } @Test public void testRejectIllegalRetainProgressValue() { final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(); - final Map attributes = newIncrementalTableTopicAttributes(); - attributes.put(TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, "yes"); - - assertCreateRejected(subscriptionInfo, attributes, "expected true or false"); + for (final String mode : + new String[] { + TopicConstant.MODE_INITIAL_VALUE, + TopicConstant.MODE_SNAPSHOT_VALUE, + TopicConstant.MODE_INCREMENTAL_VALUE + }) { + final Map attributes = newIncrementalTableTopicAttributes(); + attributes.put(TopicConstant.MODE_KEY, mode); + attributes.put(TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, "yes"); + assertCreateRejected(subscriptionInfo, attributes, "expected true or false"); + } } @Test diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/consumer/DropConsumerProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/consumer/DropConsumerProcedureTest.java index 6d9c16a7dd621..87e4b0abd1c3b 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/consumer/DropConsumerProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/consumer/DropConsumerProcedureTest.java @@ -19,23 +19,85 @@ package org.apache.iotdb.confignode.procedure.impl.subscription.consumer; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerMeta; +import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; +import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.AlterConsumerGroupPlan; +import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.CreateTopicPlan; +import org.apache.iotdb.confignode.persistence.subscription.SubscriptionInfo; +import org.apache.iotdb.confignode.procedure.impl.subscription.AbstractOperateSubscriptionProcedure; import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.tsfile.utils.PublicBAOS; import org.junit.Test; import java.io.DataOutputStream; +import java.lang.reflect.Field; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class DropConsumerProcedureTest { + @Test + public void testClosingLastConsumerPersistsConfiguredInitialRuntimeMarker() throws Exception { + for (final boolean retain : new boolean[] {false, true}) { + for (final String dialect : new String[] {"tree", "table"}) { + for (final String mode : + new String[] {TopicConstant.MODE_INITIAL_VALUE, TopicConstant.MODE_SNAPSHOT_VALUE}) { + final SubscriptionInfo subscriptions = new SubscriptionInfo(); + final Map attributes = new HashMap<>(); + attributes.put(SystemConstant.SQL_DIALECT_KEY, dialect); + attributes.put(TopicConstant.MODE_KEY, mode); + attributes.put( + TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, String.valueOf(retain)); + subscriptions.createTopic(new CreateTopicPlan(new TopicMeta("topic", 1L, attributes))); + final ConsumerGroupMeta group = + new ConsumerGroupMeta( + "group", + 1L, + new ConsumerMeta( + "consumer", + 1L, + Collections.singletonMap(ConsumerConstant.SQL_DIALECT_KEY, dialect))); + group.addSubscription("consumer", Collections.singleton("topic")); + subscriptions.alterConsumerGroup(new AlterConsumerGroupPlan(group)); + final DropConsumerProcedure procedure = + new DropConsumerProcedure(new TCloseConsumerReq("consumer", "group")); + final Field field = + AbstractOperateSubscriptionProcedure.class.getDeclaredField("subscriptionInfo"); + field.setAccessible(true); + field.set(procedure, new AtomicReference<>(subscriptions)); + procedure.executeFromValidate(null); + assertEquals( + retain, + procedure.getUpdatedConsumerGroupMeta().getRetainedTopicNames().contains("topic")); + assertTrue(procedure.getUpdatedConsumerGroupMeta().isEmpty()); + + final PublicBAOS bytes = new PublicBAOS(); + procedure.serialize(new DataOutputStream(bytes)); + final ByteBuffer buffer = ByteBuffer.wrap(bytes.getBuf(), 0, bytes.size()); + final DropConsumerProcedure recovered = + (DropConsumerProcedure) ProcedureFactory.getInstance().create(buffer); + assertEquals(procedure, recovered); + assertEquals(0, buffer.remaining()); + subscriptions.alterConsumerGroup( + new AlterConsumerGroupPlan(recovered.getUpdatedConsumerGroupMeta())); + assertEquals(retain, subscriptions.getConsumerGroupMeta("group") != null); + } + } + } + } + @Test public void serializeDeserializeTest() { PublicBAOS byteArrayOutputStream = new PublicBAOS(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedureTest.java index 741e3e344be25..d4087f712a5c3 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedureTest.java @@ -20,10 +20,16 @@ package org.apache.iotdb.confignode.procedure.impl.subscription.subscription; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerMeta; import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.CreatePipePlanV2; +import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.AlterConsumerGroupPlan; +import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.CreateTopicPlan; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.manager.PermissionManager; import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; @@ -40,6 +46,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import org.apache.tsfile.utils.PublicBAOS; @@ -88,6 +96,7 @@ public void serializeDeserializeTest() { new ConsumerGroupMeta( "test_consumer_group", 1, new ConsumerMeta("old_consumer", 1, consumerAttributes)); newConsumerGroupMeta.addSubscription("old_consumer", subscribeTopics); + newConsumerGroupMeta.retainTopic("retained_topic"); AlterConsumerGroupProcedure alterConsumerGroupProcedure = new AlterConsumerGroupProcedure(newConsumerGroupMeta); @@ -186,6 +195,111 @@ public void executeFromValidateShouldResetCreatePipeProceduresOnRetry() throws E Assert.assertEquals(1, proc.getCreatePipeProcedures().size()); } + @Test + public void executeFromValidateShouldReuseRetainedSnapshotAndInitialPipe() throws Exception { + for (final String mode : + new String[] {TopicConstant.MODE_INITIAL_VALUE, TopicConstant.MODE_SNAPSHOT_VALUE}) { + final Map consumerAttributes = new HashMap<>(); + consumerAttributes.put("username", "user"); + consumerAttributes.put("password", "password"); + + final ConsumerGroupMeta consumerGroupMeta = + new ConsumerGroupMeta( + "test_consumer_group", 1, new ConsumerMeta("old_consumer", 1, consumerAttributes)); + consumerGroupMeta.retainTopic("test_topic"); + + final Map topicAttributes = new HashMap<>(); + topicAttributes.put(TopicConstant.MODE_KEY, mode); + topicAttributes.put(TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, "true"); + final TopicMeta topicMeta = new TopicMeta("test_topic", 1, topicAttributes); + + final SubscriptionInfo subscriptionInfo = Mockito.mock(SubscriptionInfo.class); + Mockito.when(subscriptionInfo.getConsumerGroupMeta("test_consumer_group")) + .thenReturn(consumerGroupMeta); + Mockito.when(subscriptionInfo.deepCopyConsumerGroupMeta("test_consumer_group")) + .thenAnswer(invocation -> consumerGroupMeta.deepCopy()); + Mockito.when(subscriptionInfo.deepCopyTopicMeta("test_topic", false)).thenReturn(topicMeta); + + final PipeTaskInfo pipeTaskInfo = Mockito.mock(PipeTaskInfo.class); + Mockito.when( + pipeTaskInfo.isPipeExisted( + PipeStaticMeta.generateSubscriptionPipeName("test_topic", "test_consumer_group"), + false)) + .thenReturn(true); + + final CreateSubscriptionProcedure proc = + new CreateSubscriptionProcedure( + new TSubscribeReq( + "old_consumer", "test_consumer_group", Collections.singleton("test_topic"))); + setField(proc, "subscriptionInfo", new AtomicReference<>(subscriptionInfo)); + setField(proc, "pipeTaskInfo", new AtomicReference<>(pipeTaskInfo)); + + proc.executeFromValidate(mockCreateSubscriptionValidationEnv()); + + Assert.assertTrue(proc.getCreatePipeProcedures().isEmpty()); + Mockito.verify(pipeTaskInfo, Mockito.never()) + .checkBeforeCreatePipe(Mockito.any(TCreatePipeReq.class)); + } + } + + @Test + public void testResubscribeReusesRecoveredCompletedEmptySnapshot() throws Exception { + for (final boolean tableModel : new boolean[] {false, true}) { + final String dialect = tableModel ? "table" : "tree"; + final TopicMeta topic = + new TopicMeta( + "topic", + 1L, + Map.of( + SystemConstant.SQL_DIALECT_KEY, + dialect, + TopicConstant.MODE_KEY, + TopicConstant.MODE_SNAPSHOT_VALUE, + TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, + "true")); + final ConsumerGroupMeta group = + new ConsumerGroupMeta( + "group", + 1L, + new ConsumerMeta("consumer", 1L, Map.of(ConsumerConstant.SQL_DIALECT_KEY, dialect))); + group.retainTopic("topic"); + final SubscriptionInfo subscriptions = new SubscriptionInfo(); + subscriptions.createTopic(new CreateTopicPlan(topic)); + subscriptions.alterConsumerGroup( + new AlterConsumerGroupPlan(ConsumerGroupMeta.deserialize(group.serialize()))); + + final Map source = new HashMap<>(topic.generateExtractorAttributes("root")); + source.put(SystemConstant.SUBSCRIPTION_EMPTY_SNAPSHOT_COMPLETED_KEY, "true"); + final PipeMeta original = + new PipeMeta( + new PipeStaticMeta( + PipeStaticMeta.generateSubscriptionPipeName("topic", "group"), + 42L, + source, + Collections.emptyMap(), + topic.generateConnectorAttributes("group")), + new PipeRuntimeMeta()); + final PipeMeta recovered = PipeMeta.deserialize4Coordinator(original.serialize()); + final PipeTaskInfo pipes = new PipeTaskInfo(); + pipes.createPipe(new CreatePipePlanV2(recovered.getStaticMeta(), recovered.getRuntimeMeta())); + final CreateSubscriptionProcedure procedure = + new CreateSubscriptionProcedure( + new TSubscribeReq("consumer", "group", Collections.singleton("topic")) + .setIsTableModel(tableModel)); + setField(procedure, "subscriptionInfo", new AtomicReference<>(subscriptions)); + setField(procedure, "pipeTaskInfo", new AtomicReference<>(pipes)); + + procedure.executeFromValidate(mockCreateSubscriptionValidationEnv()); + + Assert.assertTrue(procedure.getCreatePipeProcedures().isEmpty()); + Assert.assertEquals( + original.getStaticMeta(), + pipes + .getPipeMetaByPipeName(original.getStaticMeta().getPipeName(), tableModel) + .getStaticMeta()); + } + } + @Test public void newlySubscribedConsensusTopicsShouldBeBasedOnGroupTopics() { final ConsumerMeta firstConsumer = diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedureTest.java index 1bf74f8db244a..c00a7ed5f4690 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedureTest.java @@ -33,6 +33,7 @@ import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import org.apache.tsfile.utils.PublicBAOS; @@ -165,6 +166,42 @@ public void executeFromValidateShouldResetDropPipeProceduresOnRetry() throws Exc Assert.assertEquals(1, proc.getDropPipeProcedures().size()); } + @Test + public void executeFromValidateShouldRetainInitialPipeWhenConfigured() throws Exception { + for (final String mode : + new String[] {TopicConstant.MODE_INITIAL_VALUE, TopicConstant.MODE_SNAPSHOT_VALUE}) { + final ConsumerMeta consumerMeta = new ConsumerMeta("old_consumer", 1, Collections.emptyMap()); + final ConsumerGroupMeta consumerGroupMeta = + new ConsumerGroupMeta("test_consumer_group", 1, consumerMeta); + consumerGroupMeta.addSubscription("old_consumer", Collections.singleton("test_topic")); + + final Map topicAttributes = new HashMap<>(); + topicAttributes.put(TopicConstant.MODE_KEY, mode); + topicAttributes.put(TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, "true"); + + final SubscriptionInfo subscriptionInfo = Mockito.mock(SubscriptionInfo.class); + Mockito.when(subscriptionInfo.deepCopyConsumerGroupMeta("test_consumer_group")) + .thenAnswer(invocation -> consumerGroupMeta.deepCopy()); + Mockito.when(subscriptionInfo.getConsumerGroupMeta("test_consumer_group")) + .thenReturn(consumerGroupMeta); + Mockito.when(subscriptionInfo.deepCopyTopicMeta("test_topic", false)) + .thenReturn(new TopicMeta("test_topic", 1, topicAttributes)); + + final PipeTaskInfo pipeTaskInfo = Mockito.mock(PipeTaskInfo.class); + final DropSubscriptionProcedure procedure = + new DropSubscriptionProcedure( + new TUnsubscribeReq( + "old_consumer", "test_consumer_group", Collections.singleton("test_topic"))); + setField(procedure, "subscriptionInfo", new AtomicReference<>(subscriptionInfo)); + setField(procedure, "pipeTaskInfo", new AtomicReference<>(pipeTaskInfo)); + + procedure.executeFromValidate(Mockito.mock(ConfigNodeProcedureEnv.class)); + + Assert.assertTrue(procedure.getDropPipeProcedures().isEmpty()); + Mockito.verify(pipeTaskInfo, Mockito.never()).checkBeforeDropPipe(Mockito.anyString()); + } + } + private static ConfigNodeProcedureEnv mockConsensusFailureEnv(final TSStatus response) throws Exception { final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedureTest.java index 56f16b6e14a4a..e15d7cd7c3c31 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/AlterTopicProcedureTest.java @@ -62,6 +62,12 @@ public void serializeDeserializeWithUpdatedAttributesTest() throws Exception { new TopicMeta("test_topic", 1, topicAttributes), updatedTopicAttributes)); } + @Test + public void serializeDeserializeWithRetainedProgressStateTest() throws Exception { + final AlterTopicProcedure procedure = new AlterTopicProcedure(false, true); + assertSerializeDeserialize(procedure); + } + private void assertSerializeDeserialize(final AlterTopicProcedure procedure) throws Exception { final PublicBAOS byteArrayOutputStream = new PublicBAOS(); final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); @@ -71,6 +77,7 @@ private void assertSerializeDeserialize(final AlterTopicProcedure procedure) thr final AlterTopicProcedure deserializedProcedure = (AlterTopicProcedure) ProcedureFactory.getInstance().create(buffer); assertEquals(procedure, deserializedProcedure); + assertEquals(0, buffer.remaining()); } @Test diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/DropTopicProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/DropTopicProcedureTest.java index 0ad614408a3c8..c8ccb02ece27f 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/DropTopicProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/topic/DropTopicProcedureTest.java @@ -19,18 +19,254 @@ package org.apache.iotdb.confignode.procedure.impl.subscription.topic; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; +import org.apache.iotdb.commons.subscription.meta.consumer.CommitProgressKeeper; +import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; +import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerMeta; +import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.CreatePipePlanV2; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.OperateMultiplePipesPlanV2; +import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.AlterConsumerGroupPlan; +import org.apache.iotdb.confignode.consensus.request.write.subscription.consumer.runtime.CommitProgressHandleMetaChangePlan; +import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.AlterTopicPlan; +import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.CreateTopicPlan; +import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.DropTopicPlan; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.persistence.pipe.PipeTaskInfo; +import org.apache.iotdb.confignode.persistence.subscription.SubscriptionInfo; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.tsfile.utils.PublicBAOS; import org.junit.Test; +import org.mockito.Mockito; import java.io.DataOutputStream; +import java.lang.reflect.Field; import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class DropTopicProcedureTest { + + @Test + public void testTopicSnapshotSurvivesRecoveryWithoutRetainedGroups() throws Exception { + for (final boolean retain : new boolean[] {false, true}) { + final SubscriptionInfo subscriptions = new SubscriptionInfo(); + final TopicMeta topic = + new TopicMeta( + "topic", + 1L, + Map.of( + TopicConstant.MODE_KEY, + TopicConstant.MODE_INITIAL_VALUE, + TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, + String.valueOf(retain))); + subscriptions.createTopic(new CreateTopicPlan(topic)); + final DropTopicProcedure procedure = new DropTopicProcedure("topic", false); + setField(procedure, "subscriptionInfo", new AtomicReference<>(subscriptions)); + procedure.executeFromValidate(null); + final PublicBAOS bytes = new PublicBAOS(); + procedure.serialize(new DataOutputStream(bytes)); + final ByteBuffer buffer = ByteBuffer.wrap(bytes.getBuf(), 0, bytes.size()); + final DropTopicProcedure recovered = + (DropTopicProcedure) ProcedureFactory.getInstance().create(buffer); + assertEquals(0, buffer.remaining()); + assertEquals(procedure, recovered); + subscriptions.dropTopic(new DropTopicPlan("topic", false)); + assertNull(subscriptions.deepCopyTopicMeta("topic", false)); + final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + final ConfigManager manager = Mockito.mock(ConfigManager.class); + final ConsensusManager consensus = Mockito.mock(ConsensusManager.class); + Mockito.when(env.getConfigManager()).thenReturn(manager); + Mockito.when(manager.getConsensusManager()).thenReturn(consensus); + Mockito.when(consensus.write(Mockito.any(CreateTopicPlan.class))) + .thenAnswer(invocation -> subscriptions.createTopic(invocation.getArgument(0))); + recovered.rollbackFromOperateOnConfigNodes(env); + assertEquals(topic, subscriptions.deepCopyTopicMeta("topic", false)); + } + } + + @Test + public void testRetainedRuntimeCleanupAndRollbackAreModelScoped() throws Exception { + for (final boolean dropTopic : new boolean[] {false, true}) { + for (final boolean tableModel : new boolean[] {false, true}) { + for (final String mode : + new String[] {TopicConstant.MODE_INITIAL_VALUE, TopicConstant.MODE_SNAPSHOT_VALUE}) { + assertRetainedRuntimeCleanupAndRollback(dropTopic, tableModel, mode); + } + } + } + } + + private void assertRetainedRuntimeCleanupAndRollback( + final boolean dropTopic, final boolean tableModel, final String mode) throws Exception { + final String topicName = "retained_topic"; + final SubscriptionInfo subscriptions = new SubscriptionInfo(); + final PipeTaskInfo pipes = new PipeTaskInfo(); + for (final boolean model : new boolean[] {false, true}) { + final String dialect = model ? "table" : "tree"; + final Map attributes = new HashMap<>(); + attributes.put(SystemConstant.SQL_DIALECT_KEY, dialect); + attributes.put(TopicConstant.MODE_KEY, mode); + attributes.put(TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, "true"); + subscriptions.createTopic(new CreateTopicPlan(new TopicMeta(topicName, 1L, attributes))); + final String groupId = "retained_group_" + model; + final ConsumerGroupMeta group = + new ConsumerGroupMeta( + groupId, + 1L, + new ConsumerMeta( + "consumer", + 1L, + Collections.singletonMap(ConsumerConstant.SQL_DIALECT_KEY, dialect))); + group.retainTopic(topicName); + group.removeConsumer("consumer"); + subscriptions.alterConsumerGroup(new AlterConsumerGroupPlan(group)); + final PipeRuntimeMeta runtime = new PipeRuntimeMeta(); + runtime.getStatus().set(PipeStatus.RUNNING); + runtime + .getConsensusGroupId2TaskMetaMap() + .put(1, new PipeTaskMeta(new SimpleProgressIndex(0, 42L), 1)); + pipes.createPipe( + new CreatePipePlanV2( + new PipeStaticMeta( + PipeStaticMeta.generateSubscriptionPipeName(topicName, groupId), + 1L, + Collections.singletonMap(SystemConstant.SQL_DIALECT_KEY, dialect), + Collections.emptyMap(), + Collections.emptyMap()), + runtime)); + final String regionId = "DataRegion[1]"; + subscriptions + .getCommitProgressKeeper() + .updateRegionProgress( + CommitProgressKeeper.generateKey(groupId, topicName, regionId, 1), + ByteBuffer.wrap(new byte[] {(byte) (model ? 2 : 1)})); + subscriptions + .getCommitProgressKeeper() + .updateRegionProgress( + CommitProgressKeeper.generateKey(groupId, "other_topic", regionId, 1), + ByteBuffer.wrap(new byte[] {(byte) (model ? 4 : 3)})); + } + final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + final ConfigManager manager = Mockito.mock(ConfigManager.class); + final ConsensusManager consensus = Mockito.mock(ConsensusManager.class); + Mockito.when(env.getConfigManager()).thenReturn(manager); + Mockito.when(manager.getConsensusManager()).thenReturn(consensus); + Mockito.when(consensus.write(Mockito.any(ConfigPhysicalPlan.class))) + .thenAnswer( + invocation -> { + final ConfigPhysicalPlan plan = invocation.getArgument(0); + if (plan instanceof CreatePipePlanV2) { + return pipes.createPipe((CreatePipePlanV2) plan); + } else if (plan instanceof OperateMultiplePipesPlanV2) { + return pipes.operateMultiplePipes((OperateMultiplePipesPlanV2) plan); + } else if (plan instanceof AlterConsumerGroupPlan) { + return subscriptions.alterConsumerGroup((AlterConsumerGroupPlan) plan); + } else if (plan instanceof DropTopicPlan) { + return subscriptions.dropTopic((DropTopicPlan) plan); + } else if (plan instanceof CreateTopicPlan) { + return subscriptions.createTopic((CreateTopicPlan) plan); + } else if (plan instanceof AlterTopicPlan) { + return subscriptions.alterTopic((AlterTopicPlan) plan); + } else if (plan instanceof CommitProgressHandleMetaChangePlan) { + return subscriptions.handleCommitProgressChanges( + (CommitProgressHandleMetaChangePlan) plan); + } + return new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + }); + + final String groupId = "retained_group_" + tableModel; + final String pipeName = PipeStaticMeta.generateSubscriptionPipeName(topicName, groupId); + final String progressKey = + CommitProgressKeeper.generateKey(groupId, topicName, "DataRegion[1]", 1); + final String otherProgressKey = + CommitProgressKeeper.generateKey(groupId, "other_topic", "DataRegion[1]", 1); + final ByteBuffer originalProgress = + subscriptions.getCommitProgressKeeper().getRegionProgress(progressKey); + final ByteBuffer unrelatedProgress = + subscriptions.getCommitProgressKeeper().getRegionProgress(otherProgressKey); + final PipeMeta originalPipe = pipes.getPipeMetaByPipeName(pipeName, tableModel); + final DropTopicProcedure drop = new DropTopicProcedure(topicName, tableModel); + final AlterTopicProcedure alter = + new AlterTopicProcedure( + subscriptions.deepCopyTopicMetaWithUpdatedAttributes( + topicName, + Collections.singletonMap( + TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, "false"), + tableModel)); + final Object procedure = dropTopic ? drop : alter; + setField(procedure, "subscriptionInfo", new AtomicReference<>(subscriptions)); + setField(procedure, "pipeTaskInfo", new AtomicReference<>(pipes)); + if (dropTopic) { + drop.executeFromValidate(env); + } else { + alter.executeFromValidate(env); + } + final PublicBAOS bytes = new PublicBAOS(); + if (dropTopic) { + drop.serialize(new DataOutputStream(bytes)); + } else { + alter.serialize(new DataOutputStream(bytes)); + } + final ByteBuffer buffer = ByteBuffer.wrap(bytes.getBuf(), 0, bytes.size()); + final Object recovered = ProcedureFactory.getInstance().create(buffer); + assertEquals(0, buffer.remaining()); + assertEquals(procedure, recovered); + setField(recovered, "subscriptionInfo", new AtomicReference<>(subscriptions)); + setField(recovered, "pipeTaskInfo", new AtomicReference<>(pipes)); + if (dropTopic) { + ((DropTopicProcedure) recovered).executeFromOperateOnConfigNodes(env); + } else { + ((AlterTopicProcedure) recovered).executeFromOperateOnConfigNodes(env); + } + assertFalse(pipes.isPipeExisted(pipeName, tableModel)); + assertNull(subscriptions.getCommitProgressKeeper().getRegionProgress(progressKey)); + assertTrue(subscriptions.getConsumerGroupsRetainingTopic(topicName, tableModel).isEmpty()); + assertEquals(1, subscriptions.getConsumerGroupsRetainingTopic(topicName, !tableModel).size()); + if (dropTopic) { + ((DropTopicProcedure) recovered).rollbackFromOperateOnConfigNodes(env); + } else { + ((AlterTopicProcedure) recovered).rollbackFromOperateOnConfigNodes(env); + } + assertEquals(1, subscriptions.getConsumerGroupsRetainingTopic(topicName, tableModel).size()); + assertTrue( + subscriptions + .deepCopyTopicMeta(topicName, tableModel) + .getConfig() + .isProgressRetainedAfterUnsubscribe()); + assertEquals( + originalPipe.getRuntimeMeta(), + pipes.getPipeMetaByPipeName(pipeName, tableModel).getRuntimeMeta()); + assertEquals( + originalProgress, subscriptions.getCommitProgressKeeper().getRegionProgress(progressKey)); + assertEquals( + unrelatedProgress, + subscriptions.getCommitProgressKeeper().getRegionProgress(otherProgressKey)); + } + @Test public void serializeDeserializeTest() { PublicBAOS byteArrayOutputStream = new PublicBAOS(); @@ -68,4 +304,50 @@ public void serializeDeserializeTableModelTest() { fail(); } } + + @Test + public void serializeDeserializeRetainedProgressStateTest() throws Exception { + final Map topicAttributes = new HashMap<>(); + topicAttributes.put(TopicConstant.MODE_KEY, TopicConstant.MODE_INITIAL_VALUE); + topicAttributes.put(TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, "true"); + final TopicMeta topicMeta = new TopicMeta("test_topic", 1, topicAttributes); + final ConsumerGroupMeta consumerGroupMeta = + new ConsumerGroupMeta( + "test_group", 1, new ConsumerMeta("consumer", 1, Collections.emptyMap())); + consumerGroupMeta.retainTopic("test_topic"); + + final DropTopicProcedure procedure = new DropTopicProcedure("test_topic", false); + setField(procedure, "existedTopicMeta", topicMeta); + final Field retainedConsumerGroupMetasField = + DropTopicProcedure.class.getDeclaredField("retainedConsumerGroupMetas"); + retainedConsumerGroupMetasField.setAccessible(true); + @SuppressWarnings("unchecked") + final List retainedConsumerGroupMetas = + (List) retainedConsumerGroupMetasField.get(procedure); + retainedConsumerGroupMetas.add(consumerGroupMeta); + final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + procedure.serialize(new DataOutputStream(byteArrayOutputStream)); + final ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + final DropTopicProcedure deserializedProcedure = + (DropTopicProcedure) ProcedureFactory.getInstance().create(buffer); + + assertEquals(procedure, deserializedProcedure); + assertEquals(0, buffer.remaining()); + } + + private static void setField(final Object target, final String fieldName, final Object value) + throws Exception { + for (Class type = target.getClass(); type != null; type = type.getSuperclass()) { + try { + final Field field = type.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + return; + } catch (final NoSuchFieldException ignored) { + // Look for inherited procedure state. + } + } + throw new NoSuchFieldException(fieldName); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 359bdd79c50ef..5b7775c231562 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -50,6 +50,7 @@ import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -199,6 +200,18 @@ protected void createPipeTask( .put(consensusGroupId, pipeTaskMeta); } + @Override + protected TPushPipeMetaRespExceptionMessage handleSinglePipeMetaChangesInternal( + final PipeMeta pipeMetaFromCoordinator) { + clearCompletedEmptySnapshotTasks(pipeMetaFromCoordinator); + final TPushPipeMetaRespExceptionMessage exception = + super.handleSinglePipeMetaChangesInternal(pipeMetaFromCoordinator); + if (exception == null) { + restoreCompletedEmptySnapshot(pipeMetaFromCoordinator); + } + return exception; + } + @Override public List handlePipeMetaChangesInternal( final List pipeMetaListFromCoordinator) { @@ -208,6 +221,7 @@ public List handlePipeMetaChangesInternal( } carryOverLocalProgressIndexForAlter(pipeMetaListFromCoordinator); + pipeMetaListFromCoordinator.forEach(this::clearCompletedEmptySnapshotTasks); final List exceptionMessages = super.handlePipeMetaChangesInternal(pipeMetaListFromCoordinator); @@ -224,9 +238,52 @@ public List handlePipeMetaChangesInternal( new TPushPipeMetaRespExceptionMessage("", e.getMessage(), System.currentTimeMillis())); } + for (final PipeMeta pipeMeta : pipeMetaListFromCoordinator) { + final String pipeName = pipeMeta.getStaticMeta().getPipeName(); + if (exceptionMessages.stream().noneMatch(exception -> pipeName.equals(exception.pipeName))) { + restoreCompletedEmptySnapshot(pipeMeta); + } + } + return exceptionMessages; } + private void clearCompletedEmptySnapshotTasks(final PipeMeta pipeMeta) { + final PipeStaticMeta staticMeta = pipeMeta.getStaticMeta(); + if (PipeStaticMeta.isSubscriptionPipe(staticMeta.getPipeName()) + && staticMeta + .getSourceParameters() + .getBooleanOrDefault(SystemConstant.SUBSCRIPTION_EMPTY_SNAPSHOT_COMPLETED_KEY, false)) { + // Region creation/leader changes may append tasks after this snapshot has completed. Those + // tasks must never scan data that arrived after the original empty snapshot. + pipeMeta.getRuntimeMeta().getConsensusGroupId2TaskMetaMap().clear(); + } + } + + private void restoreCompletedEmptySnapshot(final PipeMeta pipeMeta) { + final PipeStaticMeta staticMeta = pipeMeta.getStaticMeta(); + if (isShutdown() + || !SubscriptionConfig.getInstance().getSubscriptionEnabled() + || pipeMeta.getRuntimeMeta().getStatus().get() != PipeStatus.RUNNING + || !PipeStaticMeta.isSubscriptionPipe(staticMeta.getPipeName()) + || !staticMeta + .getSourceParameters() + .getBooleanOrDefault(SystemConstant.SUBSCRIPTION_EMPTY_SNAPSHOT_COMPLETED_KEY, false)) { + return; + } + + // This is a coordinator-wide completion decision, not the absence of local tasks. Reapply it + // after both single-pipe updates and full metadata recovery so the snapshot terminates after a + // DataNode restart. + final String consumerGroupId = + staticMeta.getSinkParameters().getString(PipeSinkConstant.SINK_CONSUMER_GROUP_KEY); + SubscriptionAgent.broker().createPipeBrokerIfNotExist(consumerGroupId); + SubscriptionAgent.broker() + .updateCompletedTopicNames( + consumerGroupId, + staticMeta.getSinkParameters().getString(PipeSinkConstant.SINK_TOPIC_KEY)); + } + /** * Carry the committed progress of an old local task into an altered task when it is safe to do * so. The old task is dropped before the new task is created, therefore this must run before diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java index 89d877665ebe9..6c567e6635a86 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java @@ -1061,6 +1061,14 @@ public void removePrefetchingQueue(final String consumerGroupId, final String to prefetchingQueueCount.invalidate(); } + public void resetAndRequeuePipeInFlightEvents( + final String consumerGroupId, final String topicName) { + final SubscriptionBroker broker = getPipeBroker(consumerGroupId); + if (Objects.nonNull(broker)) { + broker.resetAndRequeueInFlightEvents(topicName); + } + } + public boolean executePrefetch(final String consumerGroupId, final String topicName) { final ISubscriptionBroker broker = getBrokerForTopic(consumerGroupId, topicName); if (Objects.isNull(broker)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java index 8af488c5923b7..eccdc4ecac1c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgent.java @@ -30,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; @@ -100,14 +101,21 @@ private void handleSingleConsumerGroupMetaChangesInternal( if (metaFromCoordinator.isEmpty()) { if (Objects.nonNull(metaInAgent)) { - cleanupConsumerGroupSubscriptions(consumerGroupId, metaInAgent, true); + cleanupConsumerGroupSubscriptions( + consumerGroupId, metaInAgent, true, metaFromCoordinator.getRetainedTopicNames()); } final boolean retainProgress = ConsensusSubscriptionSetupHandler.retainConfiguredProgressForConsumerGroup( consumerGroupId, metaFromCoordinator.visibleUnder(true)); - dropConsumerGroupBroker(consumerGroupId); - if (retainProgress) { + final boolean retainTopicProgress = !metaFromCoordinator.getRetainedTopicNames().isEmpty(); + if (retainTopicProgress) { + SubscriptionAgent.broker().createPipeBrokerIfNotExist(consumerGroupId); + } else { + dropConsumerGroupBroker(consumerGroupId); + } + if (retainProgress || retainTopicProgress) { replaceConsumerGroupMeta(consumerGroupId, metaFromCoordinator); + requeueRetainedEvents(consumerGroupId, metaInAgent, metaFromCoordinator); } else { removeConsumerGroupMeta(consumerGroupId); } @@ -157,6 +165,11 @@ private void handleSingleConsumerGroupMetaChangesInternal( // remove prefetching queues for topics unsubscribed by the consumer group final Set topicsUnsubByGroup = ConsumerGroupMeta.getTopicsUnsubByGroup(metaInAgent, metaFromCoordinator); + final Set removedRetainedTopics = + new LinkedHashSet<>(metaInAgent.getRetainedTopicNames()); + removedRetainedTopics.removeAll(metaFromCoordinator.getRetainedTopicNames()); + removedRetainedTopics.removeAll(metaFromCoordinator.getSubscribedTopicNames()); + topicsUnsubByGroup.addAll(removedRetainedTopics); final Set pipeTopicsUnsubByGroup = new LinkedHashSet<>(); final Set consensusTopicsUnsubByGroup = new LinkedHashSet<>(); for (final String topicName : topicsUnsubByGroup) { @@ -165,6 +178,9 @@ private void handleSingleConsumerGroupMetaChangesInternal( consensusTopicsUnsubByGroup.add(topicName); continue; } + if (metaFromCoordinator.getRetainedTopicNames().contains(topicName)) { + continue; + } pipeTopicsUnsubByGroup.add(topicName); } // Detect newly subscribed topics (present in new meta but not in old meta) @@ -198,6 +214,7 @@ private void handleSingleConsumerGroupMetaChangesInternal( // TODO: Currently we fully replace the entire ConsumerGroupMeta without carefully // checking the changes in its fields. replaceConsumerGroupMeta(consumerGroupId, metaFromCoordinator); + requeueRetainedEvents(consumerGroupId, metaInAgent, metaFromCoordinator); }); } @@ -210,6 +227,20 @@ static void applyTopicDiff( publishMeta.run(); } + private void requeueRetainedEvents( + final String consumerGroupId, + final ConsumerGroupMeta existingMeta, + final ConsumerGroupMeta updatedMeta) { + if (Objects.isNull(existingMeta)) { + return; + } + for (final String topicName : updatedMeta.getRetainedTopicNames()) { + // No consumers remain subscribed to this topic. Include deliveries owned by consumers that + // left earlier, and rewind multipart responses so the next subscriber can read them in full. + SubscriptionAgent.broker().resetAndRequeuePipeInFlightEvents(consumerGroupId, topicName); + } + } + public TPushConsumerGroupMetaRespExceptionMessage handleConsumerGroupMetaChanges( final List consumerGroupMetasFromCoordinator) { for (final ConsumerGroupMeta consumerGroupMetaFromCoordinator : @@ -349,8 +380,22 @@ private void cleanupConsumerGroupSubscriptions( final String consumerGroupId, final ConsumerGroupMeta consumerGroupMeta, final boolean retainConfiguredProgress) { + cleanupConsumerGroupSubscriptions( + consumerGroupId, consumerGroupMeta, retainConfiguredProgress, Collections.emptySet()); + } + + private void cleanupConsumerGroupSubscriptions( + final String consumerGroupId, + final ConsumerGroupMeta consumerGroupMeta, + final boolean retainConfiguredProgress, + final Set retainedTopicNames) { final Set consensusTopics = new LinkedHashSet<>(); - for (final String topicName : consumerGroupMeta.getSubscribedTopicNames()) { + final Set topicNames = new LinkedHashSet<>(consumerGroupMeta.getSubscribedTopicNames()); + topicNames.addAll(consumerGroupMeta.getRetainedTopicNames()); + for (final String topicName : topicNames) { + if (retainedTopicNames.contains(topicName)) { + continue; + } if (ConsensusSubscriptionSetupHandler.isConsensusBasedTopic( topicName, consumerGroupMeta.visibleUnder(true))) { consensusTopics.add(topicName); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java index 3cfb34014d74b..4165f255dccd1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.subscription.metric.SubscriptionPrefetchingQueueMetrics; import org.apache.iotdb.db.subscription.resource.SubscriptionDataNodeResourceManager; import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; @@ -111,7 +112,8 @@ public boolean acceptsTopic(final String topicName) { public List poll( final String consumerId, final Set topicNames, final long maxBytes) { final List eventsToPoll = new ArrayList<>(); - final Set candidateTopicNames = prepareCandidateTopicNames(topicNames, eventsToPoll); + final Set candidateTopicNames = + prepareCandidateTopicNames(consumerId, topicNames, eventsToPoll); // Sort topic names based on the current subscription states (number of events received per // topic) @@ -159,6 +161,13 @@ public List poll( continue; } + // A poll may have captured its topic set before the last subscriber left. Retained queues + // stay alive, so check the published subscription again before handing out their events. + if (!canDeliverRetainedEvent(consumerId, topicName)) { + prefetchingQueue.requeue(consumerId, event.getCommitContext()); + continue; + } + // Try to get the current size of the event final long currentSize; try { @@ -206,7 +215,17 @@ public List poll( return eventsToPoll; } + private boolean canDeliverRetainedEvent(final String consumerId, final String topicName) { + final TopicConfig config = SubscriptionAgent.topic().getTopicConfig(topicName, isTableModel()); + return Objects.isNull(config) + || !config.isProgressRetainedAfterUnsubscribe() + || SubscriptionAgent.consumer() + .getTopicNamesSubscribedByConsumer(brokerId, consumerId) + .contains(topicName); + } + private Set prepareCandidateTopicNames( + final String consumerId, final Set topicNames, final List eventsToPoll /* output parameter */) { final Set candidateTopicNames = new HashSet<>(); @@ -214,9 +233,10 @@ private Set prepareCandidateTopicNames( // Check pipe-based queue final SubscriptionPrefetchingQueue prefetchingQueue = topicNameToPrefetchingQueue.get(topicName); - // If there is no prefetching queue for the topic, check if it's completed - if (Objects.isNull(prefetchingQueue)) { - if (completedTopicNames.containsKey(topicName)) { + // Retained snapshot Pipes stay alive after completion, so their queues also report + // termination directly instead of waiting for the Pipe to unbind them. + if (isTopicCompleted(topicName, prefetchingQueue)) { + if (canDeliverRetainedEvent(consumerId, topicName)) { LOGGER.info( DataNodePipeMessages .PIPE_LOG_SUBSCRIPTION_PREFETCHING_QUEUE_BOUND_TO_TOPIC_FOR_CONSUMER_ECB64624, @@ -233,8 +253,10 @@ private Set prepareCandidateTopicNames( topicName, brokerId, INVALID_COMMIT_ID))); - continue; } + continue; + } + if (Objects.isNull(prefetchingQueue)) { // There are two reasons for not printing logs here: // 1. There will be a delay in the creation of the prefetching queue after subscription. // 2. There is no corresponding prefetching queue on this DN: @@ -402,6 +424,13 @@ public int requeueInFlightEvents(final String consumerId) { .sum(); } + public void resetAndRequeueInFlightEvents(final String topicName) { + final SubscriptionPrefetchingQueue queue = topicNameToPrefetchingQueue.get(topicName); + if (Objects.nonNull(queue)) { + queue.resetAndRequeueInFlightEvents(); + } + } + @Override public int refreshInFlightEventLeases( final String consumerId, final List commitContexts) { @@ -437,8 +466,7 @@ public List fetchTopicNamesToUnsubscribe(final Set topicNames) { for (final String topicName : topicNames) { final SubscriptionPrefetchingQueue prefetchingQueue = topicNameToPrefetchingQueue.get(topicName); - // If there is no prefetching queue for the topic, check if it's completed - if (Objects.isNull(prefetchingQueue) && completedTopicNames.containsKey(topicName)) { + if (isTopicCompleted(topicName, prefetchingQueue)) { LOGGER.info( DataNodePipeMessages .PIPE_LOG_SUBSCRIPTION_PREFETCHING_QUEUE_BOUND_TO_TOPIC_FOR_CONSUMER_8F561EB2, @@ -451,6 +479,20 @@ public List fetchTopicNamesToUnsubscribe(final Set topicNames) { return topicNamesToUnsubscribe; } + private boolean isTopicCompleted( + final String topicName, final SubscriptionPrefetchingQueue prefetchingQueue) { + if (Objects.isNull(prefetchingQueue)) { + return completedTopicNames.containsKey(topicName); + } + if (!prefetchingQueue.isCompleted()) { + return false; + } + final TopicConfig config = SubscriptionAgent.topic().getTopicConfig(topicName, isTableModel()); + return Objects.nonNull(config) + && config.isSnapshotMode() + && config.isProgressRetainedAfterUnsubscribe(); + } + /////////////////////////////// prefetching queue /////////////////////////////// public void bindPrefetchingQueue( @@ -491,9 +533,8 @@ public void bindPrefetchingQueue( public void updateCompletedTopicNames(final String topicName) { // mark topic name completed only for topic of snapshot mode - if (SubscriptionAgent.topic() - .getTopicMode(topicName, isTableModel()) - .equals(TopicConstant.MODE_SNAPSHOT_VALUE)) { + if (TopicConstant.MODE_SNAPSHOT_VALUE.equals( + SubscriptionAgent.topic().getTopicMode(topicName, isTableModel()))) { completedTopicNames.put(topicName, topicName); } } @@ -514,9 +555,8 @@ public void unbindPrefetchingQueue(final String topicName) { prefetchingQueue.markClosed(); // mark topic name completed only for topic of snapshot mode - if (SubscriptionAgent.topic() - .getTopicMode(topicName, isTableModel()) - .equals(TopicConstant.MODE_SNAPSHOT_VALUE) + if (TopicConstant.MODE_SNAPSHOT_VALUE.equals( + SubscriptionAgent.topic().getTopicMode(topicName, isTableModel())) && prefetchingQueue.isCompleted()) { completedTopicNames.put(topicName, topicName); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java index bd0b193c5b08c..04af96ba438d0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java @@ -919,6 +919,39 @@ public int requeueInFlightEvents(final String consumerId) { return requeuedCount; } + /** Replays every uncommitted delivery when the last subscriber leaves a retained topic. */ + public void resetAndRequeueInFlightEvents() { + acquireWriteLock(); + try { + if (isClosed()) { + return; + } + // Receiver disconnects and takeovers can release deliveries before the last unsubscribe. + // Rewind those too, but leave never-delivered events (including batch continuations) alone. + for (final SubscriptionEvent event : prefetchingQueue) { + if (!event.isCommitted() && Objects.nonNull(event.getLastPolledConsumerId())) { + event.resetForRedelivery(); + } + } + for (final Pair key : + ImmutableSet.copyOf(inFlightEvents.keySet())) { + inFlightEvents.computeIfPresent( + key, + (ignored, event) -> { + if (event.isCommitted()) { + event.cleanUp(false); + } else { + event.resetForRedelivery(); + prefetchEvent(event); + } + return null; + }); + } + } finally { + releaseWriteLock(); + } + } + /** * @return {@code true} if ack successfully */ diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java index 0c5fd9327f24f..455432ed7a59a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java @@ -327,6 +327,12 @@ public synchronized void nack() { } } + /** Rewinds a delivery after unsubscribe without counting it as a poison-message retry. */ + public synchronized void resetForRedelivery() { + response.nack(); + lastPolledTimestamp.set(INVALID_TIMESTAMP); + } + /** Makes this event pollable again without treating local response-size control as a nack. */ public void resetLastPolledTimestamp() { lastPolledTimestamp.set(INVALID_TIMESTAMP); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTabletBatchEvents.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTabletBatchEvents.java index f96714921e493..55047b06e92d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTabletBatchEvents.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTabletBatchEvents.java @@ -54,8 +54,13 @@ public void ack() { @Override public void cleanUp(final boolean force) { batch.cleanUp(force); + cleanUpIterationSnapshot(); + } + + public void cleanUpIterationSnapshot() { if (Objects.nonNull(iterationSnapshot)) { iterationSnapshot.cleanUp(); + iterationSnapshot = null; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java index 3bcbc888f0188..f090c75ec2a53 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java @@ -125,6 +125,9 @@ public void fetchNextResponse(final long offset /* unused */) throws Exception { @Override public synchronized void nack() { cleanUp(); + // A completed download has already moved its parsed tablets out of the batch's current + // snapshot. Release that delivery's snapshot before rewinding, without acknowledging it. + events.cleanUpIterationSnapshot(); // should not reset the iterator of batch when init // TODO: avoid completely rewinding the iterator diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgentTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgentTest.java index 0d564ed0c7278..8307ba2bb1af7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgentTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionConsumerAgentTest.java @@ -19,12 +19,23 @@ package org.apache.iotdb.db.subscription.agent; +import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; +import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerMeta; +import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; +import org.apache.iotdb.db.subscription.broker.ISubscriptionBroker; +import org.apache.iotdb.db.subscription.broker.SubscriptionBroker; +import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingQueue; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; + import org.junit.Test; +import org.mockito.Mockito; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -33,6 +44,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -40,6 +52,134 @@ public class SubscriptionConsumerAgentTest { private static final long TEST_TIMEOUT_SECONDS = 5L; + @Test + @SuppressWarnings("unchecked") + public void testRetainedTopicQueueSurvivesUnsubscribeAndConsumerClose() throws Exception { + final SubscriptionConsumerAgent agent = SubscriptionAgent.consumer(); + final SubscriptionBrokerAgent brokers = SubscriptionAgent.broker(); + try { + for (final String mode : + new String[] {TopicConstant.MODE_INITIAL_VALUE, TopicConstant.MODE_SNAPSHOT_VALUE}) { + final String topicName = "topic_" + mode; + assertNull( + SubscriptionAgent.topic() + .handleSingleTopicMetaChanges( + new TopicMeta( + topicName, + 1L, + Map.of( + TopicConstant.MODE_KEY, + mode, + TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, + "true")))); + for (final boolean closeConsumer : new boolean[] {false, true}) { + final String groupId = "retained_" + mode + "_" + closeConsumer; + final ConsumerGroupMeta subscribed = + new ConsumerGroupMeta( + groupId, 1L, new ConsumerMeta("consumer", 1L, Collections.emptyMap())); + subscribed.addSubscription("consumer", Collections.singleton(topicName)); + try { + assertNull(agent.handleSingleConsumerGroupMetaChanges(subscribed)); + final Field brokersField = + SubscriptionBrokerAgent.class.getDeclaredField("consumerGroupIdToBrokers"); + brokersField.setAccessible(true); + final Map> brokerMap = + (Map>) brokersField.get(brokers); + final SubscriptionBroker broker = (SubscriptionBroker) brokerMap.get(groupId).get(0); + final Field queuesField = + SubscriptionBroker.class.getDeclaredField("topicNameToPrefetchingQueue"); + queuesField.setAccessible(true); + final Map queues = + (Map) queuesField.get(broker); + final SubscriptionPrefetchingQueue queue = + Mockito.mock(SubscriptionPrefetchingQueue.class); + Mockito.when(queue.getPrefetchingQueueId()).thenReturn(groupId + "_" + topicName); + queues.put(topicName, queue); + + final ConsumerGroupMeta detached = subscribed.deepCopy(); + detached.removeSubscription("consumer", Collections.singleton(topicName)); + detached.retainTopic(topicName); + if (closeConsumer) { + detached.removeConsumer("consumer"); + } + assertNull(agent.handleSingleConsumerGroupMetaChanges(detached)); + assertTrue(brokers.isBrokerExist(groupId)); + assertTrue(broker.hasQueue(topicName)); + Mockito.verify(queue).resetAndRequeueInFlightEvents(); + Mockito.verify(queue, Mockito.never()).cleanUp(); + + final ConsumerGroupMeta rejoined = detached.deepCopy(); + if (closeConsumer) { + rejoined.addConsumer(new ConsumerMeta("consumer", 2L, Collections.emptyMap())); + } + rejoined.addSubscription("consumer", Collections.singleton(topicName)); + rejoined.removeRetainedTopic(topicName); + assertNull(agent.handleSingleConsumerGroupMetaChanges(rejoined)); + assertTrue(broker.hasQueue(topicName)); + Mockito.verify(queue, Mockito.never()).cleanUp(); + + final ConsumerGroupMeta cleared = detached.deepCopy(); + cleared.removeRetainedTopic(topicName); + assertNull(agent.handleSingleConsumerGroupMetaChanges(cleared)); + assertFalse(broker.hasQueue(topicName)); + Mockito.verify(queue).cleanUp(); + } finally { + assertNull(agent.handleDropConsumerGroup(groupId)); + } + } + assertNull(SubscriptionAgent.topic().handleDropTopic(topicName, false)); + } + } finally { + for (final String mode : + new String[] {TopicConstant.MODE_INITIAL_VALUE, TopicConstant.MODE_SNAPSHOT_VALUE}) { + assertNull(SubscriptionAgent.topic().handleDropTopic("topic_" + mode, false)); + } + } + } + + @Test + public void testRetainedTopicBrokerIsRecreatedAfterDataNodeRecoveryWithoutDataRegions() { + final SubscriptionConsumerAgent agent = SubscriptionAgent.consumer(); + final SubscriptionBrokerAgent brokers = SubscriptionAgent.broker(); + final String topicName = "retained_recovery_without_data_region"; + final String groupId = "retained_recovery_without_data_region_group"; + try { + assertNull( + SubscriptionAgent.topic() + .handleSingleTopicMetaChanges( + new TopicMeta( + topicName, + 1L, + Map.of( + TopicConstant.MODE_KEY, + TopicConstant.MODE_INITIAL_VALUE, + TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, + "true")))); + + final ConsumerGroupMeta subscribed = + new ConsumerGroupMeta( + groupId, 1L, new ConsumerMeta("consumer", 1L, Collections.emptyMap())); + subscribed.addSubscription("consumer", Collections.singleton(topicName)); + assertNull(agent.handleSingleConsumerGroupMetaChanges(subscribed)); + + final ConsumerGroupMeta detached = subscribed.deepCopy(); + detached.removeSubscription("consumer", Collections.singleton(topicName)); + detached.removeConsumer("consumer"); + detached.retainTopic(topicName); + assertNull(agent.handleSingleConsumerGroupMetaChanges(detached)); + assertTrue(brokers.isBrokerExist(groupId)); + + assertTrue(brokers.dropBroker(groupId)); + assertFalse(brokers.isBrokerExist(groupId)); + + assertNull(agent.handleSingleConsumerGroupMetaChanges(detached.deepCopy())); + assertTrue(brokers.isBrokerExist(groupId)); + } finally { + assertNull(agent.handleDropConsumerGroup(groupId)); + assertNull(SubscriptionAgent.topic().handleDropTopic(topicName, false)); + } + } + @Test public void testTopicDiffSetsUpBeforeTeardownAndMetaPublication() { final List operations = new ArrayList<>(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionRetainedInitialReplayTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionRetainedInitialReplayTest.java new file mode 100644 index 0000000000000..560703794d94b --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionRetainedInitialReplayTest.java @@ -0,0 +1,596 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.agent; + +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; +import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; +import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; +import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; +import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerMeta; +import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; +import org.apache.iotdb.db.pipe.agent.task.PipeDataNodeTaskAgent; +import org.apache.iotdb.db.subscription.broker.ISubscriptionBroker; +import org.apache.iotdb.db.subscription.broker.SubscriptionBlockingPendingQueue; +import org.apache.iotdb.db.subscription.broker.SubscriptionBroker; +import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingQueue; +import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingTabletQueue; +import org.apache.iotdb.db.subscription.columnfilter.ColumnFilterMatcher; +import org.apache.iotdb.db.subscription.event.SubscriptionEvent; +import org.apache.iotdb.db.subscription.event.batch.SubscriptionPipeTabletEventBatch; +import org.apache.iotdb.db.subscription.event.pipe.SubscriptionPipeEvents; +import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; +import org.apache.iotdb.rpc.subscription.payload.poll.FilePiecePayload; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType; +import org.apache.iotdb.rpc.subscription.payload.poll.TabletsPayload; +import org.apache.iotdb.rpc.subscription.payload.poll.TerminationPayload; +import org.apache.iotdb.session.subscription.util.PollTimer; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.powermock.reflect.Whitebox; + +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class SubscriptionRetainedInitialReplayTest { + + @Rule public final TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testDownloadedTsFileReplaysFromBeginningWithoutPoisoning() throws Exception { + for (final boolean tableModel : new boolean[] {false, true}) { + for (final boolean closeConsumer : new boolean[] {false, true}) { + try (final RetainedSubscription subscription = new RetainedSubscription(tableModel)) { + final SubscriptionEvent event = + new SubscriptionEvent( + mock(SubscriptionPipeEvents.class), + folder.newFile(), + null, + subscription.queue.generateSubscriptionCommitContext()); + subscription.queue.prefetchEvent(event); + // More than ten detach/rejoin cycles must not trigger poison-message auto-commit. + for (int i = 0; i < 12; i++) { + assertSame(event, subscription.poll("first")); + assertEquals( + SubscriptionPollResponseType.FILE_INIT.getType(), + event.getCurrentResponse().getResponseType()); + event.fetchNextResponse(0L); + assertEquals( + SubscriptionPollResponseType.FILE_SEAL.getType(), + event.getCurrentResponse().getResponseType()); + subscription.detach(closeConsumer); + subscription.rejoin(); + assertFalse(event.isCommitted()); + assertEquals(0L, event.getNackCount()); + } + assertSame(event, subscription.poll("first")); + assertEquals( + SubscriptionPollResponseType.FILE_INIT.getType(), + event.getCurrentResponse().getResponseType()); + } + } + } + } + + @Test + public void testReleasedTsFileReplaysFromBeginningAfterLastUnsubscribe() throws Exception { + final byte[] fileBytes = {1, 2, 3, 4}; + for (final boolean tableModel : new boolean[] {false, true}) { + for (final boolean closeConsumer : new boolean[] {false, true}) { + try (final RetainedSubscription subscription = new RetainedSubscription(tableModel)) { + final SubscriptionEvent event = + new SubscriptionEvent( + mock(SubscriptionPipeEvents.class), + Files.write(folder.newFile().toPath(), fileBytes).toFile(), + null, + subscription.queue.generateSubscriptionCommitContext()); + subscription.queue.prefetchEvent(event); + assertSame(event, subscription.poll("first")); + event.fetchNextResponse(0L); + event.fetchNextResponse(fileBytes.length); + assertEquals( + SubscriptionPollResponseType.FILE_SEAL.getType(), + event.getCurrentResponse().getResponseType()); + + // A disconnected receiver releases its deliveries before the group unsubscribes. + assertEquals(1, subscription.broker.requeueInFlightEvents("first")); + assertEquals(0L, subscription.queue.getSubscriptionUncommittedEventCount()); + subscription.detach(closeConsumer); + subscription.rejoin(); + + assertSame(event, subscription.poll("first")); + assertEquals( + SubscriptionPollResponseType.FILE_INIT.getType(), + event.getCurrentResponse().getResponseType()); + assertEquals(0L, event.getNackCount()); + assertFalse(event.isCommitted()); + event.fetchNextResponse(0L); + assertArrayEquals( + fileBytes, + ((FilePiecePayload) event.getCurrentResponse().getPayload()).getFilePiece()); + event.fetchNextResponse(fileBytes.length); + assertTrue(subscription.queue.ack("first", event.getCommitContext())); + assertNull(subscription.poll("first")); + } + } + } + } + + @Test + public void testDownloadedTabletPagesReplayTheirRows() throws Exception { + assertDownloadedTabletPagesReplayTheirRows(false); + } + + @Test + public void testReleasedTabletPagesReplayTheirRows() throws Exception { + assertDownloadedTabletPagesReplayTheirRows(true); + } + + private void assertDownloadedTabletPagesReplayTheirRows(final boolean releaseBeforeDetach) + throws Exception { + try (final RetainedSubscription subscription = new RetainedSubscription(false)) { + final Tablet tablet = + new Tablet( + "root.retained.d", + Collections.singletonList(new MeasurementSchema("s1", TSDataType.INT64)), + 1); + tablet.addTimestamp(0, 42L); + tablet.addValue("s1", 0, 7L); + tablet.setRowSize(1); + final AtomicBoolean iterated = new AtomicBoolean(); + final SubscriptionPipeTabletEventBatch batch = mock(SubscriptionPipeTabletEventBatch.class); + when(batch.getColumnFilterMatcher()).thenReturn(ColumnFilterMatcher.matchAll()); + when(batch.hasNext()).thenAnswer(invocation -> !iterated.get()); + when(batch.next()) + .thenAnswer( + invocation -> { + iterated.set(true); + return new Pair<>("", Collections.singletonList(tablet)); + }); + doAnswer( + invocation -> { + iterated.set(false); + return null; + }) + .when(batch) + .resetForIteration(); + final SubscriptionEvent event = new SubscriptionEvent(batch, subscription.queue); + subscription.queue.prefetchEvent(event); + assertSame(event, subscription.poll("first")); + event.fetchNextResponse(1L); + assertSame( + tablet, ((TabletsPayload) event.getCurrentResponse().getPayload()).getTablets().get(0)); + event.fetchNextResponse(2L); + assertEquals(-1, ((TabletsPayload) event.getCurrentResponse().getPayload()).getNextOffset()); + if (releaseBeforeDetach) { + assertEquals(1, subscription.broker.requeueInFlightEvents("first")); + } + subscription.detach(false); + subscription.rejoin(); + assertSame(event, subscription.poll("first")); + assertEquals(1, ((TabletsPayload) event.getCurrentResponse().getPayload()).getNextOffset()); + event.fetchNextResponse(1L); + assertSame( + tablet, ((TabletsPayload) event.getCurrentResponse().getPayload()).getTablets().get(0)); + } + } + + @Test + public void testLastUnsubscribeReplaysEarlierConsumersUncommittedEvents() throws Exception { + try (final RetainedSubscription subscription = new RetainedSubscription(false)) { + subscription.meta.addConsumer(subscription.consumer("second")); + subscription.meta.addSubscription("second", Collections.singleton(subscription.topic)); + assertNull( + SubscriptionAgent.consumer() + .handleSingleConsumerGroupMetaChanges(subscription.meta.deepCopy())); + final SubscriptionEvent committed = subscription.newEvent(); + subscription.queue.prefetchEvent(committed); + assertSame(committed, subscription.poll("first")); + assertTrue(subscription.queue.ack("first", committed.getCommitContext())); + final SubscriptionEvent uncommitted = subscription.newEvent(); + subscription.queue.prefetchEvent(uncommitted); + assertSame(uncommitted, subscription.poll("first")); + subscription.meta.removeSubscription("first", Collections.singleton(subscription.topic)); + assertNull( + SubscriptionAgent.consumer() + .handleSingleConsumerGroupMetaChanges(subscription.meta.deepCopy())); + subscription.meta.removeSubscription("second", Collections.singleton(subscription.topic)); + subscription.meta.retainTopic(subscription.topic); + assertNull( + SubscriptionAgent.consumer() + .handleSingleConsumerGroupMetaChanges(subscription.meta.deepCopy())); + subscription.rejoin(); + assertEquals(0L, subscription.queue.getSubscriptionUncommittedEventCount()); + assertSame(uncommitted, subscription.poll("first")); + assertFalse(uncommitted.isCommitted()); + assertEquals(0, subscription.queue.getPrefetchedEventCount()); + } + } + + @Test + public void testRetainFalseStillDiscardsQueueOnConsumerClose() throws Exception { + try (final RetainedSubscription subscription = new RetainedSubscription(false, false)) { + final SubscriptionEvent event = subscription.newEvent(); + subscription.queue.prefetchEvent(event); + assertSame(event, subscription.poll("first")); + subscription.detach(true); + assertTrue(subscription.queue.isClosed()); + assertEquals(0L, subscription.queue.getSubscriptionUncommittedEventCount()); + assertEquals(0, subscription.queue.getPrefetchedEventCount()); + assertFalse(SubscriptionAgent.consumer().containsConsumerGroup(subscription.group)); + assertFalse(SubscriptionAgent.broker().isBrokerExist(subscription.group)); + } + } + + @Test + public void testPollWithStaleTopicSetCannotStrandRetainedEvents() throws Exception { + for (final String mode : + new String[] {TopicConstant.MODE_INITIAL_VALUE, TopicConstant.MODE_SNAPSHOT_VALUE}) { + for (final boolean closeConsumer : new boolean[] {false, true}) { + try (final RetainedSubscription subscription = + new RetainedSubscription(false, true, mode)) { + final SubscriptionEvent event = subscription.newEvent(); + subscription.queue.prefetchEvent(event); + assertSame(event, subscription.poll("first")); + doAnswer( + invocation -> { + invocation.callRealMethod(); + // Model a request admitted before unsubscribe that reaches the queue as soon as + // the uncommitted event is released, before the meta-change handler returns. + assertTrue( + subscription + .broker + .poll( + "first", Collections.singleton(subscription.topic), Long.MAX_VALUE) + .isEmpty()); + return null; + }) + .when(subscription.queue) + .resetAndRequeueInFlightEvents(); + subscription.detach(closeConsumer); + assertEquals(0L, subscription.queue.getSubscriptionUncommittedEventCount()); + assertEquals(1, subscription.queue.getPrefetchedEventCount()); + subscription.rejoin(); + assertSame(event, subscription.poll("first")); + } + } + } + } + + @Test + public void testCompletedRetainedSnapshotReportsTerminationAfterRejoin() throws Exception { + for (final boolean tableModel : new boolean[] {false, true}) { + for (final boolean closeConsumer : new boolean[] {false, true}) { + try (final RetainedSubscription subscription = + new RetainedSubscription(tableModel, true, TopicConstant.MODE_SNAPSHOT_VALUE)) { + final Set topicNames = Collections.singleton(subscription.topic); + assertTrue(subscription.broker.fetchTopicNamesToUnsubscribe(topicNames).isEmpty()); + subscription.queue.markCompleted(); + assertEquals( + Collections.singletonList(subscription.topic), + subscription.broker.fetchTopicNamesToUnsubscribe(topicNames)); + assertEquals( + SubscriptionPollResponseType.TERMINATION.getType(), + subscription + .broker + .poll("first", topicNames, Long.MAX_VALUE) + .get(0) + .getCurrentResponse() + .getResponseType()); + + subscription.detach(closeConsumer); + assertFalse(subscription.queue.isClosed()); + assertTrue(subscription.broker.poll("first", topicNames, Long.MAX_VALUE).isEmpty()); + subscription.rejoin(); + assertTrue(subscription.broker.hasQueue(subscription.topic)); + final List events = + subscription.broker.poll("first", topicNames, Long.MAX_VALUE); + assertEquals(1, events.size()); + assertEquals( + SubscriptionPollResponseType.TERMINATION.getType(), + events.get(0).getCurrentResponse().getResponseType()); + assertEquals(0L, subscription.queue.getSubscriptionUncommittedEventCount()); + assertEquals(0, subscription.queue.getPrefetchedEventCount()); + } + } + } + } + + @Test + @SuppressWarnings("unchecked") + public void testEmptyRetainedSnapshotCompletionSurvivesRejoinAndBrokerRecovery() + throws Exception { + final CommonConfig originalConfig = + Whitebox.getInternalState(SubscriptionConfig.class, "COMMON_CONFIG"); + final CommonConfig enabledConfig = spy(originalConfig); + when(enabledConfig.getSubscriptionEnabled()).thenReturn(true); + try { + Whitebox.setInternalState(SubscriptionConfig.class, "COMMON_CONFIG", enabledConfig); + for (final boolean tableModel : new boolean[] {false, true}) { + for (final boolean closeConsumer : new boolean[] {false, true}) { + try (final RetainedSubscription subscription = + new RetainedSubscription(tableModel, true, TopicConstant.MODE_SNAPSHOT_VALUE)) { + subscription.broker.removePrefetchingQueue(subscription.topic); + final Set topics = Collections.singleton(subscription.topic); + final Map source = + new HashMap<>( + Map.of( + SystemConstant.SQL_DIALECT_KEY, + tableModel ? "table" : "tree", + "source.mode", + "snapshot")); + final PipeRuntimeMeta runtime = new PipeRuntimeMeta(); + runtime.getStatus().set(PipeStatus.RUNNING); + final PipeMeta pipe = + new PipeMeta( + new PipeStaticMeta( + PipeStaticMeta.generateSubscriptionPipeName( + subscription.topic, subscription.group), + 1L, + source, + Collections.emptyMap(), + Map.of( + PipeSinkConstant.SINK_TOPIC_KEY, subscription.topic, + PipeSinkConstant.SINK_CONSUMER_GROUP_KEY, subscription.group)), + runtime); + final PipeDataNodeTaskAgent taskAgent = + new PipeDataNodeTaskAgent() { + @Override + protected boolean isShutdown() { + return false; + } + + @Override + protected void createPipeTask( + final int regionId, + final PipeStaticMeta staticMeta, + final PipeTaskMeta taskMeta) { + throw new AssertionError( + "A completed empty snapshot must not accept new tasks"); + } + }; + try { + assertNull( + taskAgent.handleSinglePipeMetaChanges( + PipeMeta.deserialize4TaskAgent(pipe.serialize()))); + assertTrue(subscription.broker.poll("first", topics, Long.MAX_VALUE).isEmpty()); + + source.put(SystemConstant.SUBSCRIPTION_EMPTY_SNAPSHOT_COMPLETED_KEY, "true"); + final PipeMeta completed = PipeMeta.deserialize4TaskAgent(pipe.serialize()); + assertNull(taskAgent.handleSinglePipeMetaChanges(completed)); + // Later DataRegions must not change the original snapshot's empty result. + runtime + .getConsensusGroupId2TaskMetaMap() + .put(7, new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 0)); + assertNull( + taskAgent.handleSinglePipeMetaChanges( + PipeMeta.deserialize4TaskAgent(pipe.serialize()))); + assertEquals( + Collections.singletonList(subscription.topic), + subscription.broker.fetchTopicNamesToUnsubscribe(topics)); + subscription.detach(closeConsumer); + assertTrue(subscription.broker.poll("first", topics, Long.MAX_VALUE).isEmpty()); + subscription.rejoin(); + assertEquals( + SubscriptionPollResponseType.TERMINATION.getType(), + subscription + .broker + .poll("first", topics, Long.MAX_VALUE) + .get(0) + .getCurrentResponse() + .getResponseType()); + + // Restore an unsubscribed group and its persisted Pipe after losing all local + // brokers. + subscription.detach(closeConsumer); + assertNull(SubscriptionAgent.consumer().handleDropConsumerGroup(subscription.group)); + assertNull( + SubscriptionAgent.consumer() + .handleSingleConsumerGroupMetaChanges(subscription.meta.deepCopy())); + final PipeDataNodeTaskAgent recoveredTaskAgent = + new PipeDataNodeTaskAgent() { + @Override + protected boolean isShutdown() { + return false; + } + + @Override + protected void createPipeTask( + final int regionId, + final PipeStaticMeta staticMeta, + final PipeTaskMeta taskMeta) { + throw new AssertionError( + "A completed empty snapshot must not accept new tasks"); + } + }; + assertTrue( + recoveredTaskAgent + .handlePipeMetaChangesInternal( + Collections.singletonList( + PipeMeta.deserialize4TaskAgent(pipe.serialize()))) + .isEmpty()); + final Field brokersField = + SubscriptionBrokerAgent.class.getDeclaredField("consumerGroupIdToBrokers"); + brokersField.setAccessible(true); + final Map> brokers = + (Map>) + brokersField.get(SubscriptionAgent.broker()); + final SubscriptionBroker recovered = + (SubscriptionBroker) brokers.get(subscription.group).get(0); + assertFalse(recovered.hasQueue(subscription.topic)); + assertTrue(recovered.poll("first", topics, Long.MAX_VALUE).isEmpty()); + subscription.rejoin(); + assertEquals( + SubscriptionPollResponseType.TERMINATION.getType(), + recovered + .poll("first", topics, Long.MAX_VALUE) + .get(0) + .getCurrentResponse() + .getResponseType()); + assertTrue( + recoveredTaskAgent + .handlePipeMetaChangesInternal(Collections.emptyList()) + .isEmpty()); + } finally { + taskAgent.handlePipeMetaChangesInternal(Collections.emptyList()); + } + } + } + } + } finally { + Whitebox.setInternalState(SubscriptionConfig.class, "COMMON_CONFIG", originalConfig); + } + } + + private static final class RetainedSubscription implements AutoCloseable { + private final String topic = "retained_replay"; + private final String group = "retained_replay_group"; + private final boolean tableModel; + private final boolean retain; + private final SubscriptionPrefetchingQueue queue; + private final SubscriptionBroker broker; + private ConsumerGroupMeta meta; + + private RetainedSubscription(final boolean tableModel) throws Exception { + this(tableModel, true, TopicConstant.MODE_INITIAL_VALUE); + } + + private RetainedSubscription(final boolean tableModel, final boolean retain) throws Exception { + this(tableModel, retain, TopicConstant.MODE_INITIAL_VALUE); + } + + @SuppressWarnings("unchecked") + private RetainedSubscription(final boolean tableModel, final boolean retain, final String mode) + throws Exception { + this.tableModel = tableModel; + this.retain = retain; + assertNull( + SubscriptionAgent.topic() + .handleSingleTopicMetaChanges( + new TopicMeta( + topic, + 1L, + Map.of( + SystemConstant.SQL_DIALECT_KEY, + tableModel ? "table" : "tree", + TopicConstant.MODE_KEY, + mode, + TopicConstant.RETAIN_PROGRESS_AFTER_UNSUBSCRIBE_KEY, + String.valueOf(retain), + TopicConstant.RECYCLE_UNCOMMITTED_EVENT_INTERVAL_MS_KEY, + "0")))); + meta = new ConsumerGroupMeta(group, 1L, consumer("first")); + meta.addSubscription("first", Collections.singleton(topic)); + assertNull( + SubscriptionAgent.consumer().handleSingleConsumerGroupMetaChanges(meta.deepCopy())); + final SubscriptionBlockingPendingQueue input = mock(SubscriptionBlockingPendingQueue.class); + when(input.isEmpty()).thenReturn(true); + queue = spy(new SubscriptionPrefetchingTabletQueue(group, topic, input, new AtomicLong())); + final Field brokersField = + SubscriptionBrokerAgent.class.getDeclaredField("consumerGroupIdToBrokers"); + brokersField.setAccessible(true); + final Map> brokers = + (Map>) brokersField.get(SubscriptionAgent.broker()); + broker = (SubscriptionBroker) brokers.get(group).get(0); + final Field queuesField = + SubscriptionBroker.class.getDeclaredField("topicNameToPrefetchingQueue"); + queuesField.setAccessible(true); + ((Map) queuesField.get(broker)).put(topic, queue); + } + + private ConsumerMeta consumer(final String id) { + return new ConsumerMeta( + id, + 1L, + Collections.singletonMap( + ConsumerConstant.SQL_DIALECT_KEY, tableModel ? "table" : "tree")); + } + + private SubscriptionEvent newEvent() { + return new SubscriptionEvent( + SubscriptionPollResponseType.TERMINATION.getType(), + new TerminationPayload(), + queue.generateSubscriptionCommitContext()); + } + + private SubscriptionEvent poll(final String consumerId) { + return queue.pollV2(consumerId, new PollTimer(System.currentTimeMillis(), 0L)); + } + + private void detach(final boolean closeConsumer) { + meta.removeSubscription("first", Collections.singleton(topic)); + if (retain) { + meta.retainTopic(topic); + } + if (closeConsumer) { + meta.removeConsumer("first"); + } + assertNull( + SubscriptionAgent.consumer().handleSingleConsumerGroupMetaChanges(meta.deepCopy())); + } + + private void rejoin() { + if (!meta.containsConsumer("first")) { + meta.addConsumer(consumer("first")); + } + meta.addSubscription("first", Collections.singleton(topic)); + meta.removeRetainedTopic(topic); + assertNull( + SubscriptionAgent.consumer().handleSingleConsumerGroupMetaChanges(meta.deepCopy())); + } + + @Override + public void close() { + assertNull(SubscriptionAgent.consumer().handleDropConsumerGroup(group)); + assertNull(SubscriptionAgent.topic().handleDropTopic(topic, tableModel)); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatchTest.java index d6294ff227b4f..7dbc5b0b18887 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatchTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatchTest.java @@ -34,8 +34,11 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.db.subscription.broker.SubscriptionPrefetchingTabletQueue; +import org.apache.iotdb.db.subscription.event.SubscriptionEvent; import org.apache.iotdb.db.subscription.event.pipe.SubscriptionPipeTabletBatchEvents; import org.apache.iotdb.rpc.subscription.config.ConsumerConstant; +import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; +import org.apache.iotdb.rpc.subscription.payload.poll.TabletsPayload; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; @@ -129,6 +132,39 @@ public void tearDown() throws Exception { } } + @Test(timeout = 10000) + public void testRetainedRedeliveryReleasesPreviousTabletSnapshot() throws Exception { + final long baselineTabletMemory = memoryManager.getUsedMemorySizeInBytesOfTablets(); + batch.enrichedEvents.add(createTsFileEvent("retained.tsfile", 42, false, false)); + batch.resetForIteration(); + Mockito.when(batch.prefetchingQueue.generateSubscriptionCommitContext()) + .thenReturn(new SubscriptionCommitContext(1, 0, "topic", "group", 1L)); + final SubscriptionEvent event = new SubscriptionEvent(batch, batch.prefetchingQueue); + try { + for (int i = 0; i < 3; i++) { + event.fetchNextResponse(1L); + final List tablets = + ((TabletsPayload) event.getCurrentResponse().getPayload()).getTablets(); + Assert.assertEquals(1, tablets.size()); + Assert.assertEquals(42L, tablets.get(0).getTimestamp(0)); + event.fetchNextResponse(2L); + Assert.assertEquals( + -1, ((TabletsPayload) event.getCurrentResponse().getPayload()).getNextOffset()); + if (i < 2) { + event.resetForRedelivery(); + } + } + event.ack(); + event.cleanUp(false); + Assert.assertEquals( + "Acknowledging a retained replay must release every parsed-tablet snapshot", + baselineTabletMemory, + memoryManager.getUsedMemorySizeInBytesOfTablets()); + } finally { + event.cleanUp(true); + } + } + @Test(timeout = 10000) public void testSameRegionFilesReleaseParserBeforeBatchAck() throws Exception { final long baselineTabletMemory = memoryManager.getUsedMemorySizeInBytesOfTablets(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java index f1ca66be0988e..2471e434ad6db 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java @@ -50,6 +50,8 @@ public class SystemConstant { "__system.sink-authentication-injected"; public static final String SINK_WAIT_FOR_SCHEMA_BEFORE_LOAD_KEY = "__system.sink-wait-for-schema-before-load"; + public static final String SUBSCRIPTION_EMPTY_SNAPSHOT_COMPLETED_KEY = + "__system.subscription-empty-snapshot-completed"; /////////////////////////////////// Utility /////////////////////////////////// @@ -62,6 +64,7 @@ public class SystemConstant { SYSTEM_KEYS.add(SOURCE_AUTHENTICATION_INJECTED_KEY); SYSTEM_KEYS.add(SINK_AUTHENTICATION_INJECTED_KEY); SYSTEM_KEYS.add(SINK_WAIT_FOR_SCHEMA_BEFORE_LOAD_KEY); + SYSTEM_KEYS.add(SUBSCRIPTION_EMPTY_SNAPSHOT_COMPLETED_KEY); } public static PipeParameters addSystemKeysIfNecessary(final PipeParameters givenPipeParameters) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/CommitProgressKeeper.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/CommitProgressKeeper.java index 934ef58dd204f..e36c4a0633e90 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/CommitProgressKeeper.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/CommitProgressKeeper.java @@ -119,6 +119,20 @@ public synchronized boolean hasTopicProgress( .anyMatch(prefix -> isValidRegionAndDataNodeProgressKey(key, prefix))); } + public synchronized Map getTopicProgress( + final String consumerGroupId, final String topicName) { + final Set prefixes = getTopicProgressKeyPrefixes(consumerGroupId, topicName); + final Map result = new HashMap<>(); + regionProgressMap.forEach( + (key, value) -> { + if (prefixes.stream() + .anyMatch(prefix -> isValidRegionAndDataNodeProgressKey(key, prefix))) { + result.put(key, copyBuffer(value)); + } + }); + return result; + } + private static Set getTopicProgressKeyPrefixes( final String consumerGroupId, final String topicName) { final Set prefixes = new HashSet<>(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java index a129c5ae70eae..1c46ab144ae8d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java @@ -54,6 +54,7 @@ public class ConsumerGroupMeta { private Map consumerIdToConsumerMeta; private boolean tableModel; private Map topicNameToSubscriptionCreationTime; // used when creationTime < 0 + private final Set retainedTopicNames = ConcurrentHashMap.newKeySet(); public ConsumerGroupMeta() { this.topicNameToSubscribedConsumerIdSet = new ConcurrentHashMap<>(); @@ -85,6 +86,7 @@ public ConsumerGroupMeta deepCopy() { copied.tableModel = tableModel; copied.topicNameToSubscriptionCreationTime = new ConcurrentHashMap<>(topicNameToSubscriptionCreationTime); + copied.retainedTopicNames.addAll(retainedTopicNames); return copied; } @@ -218,6 +220,18 @@ public Set getSubscribedTopicNames() { return Collections.unmodifiableSet(topicNameToSubscribedConsumerIdSet.keySet()); } + public Set getRetainedTopicNames() { + return Collections.unmodifiableSet(retainedTopicNames); + } + + public void retainTopic(final String topicName) { + retainedTopicNames.add(topicName); + } + + public void removeRetainedTopic(final String topicName) { + retainedTopicNames.remove(topicName); + } + /** * Get the consumers subscribing the given topic in this group. * @@ -338,7 +352,14 @@ public void serialize(final OutputStream outputStream) throws IOException { ReadWriteIOUtils.write(consumerGroupId, outputStream); ReadWriteIOUtils.write(creationTime, outputStream); - ReadWriteIOUtils.write(topicNameToSubscribedConsumerIdSet.size(), outputStream); + // A negative topic count identifies the extended, self-delimiting format. Do not infer the + // presence of retained topics from remaining bytes: this meta is also nested in procedures. + final boolean hasRetainedTopics = !retainedTopicNames.isEmpty(); + ReadWriteIOUtils.write( + hasRetainedTopics + ? -topicNameToSubscribedConsumerIdSet.size() - 1 + : topicNameToSubscribedConsumerIdSet.size(), + outputStream); for (final Map.Entry> entry : topicNameToSubscribedConsumerIdSet.entrySet()) { ReadWriteIOUtils.write(entry.getKey(), outputStream); @@ -365,6 +386,12 @@ public void serialize(final OutputStream outputStream) throws IOException { ReadWriteIOUtils.write(entry.getValue(), outputStream); } } + if (hasRetainedTopics) { + ReadWriteIOUtils.write(retainedTopicNames.size(), outputStream); + for (final String topicName : retainedTopicNames) { + ReadWriteIOUtils.write(topicName, outputStream); + } + } } public static ConsumerGroupMeta deserialize(final InputStream inputStream) throws IOException { @@ -375,6 +402,10 @@ public static ConsumerGroupMeta deserialize(final InputStream inputStream) throw consumerGroupMeta.topicNameToSubscribedConsumerIdSet = new ConcurrentHashMap<>(); int size = ReadWriteIOUtils.readInt(inputStream); + final boolean hasRetainedTopics = size < 0; + if (hasRetainedTopics) { + size = -size - 1; + } for (int i = 0; i < size; ++i) { final String key = ReadWriteIOUtils.readString(inputStream); @@ -407,6 +438,12 @@ public static ConsumerGroupMeta deserialize(final InputStream inputStream) throw } } + if (hasRetainedTopics) { + size = ReadWriteIOUtils.readInt(inputStream); + for (int i = 0; i < size; ++i) { + consumerGroupMeta.retainedTopicNames.add(ReadWriteIOUtils.readString(inputStream)); + } + } return consumerGroupMeta; } @@ -418,6 +455,10 @@ public static ConsumerGroupMeta deserialize(final ByteBuffer byteBuffer) { consumerGroupMeta.topicNameToSubscribedConsumerIdSet = new ConcurrentHashMap<>(); int size = ReadWriteIOUtils.readInt(byteBuffer); + final boolean hasRetainedTopics = size < 0; + if (hasRetainedTopics) { + size = -size - 1; + } for (int i = 0; i < size; ++i) { final String key = ReadWriteIOUtils.readString(byteBuffer); @@ -450,6 +491,12 @@ public static ConsumerGroupMeta deserialize(final ByteBuffer byteBuffer) { } } + if (hasRetainedTopics) { + size = ReadWriteIOUtils.readInt(byteBuffer); + for (int i = 0; i < size; ++i) { + consumerGroupMeta.retainedTopicNames.add(ReadWriteIOUtils.readString(byteBuffer)); + } + } return consumerGroupMeta; } @@ -470,6 +517,7 @@ public boolean equals(final Object obj) { && Objects.equals( this.topicNameToSubscribedConsumerIdSet, that.topicNameToSubscribedConsumerIdSet) && Objects.equals(this.consumerIdToConsumerMeta, that.consumerIdToConsumerMeta) + && Objects.equals(this.retainedTopicNames, that.retainedTopicNames) && Objects.equals( this.topicNameToSubscriptionCreationTime, that.topicNameToSubscriptionCreationTime); } @@ -482,6 +530,7 @@ public int hashCode() { tableModel, topicNameToSubscribedConsumerIdSet, consumerIdToConsumerMeta, + retainedTopicNames, topicNameToSubscriptionCreationTime); } @@ -498,6 +547,8 @@ public String toString() { + consumerIdToConsumerMeta + ", topicNameToSubscriptionCreationTime=" + topicNameToSubscriptionCreationTime + + ", retainedTopicNames=" + + retainedTopicNames + "}"; } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/consumer/ConsumerGroupDeSerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/consumer/ConsumerGroupDeSerTest.java index 2d2014d7e976b..bc53d231d9a30 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/consumer/ConsumerGroupDeSerTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/consumer/ConsumerGroupDeSerTest.java @@ -29,6 +29,8 @@ import org.junit.Test; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; @@ -39,6 +41,39 @@ public class ConsumerGroupDeSerTest { + @Test + public void testRetainedTopicsAreSelfDelimitedAndPreserveModel() throws Exception { + for (final String dialect : new String[] {"tree", "table"}) { + final ConsumerGroupMeta group = + new ConsumerGroupMeta( + "retained", + 1L, + new ConsumerMeta( + "consumer", + 1L, + Collections.singletonMap(ConsumerConstant.SQL_DIALECT_KEY, dialect))); + group.retainTopic("topic"); + group.removeConsumer("consumer"); + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + final DataOutputStream output = new DataOutputStream(bytes); + group.serialize(output); + output.writeInt(123456); + + final ByteBuffer buffer = ByteBuffer.wrap(bytes.toByteArray()); + final ConsumerGroupMeta restored = ConsumerGroupMeta.deserialize(buffer); + Assert.assertEquals(group, restored); + Assert.assertEquals(123456, buffer.getInt()); + final ByteArrayInputStream input = new ByteArrayInputStream(bytes.toByteArray()); + Assert.assertEquals(group, ConsumerGroupMeta.deserialize(input)); + Assert.assertEquals(Integer.BYTES, input.available()); + final ConsumerGroupMeta copied = group.deepCopy(); + copied.removeRetainedTopic("topic"); + Assert.assertEquals(Collections.singleton("topic"), group.getRetainedTopicNames()); + Assert.assertTrue(copied.getRetainedTopicNames().isEmpty()); + Assert.assertEquals("table".equals(dialect), restored.visibleUnder(true)); + } + } + @Test public void testEmptyGroupPreservesDialectAcrossSerializationAndRejoin() throws Exception { for (final String dialect : new String[] {"tree", "table"}) {