From ed8676e81f7c08fa9b13d2a940971428c1219d38 Mon Sep 17 00:00:00 2001 From: Di Wang Date: Mon, 28 Sep 2026 10:31:44 +0800 Subject: [PATCH 1/5] feat: add and persist Stopped node status Report graceful shutdown as Stopped, apply shared node status transition rules, and persist Stopped/Removing statuses through ConfigNode consensus for recovery after leadership changes. --- .../confignode/i18n/ConfigNodeMessages.java | 3 + .../confignode/i18n/ManagerMessages.java | 8 +- .../confignode/i18n/ProcedureMessages.java | 7 + .../confignode/i18n/ConfigNodeMessages.java | 3 + .../confignode/i18n/ManagerMessages.java | 8 +- .../confignode/i18n/ProcedureMessages.java | 7 + .../heartbeat/AINodeHeartbeatHandler.java | 4 +- .../heartbeat/ConfigNodeHeartbeatHandler.java | 4 +- .../heartbeat/DataNodeHeartbeatHandler.java | 7 +- .../consensus/request/ConfigPhysicalPlan.java | 4 + .../request/ConfigPhysicalPlanType.java | 2 + .../confignode/UpdateNodeStatusPlan.java | 101 +++++++++++ .../confignode/manager/ConfigManager.java | 35 ++-- .../iotdb/confignode/manager/IManager.java | 18 +- .../confignode/manager/ProcedureManager.java | 13 +- .../manager/RetryFailedTasksThread.java | 3 +- .../confignode/manager/load/LoadManager.java | 50 +++--- .../manager/load/balancer/RegionBalancer.java | 12 +- .../manager/load/cache/AbstractLoadCache.java | 9 +- .../manager/load/cache/LoadCache.java | 115 ++++++++++--- .../cache/consensus/ConsensusGroupCache.java | 3 +- .../load/cache/node/AINodeHeartbeatCache.java | 48 ------ .../load/cache/node/BaseNodeCache.java | 75 ++++++++ .../cache/node/ConfigNodeHeartbeatCache.java | 39 +---- .../cache/node/DataNodeHeartbeatCache.java | 57 +----- .../load/cache/node/NodeStatistics.java | 17 ++ .../load/cache/region/RegionCache.java | 3 +- .../load/cache/region/RegionGroupCache.java | 2 +- .../load/service/StatisticsService.java | 2 +- .../confignode/manager/node/NodeManager.java | 4 + .../SubscriptionRuntimeCoordinator.java | 7 +- .../executor/ConfigPlanExecutor.java | 3 + .../confignode/persistence/node/NodeInfo.java | 111 +++++++++++- .../procedure/env/ConfigNodeProcedureEnv.java | 7 +- .../procedure/env/RegionMaintainHandler.java | 2 +- .../procedure/env/RemoveDataNodeHandler.java | 67 ++++++-- .../impl/node/RemoveAINodeProcedure.java | 1 + .../impl/node/RemoveDataNodesProcedure.java | 6 +- .../procedure/impl/schema/SchemaUtils.java | 9 +- .../service/ConfigNodeShutdownHook.java | 162 ++++++++++++++---- .../iotdb/db/i18n/DataNodeMiscMessages.java | 3 +- .../iotdb/db/i18n/DataNodeMiscMessages.java | 2 +- .../iotdb/commons/cluster/NodeStatus.java | 102 +++++++++-- .../iotdb/commons/cluster/RegionStatus.java | 5 + .../src/main/thrift/confignode.thrift | 12 +- 45 files changed, 810 insertions(+), 352 deletions(-) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateNodeStatusPlan.java diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java index 800d59bf727ba..6fae63f6536f5 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java @@ -687,4 +687,7 @@ private ConfigNodeMessages() {} public static final String EXCEPTION_IOTDB_EDGE_CONFIGNODE_INTERNAL_PORT_ARG_IS_NOT_READY_WITHIN_03697FF5 = "IoTDB Edge: ConfigNode internal port %s is not ready within %s ms"; + public static final String + LOG_FAILED_TO_TRANSFER_CONFIGNODE_LEADERSHIP_BEFORE_SHUTDOWN_CONTINUING_SHUTDOWN_2B9364D5 = + "Failed to transfer ConfigNode leadership before shutdown; continuing shutdown."; } diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 0723aa05e2a5f..183ab36c47816 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -483,8 +483,8 @@ public final class ManagerMessages { "Successfully transferred config region snapshot {}."; public static final String THERE_IS_NO_RUNNING_DATANODE_TO_EXECUTE_CQ = "There is no RUNNING DataNode to execute CQ {}"; - public static final String THE_CONFIGNODE_WILL_BE_SHUTDOWN_SOON_MARK_IT_AS_UNKNOWN = - "The ConfigNode-{} will be shutdown soon, mark it as Unknown"; + public static final String LOG_NODE_ARG_REPORTED_SHUTDOWN_CURRENT_STATUS_IS_ARG_A375D665 = + "Node {} reported shutdown; current status is {}."; public static final String THE_CONFIG_REGION_AIR_GAP_CONNECTOR_DOES_NOT_SUPPORT_TRANSFERRING = "The config region air gap connector does not support transferring single file piece bytes."; public static final String THE_CONFIG_REGION_SINK_DOES_NOT_SUPPORT_TRANSFERRING_SINGLE_FILE = @@ -493,8 +493,6 @@ public final class ManagerMessages { "The config region snapshots %s cannot be parsed."; public static final String THE_DATABASE_DOESN_T_EXIST_MAYBE_IT_HAS_BEEN_PRE = "The Database: {} doesn't exist. Maybe it has been pre-deleted."; - public static final String THE_DATANODE_WILL_BE_SHUTDOWN_SOON_MARK_IT_AS_UNKNOWN = - "The DataNode-{} will be shutdown soon, mark it as Unknown"; public static final String THE_REMOVENODEREPLICASELECT_METHOD_OF_GREEDYREGIONGROUPALLOCATOR_IS_YET = "The removeNodeReplicaSelect method of GreedyRegionGroupAllocator is yet to be implemented."; public static final String THE_REMOVENODEREPLICASELECT_METHOD_OF_PARTITEGRAPHPLACEMENTREGIONGROUPALLOCATOR = @@ -717,4 +715,6 @@ private ManagerMessages() {} MESSAGE_ARG_PLEASE_MANUALLY_CHECK_LATER_WHETHER_THE_PROCEDURE_IS_EXECUTED_SUCCESSFULLY_A82B739D = "%s Please manually check later whether the procedure is executed successfully."; + public static final String MESSAGE_CONFIGNODE_LEADER_IS_WAITING_FOR_NODE_STATUS_PERSISTENCE_8CA96809 = + "ConfigNode leader is waiting for node status persistence."; } diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java index e04460375f9b4..995a252148ef2 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java @@ -1218,6 +1218,9 @@ private ProcedureMessages() {} + " cache status is {}"; public static final String MESSAGE_REMOVE_CONFIGNODE_FAILED_BECAUSE_UPDATE_CONSENSUSGROUP_PEER_INFORMATION_FAILED_FCE5302B = "Remove ConfigNode failed because update ConsensusGroup peer information failed."; public static final String MESSAGE_CAN_T_REMOVE_DATANODE_LIMIT_REPLICATION_FACTOR_D960E3A6 = "Can't remove datanode due to the limit of replication factor, "; + public static final String + MESSAGE_SUBMIT_REMOVEDATANODESPROCEDURE_FAILED_BECAUSE_WHEN_THERE_ARE_OTHER_UNKNOWN_STOPPED_OR_READONLY_NODES_IN_THE_CONSENSUS_GROUP_THAT_ARE_NOT_REMOVE_NODES_THE_REMOVE_OPERATION_CANNOT_BE_PERFORMED_FOR_SECURITY_REASONS_PLEASE_CHECK_THE_STATUS_OF_THE_NODE_ARG_AND_ENSURE_IT_IS_RUNNING_5063B3F6 = + "Submit RemoveDataNodesProcedure failed, because when there are other unknown, stopped or readonly nodes in the consensus group that are not remove nodes, the remove operation cannot be performed for security reasons. Please check the status of the node %s and ensure it is running."; public static final String MESSAGE_AVAILABLEDATANODESIZE_ARG_MAXREPLICAFACTOR_ARG_MAX_ALLOWED_REMOVED_DATA_NODE_SIZE_FB8C382C = "availableDataNodeSize: %s, maxReplicaFactor: %s, max allowed removed Data Node size is: %s"; public static final String EXCEPTION_NOT_SUPPORTED_0A83F963 = " is not supported"; public static final String LOG_START_ADD_TRIGGER_ARG_TRIGGERTABLE_CONFIG_NODES_NEEDTOSAVEJAR_ARG_0C23D81E = "Start to add trigger [{}] in TriggerTable on Config Nodes, needToSaveJar[{}]"; @@ -1421,4 +1424,8 @@ private ProcedureMessages() {} "rolling back after failure: %s."; public static final String MESSAGE_ROLLING_BACK_AFTER_AN_EARLIER_FAILURE_850D0AF5 = "rolling back after an earlier failure."; + public static final String EXCEPTION_DATANODE_SETSYSTEMSTATUS_RPC_FAILED_ARG_ARG_ARG_7E8E5F9F = + "DataNode setSystemStatus RPC failed: %s: %s: %s; "; + public static final String EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4 = + "ConfigNode trySetNodeStatus failed: %s: %s: %s; "; } diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java index 14713cbb8011b..20787786a74eb 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java @@ -730,4 +730,7 @@ private ConfigNodeMessages() {} public static final String EXCEPTION_IOTDB_EDGE_CONFIGNODE_INTERNAL_PORT_ARG_IS_NOT_READY_WITHIN_03697FF5 = "IoTDB Edge：ConfigNode 内部端口 %s 在 %s ms 内未就绪"; + public static final String + LOG_FAILED_TO_TRANSFER_CONFIGNODE_LEADERSHIP_BEFORE_SHUTDOWN_CONTINUING_SHUTDOWN_2B9364D5 = + "关闭前转移 ConfigNode leader 失败，继续关闭。"; } diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java index a140efc2f4d40..fa3d63dddb3d2 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -478,8 +478,8 @@ public final class ManagerMessages { "成功传输 config region 快照 {}。"; public static final String THERE_IS_NO_RUNNING_DATANODE_TO_EXECUTE_CQ = "没有处于 RUNNING 状态的 DataNode 可用于执行 CQ {}"; - public static final String THE_CONFIGNODE_WILL_BE_SHUTDOWN_SOON_MARK_IT_AS_UNKNOWN = - "ConfigNode-{} 即将关闭，将其标记为 Unknown"; + public static final String LOG_NODE_ARG_REPORTED_SHUTDOWN_CURRENT_STATUS_IS_ARG_A375D665 = + "节点 {} 已报告停止；当前状态为 {}。"; public static final String THE_CONFIG_REGION_AIR_GAP_CONNECTOR_DOES_NOT_SUPPORT_TRANSFERRING = "config region air gap connector 不支持传输单文件分片字节。"; public static final String THE_CONFIG_REGION_SINK_DOES_NOT_SUPPORT_TRANSFERRING_SINGLE_FILE = @@ -488,8 +488,6 @@ public final class ManagerMessages { "无法解析 config region 快照 %s。"; public static final String THE_DATABASE_DOESN_T_EXIST_MAYBE_IT_HAS_BEEN_PRE = "Database: {} 不存在，可能已被预删除。"; - public static final String THE_DATANODE_WILL_BE_SHUTDOWN_SOON_MARK_IT_AS_UNKNOWN = - "DataNode-{} 即将关闭，将其标记为 Unknown"; public static final String THE_REMOVENODEREPLICASELECT_METHOD_OF_GREEDYREGIONGROUPALLOCATOR_IS_YET = "GreedyRegionGroupAllocator 的 removeNodeReplicaSelect 方法尚未实现。"; public static final String THE_REMOVENODEREPLICASELECT_METHOD_OF_PARTITEGRAPHPLACEMENTREGIONGROUPALLOCATOR = @@ -696,4 +694,6 @@ private ManagerMessages() {} MESSAGE_ARG_PLEASE_MANUALLY_CHECK_LATER_WHETHER_THE_PROCEDURE_IS_EXECUTED_SUCCESSFULLY_A82B739D = "%s 请稍后手动检查该 Procedure 是否执行成功。"; + public static final String MESSAGE_CONFIGNODE_LEADER_IS_WAITING_FOR_NODE_STATUS_PERSISTENCE_8CA96809 = + "ConfigNode leader 正在等待节点状态持久化。"; } diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java index 3014e1c0076d8..d1916dcf870f6 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java @@ -1167,6 +1167,9 @@ private ProcedureMessages() {} "移除 ConfigNode 失败，原因：更新 ConsensusGroup peer 信息失败。"; public static final String MESSAGE_CAN_T_REMOVE_DATANODE_LIMIT_REPLICATION_FACTOR_D960E3A6 = "无法移除 DataNode，原因：受副本因子限制，"; + public static final String + MESSAGE_SUBMIT_REMOVEDATANODESPROCEDURE_FAILED_BECAUSE_WHEN_THERE_ARE_OTHER_UNKNOWN_STOPPED_OR_READONLY_NODES_IN_THE_CONSENSUS_GROUP_THAT_ARE_NOT_REMOVE_NODES_THE_REMOVE_OPERATION_CANNOT_BE_PERFORMED_FOR_SECURITY_REASONS_PLEASE_CHECK_THE_STATUS_OF_THE_NODE_ARG_AND_ENSURE_IT_IS_RUNNING_5063B3F6 = + "提交 RemoveDataNodesProcedure 失败：共识组内存在非移除目标且状态为 Unknown、Stopped 或 ReadOnly 的其他节点时，出于安全考虑无法执行移除操作。请检查节点 %s 的状态并确保其处于 Running 状态。"; public static final String MESSAGE_AVAILABLEDATANODESIZE_ARG_MAXREPLICAFACTOR_ARG_MAX_ALLOWED_REMOVED_DATA_NODE_SIZE_FB8C382C = "availableDataNodeSize：%s，maxReplicaFactor：%s，允许移除的最大 DataNode 数量为：%s"; @@ -1507,4 +1510,8 @@ private ProcedureMessages() {} "正在回滚，失败原因：%s。"; public static final String MESSAGE_ROLLING_BACK_AFTER_AN_EARLIER_FAILURE_850D0AF5 = "正在回滚此前发生的失败。"; + public static final String EXCEPTION_DATANODE_SETSYSTEMSTATUS_RPC_FAILED_ARG_ARG_ARG_7E8E5F9F = + "DataNode setSystemStatus RPC 失败：%s：%s：%s； "; + public static final String EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4 = + "ConfigNode trySetNodeStatus 失败：%s：%s：%s； "; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/AINodeHeartbeatHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/AINodeHeartbeatHandler.java index 03e6c0bfe3111..8e734ed635dad 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/AINodeHeartbeatHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/AINodeHeartbeatHandler.java @@ -22,7 +22,6 @@ import org.apache.iotdb.ainode.rpc.thrift.TAIHeartbeatResp; import org.apache.iotdb.commons.client.ThriftClient; import org.apache.iotdb.commons.cluster.NodeStatus; -import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.confignode.manager.load.LoadManager; import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample; @@ -49,8 +48,7 @@ public void onComplete(TAIHeartbeatResp aiHeartbeatResp) { @Override public void onError(Exception e) { if (ThriftClient.isConnectionBroken(e)) { - loadManager.forceUpdateNodeCache( - NodeType.AINode, nodeId, new NodeHeartbeatSample(NodeStatus.Unknown)); + loadManager.trySetNodeStatus(nodeId, NodeStatus.Unknown, false); } loadManager.getLoadCache().resetHeartbeatProcessing(nodeId); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/ConfigNodeHeartbeatHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/ConfigNodeHeartbeatHandler.java index 96225e4fb7bdb..df3dad7184052 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/ConfigNodeHeartbeatHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/ConfigNodeHeartbeatHandler.java @@ -21,7 +21,6 @@ import org.apache.iotdb.commons.client.ThriftClient; import org.apache.iotdb.commons.cluster.NodeStatus; -import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.confignode.manager.load.LoadManager; import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeHeartbeatResp; @@ -48,8 +47,7 @@ public void onComplete(TConfigNodeHeartbeatResp resp) { @Override public void onError(Exception e) { if (ThriftClient.isConnectionBroken(e)) { - loadManager.forceUpdateNodeCache( - NodeType.ConfigNode, nodeId, new NodeHeartbeatSample(NodeStatus.Unknown)); + loadManager.trySetNodeStatus(nodeId, NodeStatus.Unknown, false); } loadManager.getLoadCache().resetHeartbeatProcessing(nodeId); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java index 14e1dcbb86fd4..d2d84f90d32df 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java @@ -23,7 +23,6 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.commons.client.ThriftClient; import org.apache.iotdb.commons.cluster.NodeStatus; -import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.commons.cluster.RegionStatus; import org.apache.iotdb.confignode.conf.ConfigNodeConfig; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; @@ -105,7 +104,8 @@ private void cacheNodeHeartbeatSample(TDataNodeHeartbeatResp heartbeatResp) { } private void cacheRegionGroupHeartbeatSamples(TDataNodeHeartbeatResp heartbeatResp) { - RegionStatus regionStatus = RegionStatus.valueOf(heartbeatResp.getStatus()); + RegionStatus regionStatus = + RegionStatus.fromNodeStatus(NodeStatus.parse(heartbeatResp.getStatus())); Map judgedLeaders = heartbeatResp.isSetJudgedLeaders() @@ -220,8 +220,7 @@ private void cacheRegionSizeSamples(TDataNodeHeartbeatResp heartbeatResp) { @Override public void onError(Exception e) { if (ThriftClient.isConnectionBroken(e)) { - loadManager.forceUpdateNodeCache( - NodeType.DataNode, nodeId, new NodeHeartbeatSample(NodeStatus.Unknown)); + loadManager.trySetNodeStatus(nodeId, NodeStatus.Unknown, false); } loadManager.getLoadCache().resetHeartbeatProcessing(nodeId); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index c70b8c12fcf93..9c01288d287f6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -32,6 +32,7 @@ import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateClusterIdPlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateVersionInfoPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.ActiveCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.AddCQPlan; @@ -359,6 +360,9 @@ public static ConfigPhysicalPlan create(final ByteBuffer buffer) throws IOExcept case UpdateVersionInfo: plan = new UpdateVersionInfoPlan(); break; + case UpdateNodeStatus: + plan = new UpdateNodeStatusPlan(); + break; case UpdateClusterId: plan = new UpdateClusterIdPlan(); break; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 144cb8d897aa1..eed13c917158d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -30,6 +30,8 @@ public enum ConfigPhysicalPlanType { UpdateVersionInfo((short) 2), UpdateClusterId((short) 3), + UpdateNodeStatus((short) 4), + /** DataNode. */ RegisterDataNode((short) 100), GetDataNodeConfiguration((short) 101), diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateNodeStatusPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateNodeStatusPlan.java new file mode 100644 index 0000000000000..662503607029b --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateNodeStatusPlan.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.confignode; + +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class UpdateNodeStatusPlan extends ConfigPhysicalPlan { + + public enum Operation { + SET_STOPPED, + SET_REMOVING, + CLEAR; + + /** Maps the final node status to a persistence command without applying transition rules. */ + public static Operation fromNodeStatus(NodeStatus status) { + return switch (status) { + case Stopped -> SET_STOPPED; + case Removing -> SET_REMOVING; + case Running, ReadOnly, Unknown -> CLEAR; + }; + } + } + + private int nodeId; + + private Operation operation; + + public UpdateNodeStatusPlan() { + super(ConfigPhysicalPlanType.UpdateNodeStatus); + } + + public UpdateNodeStatusPlan(int nodeId, Operation operation) { + this(); + this.nodeId = nodeId; + this.operation = operation; + } + + public int getNodeId() { + return nodeId; + } + + public Operation getOperation() { + return operation; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(getType().getPlanType(), stream); + ReadWriteIOUtils.write(nodeId, stream); + ReadWriteIOUtils.write(operation.name(), stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) { + nodeId = ReadWriteIOUtils.readInt(buffer); + operation = Operation.valueOf(ReadWriteIOUtils.readString(buffer)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + UpdateNodeStatusPlan that = (UpdateNodeStatusPlan) o; + return nodeId == that.nodeId && operation == that.operation; + } + + @Override + public int hashCode() { + return Objects.hash(nodeId, operation); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 2a6b7e0f68cfb..af55c953b777c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -120,7 +120,6 @@ import org.apache.iotdb.confignode.manager.externalservice.ExternalServiceInfo; import org.apache.iotdb.confignode.manager.externalservice.ExternalServiceManager; import org.apache.iotdb.confignode.manager.load.LoadManager; -import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample; import org.apache.iotdb.confignode.manager.node.ClusterNodeStartUtils; import org.apache.iotdb.confignode.manager.node.NodeManager; import org.apache.iotdb.confignode.manager.node.NodeMetrics; @@ -575,17 +574,19 @@ public TSStatus removeAINode() { @Override public TSStatus reportDataNodeShutdown(TDataNodeLocation dataNodeLocation) { + return reportNodeShutdown(dataNodeLocation.getDataNodeId()); + } + + private TSStatus reportNodeShutdown(int nodeId) { TSStatus status = confirmLeader(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - // Force updating the target DataNode's status to Unknown - getLoadManager() - .forceUpdateNodeCache( - NodeType.DataNode, - dataNodeLocation.getDataNodeId(), - new NodeHeartbeatSample(NodeStatus.Unknown)); - LOGGER.info( - ManagerMessages.THE_DATANODE_WILL_BE_SHUTDOWN_SOON_MARK_IT_AS_UNKNOWN, - dataNodeLocation.getDataNodeId()); + status = getLoadManager().trySetNodeStatus(nodeId, NodeStatus.Stopped, false); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.info( + ManagerMessages.LOG_NODE_ARG_REPORTED_SHUTDOWN_CURRENT_STATUS_IS_ARG_A375D665, + nodeId, + getLoadManager().getNodeStatus(nodeId)); + } } return status; } @@ -1629,19 +1630,7 @@ public TSStatus removeConfigNode(RemoveConfigNodePlan removeConfigNodePlan) { @Override public TSStatus reportConfigNodeShutdown(TConfigNodeLocation configNodeLocation) { - TSStatus status = confirmLeader(); - if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - // Force updating the target ConfigNode's status to Unknown - getLoadManager() - .forceUpdateNodeCache( - NodeType.ConfigNode, - configNodeLocation.getConfigNodeId(), - new NodeHeartbeatSample(NodeStatus.Unknown)); - LOGGER.info( - ManagerMessages.THE_CONFIGNODE_WILL_BE_SHUTDOWN_SOON_MARK_IT_AS_UNKNOWN, - configNodeLocation.getConfigNodeId()); - } - return status; + return reportNodeShutdown(configNodeLocation.getConfigNodeId()); } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index bae45596812e1..cc203caeebab0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -345,9 +345,13 @@ public interface IManager { /** * Report that the specified DataNode will be shutdown. * - *

The ConfigNode-leader will mark it as {@link NodeStatus#Unknown} + *

The ConfigNode-leader requests {@link NodeStatus#Stopped}, retaining an existing {@link + * NodeStatus#Removing}. Success confirms that the resulting status is persisted and can be + * restored by a new leader. If the report is not received, heartbeat timeout produces {@link + * NodeStatus#Unknown} only when there is no Stopped or Removing status to retain, including one + * restored from persistence. A failed write may still update the current leader's memory. * - * @return {@link TSStatusCode#SUCCESS_STATUS} if reporting successfully + * @return {@link TSStatusCode#SUCCESS_STATUS} if the resulting status is persisted */ TSStatus reportDataNodeShutdown(TDataNodeLocation dataNodeLocation); @@ -538,10 +542,14 @@ TPermissionInfoResp login( TSStatus removeConfigNode(RemoveConfigNodePlan removeConfigNodePlan); /** - * Report that the specified ConfigNode will be shutdown. The ConfigNode-leader will mark it as - * Unknown. + * Report that the specified ConfigNode will be shutdown. The leader requests {@link + * NodeStatus#Stopped}, retaining an existing {@link NodeStatus#Removing}. Success confirms that + * the resulting status is persisted and can be restored by a new leader. If the report is not + * received, heartbeat timeout produces {@link NodeStatus#Unknown} only when there is no Stopped + * or Removing status to retain, including one restored from persistence. A failed write may still + * update the current leader's memory. * - * @return {@link TSStatusCode#SUCCESS_STATUS} if reporting successfully + * @return {@link TSStatusCode#SUCCESS_STATUS} if the resulting status is persisted */ TSStatus reportConfigNodeShutdown(TConfigNodeLocation configNodeLocation); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index c97654b649e62..f063a933ebc35 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -58,6 +58,7 @@ import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan; import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; import org.apache.iotdb.confignode.i18n.ManagerMessages; +import org.apache.iotdb.confignode.i18n.ProcedureMessages; import org.apache.iotdb.confignode.manager.partition.PartitionManager; import org.apache.iotdb.confignode.manager.subscription.SubscriptionCoordinator; import org.apache.iotdb.confignode.persistence.ProcedureInfo; @@ -776,8 +777,8 @@ public TSStatus checkRemoveDataNodes(List dataNodeLocations) removedDataNodesRegionSet.add(regionMigrationPlan.getRegionId()); } - // 4. Check if there are any other unknown or readonly DataNodes in the consensus group that are - // not the remove DataNodes + // 4. Check if there are any other unknown, stopped or readonly DataNodes in the consensus + // group that are not the remove DataNodes for (TDataNodeLocation removeDataNode : dataNodeLocations) { Set relatedDataNodes = @@ -787,13 +788,11 @@ public TSStatus checkRemoveDataNodes(List dataNodeLocations) for (TDataNodeLocation relatedDataNode : relatedDataNodes) { NodeStatus nodeStatus = getConfigManager().getLoadManager().getNodeStatus(relatedDataNode.getDataNodeId()); - if (nodeStatus == NodeStatus.Unknown || nodeStatus == NodeStatus.ReadOnly) { + if (nodeStatus.isOffline() || nodeStatus == NodeStatus.ReadOnly) { failMessage = String.format( - "Submit RemoveDataNodesProcedure failed, " - + "because when there are other unknown or readonly nodes in the consensus group that are not remove nodes, " - + "the remove operation cannot be performed for security reasons. " - + "Please check the status of the node %s and ensure it is running.", + ProcedureMessages + .MESSAGE_SUBMIT_REMOVEDATANODESPROCEDURE_FAILED_BECAUSE_WHEN_THERE_ARE_OTHER_UNKNOWN_STOPPED_OR_READONLY_NODES_IN_THE_CONSENSUS_GROUP_THAT_ARE_NOT_REMOVE_NODES_THE_REMOVE_OPERATION_CANNOT_BE_PERFORMED_FOR_SECURITY_REASONS_PLEASE_CHECK_THE_STATUS_OF_THE_NODE_ARG_AND_ENSURE_IT_IS_RUNNING_5063B3F6, relatedDataNode.getDataNodeId()); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThread.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThread.java index 2f222a93ac6d4..f4fe41fe65663 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThread.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThread.java @@ -139,8 +139,7 @@ private void triggerDetectTask() { NodeStatus nodeStatus = loadManager.getNodeStatus(dataNodeLocation.getDataNodeId()); if (nodeStatus == NodeStatus.Running) { oldUnknownNodes.remove(dataNodeLocation); - } else if (!oldUnknownNodes.contains(dataNodeLocation) - && nodeStatus == NodeStatus.Unknown) { + } else if (!oldUnknownNodes.contains(dataNodeLocation) && nodeStatus.isOffline()) { newUnknownNodes.add(dataNodeLocation); } }); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java index 5a424ea5a5e73..17d0f6494dbef 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java @@ -22,9 +22,9 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; import org.apache.iotdb.commons.cluster.NodeStatus; -import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.commons.cluster.RegionStatus; import org.apache.iotdb.commons.partition.DataPartitionTable; import org.apache.iotdb.commons.partition.SchemaPartitionTable; @@ -33,13 +33,13 @@ import org.apache.iotdb.confignode.exception.DatabaseNotExistsException; import org.apache.iotdb.confignode.exception.NoAvailableRegionGroupException; import org.apache.iotdb.confignode.exception.NotEnoughDataNodeException; +import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.manager.IManager; import org.apache.iotdb.confignode.manager.load.balancer.PartitionBalancer; import org.apache.iotdb.confignode.manager.load.balancer.RegionBalancer; import org.apache.iotdb.confignode.manager.load.balancer.RouteBalancer; import org.apache.iotdb.confignode.manager.load.cache.LoadCache; import org.apache.iotdb.confignode.manager.load.cache.consensus.ConsensusGroupHeartbeatSample; -import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample; import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample; import org.apache.iotdb.confignode.manager.load.service.EventService; import org.apache.iotdb.confignode.manager.load.service.HeartbeatService; @@ -92,6 +92,10 @@ public LoadManager(IManager configManager) { this.routeBalancer = new RouteBalancer(configManager); this.loadCache = new LoadCache(); + this.loadCache.setNodeStatusPersistence( + configManager.getNodeManager().getNodeInfo()::getPersistedNodeStatus, + // Consensus is initialized after construction; obtain the manager when submitting the plan. + plan -> configManager.getConsensusManager().write(plan)); setHeartbeatService(configManager, loadCache); this.statisticsService = new StatisticsService(loadCache); this.topologyService = new TopologyService(configManager, loadCache::updateTopology); @@ -209,7 +213,13 @@ private synchronized boolean tryUpdateLoadReady() { return false; } - loadCache.updateNodeStatistics(false); + if (!loadCache.updateNodeStatistics()) { + // In particular, do not serve as leader until its own old Stopped marker is cleared. + loadReadyReason = + ManagerMessages.MESSAGE_CONFIGNODE_LEADER_IS_WAITING_FOR_NODE_STATUS_PERSISTENCE_8CA96809; + return false; + } + eventService.checkAndBroadcastNodeStatisticsChangeEventIfNecessary(); List unreadyReasons = loadCache.getNodeHeartbeatUnreadyReasons(); @@ -337,41 +347,29 @@ public int getLowestLoadDataNode(List dataNodeIds) { } /** - * Force update the specified Node's cache, update statistics and broadcast statistics change - * event if necessary. + * Try to set the specified Node's status according to transition rules and broadcast statistics + * changes, including when persistence fails. Does not send a status-change RPC to the node. * - * @param nodeType Specified NodeType * @param nodeId Specified NodeId - * @param heartbeatSample Specified NodeHeartbeatSample + * @param nodeStatus requested status + * @param force whether an administrative update may replace Stopped or Removing + * @return processing result; success does not imply that the requested status replaced the old + * one */ - public void forceUpdateNodeCache( - NodeType nodeType, int nodeId, NodeHeartbeatSample heartbeatSample) { - switch (nodeType) { - case ConfigNode: - loadCache.cacheConfigNodeHeartbeatSample(nodeId, heartbeatSample); - break; - case DataNode: - loadCache.cacheDataNodeHeartbeatSample(nodeId, heartbeatSample); - break; - case AINode: - loadCache.cacheAINodeHeartbeatSample(nodeId, heartbeatSample); - break; - default: - break; - } - loadCache.updateNodeStatistics(true); + public TSStatus trySetNodeStatus(int nodeId, NodeStatus nodeStatus, boolean force) { + TSStatus status = loadCache.trySetNodeStatus(nodeId, nodeStatus, force); eventService.checkAndBroadcastNodeStatisticsChangeEventIfNecessary(); + return status; } /** - * Remove the NodeHeartbeatCache of the specified Node, update statistics and broadcast statistics - * change event if necessary. + * Remove the NodeHeartbeatCache of the specified Node and broadcast statistics changes if + * necessary. * * @param nodeId the index of the specified Node */ public void removeNodeCache(int nodeId) { loadCache.removeNodeCache(nodeId); - loadCache.updateNodeStatistics(true); eventService.checkAndBroadcastNodeStatisticsChangeEventIfNecessary(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java index 73583151f9819..44dbcc3791243 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java @@ -84,17 +84,15 @@ public CreateRegionGroupsPlan genRegionGroupsAllocationPlan( throws NotEnoughDataNodeException, DatabaseNotExistsException { // Some new RegionGroups will have to occupy unknown DataNodes if the number of online - // DataNodes is insufficient (Unknown DataNodes are intentionally kept as candidates). - // However, DataNodes that an in-progress RemoveDataNodesProcedure is removing must be - // excluded: placing a new replica on a node that is about to disappear would strand that - // replica and stall the removal forever. A status filter is not enough here, because a + // DataNodes is insufficient (Unknown and Stopped DataNodes are intentionally kept as + // candidates). However, DataNodes that an in-progress RemoveDataNodesProcedure is removing + // must be excluded: placing a new replica on a node that is about to disappear would strand + // that replica and stall the removal forever. A status filter is not enough here, because a // DataNode killed (e.g. kill -9) before removal is reported as Unknown (not Removing) by the // failure detector, so we additionally drop every DataNode that is currently being removed. final Set removingDataNodeIds = getProcedureManager().getRemovingDataNodeIds(); final List availableDataNodes = - getNodeManager() - .filterDataNodeThroughStatus(NodeStatus.Running, NodeStatus.Unknown) - .stream() + getNodeManager().filterDataNodeThroughStatus(NodeStatus::isRegionCandidate).stream() .filter( dataNode -> !removingDataNodeIds.contains(dataNode.getLocation().getDataNodeId())) .collect(Collectors.toList()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/AbstractLoadCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/AbstractLoadCache.java index 4c5ee322aa19f..da22a140cd497 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/AbstractLoadCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/AbstractLoadCache.java @@ -94,18 +94,15 @@ public void cacheHeartbeatSample(AbstractHeartbeatSample newHeartbeatSample) { * @return The latest heartbeat sample. */ public AbstractHeartbeatSample getLastSample() { - return slidingWindow.isEmpty() ? null : slidingWindow.get(slidingWindow.size() - 1); + synchronized (slidingWindow) { + return slidingWindow.isEmpty() ? null : slidingWindow.get(slidingWindow.size() - 1); + } } public boolean hasHeartbeatSample() { return getLastSample() != null; } - /** - * Update currentStatistics based on the latest heartbeat sample that cached in the slidingWindow. - */ - public abstract void updateCurrentStatistics(boolean forceUpdate); - public AbstractStatistics getCurrentStatistics() { return currentStatistics.get(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java index 3416246811edc..3407ec33e8b81 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java @@ -27,11 +27,15 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.commons.cluster.RegionStatus; import org.apache.iotdb.confignode.conf.ConfigNodeConfig; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan.Operation; +import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.manager.IManager; import org.apache.iotdb.confignode.manager.ProcedureManager; @@ -49,7 +53,10 @@ import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample; import org.apache.iotdb.confignode.manager.load.cache.region.RegionStatistics; import org.apache.iotdb.confignode.manager.partition.RegionGroupStatus; +import org.apache.iotdb.consensus.exception.ConsensusException; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.ratis.util.function.CheckedFunction; import org.apache.thrift.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,6 +77,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; +import java.util.function.IntFunction; import java.util.stream.Collectors; /** Maintain all kinds of heartbeat samples and statistics. */ @@ -104,6 +112,9 @@ public class LoadCache { private Map> topologyGraph; private final AtomicBoolean topologyUpdated; + private IntFunction persistedNodeStatusReader; + private CheckedFunction nodeStatusWriter; + public LoadCache() { this.nodeCacheMap = new ConcurrentHashMap<>(); this.heartbeatProcessingMap = new ConcurrentHashMap<>(); @@ -116,6 +127,19 @@ public LoadCache() { this.topologyUpdated = new AtomicBoolean(false); } + public void setNodeStatusPersistence( + IntFunction persistedNodeStatusReader, + CheckedFunction nodeStatusWriter) { + Objects.requireNonNull(persistedNodeStatusReader); + Objects.requireNonNull(nodeStatusWriter); + this.persistedNodeStatusReader = persistedNodeStatusReader; + this.nodeStatusWriter = nodeStatusWriter; + } + + private boolean hasPersistence() { + return persistedNodeStatusReader != null && nodeStatusWriter != null; + } + public void initHeartbeatCache(final IManager configManager) { initNodeHeartbeatCache( configManager.getNodeManager().getRegisteredConfigNodes(), @@ -148,10 +172,13 @@ private void initNodeHeartbeatCache( } }); // Force set itself and never update - nodeCacheMap.put( - ConfigNodeHeartbeatCache.CURRENT_NODE_ID, + ConfigNodeHeartbeatCache currentNodeCache = new ConfigNodeHeartbeatCache( - CURRENT_NODE_ID, ConfigNodeHeartbeatCache.CURRENT_NODE_STATISTICS)); + CURRENT_NODE_ID, ConfigNodeHeartbeatCache.CURRENT_NODE_STATISTICS); + if (hasPersistence()) { + currentNodeCache.initPersistence(null, this::persistNodeStatus); + } + nodeCacheMap.put(CURRENT_NODE_ID, currentNodeCache); // Init DataNodeHeartbeatCache registeredDataNodes.forEach( @@ -216,24 +243,56 @@ public boolean checkAndSetHeartbeatProcessing(int nodeId) { } /** - * Create a new NodeHeartbeatCache for the specified Node. + * Create a NodeHeartbeatCache for the specified Node if it does not already exist. * * @param nodeType The specified NodeType * @param nodeId The specified NodeId */ public void createNodeHeartbeatCache(NodeType nodeType, int nodeId) { - switch (nodeType) { - case ConfigNode: - nodeCacheMap.put(nodeId, new ConfigNodeHeartbeatCache(nodeId)); - break; - case DataNode: - nodeCacheMap.put(nodeId, new DataNodeHeartbeatCache(nodeId)); - break; - case AINode: - nodeCacheMap.put(nodeId, new AINodeHeartbeatCache(nodeId)); - break; + // A recovered registration procedure can repeat this call while a status write is in progress. + // Keep the existing cache and its heartbeat processing flag so that pending updates stay + // visible. + nodeCacheMap.computeIfAbsent( + nodeId, + id -> { + BaseNodeCache cache = + switch (nodeType) { + case ConfigNode -> new ConfigNodeHeartbeatCache(id); + case DataNode -> new DataNodeHeartbeatCache(id); + case AINode -> new AINodeHeartbeatCache(id); + }; + if (hasPersistence()) { + cache.initPersistence(persistedNodeStatusReader.apply(id), this::persistNodeStatus); + } + return cache; + }); + heartbeatProcessingMap.putIfAbsent(nodeId, new AtomicBoolean(false)); + } + + private TSStatus persistNodeStatus(BaseNodeCache cache, NodeStatus status) { + int nodeId = cache.getNodeId(); + if (nodeCacheMap.get(nodeId) != cache) { + return new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + } + Operation operation = Operation.fromNodeStatus(status); + NodeStatus persistedStatus = operation == Operation.CLEAR ? null : status; + if (persistedNodeStatusReader.apply(nodeId) == persistedStatus) { + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } + try { + return nodeStatusWriter.apply(new UpdateNodeStatusPlan(nodeId, operation)); + } catch (ConsensusException e) { + LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e); + return new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()) + .setMessage(e.getMessage()); } - heartbeatProcessingMap.put(nodeId, new AtomicBoolean(false)); + } + + public TSStatus trySetNodeStatus(int nodeId, NodeStatus status, boolean force) { + BaseNodeCache cache = nodeCacheMap.get(nodeId); + return cache == null + ? new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode()) + : cache.trySetNodeStatus(status, force); } /** @@ -269,9 +328,9 @@ public void cacheDataNodeHeartbeatSample(int nodeId, NodeHeartbeatSample sample) * @param sample the latest heartbeat sample */ public void cacheAINodeHeartbeatSample(int nodeId, NodeHeartbeatSample sample) { - nodeCacheMap - .computeIfAbsent(nodeId, empty -> new AINodeHeartbeatCache(nodeId)) - .cacheHeartbeatSample(sample); + // Only cache sample when the corresponding loadCache exists + Optional.ofNullable(nodeCacheMap.get(nodeId)) + .ifPresent(node -> node.cacheHeartbeatSample(sample)); Optional.ofNullable(heartbeatProcessingMap.get(nodeId)).ifPresent(node -> node.set(false)); } @@ -366,10 +425,13 @@ public void cacheConsensusSample( } /** Update the NodeStatistics of all Nodes. */ - public void updateNodeStatistics(boolean forceUpdate) { - nodeCacheMap - .values() - .forEach(baseNodeCache -> baseNodeCache.updateCurrentStatistics(forceUpdate)); + public boolean updateNodeStatistics() { + boolean success = true; + for (BaseNodeCache cache : nodeCacheMap.values()) { + success &= + cache.updateNodeStatistics().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode(); + } + return success; } /** Update the RegionGroupStatistics of all RegionGroups. */ @@ -379,9 +441,7 @@ public void updateRegionGroupStatistics() { /** Update the ConsensusGroupStatistics of all RegionGroups. */ public void updateConsensusGroupStatistics() { - consensusGroupCacheMap - .values() - .forEach(consensusGroupCache -> consensusGroupCache.updateCurrentStatistics(false)); + consensusGroupCacheMap.values().forEach(ConsensusGroupCache::updateCurrentStatistics); } /** @@ -506,7 +566,10 @@ public List getNodeHeartbeatUnreadyReasons() { } if ((nodeCache instanceof ConfigNodeHeartbeatCache || nodeCache instanceof DataNodeHeartbeatCache) - && !nodeCache.hasHeartbeatSample()) { + && !nodeCache.hasHeartbeatSample() + // Stopped/Removing can be restored from persisted records without waiting for a + // heartbeat. + && !nodeCache.getNodeStatus().isPersistentStatus()) { unreadyNodes.add(nodeId); } }); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/consensus/ConsensusGroupCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/consensus/ConsensusGroupCache.java index 7dcacc4fd805c..2524673ff5689 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/consensus/ConsensusGroupCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/consensus/ConsensusGroupCache.java @@ -35,8 +35,7 @@ public ConsensusGroupCache() { this.currentStatistics.set(ConsensusGroupStatistics.generateDefaultConsensusGroupStatistics()); } - @Override - public synchronized void updateCurrentStatistics(boolean forceUpdate) { + public synchronized void updateCurrentStatistics() { ConsensusGroupHeartbeatSample lastSample; synchronized (slidingWindow) { lastSample = (ConsensusGroupHeartbeatSample) getLastSample(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/AINodeHeartbeatCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/AINodeHeartbeatCache.java index ca15f9cb5150c..8a8a765a18a9c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/AINodeHeartbeatCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/AINodeHeartbeatCache.java @@ -19,57 +19,9 @@ package org.apache.iotdb.confignode.manager.load.cache.node; -import org.apache.iotdb.common.rpc.thrift.TLoadSample; -import org.apache.iotdb.commons.cluster.NodeStatus; -import org.apache.iotdb.confignode.manager.load.cache.AbstractHeartbeatSample; - -import java.util.Collections; -import java.util.List; -import java.util.concurrent.atomic.AtomicReference; - public class AINodeHeartbeatCache extends BaseNodeCache { - private final AtomicReference latestLoadSample; - public AINodeHeartbeatCache(int aiNodeId) { super(aiNodeId); - this.latestLoadSample = new AtomicReference<>(new TLoadSample()); - } - - @Override - public void updateCurrentStatistics(boolean forceUpdate) { - NodeHeartbeatSample lastSample; - final List heartbeatHistory; - /* Update Node status */ - NodeStatus status = null; - String statusReason = null; - long currentNanoTime = System.nanoTime(); - synchronized (slidingWindow) { - lastSample = (NodeHeartbeatSample) getLastSample(); - heartbeatHistory = Collections.unmodifiableList(slidingWindow); - /* Update load sample */ - if (lastSample != null && lastSample.isSetLoadSample()) { - latestLoadSample.set((lastSample.getLoadSample())); - } - - if (lastSample != null && NodeStatus.Removing.equals(lastSample.getStatus())) { - status = NodeStatus.Removing; - } else if (!failureDetector.isAvailable(nodeId, heartbeatHistory)) { - /* Failure detector decides that this AINode is UNKNOWN */ - status = NodeStatus.Unknown; - } else if (lastSample != null) { - status = lastSample.getStatus(); - statusReason = lastSample.getStatusReason(); - } - } - - long loadScore = NodeStatus.isNormalStatus(status) ? 0 : Long.MAX_VALUE; - - NodeStatistics newStatistics = - new NodeStatistics(currentNanoTime, status, statusReason, loadScore); - if (!currentStatistics.get().equals(newStatistics)) { - // Update the current NodeStatistics if necessary - currentStatistics.set(newStatistics); - } } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java index 4eeb96344ccd1..10a3e939c6275 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java @@ -19,8 +19,15 @@ package org.apache.iotdb.confignode.manager.load.cache.node; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.manager.load.cache.AbstractHeartbeatSample; import org.apache.iotdb.confignode.manager.load.cache.AbstractLoadCache; +import org.apache.iotdb.rpc.TSStatusCode; + +import java.util.Collections; +import java.util.List; +import java.util.function.BiFunction; /** * NodeCache caches the NodeHeartbeatSamples of a Node. Update and cache the current statistics of @@ -30,6 +37,8 @@ public abstract class BaseNodeCache extends AbstractLoadCache { protected final int nodeId; + private BiFunction nodeStatusPersister; + /** Constructor for NodeCache with default NodeStatistics. */ protected BaseNodeCache(int nodeId) { super(); @@ -41,6 +50,72 @@ public int getNodeId() { return nodeId; } + /** Initialize a new cache before adding it to LoadCache's nodeCacheMap. */ + public void initPersistence( + NodeStatus persistedStatus, + BiFunction nodeStatusPersister) { + this.nodeStatusPersister = nodeStatusPersister; + if (persistedStatus != null) { + currentStatistics.set( + new NodeStatistics(System.nanoTime(), persistedStatus, null, Long.MAX_VALUE)); + } + } + + public TSStatus updateNodeStatistics() { + synchronized (slidingWindow) { + return applyNodeStatistics(calculateCurrentStatistics(), false); + } + } + + /** + * Try to set the requested status and record a sample for later statistics refreshes. Transition + * rules may retain the previous status even when this method returns success. A persistence + * failure is returned without discarding the new statistics. + */ + public TSStatus trySetNodeStatus(NodeStatus status, boolean force) { + synchronized (slidingWindow) { + NodeHeartbeatSample sample = new NodeHeartbeatSample(status); + cacheHeartbeatSample(sample); + return applyNodeStatistics( + new NodeStatistics( + sample.getSampleLogicalTimestamp(), + status, + null, + NodeStatus.isNormalStatus(status) ? 0 : Long.MAX_VALUE), + force); + } + } + + private TSStatus applyNodeStatistics(NodeStatistics newStats, boolean force) { + newStats = NodeStatistics.transition((NodeStatistics) currentStatistics.get(), newStats, force); + TSStatus result = + nodeStatusPersister == null + ? new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()) + : nodeStatusPersister.apply(this, newStats.getStatus()); + currentStatistics.set(newStats); + return result; + } + + /** Called with the slidingWindow lock held through calculation, persistence and publication. */ + protected NodeStatistics calculateCurrentStatistics() { + NodeStatus status; + String statusReason = null; + long currentNanoTime = System.nanoTime(); + NodeHeartbeatSample lastSample = (NodeHeartbeatSample) getLastSample(); + List heartbeatHistory = Collections.unmodifiableList(slidingWindow); + if (lastSample == null || !failureDetector.isAvailable(nodeId, heartbeatHistory)) { + status = NodeStatus.Unknown; + } else { + status = lastSample.getStatus(); + statusReason = lastSample.getStatusReason(); + } + return new NodeStatistics( + currentNanoTime, + status, + statusReason, + NodeStatus.isNormalStatus(status) ? 0 : Long.MAX_VALUE); + } + /** * TODO: The loadScore of each Node will be changed to Double * diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/ConfigNodeHeartbeatCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/ConfigNodeHeartbeatCache.java index 4d675e1e8a442..ae0b20e3a6261 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/ConfigNodeHeartbeatCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/ConfigNodeHeartbeatCache.java @@ -21,10 +21,6 @@ import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; -import org.apache.iotdb.confignode.manager.load.cache.AbstractHeartbeatSample; - -import java.util.Collections; -import java.util.List; /** Heartbeat cache for cluster ConfigNodes. */ public class ConfigNodeHeartbeatCache extends BaseNodeCache { @@ -48,37 +44,10 @@ public ConfigNodeHeartbeatCache(int configNodeId, NodeStatistics statistics) { } @Override - public synchronized void updateCurrentStatistics(boolean forceUpdate) { - // Skip itself and the Removing status can not be updated - if (nodeId == CURRENT_NODE_ID || NodeStatus.Removing.equals(getNodeStatus())) { - return; - } - - NodeHeartbeatSample lastSample; - // Update Node status - NodeStatus status; - long currentNanoTime = System.nanoTime(); - final List heartbeatHistory; - synchronized (slidingWindow) { - lastSample = (NodeHeartbeatSample) getLastSample(); - heartbeatHistory = Collections.unmodifiableList(slidingWindow); - - if (lastSample == null) { - /* First heartbeat not received from this ConfigNode, status is UNKNOWN */ - status = NodeStatus.Unknown; - } else if (!failureDetector.isAvailable(nodeId, heartbeatHistory)) { - /* Failure detector decides that this ConfigNode is UNKNOWN */ - status = NodeStatus.Unknown; - } else { - status = lastSample.getStatus(); - } + protected NodeStatistics calculateCurrentStatistics() { + if (nodeId == CURRENT_NODE_ID) { + return (NodeStatistics) currentStatistics.get(); } - - /* Update loadScore */ - // Only consider Running ConfigNode as available currently - // TODO: Construct load score module - long loadScore = NodeStatus.isNormalStatus(status) ? 0 : Long.MAX_VALUE; - - currentStatistics.set(new NodeStatistics(currentNanoTime, status, null, loadScore)); + return super.calculateCurrentStatistics(); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/DataNodeHeartbeatCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/DataNodeHeartbeatCache.java index e3d6d15c8356b..97451c36579b2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/DataNodeHeartbeatCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/DataNodeHeartbeatCache.java @@ -20,22 +20,12 @@ package org.apache.iotdb.confignode.manager.load.cache.node; import org.apache.iotdb.common.rpc.thrift.TLoadSample; -import org.apache.iotdb.commons.cluster.NodeStatus; -import org.apache.iotdb.confignode.i18n.ManagerMessages; -import org.apache.iotdb.confignode.manager.load.cache.AbstractHeartbeatSample; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.List; import java.util.concurrent.atomic.AtomicReference; /** Heartbeat cache for cluster DataNodes. */ public class DataNodeHeartbeatCache extends BaseNodeCache { - private static final Logger LOGGER = LoggerFactory.getLogger(DataNodeHeartbeatCache.class); - // TODO: The load sample may be moved into NodeStatistics in the future private final AtomicReference latestLoadSample; @@ -46,49 +36,12 @@ public DataNodeHeartbeatCache(int dataNodeId) { } @Override - public synchronized void updateCurrentStatistics(boolean forceUpdate) { - // The Removing status can not be updated - if (!forceUpdate && NodeStatus.Removing.equals(getNodeStatus())) { - return; - } - - NodeHeartbeatSample lastSample; - final List heartbeatHistory; - /* Update Node status */ - NodeStatus status; - String statusReason = null; - long currentNanoTime = System.nanoTime(); - synchronized (slidingWindow) { - lastSample = (NodeHeartbeatSample) getLastSample(); - heartbeatHistory = Collections.unmodifiableList(slidingWindow); - /* Update load sample */ - if (lastSample != null && lastSample.isSetLoadSample()) { - latestLoadSample.set(lastSample.getLoadSample()); - } - - if (lastSample == null) { - /* First heartbeat not received from this DataNode, status is UNKNOWN */ - status = NodeStatus.Unknown; - } else if (!failureDetector.isAvailable(nodeId, heartbeatHistory)) { - /* Failure detector decides that this DataNode is UNKNOWN */ - status = NodeStatus.Unknown; - } else { - status = lastSample.getStatus(); - statusReason = lastSample.getStatusReason(); - } - } - - /* Update loadScore */ - // Only consider Running DataNode as available currently - // TODO: Construct load score module - long loadScore = NodeStatus.isNormalStatus(status) ? 0 : Long.MAX_VALUE; - - currentStatistics.set(new NodeStatistics(currentNanoTime, status, statusReason, loadScore)); - - if (forceUpdate) { - LOGGER.debug( - ManagerMessages.FORCE_UPDATE_NODECACHE_STATUS_CURRENTNANOTIME, status, currentNanoTime); + protected NodeStatistics calculateCurrentStatistics() { + NodeHeartbeatSample lastSample = (NodeHeartbeatSample) getLastSample(); + if (lastSample != null && lastSample.isSetLoadSample()) { + latestLoadSample.set(lastSample.getLoadSample()); } + return super.calculateCurrentStatistics(); } public double getFreeDiskSpace() { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeStatistics.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeStatistics.java index 12c554d51e8d8..04ed6c5028901 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeStatistics.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeStatistics.java @@ -55,6 +55,23 @@ public static NodeStatistics generateDefaultNodeStatistics() { return new NodeStatistics(Long.MIN_VALUE, NodeStatus.Unknown, null, Long.MAX_VALUE); } + /** + * Resolves the requested statistics through {@link NodeStatus#transition}. If the requested + * status is accepted, its reason and load score are kept. Otherwise, the final status must not + * carry the rejected status's reason or load score, e.g. Removing must not inherit ReadOnly's + * DiskFull reason. The requested statistics timestamp is retained in both cases. + */ + public static NodeStatistics transition( + NodeStatistics previous, NodeStatistics requested, boolean force) { + NodeStatus status = NodeStatus.transition(previous.status, requested.status, force); + if (status == requested.status) { + return requested; + } + // if the transition output is not the same as requested, it could be Stopped or Removing status + // both do not have status reason, set reason to null. + return new NodeStatistics(requested.getStatisticsNanoTimestamp(), status, null, Long.MAX_VALUE); + } + public NodeStatus getStatus() { return status; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionCache.java index a374c923f2405..23a777b7fb63e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionCache.java @@ -42,8 +42,7 @@ public RegionCache(int dataNodeId, TConsensusGroupId gid) { this.currentStatistics.set(RegionStatistics.generateDefaultRegionStatistics()); } - @Override - public synchronized void updateCurrentStatistics(boolean forceUpdate) { + public synchronized void updateCurrentStatistics() { RegionHeartbeatSample lastSample; List history; synchronized (slidingWindow) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionGroupCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionGroupCache.java index 2cdecc757fd26..e6d65bd5003c7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionGroupCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionGroupCache.java @@ -101,7 +101,7 @@ public synchronized void removeRegionCache(int dataNodeId) { * slidingWindow. */ public synchronized void updateCurrentStatistics() { - regionCacheMap.values().forEach(regionCache -> regionCache.updateCurrentStatistics(false)); + regionCacheMap.values().forEach(RegionCache::updateCurrentStatistics); Map regionStatisticsMap = regionCacheMap.entrySet().stream() .collect( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/service/StatisticsService.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/service/StatisticsService.java index 620276767f096..8d61102ac59cc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/service/StatisticsService.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/service/StatisticsService.java @@ -98,7 +98,7 @@ public void reloadHeartbeatInterval() { } private void updateLoadStatistics() { - loadCache.updateNodeStatistics(false); + loadCache.updateNodeStatistics(); loadCache.updateRegionGroupStatistics(); loadCache.updateConsensusGroupStatistics(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 64d4cf741e159..3a438b3866acb 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -151,6 +151,10 @@ public NodeManager(IManager configManager, NodeInfo nodeInfo) { this.removeConfigNodeLock = new ReentrantLock(); } + public NodeInfo getNodeInfo() { + return nodeInfo; + } + /** * Get system configurations. * diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinator.java index dd8fe2e70cf7e..cfe16f2926875 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinator.java @@ -84,8 +84,7 @@ public synchronized void handleNodeStatisticsChange(final NodeStatisticsChangeEv final NodeStatus oldStatus = getNodeStatus(pair.getLeft()); final NodeStatus newStatus = getNodeStatus(pair.getRight()); return oldStatus != newStatus - && (isRuntimeSensitiveStatus(oldStatus) - || isRuntimeSensitiveStatus(newStatus)); + && (oldStatus.mayBeOffline() || newStatus.mayBeOffline()); }); if (!shouldRefreshRuntime) { return; @@ -153,8 +152,4 @@ private void submitRuntimeRefresh( private static NodeStatus getNodeStatus(final NodeStatistics statistics) { return statistics == null ? NodeStatus.Unknown : statistics.getStatus(); } - - private static boolean isRuntimeSensitiveStatus(final NodeStatus status) { - return status == NodeStatus.Unknown || status == NodeStatus.Removing; - } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index a09c283a48d35..8919308a159c5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -66,6 +66,7 @@ import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateClusterIdPlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateVersionInfoPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.ActiveCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.AddCQPlan; @@ -523,6 +524,8 @@ public TSStatus executeNonQueryPlan(ConfigPhysicalPlan physicalPlan) return nodeInfo.removeConfigNode((RemoveConfigNodePlan) physicalPlan); case UpdateVersionInfo: return nodeInfo.updateVersionInfo((UpdateVersionInfoPlan) physicalPlan); + case UpdateNodeStatus: + return nodeInfo.applyNodeStatusPlan((UpdateNodeStatusPlan) physicalPlan); case UpdateClusterId: return clusterInfo.updateClusterId((UpdateClusterIdPlan) physicalPlan); case CreateFunction: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java index d98a05c8a8cdc..aa2535bf48d83 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java @@ -36,6 +36,7 @@ import org.apache.iotdb.confignode.consensus.request.write.ainode.UpdateAINodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateVersionInfoPlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan; @@ -109,6 +110,9 @@ public class NodeInfo implements SnapshotProcessor { private final Map nodeVersionInfo; private final ReentrantReadWriteLock versionInfoReadWriteLock; + private final Map persistedNodeStatuses; + private final ReentrantReadWriteLock persistedNodeStatusReadWriteLock; + private static final String SNAPSHOT_FILENAME = "node_info.bin"; public NodeInfo() { @@ -123,6 +127,9 @@ public NodeInfo() { this.nodeVersionInfo = new ConcurrentHashMap<>(); this.versionInfoReadWriteLock = new ReentrantReadWriteLock(); + + this.persistedNodeStatuses = new HashMap<>(); + this.persistedNodeStatusReadWriteLock = new ReentrantReadWriteLock(); } /** @@ -177,15 +184,18 @@ public TSStatus removeDataNode(RemoveDataNodePlan req) { dataNodeInfoReadWriteLock.writeLock().lock(); versionInfoReadWriteLock.writeLock().lock(); + persistedNodeStatusReadWriteLock.writeLock().lock(); try { req.getDataNodeLocations() .forEach( removeDataNodes -> { registeredDataNodes.remove(removeDataNodes.getDataNodeId()); nodeVersionInfo.remove(removeDataNodes.getDataNodeId()); + persistedNodeStatuses.remove(removeDataNodes.getDataNodeId()); LOGGER.info(ConfigNodeMessages.REMOVED_THE_DATANODE_FROM_CLUSTER, removeDataNodes); }); } finally { + persistedNodeStatusReadWriteLock.writeLock().unlock(); versionInfoReadWriteLock.writeLock().unlock(); dataNodeInfoReadWriteLock.writeLock().unlock(); } @@ -461,9 +471,11 @@ public TSStatus removeConfigNode(RemoveConfigNodePlan removeConfigNodePlan) { TSStatus status = new TSStatus(); configNodeInfoReadWriteLock.writeLock().lock(); versionInfoReadWriteLock.writeLock().lock(); + persistedNodeStatusReadWriteLock.writeLock().lock(); try { registeredConfigNodes.remove(removeConfigNodePlan.getConfigNodeLocation().getConfigNodeId()); nodeVersionInfo.remove(removeConfigNodePlan.getConfigNodeLocation().getConfigNodeId()); + persistedNodeStatuses.remove(removeConfigNodePlan.getConfigNodeLocation().getConfigNodeId()); SystemPropertiesUtils.storeConfigNodeList(new ArrayList<>(registeredConfigNodes.values())); LOGGER.info( ConfigNodeMessages.SUCCESSFULLY_REMOVE_CONFIGNODE_CURRENT_CONFIGNODEGROUP, @@ -477,6 +489,7 @@ public TSStatus removeConfigNode(RemoveConfigNodePlan removeConfigNodePlan) { ConfigNodeMessages .MESSAGE_REMOVE_CONFIGNODE_FAILED_BECAUSE_CURRENT_CONFIGNODE_CAN_T_STORE_CONFIGNODE_8AB3BCB4); } finally { + persistedNodeStatusReadWriteLock.writeLock().unlock(); versionInfoReadWriteLock.writeLock().unlock(); configNodeInfoReadWriteLock.writeLock().unlock(); } @@ -525,9 +538,9 @@ public TSStatus updateAINode(UpdateAINodePlan updateAINodePlan) { } /** - * Persist Information about remove dataNode. + * Persist information about removing an AINode. * - * @param req RemoveDataNodePlan + * @param req RemoveAINodePlan * @return {@link TSStatus} */ public TSStatus removeAINode(RemoveAINodePlan req) { @@ -538,12 +551,15 @@ public TSStatus removeAINode(RemoveAINodePlan req) { aiNodeInfoReadWriteLock.writeLock().lock(); versionInfoReadWriteLock.writeLock().lock(); + persistedNodeStatusReadWriteLock.writeLock().lock(); TAINodeLocation removedAINode = req.getAINodeLocation(); try { registeredAINodes.remove(removedAINode.getAiNodeId()); nodeVersionInfo.remove(removedAINode.getAiNodeId()); + persistedNodeStatuses.remove(removedAINode.getAiNodeId()); LOGGER.info(ConfigNodeMessages.REMOVED_THE_AINODE_FROM_CLUSTER, removedAINode); } finally { + persistedNodeStatusReadWriteLock.writeLock().unlock(); versionInfoReadWriteLock.writeLock().unlock(); aiNodeInfoReadWriteLock.writeLock().unlock(); } @@ -633,6 +649,53 @@ public int generateNextNodeId() { return nextNodeId.incrementAndGet(); } + public TSStatus applyNodeStatusPlan(UpdateNodeStatusPlan plan) { + // Registration maps support concurrent reads. Node removal also holds this lock, so it cannot + // interleave with the existence check and status update. + persistedNodeStatusReadWriteLock.writeLock().lock(); + try { + int nodeId = plan.getNodeId(); + + if (registeredConfigNodes.containsKey(nodeId) + || registeredDataNodes.containsKey(nodeId) + || registeredAINodes.containsKey(nodeId)) { + switch (plan.getOperation()) { + case SET_STOPPED: + persistedNodeStatuses.put(nodeId, NodeStatus.Stopped); + break; + case SET_REMOVING: + persistedNodeStatuses.put(nodeId, NodeStatus.Removing); + break; + case CLEAR: + persistedNodeStatuses.remove(nodeId); + break; + } + } + + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } finally { + persistedNodeStatusReadWriteLock.writeLock().unlock(); + } + } + + public NodeStatus getPersistedNodeStatus(int nodeId) { + persistedNodeStatusReadWriteLock.readLock().lock(); + try { + return persistedNodeStatuses.get(nodeId); + } finally { + persistedNodeStatusReadWriteLock.readLock().unlock(); + } + } + + public Map getPersistedNodeStatuses() { + persistedNodeStatusReadWriteLock.readLock().lock(); + try { + return new HashMap<>(persistedNodeStatuses); + } finally { + persistedNodeStatusReadWriteLock.readLock().unlock(); + } + } + @Override public boolean processTakeSnapshot(File snapshotDir) throws IOException, TException { File snapshotFile = new File(snapshotDir, SNAPSHOT_FILENAME); @@ -648,6 +711,7 @@ public boolean processTakeSnapshot(File snapshotDir) throws IOException, TExcept dataNodeInfoReadWriteLock.readLock().lock(); aiNodeInfoReadWriteLock.readLock().lock(); versionInfoReadWriteLock.readLock().lock(); + persistedNodeStatusReadWriteLock.readLock().lock(); try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile); TIOStreamTransport tioStreamTransport = new TIOStreamTransport(fileOutputStream)) { @@ -663,6 +727,8 @@ public boolean processTakeSnapshot(File snapshotDir) throws IOException, TExcept serializeVersionInfo(fileOutputStream); + serializePersistedNodeStatuses(fileOutputStream); + tioStreamTransport.flush(); fileOutputStream.getFD().sync(); @@ -671,6 +737,7 @@ public boolean processTakeSnapshot(File snapshotDir) throws IOException, TExcept return tmpFile.renameTo(snapshotFile); } finally { + persistedNodeStatusReadWriteLock.readLock().unlock(); versionInfoReadWriteLock.readLock().unlock(); aiNodeInfoReadWriteLock.readLock().unlock(); dataNodeInfoReadWriteLock.readLock().unlock(); @@ -724,6 +791,14 @@ private void serializeVersionInfo(OutputStream outputStream) throws IOException } } + private void serializePersistedNodeStatuses(OutputStream outputStream) throws IOException { + ReadWriteIOUtils.write(persistedNodeStatuses.size(), outputStream); + for (Entry entry : persistedNodeStatuses.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), outputStream); + ReadWriteIOUtils.write(entry.getValue().getStatus(), outputStream); + } + } + @Override public void processLoadSnapshot(File snapshotDir) throws IOException, TException { @@ -739,6 +814,7 @@ public void processLoadSnapshot(File snapshotDir) throws IOException, TException dataNodeInfoReadWriteLock.writeLock().lock(); aiNodeInfoReadWriteLock.writeLock().lock(); versionInfoReadWriteLock.writeLock().lock(); + persistedNodeStatusReadWriteLock.writeLock().lock(); try (ByteArrayInputStream inputStream = new ByteArrayInputStream(Files.readAllBytes(snapshotFile.toPath())); @@ -759,7 +835,9 @@ public void processLoadSnapshot(File snapshotDir) throws IOException, TException deserializeBuildInfo(inputStream); + deserializePersistedNodeStatuses(inputStream); } finally { + persistedNodeStatusReadWriteLock.writeLock().unlock(); versionInfoReadWriteLock.writeLock().unlock(); aiNodeInfoReadWriteLock.writeLock().unlock(); dataNodeInfoReadWriteLock.writeLock().unlock(); @@ -830,6 +908,18 @@ private void deserializeBuildInfo(InputStream inputStream) throws IOException { } } + private void deserializePersistedNodeStatuses(InputStream inputStream) throws IOException { + if (inputStream.available() == 0) { + return; + } + + int size = ReadWriteIOUtils.readInt(inputStream); + for (int i = 0; i < size; i++) { + int nodeId = ReadWriteIOUtils.readInt(inputStream); + persistedNodeStatuses.put(nodeId, NodeStatus.parse(ReadWriteIOUtils.readString(inputStream))); + } + } + public static int getMinimumDataNode() { return MINIMUM_DATANODE; } @@ -840,6 +930,13 @@ public void clear() { registeredConfigNodes.clear(); registeredAINodes.clear(); nodeVersionInfo.clear(); + + persistedNodeStatusReadWriteLock.writeLock().lock(); + try { + persistedNodeStatuses.clear(); + } finally { + persistedNodeStatusReadWriteLock.writeLock().unlock(); + } } @Override @@ -855,11 +952,17 @@ public boolean equals(Object o) { && nextNodeId.get() == nodeInfo.nextNodeId.get() && registeredDataNodes.equals(nodeInfo.registeredDataNodes) && registeredAINodes.equals(nodeInfo.registeredAINodes) - && nodeVersionInfo.equals(nodeInfo.nodeVersionInfo); + && nodeVersionInfo.equals(nodeInfo.nodeVersionInfo) + && getPersistedNodeStatuses().equals(nodeInfo.getPersistedNodeStatuses()); } @Override public int hashCode() { - return Objects.hash(registeredConfigNodes, nextNodeId, registeredDataNodes, nodeVersionInfo); + return Objects.hash( + registeredConfigNodes, + nextNodeId, + registeredDataNodes, + nodeVersionInfo, + getPersistedNodeStatuses()); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java index 6f2f90b739265..d1cfe16246796 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java @@ -1032,10 +1032,9 @@ && isRuntimeActiveWriterNode(oldLeaderNodeId)) { return responseMap; } - private boolean isRuntimeActiveWriterNode(final int dataNodeId) { - return dataNodeId >= 0 - && getLoadManager().getNodeStatus(dataNodeId) != NodeStatus.Unknown - && getLoadManager().getNodeStatus(dataNodeId) != NodeStatus.Removing; + boolean isRuntimeActiveWriterNode(final int dataNodeId) { + final NodeStatus nodeStatus = getLoadManager().getNodeStatus(dataNodeId); + return dataNodeId >= 0 && !nodeStatus.mayBeOffline(); } private static Map sendPipeMetaRequest( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java index 22c65e1798baa..f378ef132182d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java @@ -310,7 +310,7 @@ public TSStatus submitDeleteOldRegionPeerTask( new TMaintainPeerReq(regionId, originalDataNode, procedureId); final NodeStatus nodeStatus = getDataNodeStatus(originalDataNode.getDataNodeId()); - final boolean useFullRetry = !NodeStatus.Unknown.equals(nodeStatus); + final boolean useFullRetry = !nodeStatus.isOffline(); if (!useFullRetry) { LOGGER.info( ProcedureMessages.DATANODE_IS_SUBMIT_DELETE_OLD_REGION_PEER_WITH_A_SINGLE, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java index 9455d34a0ddb4..d57b4a7bb32b0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java @@ -26,7 +26,6 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cluster.NodeStatus; -import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.commons.cluster.RegionStatus; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.utils.NodeUrlUtils; @@ -43,7 +42,6 @@ import org.apache.iotdb.confignode.manager.lease.DataNodeContactTracker; import org.apache.iotdb.confignode.manager.load.balancer.region.GreedyCopySetRegionGroupAllocator; import org.apache.iotdb.confignode.manager.load.balancer.region.IRegionGroupAllocator; -import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample; import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample; import org.apache.iotdb.confignode.manager.partition.PartitionMetrics; import org.apache.iotdb.confignode.persistence.node.NodeInfo; @@ -55,6 +53,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -114,9 +113,11 @@ public boolean checkEnoughDataNodeAfterRemoving(List removedD (int) removedDataNodes.stream() .filter( - x -> - configManager.getLoadManager().getNodeStatus(x.getDataNodeId()) - != NodeStatus.Unknown) + node -> + !configManager + .getLoadManager() + .getNodeStatus(node.getDataNodeId()) + .isOffline()) .count(); return availableDatanodeSize - removedDataNodeSize >= NodeInfo.getMinimumDataNode(); @@ -132,7 +133,8 @@ public boolean checkEnoughDataNodeAfterRemoving(List removedD * Removing, Running, etc.) */ public void changeDataNodeStatus( - List removedDataNodes, Map nodeStatusMap) { + List removedDataNodes, Map nodeStatusMap) + throws IOException { LOGGER.info( ProcedureMessages.BEGIN_TO_CHANGE_DATANODE_STATUS_NODESTATUSMAP, REMOVE_DATANODE_PROCESS, @@ -150,27 +152,44 @@ public void changeDataNodeStatus( CnToDnInternalServiceAsyncRequestManager.getInstance() .sendAsyncRequestWithRetry(changeDataNodeStatusContext); + StringBuilder statusChangeFailures = new StringBuilder(); for (Map.Entry entry : changeDataNodeStatusContext.getResponseMap().entrySet()) { int dataNodeId = entry.getKey(); NodeStatus nodeStatus = nodeStatusMap.get(dataNodeId); - RegionStatus regionStatus = RegionStatus.valueOf(nodeStatus.getStatus()); - if (!isSucceed(entry.getValue())) { + // Removing or restoring an offline status does not require a DataNode acknowledgement. + // Persistence is still required below, even if the RPC failed. + if (!isSucceed(entry.getValue()) && !nodeStatus.mayBeOffline()) { LOGGER.error( ProcedureMessages.FAILED_TO_CHANGE_DATANODE_STATUS_DATANODEID_NODESTATUS, REMOVE_DATANODE_PROCESS, dataNodeId, nodeStatus); + statusChangeFailures.append( + String.format( + ProcedureMessages + .EXCEPTION_DATANODE_SETSYSTEMSTATUS_RPC_FAILED_ARG_ARG_ARG_7E8E5F9F, + dataNodeId, + nodeStatus, + entry.getValue())); continue; } // Force updating NodeStatus long currentTime = System.nanoTime(); - configManager - .getLoadManager() - .forceUpdateNodeCache( - NodeType.DataNode, dataNodeId, new NodeHeartbeatSample(currentTime, nodeStatus)); + TSStatus status = + configManager.getLoadManager().trySetNodeStatus(dataNodeId, nodeStatus, true); + if (!isSucceed(status)) { + // Do not advance the persisted procedure state before its node status is durable. + statusChangeFailures.append( + String.format( + ProcedureMessages.EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4, + dataNodeId, + nodeStatus, + status)); + continue; + } LOGGER.info( ProcedureMessages.FORCE_UPDATE_NODECACHE_DATANODEID_NODESTATUS_CURRENTTIME, @@ -180,6 +199,7 @@ public void changeDataNodeStatus( currentTime); // Force update RegionStatus + RegionStatus regionStatus = RegionStatus.fromNodeStatus(nodeStatus); if (regionStatus != RegionStatus.Removing) { Map> heartbeatSampleMap = new TreeMap<>(); @@ -195,6 +215,9 @@ public void changeDataNodeStatus( configManager.getLoadManager().forceUpdateRegionGroupCache(heartbeatSampleMap); } } + if (statusChangeFailures.length() > 0) { + throw new IOException(statusChangeFailures.toString()); + } } /** @@ -234,7 +257,7 @@ public List selectedRegionMigrationPlans( final List availableDataNodes = configManager .getNodeManager() - .filterDataNodeThroughStatus(NodeStatus.Running, NodeStatus.Unknown) + .filterDataNodeThroughStatus(NodeStatus::isRegionCandidate) .stream() .filter(node -> !removedDataNodesSet.contains(node.getLocation().getDataNodeId())) .collect(Collectors.toList()); @@ -440,12 +463,18 @@ public void broadcastDataNodeStatusChange(List dataNodes) { * * @param removedDataNodes the list of DataNodeLocations to be removed */ - public void removeDataNodePersistence(List removedDataNodes) { + public void removeDataNodePersistence(List removedDataNodes) + throws IOException { // Remove consensus record try { - configManager.getConsensusManager().write(new RemoveDataNodePlan(removedDataNodes)); + TSStatus status = + configManager.getConsensusManager().write(new RemoveDataNodePlan(removedDataNodes)); + if (!isSucceed(status)) { + throw new IOException(status.toString()); + } } catch (ConsensusException e) { LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e); + throw new IOException(e); } // Adjust maxRegionGroupNum @@ -596,9 +625,11 @@ public TSStatus checkRegionReplication(RemoveDataNodePlan removeDataNodePlan) { (int) removeDataNodePlan.getDataNodeLocations().stream() .filter( - x -> - configManager.getLoadManager().getNodeStatus(x.getDataNodeId()) - != NodeStatus.Unknown) + node -> + !configManager + .getLoadManager() + .getNodeStatus(node.getDataNodeId()) + .isOffline()) .count(); if (availableDatanodeSize - removedDataNodeSize < NodeInfo.getMinimumDataNode()) { status.setCode(TSStatusCode.NO_ENOUGH_DATANODE.getStatusCode()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveAINodeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveAINodeProcedure.java index 95c320ef87853..23b041c2587e9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveAINodeProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveAINodeProcedure.java @@ -98,6 +98,7 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, RemoveAINodeState st removedAINode, response.getMessage())); } + env.getConfigManager().getLoadManager().removeNodeCache(removedAINode.getAiNodeId()); return Flow.NO_MORE_STATE; default: throw new UnsupportedOperationException( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodesProcedure.java index 400d8f8facd66..ff63a8531fbea 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodesProcedure.java @@ -120,6 +120,8 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, RemoveDataNodeState removedDataNodes); return Flow.NO_MORE_STATE; } + // Retry preparation separately: changing nodes to Removing affects the capacity check. + break; case REMOVE_DATA_NODE_PREPARE: Map removedNodeStatusMap = new HashMap<>(); removedDataNodes.forEach( @@ -159,6 +161,8 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, RemoveDataNodeState e); if (getCycles() > RETRY_THRESHOLD) { setFailure(new ProcedureException(ProcedureMessages.STATE_STUCK_AT + state)); + } else { + setNextState(state); } } } @@ -211,7 +215,7 @@ private String simplifyTDataNodeLocation(TDataNodeLocation dataNodeLocation) { dataNodeLocation.getDataNodeId(), dataNodeLocation.getInternalEndPoint().getIp()); } - private void checkRegionStatusAndStopDataNode(ConfigNodeProcedureEnv env) { + private void checkRegionStatusAndStopDataNode(ConfigNodeProcedureEnv env) throws IOException { List replicaSets = env.getConfigManager().getPartitionManager().getAllReplicaSets(); List rollBackDataNodes = new ArrayList<>(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java index 69a3cec44005d..7714d4cc53461 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java @@ -282,9 +282,12 @@ public static Map filterFencedDataNode( final ConfigManager configManager) { return configManager.getNodeManager().getRegisteredDataNodeLocations().entrySet().stream() .filter( - entry -> - configManager.getLoadManager().getNodeStatus(entry.getKey()) != NodeStatus.Unknown - || !DataNodeContactTracker.getInstance().isDataNodeFenced(entry.getKey())) + entry -> { + final NodeStatus status = + configManager.getLoadManager().getNodeStatus(entry.getKey()); + return !status.isOffline() + || !DataNodeContactTracker.getInstance().isDataNodeFenced(entry.getKey()); + }) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java index 6b894804dc53f..2296d54cd0292 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java @@ -30,6 +30,10 @@ import org.apache.iotdb.confignode.conf.ConfigNodeConstant; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.consensus.common.Peer; +import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.db.utils.MemUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -37,6 +41,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeUnit; public class ConfigNodeShutdownHook extends Thread { @@ -45,49 +51,59 @@ public class ConfigNodeShutdownHook extends Thread { private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf(); private static final int SHUTDOWN_REPORT_RETRY_NUM = 2; + private final ConfigNode configNode; + private final SyncConfigNodeClientPool clientPool; + + public ConfigNodeShutdownHook() { + this(ConfigNode.getInstance(), SyncConfigNodeClientPool.getInstance()); + } + + ConfigNodeShutdownHook(ConfigNode configNode, SyncConfigNodeClientPool clientPool) { + this.configNode = configNode; + this.clientPool = clientPool; + } + @Override public void run() { LOGGER.info(ConfigNodeMessages.CONFIGNODE_EXITING); - boolean isLeader = getConfigNodeInstance().getConfigManager().getConsensusManager().isLeader(); + TConfigNodeLocation localNode = + new TConfigNodeLocation( + CONF.getConfigNodeId(), + new TEndPoint(CONF.getInternalAddress(), CONF.getInternalPort()), + new TEndPoint(CONF.getInternalAddress(), CONF.getConsensusPort())); + // Save the same peer list for leadership transfer and the later shutdown report. + ConfigManager configManager = configNode.getConfigManager(); + List otherNodes = + configManager == null + ? List.of() + : configManager.getNodeManager().getRegisteredConfigNodes().stream() + .filter(node -> !node.getInternalEndPoint().equals(localNode.getInternalEndPoint())) + .toList(); + List reportTargets = + otherNodes.stream().map(TConfigNodeLocation::getInternalEndPoint).toList(); + TEndPoint seed = CONF.getSeedConfigNode(); + if (reportTargets.isEmpty() && seed != null && !seed.equals(localNode.getInternalEndPoint())) { + reportTargets = List.of(seed); + } + if (configManager != null) { + transferLeader(configManager, otherNodes); + } try { - ConfigNode.getInstance().deactivate(); + configNode.deactivate(); } catch (IOException e) { LOGGER.error(ConfigNodeMessages.MEET_ERROR_WHEN_DEACTIVATE_CONFIGNODE, e); } - if (!isLeader) { - // Set and report shutdown to cluster ConfigNode-leader - CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.Unknown); - boolean isReportSuccess = false; - TEndPoint seedConfigNode = CONF.getSeedConfigNode(); - for (int retry = 0; retry < SHUTDOWN_REPORT_RETRY_NUM; retry++) { - TSStatus result = - (TSStatus) - SyncConfigNodeClientPool.getInstance() - .sendSyncRequestToConfigNodeWithRetry( - seedConfigNode, - new TConfigNodeLocation( - CONF.getConfigNodeId(), - new TEndPoint(CONF.getInternalAddress(), CONF.getInternalPort()), - new TEndPoint(CONF.getInternalAddress(), CONF.getConsensusPort())), - CnToCnNodeRequestType.REPORT_CONFIG_NODE_SHUTDOWN); - - if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - // Report success - isReportSuccess = true; - break; - } else if (result.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { - // Redirect - seedConfigNode = result.getRedirectNode(); - } - } - if (!isReportSuccess) { - LOGGER.error( - ConfigNodeMessages - .REPORTING_CONFIGNODE_SHUTDOWN_FAILED_THE_CLUSTER_WILL_STILL_TAKE_THE); - } + // Set and report shutdown to the cluster ConfigNode-leader best-effort, regardless of + // leadership: a leader that just stepped down may still reach the newly elected leader via + // redirect. If no leader is reachable, the new leader will mark this node Unknown by + // heartbeat timeout instead. + CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.Stopped); + if (!reportShutdown(localNode, reportTargets)) { + LOGGER.error( + ConfigNodeMessages.REPORTING_CONFIGNODE_SHUTDOWN_FAILED_THE_CLUSTER_WILL_STILL_TAKE_THE); } if (LOGGER.isInfoEnabled()) { @@ -99,7 +115,83 @@ public void run() { } } - protected ConfigNode getConfigNodeInstance() { - return ConfigNode.getInstance(); + private void transferLeader(ConfigManager configManager, List otherNodes) { + ConsensusManager consensusManager = configManager.getConsensusManager(); + if (consensusManager == null || !consensusManager.isLeader()) { + return; + } + TConfigNodeLocation target = + otherNodes.stream() + .filter( + node -> + configManager.getLoadManager().getNodeStatus(node.getConfigNodeId()) + == NodeStatus.Running) + .findFirst() + .orElse(null); + if (target == null) { + return; + } + try { + // The local consensus service must remain open until the transfer attempt finishes. + consensusManager + .getConsensusImpl() + .transferLeader( + consensusManager.getConsensusGroupId(), + new Peer( + consensusManager.getConsensusGroupId(), + target.getConfigNodeId(), + target.getConsensusEndPoint())); + } catch (ConsensusException e) { + LOGGER.warn( + ConfigNodeMessages + .LOG_FAILED_TO_TRANSFER_CONFIGNODE_LEADERSHIP_BEFORE_SHUTDOWN_CONTINUING_SHUTDOWN_2B9364D5, + e); + } + } + + private boolean reportShutdown(TConfigNodeLocation localNode, List reportTargets) { + if (reportTargets.isEmpty()) { + return false; + } + long deadline = + System.nanoTime() + + TimeUnit.MILLISECONDS.toNanos( + CommonDescriptor.getInstance().getConfig().getCnConnectionTimeoutInMS()); + while (System.nanoTime() < deadline && !Thread.currentThread().isInterrupted()) { + for (TEndPoint target : reportTargets) { + for (int retry = 0; retry < SHUTDOWN_REPORT_RETRY_NUM; retry++) { + if (System.nanoTime() >= deadline || Thread.currentThread().isInterrupted()) { + return false; + } + TSStatus result = + (TSStatus) + clientPool.sendSyncRequestToConfigNodeWithRetry( + target, localNode, CnToCnNodeRequestType.REPORT_CONFIG_NODE_SHUTDOWN); + if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return true; + } + if (result.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode() + && result.isSetRedirectNode()) { + target = result.getRedirectNode(); + if (target.equals(localNode.getInternalEndPoint())) { + break; + } + } + } + } + // A newly elected leader may still be initializing its services after the transfer. + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + break; + } + try { + TimeUnit.NANOSECONDS.sleep(Math.min(remaining, TimeUnit.SECONDS.toNanos(1))); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOGGER.warn(ConfigNodeMessages.RETRY_WAIT_FAILED, e); + return false; + } + } + return false; } } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index 405a5139e3466..fc241312e51c2 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1081,8 +1081,7 @@ private DataNodeMiscMessages() {} public static final String MISC_LOG_DATANODE_EXITS_JVM_MEMORY_USAGE_BE69D1F5 = "DataNode exits. Jvm memory usage: {}"; public static final String MISC_LOG_FAILED_TO_REPORT_DATANODE_S_SHUTDOWN_TO_CONFIGNODE_THE_CLUSTER_E6727497 = - "Failed to report DataNode's shutdown to ConfigNode. The cluster will still take the current " - + "DataNode as Running for a few seconds."; + "Failed to confirm the DataNode shutdown report with the ConfigNode leader."; public static final String MISC_LOG_SOMETHING_WRONG_HAPPENED_WHILE_CALLING_CONSENSUS_LAYER_S_8B8FBB16 = "Something wrong happened while calling consensus layer's triggerSnapshot API."; public static final String MISC_LOG_THE_ADDREGIONPEERTASK_HAS_ALREADY_BEEN_SUBMITTED_AND_WILL_4D398F73 = diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index a87a49eaaf5f4..b0266763747a8 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1080,7 +1080,7 @@ private DataNodeMiscMessages() {} public static final String MISC_LOG_DATANODE_EXITS_JVM_MEMORY_USAGE_BE69D1F5 = "DataNode 退出。JVM 内存使用情况：{}"; public static final String MISC_LOG_FAILED_TO_REPORT_DATANODE_S_SHUTDOWN_TO_CONFIGNODE_THE_CLUSTER_E6727497 = - "无法向 ConfigNode 上报 DataNode 关闭状态。集群在接下来的几秒内仍会将当前 DataNode 视为 Running。"; + "未能获得 ConfigNode leader 对 DataNode 关闭报告的确认。"; public static final String MISC_LOG_SOMETHING_WRONG_HAPPENED_WHILE_CALLING_CONSENSUS_LAYER_S_8B8FBB16 = "调用共识层的 triggerSnapshot API 时发生错误。"; public static final String MISC_LOG_THE_ADDREGIONPEERTASK_HAS_ALREADY_BEEN_SUBMITTED_AND_WILL_4D398F73 = diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java index 518a9faaed2ec..8df310329247a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java @@ -33,9 +33,22 @@ public enum NodeStatus { Removing("Removing"), /** Only query statements are permitted */ - ReadOnly("ReadOnly"); + ReadOnly("ReadOnly"), + + /** Node was stopped intentionally and reported its shutdown */ + Stopped("Stopped"); + + /** + * Reasons for entering ReadOnly. These strings cross node RPCs and are compared literally (e.g. + * the DiskFull auto-recovery in sampleDiskLoad), so they must stay locale-independent plain + * constants instead of i18n messages. + */ public static final String DISK_FULL = "DiskFull"; + public static final String MANUAL = "Manual"; + public static final String STOPPING = "Stopping"; + public static final String UNRECOVERABLE_ERROR = "UnrecoverableError"; + private final String status; NodeStatus(String status) { @@ -55,22 +68,87 @@ public static NodeStatus parse(String status) { throw new RuntimeException(String.format(CommonMessages.NODE_STATUS_NOT_EXIST, status)); } + /** + * Resolves a requested status. Ordinary updates retain Removing and do not replace an explicit + * Stopped report with Unknown from a heartbeat failure or timeout. A live heartbeat, such as + * Running after a restart, can revive Stopped. Explicit management updates with force bypass + * these rules, including when rolling back a removal. + */ + public static NodeStatus transition(NodeStatus previous, NodeStatus requested, boolean force) { + if (force) { + return requested; + } + if (previous == Removing) { + return Removing; + } + + if (previous == Stopped && requested == Unknown) { + return Stopped; + } + return requested; + } + + /** + * Selects the reason for a local ReadOnly-to-ReadOnly update: Stopping > Manual > + * UnrecoverableError > DiskFull > null or unclassified reasons. Equal priority keeps the previous + * reason, including the first unrecoverable error's details. + * + *

Do not apply this priority to received heartbeat reasons, which already reflect the remote + * node's decision. + */ + public static String transitionReadOnlyReason(String previous, String requested) { + return getReadOnlyReasonPriority(previous) >= getReadOnlyReasonPriority(requested) + ? previous + : requested; + } + + private static int getReadOnlyReasonPriority(String reason) { + if (reason == null) { + return 0; + } + if (reason.startsWith(UNRECOVERABLE_ERROR)) { + return 2; + } + return switch (reason) { + case STOPPING -> 4; + case MANUAL -> 3; + case DISK_FULL -> 1; + default -> 0; + }; + } + public static boolean isNormalStatus(NodeStatus status) { // Currently, the only normal status is Running return status != null && status.equals(NodeStatus.Running); } + /** + * Whether this status has a persistent record. Other statuses may still require clearing an + * existing record when the node recovers. + */ + public boolean isPersistentStatus() { + return this == Stopped || this == Removing; + } + + /** No available heartbeat, or an explicit shutdown report. Removing alone is not offline. */ + public boolean isOffline() { + return this == Unknown || this == Stopped; + } + + /** These statuses allow an offline node; they do not confirm that the node is offline. */ + public boolean mayBeOffline() { + return this == Removing || isOffline(); + } + + /** + * Eligible for a new or migrated Region replica. Offline nodes remain candidates when there are + * too few online nodes; ReadOnly and Removing nodes must not receive new replicas. + */ + public boolean isRegionCandidate() { + return this == Running || isOffline(); + } + public static boolean isReadable(NodeStatus status) { - switch (status) { - case Running: - case Removing: - case ReadOnly: - return true; - case Unknown: - return false; - default: - throw new UnsupportedOperationException( - String.format(CommonMessages.UNKNOWN_NODE_STATUS, status)); - } + return !status.isOffline(); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/cluster/RegionStatus.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/cluster/RegionStatus.java index b61a2d6a5f785..e44910d14474b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/cluster/RegionStatus.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/cluster/RegionStatus.java @@ -46,6 +46,11 @@ public String getStatus() { return status; } + public static RegionStatus fromNodeStatus(NodeStatus status) { + // Regions do not distinguish an explicitly stopped node from an unreachable node. + return status == NodeStatus.Stopped ? Unknown : valueOf(status.getStatus()); + } + public static boolean isNormalStatus(RegionStatus status) { // Currently, the only normal status is Running return status != null && status.equals(RegionStatus.Running); diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 512e2e05f05fc..b78e6ed9dff7b 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1422,9 +1422,11 @@ service IConfigNodeRPCService { /** * Report that the specified DataNode will be shutdown. - * The ConfigNode-leader will mark it as Unknown. + * The leader requests Stopped, retaining Removing if set. Success confirms persistence. + * Without a report, heartbeat timeout produces Unknown only if no Stopped or Removing + * status can be retained, including a status restored from persistence. * - * @return SUCCESS_STATUS if reporting successfully + * @return SUCCESS_STATUS if the resulting status is persisted */ common.TSStatus reportDataNodeShutdown(common.TDataNodeLocation dataNodeLocation) @@ -1657,9 +1659,11 @@ service IConfigNodeRPCService { /** * Report that the specified ConfigNode will be shutdown. - * The ConfigNode-leader will mark it as Unknown. + * The leader requests Stopped, retaining Removing if set. Success confirms persistence. + * Without a report, heartbeat timeout produces Unknown only if no Stopped or Removing + * status can be retained, including a status restored from persistence. * - * @return SUCCESS_STATUS if reporting successfully + * @return SUCCESS_STATUS if the resulting status is persisted */ common.TSStatus reportConfigNodeShutdown(common.TConfigNodeLocation configNodeLocation) From 0172767a96c1c98ee77a2dd21b4a7914839e4886 Mon Sep 17 00:00:00 2001 From: Di Wang Date: Mon, 28 Sep 2026 10:31:44 +0800 Subject: [PATCH 2/5] test: cover Stopped node status and persistence Cover shutdown reporting, status transitions, scheduling and removal behavior, and node status persistence and recovery with unit and integration tests. --- .../apache/iotdb/it/env/cluster/EnvUtils.java | 12 + .../IoTDBClusterNodeErrorStartUpIT.java | 31 +- .../IoTDBClusterNodeShutdownHookIT.java | 60 +- .../it/cluster/IoTDBNodeStatusFailoverIT.java | 673 +++++++++++++++++ .../cluster/IoTDBNodeStatusPersistenceIT.java | 289 +++++++ .../it/cluster/NodeStatusTestAgent.java | 197 +++++ .../IoTDBRegionGroupLeaderDistributionIT.java | 3 +- .../partition/IoTDBPartitionCreationIT.java | 58 +- .../it/partition/IoTDBPartitionDurableIT.java | 49 +- ...ggregationQueryWithNetworkPartitionIT.java | 5 +- ...IoTDBTableViewQueryWithCachedDeviceIT.java | 3 +- .../iotdb/session/it/IoTDBConnectionsIT.java | 53 +- .../iotdb/session/it/pool/SessionPoolIT.java | 21 +- .../DataNodeHeartbeatHandlerTest.java | 100 +++ .../request/ConfigPhysicalPlanSerDeTest.java | 15 + .../ConfigManagerShutdownReportTest.java | 161 ++++ .../manager/ProcedureManagerTest.java | 25 + .../manager/RetryFailedTasksThreadTest.java | 107 +++ .../manager/load/LoadManagerTest.java | 80 +- .../load/balancer/RegionBalancerTest.java | 144 ++++ .../router/priority/GreedyPriorityTest.java | 2 +- .../priority/LeaderPriorityBalancerTest.java | 2 +- .../load/cache/ConsensusGroupCacheTest.java | 2 +- .../load/cache/LoadCacheNodeStatusTest.java | 269 +++++++ .../LoadCachePersistedNodeStatusTest.java | 703 ++++++++++++++++++ .../manager/load/cache/NodeCacheTest.java | 137 +++- .../SubscriptionRuntimeCoordinatorTest.java | 119 +++ .../confignode/persistence/NodeInfoTest.java | 176 +++++ .../env/ConfigNodeProcedureEnvTest.java | 93 +++ .../env/RegionMaintainHandlerTest.java | 68 ++ .../node/RemoveDataNodePersistenceTest.java | 572 ++++++++++++++ .../impl/schema/SchemaUtilsTest.java | 91 +++ .../service/ConfigNodeShutdownHookTest.java | 347 +++++++++ .../iotdb/commons/cluster/NodeStatusTest.java | 91 +++ 34 files changed, 4642 insertions(+), 116 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusFailoverIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/NodeStatusTestAgent.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandlerTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerShutdownReportTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThreadTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancerTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinatorTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnvTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtilsTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHookTest.java create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/cluster/NodeStatusTest.java diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java index 63c2041199d82..b69e86c12bd42 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java @@ -19,6 +19,7 @@ package org.apache.iotdb.it.env.cluster; +import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.it.framework.IoTDBTestLogger; import org.apache.tsfile.external.commons.lang3.SystemUtils; @@ -69,6 +70,17 @@ public class EnvUtils { + /** + * The status a node locally stopped via {@code AbstractNodeWrapper.stop()} is expected to be in. + * On Windows, {@code Process.destroy()} terminates the node process without running the JVM + * shutdown hooks, so the graceful-shutdown report is never sent and the ConfigNode marks the node + * Unknown by heartbeat timeout. On Unix, the shutdown hook reports the stop and the node becomes + * Stopped. + */ + public static NodeStatus getNodeStatusAfterLocalStop() { + return SystemUtils.IS_OS_WINDOWS ? NodeStatus.Unknown : NodeStatus.Stopped; + } + public static int[] searchAvailablePorts() { int length = 10; while (true) { diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeErrorStartUpIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeErrorStartUpIT.java index e12ab129d21c7..3f477c1c95fad 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeErrorStartUpIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeErrorStartUpIT.java @@ -34,6 +34,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.EnvUtils; import org.apache.iotdb.it.env.cluster.config.MppBaseConfig; import org.apache.iotdb.it.env.cluster.config.MppCommonConfig; import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper; @@ -212,15 +213,35 @@ public void testIllegalNodeRestart() dataNodeRestartResp.getStatus().getCode()); Assert.assertTrue(dataNodeRestartResp.getStatus().getMessage().contains("whose nodeId=")); - // Shutdown and check + // Local stop runs shutdown hooks on Unix, but terminates the process directly on Windows. + // A ConfigNode that was the leader at shutdown time can also remain Unknown if its report + // cannot reach another leader. EnvFactory.getEnv().shutdownConfigNode(1); EnvFactory.getEnv().shutdownDataNode(0); EnvFactory.getEnv() .ensureNodeStatus( - Arrays.asList( - EnvFactory.getEnv().getConfigNodeWrapper(1), - EnvFactory.getEnv().getDataNodeWrapper(0)), - Arrays.asList(NodeStatus.Unknown, NodeStatus.Unknown)); + Arrays.asList(EnvFactory.getEnv().getDataNodeWrapper(0)), + Arrays.asList(EnvUtils.getNodeStatusAfterLocalStop())); + boolean isConfigNodeDown = false; + for (int retry = 0; retry < 30; retry++) { + TShowClusterResp showClusterResp = client.showCluster(); + for (TConfigNodeLocation configNodeLocation : showClusterResp.getConfigNodeList()) { + if (configNodeLocation.getConsensusEndPoint().getPort() + == registeredConfigNodeWrapper.getConsensusPort()) { + String configNodeStatus = + showClusterResp.getNodeStatus().get(configNodeLocation.getConfigNodeId()); + if (NodeStatus.Stopped.getStatus().equals(configNodeStatus) + || NodeStatus.Unknown.getStatus().equals(configNodeStatus)) { + isConfigNodeDown = true; + } + } + } + if (isConfigNodeDown) { + break; + } + Thread.sleep(1000); + } + Assert.assertTrue(isConfigNodeDown); /* Restart and updatePeer */ // TODO: Delete this IT after enable modify internal TEndPoints diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeShutdownHookIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeShutdownHookIT.java index 8917f163557be..e947a7c527379 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeShutdownHookIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeShutdownHookIT.java @@ -19,12 +19,15 @@ package org.apache.iotdb.confignode.it.cluster; +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.EnvUtils; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.rpc.TSStatusCode; @@ -39,7 +42,6 @@ import java.io.IOException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; @RunWith(IoTDBTestRunner.class) @Category({ClusterIT.class}) @@ -75,29 +77,57 @@ public void testNodeShutdownReporter() try (SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - // The unknown Nodes should be detected immediately with the help of shutdown hook + // Unix shutdown hooks report Stopped; Windows local stop is detected as Unknown. + // A ConfigNode whose report cannot reach the newly elected leader remains Unknown too. + TShowClusterResp showClusterResp = client.showCluster(); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), showClusterResp.getStatus().getCode()); + + int stoppedDataNodeId = -1; + for (TDataNodeLocation dataNodeLocation : showClusterResp.getDataNodeList()) { + if (dataNodeLocation.getInternalEndPoint().getPort() + == EnvFactory.getEnv().getDataNodeWrapper(0).getInternalPort()) { + stoppedDataNodeId = dataNodeLocation.getDataNodeId(); + } + } + Assert.assertNotEquals(-1, stoppedDataNodeId); + + int stoppedConfigNodeId = -1; + for (TConfigNodeLocation configNodeLocation : showClusterResp.getConfigNodeList()) { + if (configNodeLocation.getConsensusEndPoint().getPort() + == EnvFactory.getEnv().getConfigNodeWrapper(1).getConsensusPort()) { + stoppedConfigNodeId = configNodeLocation.getConfigNodeId(); + } + } + Assert.assertNotEquals(-1, stoppedConfigNodeId); + boolean isDetected = false; - for (int retry = 0; retry < 5; retry++) { - TShowClusterResp showClusterResp = client.showCluster(); + for (int retry = 0; retry < 30; retry++) { + showClusterResp = client.showCluster(); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), showClusterResp.getStatus().getCode()); - AtomicInteger unknownNum = new AtomicInteger(0); - showClusterResp - .getNodeStatus() - .forEach( - (nodeId, nodeStatus) -> { - if (NodeStatus.Unknown.getStatus().equals(nodeStatus)) { - unknownNum.getAndIncrement(); - } - }); - if (unknownNum.get() == 2) { + + // Windows does not run the shutdown hook, so allow heartbeat failure detection to finish. + final String dataNodeStatus = showClusterResp.getNodeStatus().get(stoppedDataNodeId); + final boolean isDataNodeStopped = + EnvUtils.getNodeStatusAfterLocalStop().getStatus().equals(dataNodeStatus); + + // The stopped ConfigNode is Stopped when its report reached a leader, and may otherwise + // remain Unknown until heartbeat timeout + final String configNodeStatus = showClusterResp.getNodeStatus().get(stoppedConfigNodeId); + final boolean isConfigNodeDetected = + NodeStatus.Stopped.getStatus().equals(configNodeStatus) + || NodeStatus.Unknown.getStatus().equals(configNodeStatus); + + if (isDataNodeStopped && isConfigNodeDetected) { isDetected = true; break; } TimeUnit.SECONDS.sleep(1); } - Assert.assertTrue(isDetected); + Assert.assertTrue( + "Timed out waiting for the stopped DataNode and ConfigNode to be detected", isDetected); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusFailoverIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusFailoverIT.java new file mode 100644 index 0000000000000..474fe9148d66a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusFailoverIT.java @@ -0,0 +1,673 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.confignode.it.cluster; + +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.client.ClientPoolFactory; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import com.sun.tools.attach.VirtualMachine; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.ErrorCollector; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** Process-level status persistence checks with three actual ConfigRegion voters. */ +@RunWith(IoTDBTestRunner.class) +@Category(ClusterIT.class) +public class IoTDBNodeStatusFailoverIT { + @Rule public TemporaryFolder temporary = new TemporaryFolder(); + @Rule public ErrorCollector failures = new ErrorCollector(); + private Path agentJar; + private int queryNode; + + @After + public void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testGracefulLeaderShutdownPreservesStoppedAndRemoving() throws Exception { + init(3); + int stopped = stoppedFixture(0); + int removing = removingFixture(1); + int leader = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + Path marker = submit(leader, "shutdown"); + awaitFile(marker); + assertTrue( + "System.exit must finish all JVM shutdown hooks", + EnvFactory.getEnv() + .getConfigNodeWrapper(leader) + .getInstance() + .waitFor(60, TimeUnit.SECONDS)); + assertNotEquals(leader, EnvFactory.getEnv().getLeaderConfigNodeIndex()); + assertStatus(0, stopped, NodeStatus.Stopped); + assertStatus(1, removing, NodeStatus.Removing); + EnvFactory.getEnv().startConfigNode(leader); + awaitConfigNodeRunning(leader); + awaitPersisted(leader, stopped, NodeStatus.Stopped); + transfer(leader); + assertStatus(0, stopped, NodeStatus.Stopped); + assertStatus(1, removing, NodeStatus.Removing); + } + + @Test + public void testGracefulSeedShutdownReportsStopped() throws Exception { + init(1); + transfer(1); + int seedId = configLocation(cluster(), 0).getConfigNodeId(); + // Use System.exit so the actual shutdown hook runs on Windows as well as Unix. + awaitFile(submit(0, "shutdown")); + assertTrue( + EnvFactory.getEnv().getConfigNodeWrapper(0).getInstance().waitFor(60, TimeUnit.SECONDS)); + EnvFactory.getEnv() + .ensureNodeStatus( + Collections.singletonList(EnvFactory.getEnv().getConfigNodeWrapper(0)), + Collections.singletonList(NodeStatus.Stopped)); + awaitPersisted(1, seedId, NodeStatus.Stopped); + awaitPersisted(2, seedId, NodeStatus.Stopped); + + EnvFactory.getEnv().startConfigNode(0); + awaitConfigNodeRunning(0); + awaitPersisted(0, seedId, null); + awaitPersisted(1, seedId, null); + awaitPersisted(2, seedId, null); + } + + @Test + public void testForcedLeaderShutdownWithTwoDataNodes() throws Exception { + init(2); + int stopped = stoppedFixture(0); + int leader = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + EnvFactory.getEnv().getConfigNodeWrapper(leader).stopForcibly(); + assertFalse(EnvFactory.getEnv().getConfigNodeWrapper(leader).isAlive()); + assertNotEquals(leader, EnvFactory.getEnv().getLeaderConfigNodeIndex()); + assertStatus(0, stopped, NodeStatus.Stopped); + EnvFactory.getEnv() + .ensureNodeStatus( + Collections.singletonList(EnvFactory.getEnv().getConfigNodeWrapper(leader)), + Collections.singletonList(NodeStatus.Unknown)); + EnvFactory.getEnv().startConfigNode(leader); + awaitConfigNodeRunning(leader); + awaitPersisted(leader, stopped, NodeStatus.Stopped); + transfer(leader); + assertStatus(0, stopped, NodeStatus.Stopped); + } + + @Test + public void testNewLeaderChangesSurviveReturnToOriginalLeader() throws Exception { + init(3); + int stopped = stoppedFixture(0); + int removing = removingFixture(1); + int first = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + int second = (first + 1) % 3; + transfer(second); + assertStatus(0, stopped, NodeStatus.Stopped); + assertStatus(1, removing, NodeStatus.Removing); + EnvFactory.getEnv().startDataNode(0); + assertStatus(0, stopped, NodeStatus.Running); + assertSuccess(control(second, "update:" + removing + ":Running")); + awaitPersisted(first, stopped, null); + awaitPersisted(first, removing, null); + transfer((first + 2) % 3); + transfer(first); + EnvFactory.getEnv().getDataNodeWrapper(0).stopForcibly(); + assertStatus(0, stopped, NodeStatus.Unknown); + transfer(second); + assertStatus(0, stopped, NodeStatus.Unknown); + assertStatus(1, removing, NodeStatus.Unknown); + } + + @Test + public void testLiveOldLeaderPartitionAndQuorumRecovery() throws Exception { + init(3); + int stopped = stoppedFixture(0); + int first = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + TConfigNodeLocation firstLocation = configLocation(cluster(), first); + for (int i = 0; i < 3; i++) { + control(i, "partition:" + firstLocation.getConfigNodeId()); + } + int majorityLeader = awaitOtherLeader(first); + assertTrue( + "Partition must leave the old leader process alive", + EnvFactory.getEnv().getConfigNodeWrapper(first).isAlive()); + assertSuccess(control(majorityLeader, "update:" + stopped + ":Removing")); + try { + assertStatus(0, stopped, NodeStatus.Removing); + } catch (SQLException e) { + // Preserve a failure for SQL routing without skipping consensus recovery assertions. + failures.addError( + new AssertionError( + "SHOW through the healthy DataNode cannot reach the elected majority", e)); + } + heal(); + awaitPersisted(first, stopped, NodeStatus.Removing); + transfer(first); + assertStatus(0, stopped, NodeStatus.Removing); + + // Keep every process alive but prevent any pair of ConfigRegion voters communicating. + for (int i = 0; i < 3; i++) { + control(i, "partition:*"); + } + Properties noQuorum = control(first, "update:" + stopped + ":Running"); + assertNotEquals( + "No quorum cannot confirm clearing Removing", "200", noQuorum.getProperty("code")); + assertEquals( + "Removing", control(first, "inspect:" + stopped).getProperty("persisted." + stopped)); + heal(); + int recovered = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + assertSuccess(control(recovered, "update:" + stopped + ":Running")); + for (int i = 0; i < 3; i++) { + awaitPersisted(i, stopped, null); + } + transfer((recovered + 1) % 3); + assertStatus(0, stopped, NodeStatus.Unknown); + } + + private void init(int dataNodes) throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setConfigNodeRatisSnapshotTriggerThreshold(Integer.MAX_VALUE) + .setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(4000); + EnvFactory.getEnv().initClusterEnvironment(3, dataNodes); + queryNode = dataNodes - 1; + agentJar = temporary.getRoot().toPath().resolve("node-status-agent.jar"); + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + manifest.getMainAttributes().putValue("Agent-Class", NodeStatusTestAgent.class.getName()); + String resource = NodeStatusTestAgent.class.getName().replace('.', '/') + ".class"; + try (JarOutputStream jar = new JarOutputStream(Files.newOutputStream(agentJar), manifest); + InputStream bytecode = getClass().getClassLoader().getResourceAsStream(resource)) { + assertNotNull(bytecode); + jar.putNextEntry(new JarEntry(resource)); + bytecode.transferTo(jar); + jar.closeEntry(); + } + } + + @Test + public void testNoQuorumTransitionMatrix() throws Exception { + init(2); + int nodeId = stoppedFixture(0); + NodeStatus[][] transitions = { + {NodeStatus.Running, NodeStatus.Stopped}, + {NodeStatus.Stopped, NodeStatus.Running}, + {NodeStatus.Removing, NodeStatus.Running}, + {NodeStatus.Removing, NodeStatus.Stopped} + }; + for (NodeStatus[] transition : transitions) { + int leader = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + assertSuccess(control(leader, "update:" + nodeId + ":" + transition[0])); + NodeStatus persisted = transition[0] == NodeStatus.Running ? null : transition[0]; + for (int i = 0; i < 3; i++) { + awaitPersisted(i, nodeId, persisted); + } + try { + for (int i = 0; i < 3; i++) { + control(i, "partition:*"); + } + Properties result = awaitFile(submit(leader, "update:" + nodeId + ":" + transition[1])); + assertNotEquals( + "No quorum: " + transition[0] + " -> " + transition[1] + " " + result, + "200", + result.getProperty("code")); + for (int i = 0; i < 3; i++) { + assertEquals( + String.valueOf(persisted), + control(i, "inspect:" + nodeId).getProperty("persisted." + nodeId)); + } + // A read may reject or return a local cache before the old leader steps down. Record + // the actual direct RPC response rather than assert an undocumented linearizable read. + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getConfigNodeConnection(leader)) { + TShowClusterResp read = client.showCluster(); + System.out.println( + "No-quorum SHOW RPC: " + transition[0] + " -> " + transition[1] + ", " + read); + } + try (Connection connection = queryConnection(); + Statement statement = connection.createStatement()) { + try (ResultSet rows = statement.executeQuery("SHOW DATANODES")) { + while (rows.next()) { + if (rows.getInt(1) == nodeId) { + System.out.println("No-quorum SHOW SQL cached status: " + rows.getString("Status")); + } + } + } + } catch (SQLException unavailable) { + System.out.println("No-quorum SHOW SQL unavailable: " + unavailable.getMessage()); + } + } finally { + heal(); + } + int recovered = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + assertSuccess(control(recovered, "update:" + nodeId + ":" + transition[1])); + NodeStatus finalPersisted = transition[1] == NodeStatus.Running ? null : transition[1]; + for (int i = 0; i < 3; i++) { + awaitPersisted(i, nodeId, finalPersisted); + } + } + } + + @Test + public void testSnapshotAndFollowingClearLogRecoverTogether() throws Exception { + init(3); + int stopped = stoppedFixture(0); + int removing = removingFixture(1); + for (int i = 0; i < 3; i++) { + awaitPersisted(i, stopped, NodeStatus.Stopped); + awaitPersisted(i, removing, NodeStatus.Removing); + control(i, "snapshot"); + assertSnapshot(i, stopped, NodeStatus.Stopped, removing, NodeStatus.Removing); + } + int leader = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + assertSuccess(control(leader, "update:" + stopped + ":Running")); + assertSuccess(control(leader, "update:" + removing + ":Stopped")); + for (int i = 0; i < 3; i++) { + awaitPersisted(i, stopped, null); + awaitPersisted(i, removing, NodeStatus.Stopped); + } + restartConfigNodes(); + assertStatus(0, stopped, NodeStatus.Unknown); + assertStatus(1, removing, NodeStatus.Stopped); + for (int i = 0; i < 3; i++) { + control(i, "snapshot"); + assertSnapshot(i, stopped, null, removing, NodeStatus.Stopped); + } + restartConfigNodes(); + assertStatus(0, stopped, NodeStatus.Unknown); + assertStatus(1, removing, NodeStatus.Stopped); + } + + private void assertSnapshot( + int configNode, int firstId, NodeStatus first, int secondId, NodeStatus second) + throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + do { + List snapshots; + // Stop at sm/_, without opening temporary snapshot directories on Windows. + try (Stream paths = + Files.walk( + Paths.get( + EnvFactory.getEnv().getConfigNodeWrapper(configNode).getNodePath(), + "data", + "confignode", + "consensus"), + 3)) { + snapshots = + paths + .filter( + p -> + p.getFileName().toString().matches("\\d+_\\d+") + && p.getParent().getFileName().toString().equals("sm")) + .collect(Collectors.toList()); + } catch (IOException | UncheckedIOException e) { + // Ratis may delete an old snapshot while the directories are being enumerated. + TimeUnit.MILLISECONDS.sleep(200); + continue; + } + for (Path snapshot : snapshots) { + NodeInfo restored = new NodeInfo(); + try { + restored.processLoadSnapshot(snapshot.toFile()); + } catch (IOException e) { + // The snapshot may have been replaced since it was enumerated. + continue; + } + if (java.util.Objects.equals(first, restored.getPersistedNodeStatus(firstId)) + && java.util.Objects.equals(second, restored.getPersistedNodeStatus(secondId))) { + return; + } + } + TimeUnit.MILLISECONDS.sleep(200); + } while (System.nanoTime() < deadline); + fail("No completed snapshot has the expected durable markers on ConfigNode " + configNode); + } + + private void restartConfigNodes() throws Exception { + EnvFactory.getEnv().getConfigNodeWrapperList().forEach(n -> n.stopForcibly()); + EnvFactory.getEnv().getConfigNodeWrapperList().forEach(n -> n.start()); + EnvFactory.getEnv().getLeaderConfigNodeIndex(); + EnvFactory.getEnv() + .ensureNodeStatus( + new java.util.ArrayList<>(EnvFactory.getEnv().getConfigNodeWrapperList()), + java.util.Arrays.asList(NodeStatus.Running, NodeStatus.Running, NodeStatus.Running)); + } + + private void awaitConfigNodeRunning(int index) { + EnvFactory.getEnv() + .ensureNodeStatus( + Collections.singletonList(EnvFactory.getEnv().getConfigNodeWrapper(index)), + Collections.singletonList(NodeStatus.Running)); + } + + @Test + public void testRealRemovalPersistsIntentAndDeletesRegistration() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaReplicationFactor(2) + .setDataReplicationFactor(2); + init(3); + int leader = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + int victim = -1; + int victimIndex = -1; + try (Connection connection = queryConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.status_remove"); + statement.execute( + "CREATE TIMESERIES root.status_remove.d.s WITH DATATYPE=INT32, ENCODING=RLE"); + statement.execute("INSERT INTO root.status_remove.d(time,s) VALUES (1,1)"); + TShowClusterResp before = cluster(); + int queryId = dataLocation(before, queryNode).getDataNodeId(); + try (ResultSet regions = statement.executeQuery("SHOW REGIONS")) { + while (regions.next()) { + int candidate = regions.getInt("DataNodeId"); + if (candidate != queryId) { + victim = candidate; + break; + } + } + } + assertTrue("The victim must host a real region replica", victim >= 0); + for (int i = 0; i < 3; i++) { + if (dataLocation(before, i).getDataNodeId() == victim) { + victimIndex = i; + } + } + assertTrue(victimIndex >= 0); + Path barrier = submit(leader, "barrier"); + assertNull(awaitFile(barrier).getProperty("error")); + try { + statement.execute("REMOVE DATANODE " + victim); + awaitFile(Paths.get(barrier + ".reached")); + assertStatus(victimIndex, victim, NodeStatus.Removing); + for (int i = 0; i < 3; i++) { + awaitPersisted(i, victim, NodeStatus.Removing); + } + } finally { + Files.createFile(Paths.get(barrier + ".release")); + } + long deadline = System.nanoTime() + TimeUnit.MINUTES.toNanos(3); + boolean removed = false; + do { + final int victimId = victim; + removed = + cluster().getDataNodeList().stream().noneMatch(n -> n.getDataNodeId() == victimId); + if (!removed) { + TimeUnit.MILLISECONDS.sleep(200); + } + } while (!removed && System.nanoTime() < deadline); + assertTrue("RemoveDataNodesProcedure must unregister the node", removed); + for (int i = 0; i < 3; i++) { + awaitPersisted(i, victim, null); + } + transfer((leader + 1) % 3); + try (ResultSet rows = statement.executeQuery("SHOW DATANODES")) { + while (rows.next()) { + assertNotEquals(victim, rows.getInt(1)); + } + } + try (ResultSet rows = statement.executeQuery("SELECT s FROM root.status_remove.d")) { + assertTrue(rows.next()); + assertEquals(1, rows.getInt(2)); + assertFalse(rows.next()); + } + } + } + + private int stoppedFixture(int index) throws Exception { + TDataNodeLocation node = dataLocation(cluster(), index); + EnvFactory.getEnv().getDataNodeWrapper(index).stopForcibly(); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + assertEquals(200, client.reportDataNodeShutdown(node).getCode()); + } + assertStatus(index, node.getDataNodeId(), NodeStatus.Stopped); + return node.getDataNodeId(); + } + + private int removingFixture(int index) throws Exception { + TDataNodeLocation node = dataLocation(cluster(), index); + IClientManager pool = + new IClientManager.Factory() + .createClientManager( + new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory()); + try (SyncDataNodeInternalServiceClient client = pool.borrowClient(node.getInternalEndPoint())) { + assertEquals(200, client.setSystemStatus("Removing").getCode()); + } finally { + pool.close(); + } + assertStatus(index, node.getDataNodeId(), NodeStatus.Removing); + EnvFactory.getEnv().getDataNodeWrapper(index).stopForcibly(); + return node.getDataNodeId(); + } + + private void assertStatus(int index, int nodeId, NodeStatus expected) throws Exception { + EnvFactory.getEnv() + .ensureNodeStatus( + Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(index)), + Collections.singletonList(expected)); + assertEquals(expected.getStatus(), cluster().getNodeStatus().get(nodeId)); + // Pin the SQL connection to the healthy query node; do not fan out to stopped DataNodes. + try (Connection connection = queryConnection(); + Statement statement = connection.createStatement()) { + for (String sql : new String[] {"SHOW CLUSTER", "SHOW DATANODES"}) { + boolean found = false; + try (ResultSet rows = statement.executeQuery(sql)) { + while (rows.next()) { + if (rows.getInt(1) == nodeId) { + assertEquals(sql, expected.getStatus(), rows.getString("Status")); + found = true; + } + } + } + assertTrue(sql + " did not contain DataNode " + nodeId, found); + } + } + } + + private void transfer(int target) throws Exception { + int leader = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + if (leader == target) { + return; + } + long previousTerm = Long.parseLong(control(leader, "inspect").getProperty("term")); + TConfigNodeLocation location = configLocation(cluster(), target); + control( + leader, + "transfer:" + location.getConfigNodeId() + ":" + location.getConsensusEndPoint().getPort()); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60); + do { + if (EnvFactory.getEnv().getLeaderConfigNodeIndex() == target) { + assertTrue(Long.parseLong(control(target, "inspect").getProperty("term")) > previousTerm); + return; + } + TimeUnit.MILLISECONDS.sleep(200); + } while (System.nanoTime() < deadline); + fail("Leadership did not transfer to ConfigNode " + target); + } + + private int awaitOtherLeader(int excluded) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60); + do { + for (int i = 0; i < 3; i++) { + if (i != excluded) { + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getConfigNodeConnection(i)) { + if (client.showCluster().getStatus().getCode() == 200) { + return i; + } + } catch (Exception ignored) { + /* Election can temporarily reject a read. */ + } + } + } + TimeUnit.MILLISECONDS.sleep(200); + } while (System.nanoTime() < deadline); + throw new AssertionError("Majority did not elect a ready leader"); + } + + private void heal() throws Exception { + for (int i = 0; i < 3; i++) { + control(i, "heal"); + } + } + + private void awaitPersisted(int node, int dataNode, NodeStatus expected) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60); + do { + Properties state = control(node, "inspect:" + dataNode); + if (String.valueOf(expected).equals(state.getProperty("persisted." + dataNode))) { + return; + } + TimeUnit.MILLISECONDS.sleep(200); + } while (System.nanoTime() < deadline); + fail("ConfigNode " + node + " did not persist " + expected + " for " + dataNode); + } + + private Path submit(int node, String command) throws Exception { + Path output = + temporary.getRoot().toPath().resolve("control-" + System.nanoTime() + ".properties"); + VirtualMachine vm = + VirtualMachine.attach( + Long.toString(EnvFactory.getEnv().getConfigNodeWrapper(node).getInstance().pid())); + try { + vm.loadAgent(agentJar.toString(), command + "|" + output); + } finally { + vm.detach(); + } + return output; + } + + private Properties control(int node, String command) throws Exception { + Properties result = awaitFile(submit(node, command)); + assertNull(command + " failed: " + result, result.getProperty("error")); + return result; + } + + private Properties awaitFile(Path output) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(90); + while (!Files.exists(output) && System.nanoTime() < deadline) { + TimeUnit.MILLISECONDS.sleep(100); + } + assertTrue("Test control timed out: " + output, Files.exists(output)); + Properties result = new Properties(); + try (InputStream stream = Files.newInputStream(output)) { + result.load(stream); + } + return result; + } + + private TShowClusterResp cluster() throws Exception { + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + TShowClusterResp result = client.showCluster(); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), result.getStatus().getCode()); + return result; + } + } + + private TDataNodeLocation dataLocation(TShowClusterResp cluster, int index) { + return cluster.getDataNodeList().stream() + .filter( + n -> + n.getInternalEndPoint().getPort() + == EnvFactory.getEnv().getDataNodeWrapper(index).getInternalPort()) + .findFirst() + .orElseThrow(AssertionError::new); + } + + private TConfigNodeLocation configLocation(TShowClusterResp cluster, int index) { + return cluster.getConfigNodeList().stream() + .filter( + n -> + n.getInternalEndPoint().getPort() + == EnvFactory.getEnv().getConfigNodeWrapper(index).getPort()) + .findFirst() + .orElseThrow(AssertionError::new); + } + + private void assertSuccess(Properties result) { + assertEquals(result.toString(), "200", result.getProperty("code")); + } + + private Connection queryConnection() throws SQLException { + Properties properties = new Properties(); + properties.setProperty("user", "root"); + properties.setProperty("password", "root"); + properties.setProperty("network_timeout", "15000"); + properties.setProperty("sql_dialect", "tree"); + return DriverManager.getConnection( + "jdbc:iotdb://" + + EnvFactory.getEnv().getDataNodeWrapper(queryNode).getIp() + + ":" + + EnvFactory.getEnv().getDataNodeWrapper(queryNode).getPort() + + "/", + properties); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java new file mode 100644 index 0000000000000..04bbc101a726f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.cluster; + +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.client.ClientPoolFactory; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.thrift.TException; +import org.junit.After; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) +public class IoTDBNodeStatusPersistenceIT { + + @After + public void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testStoppedNodesSurviveLeaderFailureAndLogRecovery() throws Exception { + initCluster(5, Integer.MAX_VALUE); + final int leaderIndex = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + final int stoppedConfigNodeIndex = (leaderIndex + 1) % 5; + stopAndReportNodes(stoppedConfigNodeIndex); + assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Stopped); + + // Five ConfigNodes retain a quorum with both a stopped follower and a failed leader. + EnvFactory.getEnv().getConfigNodeWrapper(leaderIndex).stopForcibly(); + assertNotEquals(leaderIndex, EnvFactory.getEnv().getLeaderConfigNodeIndex()); + assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Stopped); + + EnvFactory.getEnv().startConfigNode(leaderIndex); + EnvFactory.getEnv() + .ensureNodeStatus( + Collections.singletonList(EnvFactory.getEnv().getConfigNodeWrapper(leaderIndex)), + Collections.singletonList(NodeStatus.Running)); + + // Restart every surviving ConfigNode and restore the stopped states from consensus logs. + restartLiveConfigNodes(); + assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Stopped); + + EnvFactory.getEnv().startDataNode(0); + assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Running); + + // Keep the DataNode offline during the next election so fresh heartbeats cannot hide a + // resurrected Stopped status. A crash without a shutdown report must become Unknown. + EnvFactory.getEnv().getDataNodeWrapper(0).stopForcibly(); + assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Unknown); + final int nextLeaderIndex = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + EnvFactory.getEnv().getConfigNodeWrapper(nextLeaderIndex).stopForcibly(); + assertNotEquals(nextLeaderIndex, EnvFactory.getEnv().getLeaderConfigNodeIndex()); + assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Unknown); + } + + @Test + public void testStoppedNodesSurviveSnapshotRecovery() throws Exception { + initCluster(3, 1); + final int stoppedConfigNodeIndex = (EnvFactory.getEnv().getLeaderConfigNodeIndex() + 1) % 3; + stopAndReportNodes(stoppedConfigNodeIndex); + assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Stopped); + + final List liveConfigNodes = getLiveConfigNodes(); + final TShowClusterResp cluster; + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + cluster = client.showCluster(); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), cluster.getStatus().getCode()); + } + final int stoppedConfigNodeId = + findConfigNodeLocation(cluster, stoppedConfigNodeIndex).getConfigNodeId(); + final int stoppedDataNodeId = cluster.getDataNodeList().get(0).getDataNodeId(); + + // Verify actual completed snapshots, including their contents, before restarting the quorum. + // A low snapshot threshold alone would not prove snapshot recovery was exercised. + for (ConfigNodeWrapper configNode : liveConfigNodes) { + awaitStoppedNodeSnapshot(configNode, stoppedConfigNodeId, stoppedDataNodeId); + } + restartLiveConfigNodes(); + assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Stopped); + } + + @Test + public void testRemovingNodeSurvivesLeaderFailure() throws Exception { + initCluster(3, Integer.MAX_VALUE); + final IClientManager clientManager = + new IClientManager.Factory() + .createClientManager( + new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory()); + try (SyncDataNodeInternalServiceClient client = + clientManager.borrowClient( + new TEndPoint( + EnvFactory.getEnv().getDataNodeWrapper(0).getIp(), + EnvFactory.getEnv().getDataNodeWrapper(0).getInternalPort()))) { + // Use the removal procedure's DataNode RPC. The leader learns Removing through heartbeats. + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.setSystemStatus(NodeStatus.Removing.getStatus()).getCode()); + } finally { + clientManager.close(); + } + EnvFactory.getEnv() + .ensureNodeStatus( + Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(0)), + Collections.singletonList(NodeStatus.Removing)); + + // The new leader must restore Removing without any help from fresh DataNode heartbeats. + EnvFactory.getEnv().getDataNodeWrapper(0).stopForcibly(); + assertFalse(EnvFactory.getEnv().getDataNodeWrapper(0).isAlive()); + final int leaderIndex = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + EnvFactory.getEnv().getConfigNodeWrapper(leaderIndex).stopForcibly(); + assertFalse(EnvFactory.getEnv().getConfigNodeWrapper(leaderIndex).isAlive()); + assertNotEquals(leaderIndex, EnvFactory.getEnv().getLeaderConfigNodeIndex()); + EnvFactory.getEnv() + .ensureNodeStatus( + Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(0)), + Collections.singletonList(NodeStatus.Removing)); + } + + private void initCluster(int configNodeCount, int snapshotThreshold) throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setConfigNodeRatisSnapshotTriggerThreshold(snapshotThreshold) + .setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(4000); + EnvFactory.getEnv().initClusterEnvironment(configNodeCount, 1); + } + + private void stopAndReportNodes(int configNodeIndex) throws Exception { + final TConfigNodeLocation configNodeLocation; + final TDataNodeLocation dataNodeLocation; + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + final TShowClusterResp cluster = client.showCluster(); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), cluster.getStatus().getCode()); + configNodeLocation = findConfigNodeLocation(cluster, configNodeIndex); + dataNodeLocation = cluster.getDataNodeList().get(0); + } + + EnvFactory.getEnv().getConfigNodeWrapper(configNodeIndex).stopForcibly(); + EnvFactory.getEnv().getDataNodeWrapper(0).stopForcibly(); + assertFalse(EnvFactory.getEnv().getConfigNodeWrapper(configNodeIndex).isAlive()); + assertFalse(EnvFactory.getEnv().getDataNodeWrapper(0).isAlive()); + + // Deliver the shutdown hooks' reports explicitly: Process.destroy() does not execute JVM + // shutdown hooks on every platform. These tests exercise the reports' durable handling. + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.reportConfigNodeShutdown(configNodeLocation).getCode()); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.reportDataNodeShutdown(dataNodeLocation).getCode()); + } + } + + private TConfigNodeLocation findConfigNodeLocation( + TShowClusterResp cluster, int configNodeIndex) { + return cluster.getConfigNodeList().stream() + .filter( + node -> + node.getInternalEndPoint().getPort() + == EnvFactory.getEnv().getConfigNodeWrapper(configNodeIndex).getPort()) + .findFirst() + .orElseThrow(AssertionError::new); + } + + private void assertNodeStatuses(int stoppedConfigNodeIndex, NodeStatus dataNodeStatus) { + EnvFactory.getEnv() + .ensureNodeStatus( + Arrays.asList( + EnvFactory.getEnv().getConfigNodeWrapper(stoppedConfigNodeIndex), + EnvFactory.getEnv().getDataNodeWrapper(0)), + Arrays.asList(NodeStatus.Stopped, dataNodeStatus)); + } + + private List getLiveConfigNodes() { + return EnvFactory.getEnv().getConfigNodeWrapperList().stream() + .filter(ConfigNodeWrapper::isAlive) + .collect(Collectors.toList()); + } + + private void restartLiveConfigNodes() { + final List liveConfigNodes = getLiveConfigNodes(); + liveConfigNodes.forEach(ConfigNodeWrapper::stopForcibly); + liveConfigNodes.forEach(node -> assertFalse(node.isAlive())); + liveConfigNodes.forEach(ConfigNodeWrapper::start); + } + + private void awaitStoppedNodeSnapshot( + ConfigNodeWrapper configNode, int stoppedConfigNodeId, int stoppedDataNodeId) + throws Exception { + for (int retry = 0; retry < 60; retry++) { + final List snapshotFiles = new ArrayList<>(); + try { + final List groups; + try (Stream paths = + Files.list(Paths.get(configNode.getNodePath(), "data", "confignode", "consensus"))) { + groups = paths.filter(Files::isDirectory).collect(Collectors.toList()); + } + // Never walk temporary snapshot directories: an open directory handle can prevent + // Ratis from atomically renaming them on Windows. + for (Path group : groups) { + try (Stream paths = Files.list(group.resolve("sm"))) { + snapshotFiles.addAll( + paths + .filter(path -> path.getFileName().toString().matches("\\d+_\\d+")) + .map(path -> path.resolve("node_info.bin")) + .collect(Collectors.toList())); + } + } + } catch (IOException | UncheckedIOException e) { + // Ratis can delete an older snapshot while this directory tree is being enumerated. + TimeUnit.SECONDS.sleep(1); + continue; + } + for (Path snapshotFile : snapshotFiles) { + final NodeInfo recovered = new NodeInfo(); + try { + recovered.processLoadSnapshot(snapshotFile.getParent().toFile()); + } catch (IOException e) { + // A newer snapshot may have replaced this one since the directory was enumerated. + continue; + } + if (NodeStatus.Stopped.equals(recovered.getPersistedNodeStatus(stoppedConfigNodeId)) + && NodeStatus.Stopped.equals(recovered.getPersistedNodeStatus(stoppedDataNodeId))) { + return; + } + } + TimeUnit.SECONDS.sleep(1); + } + fail("No completed snapshot contains both Stopped nodes on " + configNode.getId()); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/NodeStatusTestAgent.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/NodeStatusTestAgent.java new file mode 100644 index 0000000000000..a87b6722f92a4 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/NodeStatusTestAgent.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.confignode.it.cluster; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.request.AsyncRequestManager; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.utils.function.CheckedTriConsumer; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.confignode.service.ConfigNode; +import org.apache.iotdb.consensus.common.Peer; +import org.apache.iotdb.consensus.ratis.utils.Utils; + +import com.google.common.collect.ImmutableMap; +import org.apache.ratis.util.CodeInjectionForTesting; +import org.apache.thrift.TException; + +import java.io.OutputStream; +import java.lang.instrument.Instrumentation; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +/** Loaded only into test JVMs. No production endpoint or production bytecode is changed. */ +public final class NodeStatusTestAgent { + private NodeStatusTestAgent() {} + + public static void agentmain(String arguments, Instrumentation ignored) { + String[] parts = arguments.split("\\|", 2); + String command = parts[0]; + Path output = Paths.get(parts[1]); + // Return from attach immediately even if a consensus write needs to time out. + Thread worker = new Thread(() -> execute(command, output), "node-status-test-control"); + worker.setDaemon(true); + worker.start(); + } + + private static void execute(String command, Path output) { + Properties result = new Properties(); + try { + String[] args = command.split(":"); + if ("shutdown".equals(args[0])) { + Runtime.getRuntime().addShutdownHook(new Thread(() -> store(output, result))); + System.exit(0); + return; + } + ConfigManager manager = ConfigNode.getInstance().getConfigManager(); + ConsensusManager consensus = manager.getConsensusManager(); + switch (args[0]) { + case "snapshot": + consensus.getConsensusImpl().triggerSnapshot(consensus.getConsensusGroupId(), true); + break; + case "barrier": + installRemovalBarrier(output); + break; + case "partition": + String isolated = + "*".equals(args[1]) + ? "*" + : Utils.fromNodeIdToRaftPeerId(Integer.parseInt(args[1])).toString(); + for (String point : + new String[] {"requestVote", "appendEntries", "startLeaderElection"}) { + CodeInjectionForTesting.put( + "RaftServerImpl." + point, + (local, remote, values) -> { + if ("*".equals(isolated) + || isolated.equals(String.valueOf(local)) + || isolated.equals(String.valueOf(remote))) { + // Fail at receipt, before term/log processing. This drops Raft messages in both + // directions while leaving the JVM and ConfigNode business RPC server alive. + throw new IllegalStateException("Injected Raft message partition"); + } + return false; + }); + } + break; + case "heal": + for (String point : + new String[] {"requestVote", "appendEntries", "startLeaderElection"}) { + CodeInjectionForTesting.remove("RaftServerImpl." + point); + } + break; + case "transfer": + consensus + .getConsensusImpl() + .transferLeader( + consensus.getConsensusGroupId(), + new Peer( + consensus.getConsensusGroupId(), + Integer.parseInt(args[1]), + new TEndPoint("127.0.0.1", Integer.parseInt(args[2])))); + break; + case "update": + int id = Integer.parseInt(args[1]); + NodeStatus status = NodeStatus.parse(args[2]); + TSStatus response = manager.getLoadManager().trySetNodeStatus(id, status, true); + result.setProperty("code", Integer.toString(response.getCode())); + break; + case "inspect": + NodeInfo nodeInfo = manager.getNodeManager().getNodeInfo(); + for (int i = 1; i < args.length; i++) { + int nodeId = Integer.parseInt(args[i]); + result.setProperty( + "persisted." + nodeId, String.valueOf(nodeInfo.getPersistedNodeStatus(nodeId))); + } + break; + default: + throw new IllegalArgumentException(command); + } + result.setProperty( + "term", + Long.toString( + consensus.getConsensusImpl().getLogicalClock(consensus.getConsensusGroupId()))); + result.setProperty("leader", Boolean.toString(consensus.isLeader())); + } catch (Throwable e) { + result.setProperty("error", e.toString()); + } + store(output, result); + } + + private static void store(Path output, Properties result) { + try { + Path temporary = output.resolveSibling(output.getFileName() + ".tmp"); + try (OutputStream stream = Files.newOutputStream(temporary)) { + result.store(stream, "Node status test control"); + } + Files.move(temporary, output); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static void installRemovalBarrier(Path output) throws Exception { + CnToDnInternalServiceAsyncRequestManager requests = + CnToDnInternalServiceAsyncRequestManager.getInstance(); + Field field = AsyncRequestManager.class.getDeclaredField("actionMap"); + field.setAccessible(true); + Map actions = new HashMap((Map) field.get(requests)); + CheckedTriConsumer original = + (CheckedTriConsumer) + actions.get(CnToDnAsyncRequestType.CLEAN_DATA_NODE_CACHE); + Path reached = Paths.get(output + ".reached"); + Path release = Paths.get(output + ".release"); + CheckedTriConsumer gated = + (request, client, callback) -> { + if (!Files.exists(reached)) { + store(reached, new Properties()); + } + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(120); + while (!Files.exists(release)) { + if (System.nanoTime() >= deadline) { + throw new TException("Removal test barrier timed out"); + } + try { + TimeUnit.MILLISECONDS.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TException(e); + } + } + try { + original.accept(request, client, callback); + } catch (Exception e) { + throw new TException(e); + } + }; + actions.put(CnToDnAsyncRequestType.CLEAN_DATA_NODE_CACHE, gated); + field.set(requests, ImmutableMap.copyOf(actions)); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/load/IoTDBRegionGroupLeaderDistributionIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/load/IoTDBRegionGroupLeaderDistributionIT.java index fc0b73d6e1c37..6fb971e94d71a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/load/IoTDBRegionGroupLeaderDistributionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/load/IoTDBRegionGroupLeaderDistributionIT.java @@ -34,6 +34,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.EnvUtils; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.rpc.TSStatusCode; @@ -208,7 +209,7 @@ public void testCFDWithUnknownStatus() throws Exception { EnvFactory.getEnv() .ensureNodeStatus( Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(0)), - Collections.singletonList(NodeStatus.Unknown)); + Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop())); // Check leader distribution isDistributionBalanced = false; diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java index 954f54e01bd02..3e5195586bebb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java @@ -28,6 +28,7 @@ import org.apache.iotdb.commons.cluster.RegionStatus; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.confignode.it.utils.ConfigNodeTestUtils; +import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfo; import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq; import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; @@ -39,6 +40,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.EnvUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -318,36 +320,34 @@ public void testPartitionAllocation() throws Exception { // Shutdown 1 DataNode // Current cluster: 1C5D - // DataNode status: Running, Running, Removing, ReadOnly, Unknown + // DataNode status: Running, Running, Removing, ReadOnly, Stopped (Unknown on Windows). // Region distribution: [0, 1, 2], [0, 1, 2], [0], [1], [2] EnvFactory.getEnv().shutdownDataNode(4); // Wait for shutdown check - while (true) { - AtomicBoolean containUnknown = new AtomicBoolean(false); + boolean isShutdownDetected = false; + for (int retry = 0; retry < 60; retry++) { TShowDataNodesResp showDataNodesResp = client.showDataNodes(); - showDataNodesResp - .getDataNodesInfoList() - .forEach( - dataNodeInfo -> { - if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus())) { - containUnknown.set(true); - } - }); - - if (containUnknown.get()) { + for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) { + if (EnvUtils.getNodeStatusAfterLocalStop().getStatus().equals(dataNodeInfo.getStatus())) { + isShutdownDetected = true; + break; + } + } + if (isShutdownDetected) { break; } TimeUnit.SECONDS.sleep(1); } + Assert.assertTrue(isShutdownDetected); // Register 1 DataNode and Create 1 DataPartition to extend 1 DataRegionGroup // The new DataRegions wouldn't be allocated to the Removing and ReadOnly DataNode - // But the new DataRegion can be allocated to the Unknown DataNode + // But the new DataRegion can be allocated to the offline DataNode. // Current cluster: 1C6D - // Status: Running, Running, Removing, ReadOnly, Unknown, Running + // Status: Running, Running, Removing, ReadOnly, Stopped (Unknown on Windows), Running. // RegionGroup: [0, 1, 2, 3], [0, 1, 2], [0], [1], [2, 3], [3] EnvFactory.getEnv().registerNewDataNode(false); - // Use thread sleep to replace verifying because the Unknown DataNode can not pass the + // Use thread sleep to replace verifying because the offline DataNode can not pass the // connection check TimeUnit.SECONDS.sleep(25); partitionSlotsMap = @@ -446,23 +446,25 @@ public void testPartitionAllocation() throws Exception { // RegionGroup: [0, 1, 2, 3], [0, 1, 2], [0], [1], [2, 3], [3] EnvFactory.getEnv().startDataNode(4); // Wait for restart check - while (true) { - AtomicBoolean containUnknown = new AtomicBoolean(false); + boolean isRestartDetected = false; + for (int retry = 0; retry < 60; retry++) { TShowDataNodesResp showDataNodesResp = client.showDataNodes(); - showDataNodesResp - .getDataNodesInfoList() - .forEach( - dataNodeInfo -> { - if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus())) { - containUnknown.set(true); - } - }); - - if (!containUnknown.get()) { + boolean containDown = false; + for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) { + // The restarted DataNode keeps Stopped until its first heartbeat revives it + if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus()) + || NodeStatus.Stopped.getStatus().equals(dataNodeInfo.getStatus())) { + containDown = true; + break; + } + } + if (!containDown) { + isRestartDetected = true; break; } TimeUnit.SECONDS.sleep(1); } + Assert.assertTrue(isRestartDetected); // Check Region count and status for (int i = 0; i < 30; i++) { runningCnt = 0; diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java index 8d6e4a5f83113..a9262f6981856 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java @@ -40,6 +40,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.EnvUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -285,49 +286,49 @@ public void testReadOnlyDataNode() throws Exception { } @Test - public void testUnknownDataNode() throws Exception { + public void testStoppedDataNode() throws Exception { // Shutdown a DataNode, the ConfigNode should still be able to create RegionGroup + NodeStatus stoppedStatus = EnvUtils.getNodeStatusAfterLocalStop(); EnvFactory.getEnv().shutdownDataNode(testDataNodeId); EnvFactory.getEnv() .ensureNodeStatus( Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(testDataNodeId)), - Collections.singletonList(NodeStatus.Unknown)); + Collections.singletonList(stoppedStatus)); try (SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { // Wait for shutdown check TShowClusterResp showClusterResp; - while (true) { - AtomicBoolean containUnknown = new AtomicBoolean(false); + boolean isShutdownDetected = false; + for (int retry = 0; retry < 60; retry++) { TShowDataNodesResp showDataNodesResp = client.showDataNodes(); - showDataNodesResp - .getDataNodesInfoList() - .forEach( - dataNodeInfo -> { - if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus())) { - containUnknown.set(true); - } - }); - - if (containUnknown.get()) { + for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) { + if (stoppedStatus.getStatus().equals(dataNodeInfo.getStatus())) { + isShutdownDetected = true; + break; + } + } + if (isShutdownDetected) { break; } TimeUnit.SECONDS.sleep(1); } + Assert.assertTrue(isShutdownDetected); int runningCnt = 0; - int unknownCnt = 0; + int stoppedCnt = 0; showClusterResp = client.showCluster(); for (TDataNodeLocation dataNodeLocation : showClusterResp.getDataNodeList()) { if (NodeStatus.Running.getStatus() .equals(showClusterResp.getNodeStatus().get(dataNodeLocation.getDataNodeId()))) { runningCnt += 1; - } else if (NodeStatus.Unknown.getStatus() + } else if (stoppedStatus + .getStatus() .equals(showClusterResp.getNodeStatus().get(dataNodeLocation.getDataNodeId()))) { - unknownCnt += 1; + stoppedCnt += 1; } } Assert.assertEquals(2, runningCnt); - Assert.assertEquals(1, unknownCnt); + Assert.assertEquals(1, stoppedCnt); // Test getOrCreateDataPartition, ConfigNode should create DataPartition and return Map> partitionSlotsMap = ConfigNodeTestUtils.constructPartitionSlotsMap( @@ -369,7 +370,7 @@ public void testUnknownDataNode() throws Exception { // Check Region count runningCnt = 0; - unknownCnt = 0; + int regionUnknownCnt = 0; TShowRegionResp showRegionResp = client.showRegion(new TShowRegionReq()); showRegionResp .getRegionInfoList() @@ -383,12 +384,12 @@ public void testUnknownDataNode() throws Exception { if (RegionStatus.Running.getStatus().equals(regionInfo.getStatus())) { runningCnt += 1; } else if (RegionStatus.Unknown.getStatus().equals(regionInfo.getStatus())) { - unknownCnt += 1; + regionUnknownCnt += 1; } } // The runningCnt should be exactly twice as the unknownCnt // since there exists one DataNode is shutdown - Assert.assertEquals(unknownCnt * 2, runningCnt); + Assert.assertEquals(regionUnknownCnt * 2, runningCnt); // Test getOrCreateDataPartition, ConfigNode should create DataPartition and return partitionSlotsMap = @@ -430,7 +431,7 @@ public void testUnknownDataNode() throws Exception { // Check Region count and status runningCnt = 0; - unknownCnt = 0; + regionUnknownCnt = 0; showRegionResp = client.showRegion(new TShowRegionReq()); showRegionResp .getRegionInfoList() @@ -444,12 +445,12 @@ public void testUnknownDataNode() throws Exception { if (RegionStatus.Running.getStatus().equals(regionInfo.getStatus())) { runningCnt += 1; } else if (RegionStatus.Unknown.getStatus().equals(regionInfo.getStatus())) { - unknownCnt += 1; + regionUnknownCnt += 1; } } // The runningCnt should be exactly twice as the unknownCnt // since there exists one DataNode is shutdown - Assert.assertEquals(unknownCnt * 2, runningCnt); + Assert.assertEquals(regionUnknownCnt * 2, runningCnt); EnvFactory.getEnv().startDataNode(testDataNodeId); EnvFactory.getEnv() diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationQueryWithNetworkPartitionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationQueryWithNetworkPartitionIT.java index 3e8bc96f1db0d..0d62de9dd73e5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationQueryWithNetworkPartitionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationQueryWithNetworkPartitionIT.java @@ -24,6 +24,7 @@ import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.EnvUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.TableClusterIT; @@ -167,7 +168,7 @@ public void test1() throws IoTDBConnectionException, StatementExecutionException EnvFactory.getEnv() .ensureNodeStatus( Collections.singletonList(dataNodeWrapper), - Collections.singletonList(NodeStatus.Unknown)); + Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop())); } List otherNodes = new ArrayList<>(); @@ -209,7 +210,7 @@ public void test2() EnvFactory.getEnv() .ensureNodeStatus( Collections.singletonList(dataNodeWrapper), - Collections.singletonList(NodeStatus.Unknown)); + Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop())); } List otherNodes = new ArrayList<>(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryWithCachedDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryWithCachedDeviceIT.java index 74e009afb179e..b9f905c30edc7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryWithCachedDeviceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryWithCachedDeviceIT.java @@ -24,6 +24,7 @@ import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.EnvUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; @@ -97,7 +98,7 @@ public void test1() throws IoTDBConnectionException, StatementExecutionException EnvFactory.getEnv() .ensureNodeStatus( Collections.singletonList(dataNodeWrapper), - Collections.singletonList(NodeStatus.Unknown)); + Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop())); } EnvFactory.getEnv().startAllDataNodes(); for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBConnectionsIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBConnectionsIT.java index 18b5f897e5c2b..90795d969ffcd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBConnectionsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBConnectionsIT.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; @@ -49,7 +50,6 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.iotdb.db.it.utils.TestUtils.createUser; import static org.apache.iotdb.itbase.env.BaseEnv.TABLE_SQL_DIALECT; @@ -278,23 +278,24 @@ public void testClosedDataNodeGetConnections() throws Exception { (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { // Wait for shutdown check - while (true) { - AtomicBoolean containUnknown = new AtomicBoolean(false); + boolean isShutdownDetected = false; + for (int retry = 0; retry < 60; retry++) { TShowDataNodesResp showDataNodesResp = client.showDataNodes(); - showDataNodesResp - .getDataNodesInfoList() - .forEach( - dataNodeInfo -> { - if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus())) { - containUnknown.set(true); - } - }); - - if (containUnknown.get()) { + for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) { + // A gracefully stopped DataNode is reported as Stopped by its shutdown hook; if + // the report fails it becomes Unknown instead + if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus()) + || NodeStatus.Stopped.getStatus().equals(dataNodeInfo.getStatus())) { + isShutdownDetected = true; + break; + } + } + if (isShutdownDetected) { break; } TimeUnit.SECONDS.sleep(1); } + Assert.assertTrue(isShutdownDetected); } int activeDataNodeId = (int) allDataNodeId.toArray()[1]; @@ -327,23 +328,25 @@ public void testClosedDataNodeGetConnections() throws Exception { try (SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { // Wait for restart check - while (true) { - AtomicBoolean containUnknown = new AtomicBoolean(false); + boolean isRestartDetected = false; + for (int retry = 0; retry < 60; retry++) { TShowDataNodesResp showDataNodesResp = client.showDataNodes(); - showDataNodesResp - .getDataNodesInfoList() - .forEach( - dataNodeInfo -> { - if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus())) { - containUnknown.set(true); - } - }); - - if (!containUnknown.get()) { + boolean containDown = false; + for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) { + // The restarted DataNode keeps Stopped until its first heartbeat revives it + if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus()) + || NodeStatus.Stopped.getStatus().equals(dataNodeInfo.getStatus())) { + containDown = true; + break; + } + } + if (!containDown) { + isRestartDetected = true; break; } TimeUnit.SECONDS.sleep(1); } + Assert.assertTrue(isRestartDetected); } // The ConfigNode may report the restarted DataNode as Running before its client RPC service is diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java index 0a801477866f3..0ce3be416d400 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java @@ -23,6 +23,7 @@ import org.apache.iotdb.isession.pool.ISessionPool; import org.apache.iotdb.isession.pool.SessionDataSetWrapper; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.EnvUtils; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; @@ -252,7 +253,10 @@ public void tryIfTheServerIsRestart() throws InterruptedException { node.stop(); EnvFactory.getEnv() .ensureNodeStatus( - Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown)); + // A gracefully stopped DataNode is reported as Stopped by its shutdown hook (on + // Windows the process is hard-killed, so it is marked Unknown instead) + Collections.singletonList(node), + Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop())); // user does not know what happens. while (wrapper.hasNext()) { wrapper.next(); @@ -263,7 +267,10 @@ public void tryIfTheServerIsRestart() throws InterruptedException { node.stop(); EnvFactory.getEnv() .ensureNodeStatus( - Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown)); + // A gracefully stopped DataNode is reported as Stopped by its shutdown hook (on + // Windows the process is hard-killed, so it is marked Unknown instead) + Collections.singletonList(node), + Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop())); node.start(); EnvFactory.getEnv() .ensureNodeStatus( @@ -289,7 +296,10 @@ public void tryIfTheServerIsRestart() throws InterruptedException { node.stop(); EnvFactory.getEnv() .ensureNodeStatus( - Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown)); + // A gracefully stopped DataNode is reported as Stopped by its shutdown hook (on + // Windows the process is hard-killed, so it is marked Unknown instead) + Collections.singletonList(node), + Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop())); node.start(); EnvFactory.getEnv() .ensureNodeStatus( @@ -345,7 +355,10 @@ public void restart() throws InterruptedException { node.stop(); EnvFactory.getEnv() .ensureNodeStatus( - Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown)); + // A gracefully stopped DataNode is reported as Stopped by its shutdown hook (on + // Windows the process is hard-killed, so it is marked Unknown instead) + Collections.singletonList(node), + Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop())); pool = EnvFactory.getEnv().getSessionPool(1); // all this ten data will fail. write10Data(pool, false); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandlerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandlerTest.java new file mode 100644 index 0000000000000..08e50ac9f76ce --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandlerTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.client.async.handlers.heartbeat; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.cluster.NodeType; +import org.apache.iotdb.commons.cluster.RegionStatus; +import org.apache.iotdb.confignode.manager.lease.DataNodeContactTracker; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.load.cache.LoadCache; +import org.apache.iotdb.confignode.manager.pipe.coordinator.runtime.PipeRuntimeCoordinator; +import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp; + +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DataNodeHeartbeatHandlerTest { + + @Test + public void testStoppedHeartbeatUpdatesNodeAndRegions() { + int nodeId = 7; + TConsensusGroupId dataRegionId = new TConsensusGroupId(TConsensusGroupType.DataRegion, 10); + TConsensusGroupId schemaRegionId = new TConsensusGroupId(TConsensusGroupType.SchemaRegion, 20); + Map judgedLeaders = + Map.of(dataRegionId, false, schemaRegionId, false); + LoadCache loadCache = new LoadCache(); + loadCache.createNodeHeartbeatCache(NodeType.DataNode, nodeId); + judgedLeaders + .keySet() + .forEach(id -> loadCache.createRegionGroupHeartbeatCache("root.db", id, Set.of(nodeId))); + LoadManager loadManager = mock(LoadManager.class); + when(loadManager.getLoadCache()).thenReturn(loadCache); + DataNodeHeartbeatHandler handler = + new DataNodeHeartbeatHandler( + nodeId, + loadManager, + new HashMap<>(), + new HashMap<>(), + new HashMap<>(), + ignored -> {}, + ignored -> {}, + mock(PipeRuntimeCoordinator.class)); + + try { + handler.onComplete( + new TDataNodeHeartbeatResp() + .setHeartbeatTimestamp(System.nanoTime()) + .setStatus(NodeStatus.Running.getStatus()) + .setJudgedLeaders(judgedLeaders)); + loadCache.updateNodeStatistics(); + loadCache.updateRegionGroupStatistics(); + assertEquals(NodeStatus.Running, loadCache.getNodeStatus(nodeId)); + for (TConsensusGroupId regionId : judgedLeaders.keySet()) { + assertEquals(RegionStatus.Running, loadCache.getRegionStatus(regionId, nodeId)); + } + + handler.onComplete( + new TDataNodeHeartbeatResp() + .setHeartbeatTimestamp(System.nanoTime()) + .setStatus(NodeStatus.Stopped.getStatus()) + .setJudgedLeaders(judgedLeaders)); + loadCache.updateNodeStatistics(); + loadCache.updateRegionGroupStatistics(); + assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(nodeId)); + for (TConsensusGroupId regionId : judgedLeaders.keySet()) { + assertEquals( + RegionStatus.Unknown, loadCache.getRegionCacheLastSampleStatus(regionId, nodeId)); + assertEquals(RegionStatus.Unknown, loadCache.getRegionStatus(regionId, nodeId)); + } + } finally { + DataNodeContactTracker.getInstance().removeDataNode(nodeId); + } + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index 0e8585898031e..602c035750556 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -73,6 +73,8 @@ import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateClusterIdPlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan.Operation; import org.apache.iotdb.confignode.consensus.request.write.cq.ActiveCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.AddCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.DropCQPlan; @@ -214,6 +216,19 @@ public class ConfigPhysicalPlanSerDeTest { + @Test + public void testUpdateNodeStatusPlan() throws IOException { + for (Operation operation : Operation.values()) { + UpdateNodeStatusPlan original = new UpdateNodeStatusPlan(37, operation); + UpdateNodeStatusPlan restored = + (UpdateNodeStatusPlan) + ConfigPhysicalPlan.Factory.create(original.serializeToByteBuffer()); + Assert.assertEquals(original, restored); + Assert.assertEquals(37, restored.getNodeId()); + Assert.assertEquals(operation, restored.getOperation()); + } + } + @Test public void RegisterDataNodePlanTest() throws IOException { TDataNodeLocation dataNodeLocation = new TDataNodeLocation(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerShutdownReportTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerShutdownReportTest.java new file mode 100644 index 0000000000000..4a8cf8e055255 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerShutdownReportTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager; + +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class ConfigManagerShutdownReportTest { + + private static final TSStatus SUCCESS_STATUS = RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + + private static ConfigManager CONFIG_MANAGER_SPY; + private static LoadManager LOAD_MANAGER; + + private static final int DATA_NODE_ID = 1; + private static final int CONFIG_NODE_ID = 2; + + private static final TDataNodeLocation DATA_NODE_LOCATION = + new TDataNodeLocation( + DATA_NODE_ID, + new TEndPoint("127.0.0.1", 2000), + new TEndPoint("127.0.0.1", 2001), + new TEndPoint("127.0.0.1", 2002), + new TEndPoint("127.0.0.1", 2003), + new TEndPoint("127.0.0.1", 2004)); + + private static final TConfigNodeLocation CONFIG_NODE_LOCATION = + new TConfigNodeLocation( + CONFIG_NODE_ID, new TEndPoint("127.0.0.1", 1000), new TEndPoint("127.0.0.1", 1001)); + + @BeforeClass + public static void setUp() throws IOException { + final ConfigManager configManager = new ConfigManager(); + CONFIG_MANAGER_SPY = spy(configManager); + LOAD_MANAGER = mock(LoadManager.class); + when(CONFIG_MANAGER_SPY.getLoadManager()).thenReturn(LOAD_MANAGER); + doReturn(SUCCESS_STATUS).when(CONFIG_MANAGER_SPY).confirmLeader(); + } + + @Before + public void clearPreviousInteractions() { + // The shared LoadManager mock keeps interactions from previous tests; verification must only + // consider calls from the current test. + clearInvocations(LOAD_MANAGER); + when(LOAD_MANAGER.trySetNodeStatus(anyInt(), any(NodeStatus.class), eq(false))) + .thenReturn(SUCCESS_STATUS); + } + + @Test + public void testReportDataNodeShutdownUpdatesRemovingDataNodeThroughCache() { + when(LOAD_MANAGER.getNodeStatus(DATA_NODE_ID)).thenReturn(NodeStatus.Removing); + + Assert.assertEquals( + SUCCESS_STATUS.getCode(), + CONFIG_MANAGER_SPY.reportDataNodeShutdown(DATA_NODE_LOCATION).getCode()); + + verify(LOAD_MANAGER).trySetNodeStatus(eq(DATA_NODE_ID), eq(NodeStatus.Stopped), eq(false)); + } + + @Test + public void testReportDataNodeShutdownRequestsStoppedStatus() { + when(LOAD_MANAGER.getNodeStatus(DATA_NODE_ID)).thenReturn(NodeStatus.Stopped); + + Assert.assertEquals( + SUCCESS_STATUS.getCode(), + CONFIG_MANAGER_SPY.reportDataNodeShutdown(DATA_NODE_LOCATION).getCode()); + + verify(LOAD_MANAGER).trySetNodeStatus(eq(DATA_NODE_ID), eq(NodeStatus.Stopped), eq(false)); + } + + @Test + public void testReportConfigNodeShutdownUpdatesRemovingConfigNodeThroughCache() { + when(LOAD_MANAGER.getNodeStatus(CONFIG_NODE_ID)).thenReturn(NodeStatus.Removing); + + Assert.assertEquals( + SUCCESS_STATUS.getCode(), + CONFIG_MANAGER_SPY.reportConfigNodeShutdown(CONFIG_NODE_LOCATION).getCode()); + + verify(LOAD_MANAGER).trySetNodeStatus(eq(CONFIG_NODE_ID), eq(NodeStatus.Stopped), eq(false)); + } + + @Test + public void testReportConfigNodeShutdownRequestsStoppedStatus() { + when(LOAD_MANAGER.getNodeStatus(CONFIG_NODE_ID)).thenReturn(NodeStatus.Stopped); + + Assert.assertEquals( + SUCCESS_STATUS.getCode(), + CONFIG_MANAGER_SPY.reportConfigNodeShutdown(CONFIG_NODE_LOCATION).getCode()); + + verify(LOAD_MANAGER).trySetNodeStatus(eq(CONFIG_NODE_ID), eq(NodeStatus.Stopped), eq(false)); + } + + @Test + public void testReportDataNodeShutdownPropagatesPersistenceFailure() { + TSStatus failure = RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR); + when(LOAD_MANAGER.trySetNodeStatus(eq(DATA_NODE_ID), eq(NodeStatus.Stopped), eq(false))) + .thenReturn(failure); + + for (NodeStatus currentStatus : new NodeStatus[] {NodeStatus.Running, NodeStatus.Removing}) { + when(LOAD_MANAGER.getNodeStatus(DATA_NODE_ID)).thenReturn(currentStatus); + Assert.assertSame(failure, CONFIG_MANAGER_SPY.reportDataNodeShutdown(DATA_NODE_LOCATION)); + } + verify(LOAD_MANAGER, times(2)) + .trySetNodeStatus(eq(DATA_NODE_ID), eq(NodeStatus.Stopped), eq(false)); + } + + @Test + public void testReportConfigNodeShutdownPropagatesPersistenceFailure() { + TSStatus failure = RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR); + when(LOAD_MANAGER.trySetNodeStatus(eq(CONFIG_NODE_ID), eq(NodeStatus.Stopped), eq(false))) + .thenReturn(failure); + + for (NodeStatus currentStatus : new NodeStatus[] {NodeStatus.Running, NodeStatus.Removing}) { + when(LOAD_MANAGER.getNodeStatus(CONFIG_NODE_ID)).thenReturn(currentStatus); + Assert.assertSame(failure, CONFIG_MANAGER_SPY.reportConfigNodeShutdown(CONFIG_NODE_LOCATION)); + } + verify(LOAD_MANAGER, times(2)) + .trySetNodeStatus(eq(CONFIG_NODE_ID), eq(NodeStatus.Stopped), eq(false)); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java index 4de883b9c158f..0289d0b642097 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java @@ -111,6 +111,7 @@ public class ProcedureManagerTest { @BeforeClass public static void setUp() throws IOException { IManager CONFIG_MANAGER = new ConfigManager(); + IManager CONFIG_MANAGER_SPY = spy(CONFIG_MANAGER); ProcedureManager procedureManager = CONFIG_MANAGER.getProcedureManager(); PROCEDURE_MANAGER = spy(procedureManager); @@ -130,6 +131,10 @@ public static void setUp() throws IOException { when(PROCEDURE_EXECUTOR.getProcedures()).thenReturn(procedureMap); when(PROCEDURE_MANAGER.getEnv()).thenReturn(ENV); when(ENV.getRemoveDataNodeHandler()).thenReturn(REMOVE_DATA_NODE_HANDLER); + // Inject the LoadManager spy into the call chain used by checkRemoveDataNodes, so that the + // node-status stubs below take effect on the production path + when(PROCEDURE_MANAGER.getConfigManager()).thenReturn(CONFIG_MANAGER_SPY); + when(CONFIG_MANAGER_SPY.getLoadManager()).thenReturn(LOAD_MANAGER); } @Test @@ -197,6 +202,26 @@ public void testCheckRemoveDataNodeWithAnotherUnknownDataNode() { TSStatus status = PROCEDURE_MANAGER.checkRemoveDataNodes(removedDataNodes); Assert.assertTrue(isFailed(status)); + Assert.assertTrue(status.getMessage().contains("unknown")); + } + + @Test + public void testCheckRemoveDataNodeWithAnotherStoppedDataNode() { + Set relatedDataNodes = new HashSet<>(); + relatedDataNodes.add(removeDataNodeLocationA); + relatedDataNodes.add(coordinatorDataNodeLocation); + + when(REMOVE_DATA_NODE_HANDLER.getRelatedDataNodeLocations(removeDataNodeLocationA)) + .thenReturn(relatedDataNodes); + + when(LOAD_MANAGER.getNodeStatus(removeDataNodeLocationA.getDataNodeId())) + .thenReturn(NodeStatus.Running); + when(LOAD_MANAGER.getNodeStatus(coordinatorDataNodeLocation.getDataNodeId())) + .thenReturn(NodeStatus.Stopped); + + TSStatus status = PROCEDURE_MANAGER.checkRemoveDataNodes(removedDataNodes); + Assert.assertTrue(isFailed(status)); + Assert.assertTrue(status.getMessage().contains("stopped")); } @Test diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThreadTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThreadTest.java new file mode 100644 index 0000000000000..972421961ec02 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThreadTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.node.NodeManager; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Test; + +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; + +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class RetryFailedTasksThreadTest { + + private static final int STOPPED_DATA_NODE_ID = 1; + + private final IManager configManager = mock(IManager.class); + private final NodeManager nodeManager = mock(NodeManager.class); + private final LoadManager loadManager = mock(LoadManager.class); + + private final TDataNodeConfiguration stoppedDataNode = + new TDataNodeConfiguration() + .setLocation( + new TDataNodeLocation( + STOPPED_DATA_NODE_ID, + new TEndPoint("127.0.0.1", 2000), + new TEndPoint("127.0.0.1", 2001), + new TEndPoint("127.0.0.1", 2002), + new TEndPoint("127.0.0.1", 2003), + new TEndPoint("127.0.0.1", 2004))); + + private RetryFailedTasksThread retryFailedTasksThread; + + // The thread captures its managers at construction time, so it must be created after the stubs. + private void createThreadWithStoppedDataNode() { + when(configManager.getNodeManager()).thenReturn(nodeManager); + when(configManager.getLoadManager()).thenReturn(loadManager); + when(nodeManager.getRegisteredDataNodes()) + .thenReturn(Collections.singletonList(stoppedDataNode)); + when(loadManager.getNodeStatus(STOPPED_DATA_NODE_ID)).thenReturn(NodeStatus.Stopped); + when(configManager.transfer(anyList())) + .thenReturn(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + retryFailedTasksThread = new RetryFailedTasksThread(configManager); + } + + private void invokeTriggerDetectTask() throws Exception { + final Method method = RetryFailedTasksThread.class.getDeclaredMethod("triggerDetectTask"); + method.setAccessible(true); + method.invoke(retryFailedTasksThread); + } + + @Test + public void testStoppedDataNodeTriggersRegionTransfer() throws Exception { + createThreadWithStoppedDataNode(); + + invokeTriggerDetectTask(); + + // A Stopped node is handled like Unknown: its regions are transferred. + verify(configManager) + .transfer( + argThat( + (List nodes) -> + nodes.size() == 1 && nodes.get(0).getDataNodeId() == STOPPED_DATA_NODE_ID)); + } + + @Test + public void testContinuingStoppedDataNodeIsNotTransferredTwice() throws Exception { + createThreadWithStoppedDataNode(); + + invokeTriggerDetectTask(); + invokeTriggerDetectTask(); + + // Like a continuing Unknown node, a continuing Stopped node only triggers one transfer. + verify(configManager, times(1)).transfer(anyList()); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java index e2c51df9693be..df1b7fb1449ef 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java @@ -19,22 +19,30 @@ package org.apache.iotdb.confignode.manager.load; +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.commons.cluster.RegionStatus; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan.Operation; +import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.manager.IManager; import org.apache.iotdb.confignode.manager.load.cache.LoadCache; import org.apache.iotdb.confignode.manager.load.cache.consensus.ConsensusGroupHeartbeatSample; import org.apache.iotdb.confignode.manager.load.cache.consensus.ConsensusGroupStatistics; +import org.apache.iotdb.confignode.manager.load.cache.node.ConfigNodeHeartbeatCache; import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample; import org.apache.iotdb.confignode.manager.load.cache.node.NodeStatistics; import org.apache.iotdb.confignode.manager.load.cache.region.RegionGroupStatistics; import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample; import org.apache.iotdb.confignode.manager.load.cache.region.RegionStatistics; +import org.apache.iotdb.confignode.manager.node.NodeManager; import org.apache.iotdb.confignode.manager.partition.RegionGroupStatus; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.Pair; import org.junit.Assert; @@ -49,9 +57,17 @@ import java.util.TreeMap; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + public class LoadManagerTest { private static LoadManager LOAD_MANAGER; private static LoadCache LOAD_CACHE; @@ -65,6 +81,9 @@ public static void setUp() throws IOException { IManager CONFIG_MANAGER = new ConfigManager(); LOAD_MANAGER = CONFIG_MANAGER.getLoadManager(); LOAD_CACHE = LOAD_MANAGER.getLoadCache(); + // These cache/event tests use successful stub writes without starting consensus. + LOAD_CACHE.setNodeStatusPersistence( + nodeId -> null, plan -> new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); } @Before @@ -90,7 +109,7 @@ public void testNodeCache() throws InterruptedException { // Simulate update to Running status LOAD_CACHE.cacheConfigNodeHeartbeatSample(0, new NodeHeartbeatSample(NodeStatus.Running)); LOAD_CACHE.cacheDataNodeHeartbeatSample(1, new NodeHeartbeatSample(NodeStatus.Running)); - LOAD_CACHE.updateNodeStatistics(false); + LOAD_CACHE.updateNodeStatistics(); LOAD_MANAGER.getEventService().checkAndBroadcastNodeStatisticsChangeEventIfNecessary(); NODE_SEMAPHORE.acquire(); Assert.assertEquals(NodeStatus.Running, LOAD_CACHE.getNodeStatus(0)); @@ -105,8 +124,7 @@ public void testNodeCache() throws InterruptedException { differentNodeStatisticsMap.get(1)); // Force update to Removing status - LOAD_MANAGER.forceUpdateNodeCache( - NodeType.DataNode, 1, new NodeHeartbeatSample(NodeStatus.Removing)); + LOAD_MANAGER.trySetNodeStatus(1, NodeStatus.Removing, true); NODE_SEMAPHORE.acquire(); Assert.assertEquals(NodeStatus.Removing, LOAD_CACHE.getNodeStatus(1)); differentNodeStatisticsMap = FAKE_SUBSCRIBER.getDifferentNodeStatisticsMap(); @@ -118,7 +136,7 @@ public void testNodeCache() throws InterruptedException { // Removing status can't be updated to any other status automatically LOAD_CACHE.cacheDataNodeHeartbeatSample(1, new NodeHeartbeatSample(NodeStatus.ReadOnly)); - LOAD_CACHE.updateNodeStatistics(false); + LOAD_CACHE.updateNodeStatistics(); LOAD_MANAGER.getEventService().checkAndBroadcastNodeStatisticsChangeEventIfNecessary(); Assert.assertEquals(NodeStatus.Removing, LOAD_CACHE.getNodeStatus(1)); @@ -316,7 +334,7 @@ public void testLoadWarmUpRequiresOnlyConfigNodeAndDataNodeSamples() { dataNodeId -> loadCache.cacheDataNodeHeartbeatSample( dataNodeId, new NodeHeartbeatSample(NodeStatus.Running))); - loadCache.updateNodeStatistics(false); + loadCache.updateNodeStatistics(); Assert.assertTrue(loadCache.getNodeHeartbeatUnreadyReasons().isEmpty()); } @@ -335,13 +353,63 @@ public void testRegionAndConsensusGroupsDoNotBlockLoadWarmUp() { loadCache.cacheDataNodeHeartbeatSample( dataNodeId, new NodeHeartbeatSample(NodeStatus.Running)); }); - loadCache.updateNodeStatistics(false); + loadCache.updateNodeStatistics(); loadCache.updateRegionGroupStatistics(); loadCache.updateConsensusGroupStatistics(); Assert.assertTrue(loadCache.getNodeHeartbeatUnreadyReasons().isEmpty()); } + @Test + public void testLoadWarmUpWaitsForLeaderStoppedMarkerToBeCleared() throws Exception { + int leaderId = ConfigNodeHeartbeatCache.CURRENT_NODE_ID; + IManager configManager = mock(IManager.class, RETURNS_DEEP_STUBS); + NodeManager nodeManager = configManager.getNodeManager(); + when(nodeManager.getRegisteredConfigNodes()) + .thenReturn(Collections.singletonList(new TConfigNodeLocation().setConfigNodeId(leaderId))); + when(nodeManager.getRegisteredDataNodes()).thenReturn(Collections.emptyList()); + when(nodeManager.getRegisteredAINodes()).thenReturn(Collections.emptyList()); + when(configManager.getClusterSchemaManager().getDatabaseNames(null)) + .thenReturn(Collections.emptyList()); + AtomicReference persistedLeaderStatus = new AtomicReference<>(NodeStatus.Stopped); + Function nodeStatusWriter = mock(Function.class); + when(nodeStatusWriter.apply(new UpdateNodeStatusPlan(leaderId, Operation.CLEAR))) + .thenReturn(new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())) + .thenReturn(new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())) + .thenAnswer( + invocation -> { + persistedLeaderStatus.set(null); + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + }); + LoadManager loadManager = new LoadManager(configManager); + when(configManager.getLoadManager()).thenReturn(loadManager); + LoadCache loadCache = loadManager.getLoadCache(); + loadCache.setNodeStatusPersistence(id -> persistedLeaderStatus.get(), nodeStatusWriter::apply); + loadCache.initHeartbeatCache(configManager); + + try { + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(leaderId)); + Assert.assertTrue(loadCache.getNodeHeartbeatUnreadyReasons().isEmpty()); + loadManager.markLoadServicesStartedForTest( + System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(31)); + + Assert.assertFalse(loadManager.isLoadReady()); + Assert.assertEquals(NodeStatus.Stopped, persistedLeaderStatus.get()); + Assert.assertEquals( + ManagerMessages.MESSAGE_CONFIGNODE_LEADER_IS_WAITING_FOR_NODE_STATUS_PERSISTENCE_8CA96809, + loadManager.getLoadReadyReason()); + // The first-heartbeat timeout must not bypass a repeated persistence failure. + Assert.assertFalse(loadManager.isLoadReady()); + Assert.assertEquals(NodeStatus.Stopped, persistedLeaderStatus.get()); + + Assert.assertTrue(loadManager.isLoadReady()); + Assert.assertNull(persistedLeaderStatus.get()); + verify(nodeStatusWriter, times(3)).apply(new UpdateNodeStatusPlan(leaderId, Operation.CLEAR)); + } finally { + loadManager.stopLoadServices(); + } + } + @Test public void testLoadWarmUpToleratesMissingFirstHeartbeatAfterThirtySeconds() { LOAD_CACHE.clearHeartbeatCache(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancerTest.java new file mode 100644 index 0000000000000..536379d79c5f2 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancerTest.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.load.balancer; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.exception.NotEnoughDataNodeException; +import org.apache.iotdb.confignode.manager.IManager; +import org.apache.iotdb.confignode.manager.ProcedureManager; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.node.NodeManager; +import org.apache.iotdb.confignode.manager.partition.PartitionManager; +import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentMatchers; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.function.Function; + +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class RegionBalancerTest { + + private static final int STOPPED_DATA_NODE_ID = 1; + private static final int REMOVING_DATA_NODE_ID = 2; + + private final IManager configManager = mock(IManager.class); + private final NodeManager nodeManager = mock(NodeManager.class); + private final ProcedureManager procedureManager = mock(ProcedureManager.class); + private final ClusterSchemaManager clusterSchemaManager = mock(ClusterSchemaManager.class); + private final PartitionManager partitionManager = mock(PartitionManager.class); + private final LoadManager loadManager = mock(LoadManager.class); + + private final TDataNodeConfiguration stoppedDataNode = + new TDataNodeConfiguration() + .setLocation( + new TDataNodeLocation( + STOPPED_DATA_NODE_ID, + new TEndPoint("127.0.0.1", 2000), + new TEndPoint("127.0.0.1", 2001), + new TEndPoint("127.0.0.1", 2002), + new TEndPoint("127.0.0.1", 2003), + new TEndPoint("127.0.0.1", 2004))); + private final TDataNodeConfiguration removingDataNode = + new TDataNodeConfiguration() + .setLocation( + new TDataNodeLocation( + REMOVING_DATA_NODE_ID, + new TEndPoint("127.0.0.1", 3000), + new TEndPoint("127.0.0.1", 3001), + new TEndPoint("127.0.0.1", 3002), + new TEndPoint("127.0.0.1", 3003), + new TEndPoint("127.0.0.1", 3004))); + + private final RegionBalancer regionBalancer = new RegionBalancer(configManager); + + @Before + public void setUp() { + when(configManager.getNodeManager()).thenReturn(nodeManager); + when(configManager.getProcedureManager()).thenReturn(procedureManager); + when(configManager.getClusterSchemaManager()).thenReturn(clusterSchemaManager); + when(configManager.getPartitionManager()).thenReturn(partitionManager); + when(configManager.getLoadManager()).thenReturn(loadManager); + } + + @Test + public void testStoppedDataNodesAreKeptAsAllocationCandidates() throws Exception { + // With an empty allotment only the candidate filtering runs. Verify the supplied predicate + // accepts Running, Unknown and Stopped while excluding ReadOnly and Removing. + when(procedureManager.getRemovingDataNodeIds()).thenReturn(Collections.emptySet()); + when(nodeManager.filterDataNodeThroughStatus( + ArgumentMatchers.>any())) + .thenReturn(Arrays.asList(stoppedDataNode, removingDataNode)); + + regionBalancer.genRegionGroupsAllocationPlan( + Collections.emptyMap(), TConsensusGroupType.DataRegion); + + verify(nodeManager) + .filterDataNodeThroughStatus( + argThat( + (Function predicate) -> + predicate != null + && predicate.apply(NodeStatus.Running) + && predicate.apply(NodeStatus.Unknown) + && predicate.apply(NodeStatus.Stopped) + && !predicate.apply(NodeStatus.ReadOnly) + && !predicate.apply(NodeStatus.Removing))); + } + + @Test + public void testRemovingDataNodesAreExcludedFromCandidates() throws Exception { + when(procedureManager.getRemovingDataNodeIds()) + .thenReturn(new HashSet<>(Collections.singletonList(REMOVING_DATA_NODE_ID))); + // The mocked status filter returns both nodes; the in-progress removal must drop the second. + when(nodeManager.filterDataNodeThroughStatus( + ArgumentMatchers.>any())) + .thenReturn(Arrays.asList(stoppedDataNode, removingDataNode)); + when(clusterSchemaManager.getReplicationFactor(eq("db1"), eq(TConsensusGroupType.DataRegion))) + .thenReturn(3); + + try { + regionBalancer.genRegionGroupsAllocationPlan( + Collections.singletonMap("db1", 1), TConsensusGroupType.DataRegion); + Assert.fail("Expected NotEnoughDataNodeException"); + } catch (NotEnoughDataNodeException e) { + // The remaining candidates keep the Stopped node but not the node being removed. + Assert.assertTrue( + "Stopped node should be kept: " + e.getMessage(), + e.getMessage().contains("dataNodeId:" + STOPPED_DATA_NODE_ID)); + Assert.assertFalse( + "Removing node should be excluded: " + e.getMessage(), + e.getMessage().contains("dataNodeId:" + REMOVING_DATA_NODE_ID)); + } + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/GreedyPriorityTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/GreedyPriorityTest.java index 48104bf78a3e2..d601b645dbf2d 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/GreedyPriorityTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/GreedyPriorityTest.java @@ -65,7 +65,7 @@ public void testGenLoadScoreGreedyRoutingPolicy() { nodeCacheMap.put(i, new DataNodeHeartbeatCache(i)); nodeCacheMap.get(i).cacheHeartbeatSample(new NodeHeartbeatSample(currentTimeNs, statuses[i])); } - nodeCacheMap.values().forEach(baseNodeCache -> baseNodeCache.updateCurrentStatistics(false)); + nodeCacheMap.values().forEach(BaseNodeCache::updateNodeStatistics); /* Build TRegionReplicaSet */ TConsensusGroupId groupId1 = new TConsensusGroupId(TConsensusGroupType.SchemaRegion, 1); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/LeaderPriorityBalancerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/LeaderPriorityBalancerTest.java index 5af3a677aab88..90dbddc3f2716 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/LeaderPriorityBalancerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/LeaderPriorityBalancerTest.java @@ -65,7 +65,7 @@ public void testGenRealTimeRoutingPolicy() { .cacheHeartbeatSample(new NodeHeartbeatSample(currentTimeNs, NodeStatus.Running)); } } - nodeCacheMap.values().forEach(baseNodeCache -> baseNodeCache.updateCurrentStatistics(false)); + nodeCacheMap.values().forEach(BaseNodeCache::updateNodeStatistics); // Build TRegionReplicaSet TConsensusGroupId groupId1 = new TConsensusGroupId(TConsensusGroupType.SchemaRegion, 1); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/ConsensusGroupCacheTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/ConsensusGroupCacheTest.java index caa3e8394399c..7bbd366be5799 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/ConsensusGroupCacheTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/ConsensusGroupCacheTest.java @@ -31,7 +31,7 @@ public void periodicUpdateTest() { ConsensusGroupCache consensusGroupCache = new ConsensusGroupCache(); ConsensusGroupHeartbeatSample sample = new ConsensusGroupHeartbeatSample(1L, 1); consensusGroupCache.cacheHeartbeatSample(sample); - consensusGroupCache.updateCurrentStatistics(false); + consensusGroupCache.updateCurrentStatistics(); Assert.assertEquals(1, consensusGroupCache.getLeaderId()); } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java new file mode 100644 index 0000000000000..40981b07a3b6e --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.load.cache; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan.Operation; +import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.IManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.node.NodeManager; +import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.consensus.exception.ConsensusException; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class LoadCacheNodeStatusTest { + private static final int NODE_ID = 1; + + private NodeInfo nodeInfo; + private LoadCache loadCache; + private ConsensusManager consensusManager; + + @Before + public void setUp() throws Exception { + nodeInfo = new NodeInfo(); + nodeInfo.registerDataNode( + new RegisterDataNodePlan( + new TDataNodeConfiguration() + .setLocation(new TDataNodeLocation().setDataNodeId(NODE_ID)))); + IManager manager = mock(IManager.class, RETURNS_DEEP_STUBS); + consensusManager = mock(ConsensusManager.class); + when(manager.getConsensusManager()).thenReturn(consensusManager); + when(consensusManager.write(any(UpdateNodeStatusPlan.class))) + .thenAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))); + when(manager.getNodeManager()).thenReturn(new NodeManager(manager, nodeInfo)); + when(manager.getClusterSchemaManager().getDatabaseNames(null)) + .thenReturn(Collections.emptyList()); + loadCache = new LoadCache(); + loadCache.setNodeStatusPersistence(nodeInfo::getPersistedNodeStatus, consensusManager::write); + loadCache.initHeartbeatCache(manager); + } + + @Test + @SuppressWarnings("unchecked") + public void testLoadManagerSuppliesPersistenceBeforeConsensusInitialization() throws Exception { + ConfigManager manager = new ConfigManager(); + try { + NodeInfo info = manager.getNodeManager().getNodeInfo(); + info.registerDataNode( + new RegisterDataNodePlan( + new TDataNodeConfiguration() + .setLocation(new TDataNodeLocation().setDataNodeId(NODE_ID)))); + LoadCache cache = manager.getLoadManager().getLoadCache(); + cache.initHeartbeatCache(manager); + + // Use the real methods supplied by LoadManager, replacing only consensus submission. + Field consensusField = ConfigManager.class.getDeclaredField("consensusManager"); + consensusField.setAccessible(true); + ((AtomicReference) consensusField.get(manager)).set(consensusManager); + doThrow(new ConsensusException("quorum unavailable")) + .when(consensusManager) + .write(any(UpdateNodeStatusPlan.class)); + TSStatus result = cache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, false); + assertFailure(result); + assertEquals("quorum unavailable", result.getMessage()); + assertEquals(NodeStatus.Stopped, cache.getNodeStatus(NODE_ID)); + assertNull(info.getPersistedNodeStatus(NODE_ID)); + + doAnswer(invocation -> info.applyNodeStatusPlan(invocation.getArgument(0))) + .when(consensusManager) + .write(any(UpdateNodeStatusPlan.class)); + assertTrue(cache.updateNodeStatistics()); + assertEquals(NodeStatus.Stopped, info.getPersistedNodeStatus(NODE_ID)); + assertSuccess(cache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, false)); + verify(consensusManager, times(2)) + .write(new UpdateNodeStatusPlan(NODE_ID, Operation.SET_STOPPED)); + } finally { + manager.close(); + } + } + + @Test + public void testOnlyDurableTransitionsWriteConsensus() throws Exception { + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, false)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, false)); + verify(consensusManager, never()).write(any()); + + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, false)); + assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, false)); + verify(consensusManager, times(1)).write(any()); + + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, true)); + assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, true)); + assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID)); + verify(consensusManager, times(3)).write(any()); + } + + @Test + public void testFailedWriteIsRetriedAndClearIsDurable() throws Exception { + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, false)); + doReturn(new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())) + .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) + .when(consensusManager) + .write(any(UpdateNodeStatusPlan.class)); + assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, false)); + assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); + // A later connection failure must retain the stop and retry its uncommitted record. + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, false)); + assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); + assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); + verify(consensusManager, times(2)) + .write(new UpdateNodeStatusPlan(NODE_ID, Operation.SET_STOPPED)); + + doThrow(new ConsensusException("test consensus failure")) + .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) + .when(consensusManager) + .write(any(UpdateNodeStatusPlan.class)); + assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, false)); + assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); + assertTrue(loadCache.updateNodeStatistics()); + assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); + assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID)); + } + + private static void assertSuccess(TSStatus status) { + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } + + @Test + public void testCommittedWriteWithLostResponseIsNotClearedByUnknown() throws Exception { + doAnswer( + i -> { + nodeInfo.applyNodeStatusPlan(i.getArgument(0)); + throw new ConsensusException("response lost after commit"); + }) + .when(consensusManager) + .write(any(UpdateNodeStatusPlan.class)); + assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, false)); + assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, false)); + assertTrue(loadCache.updateNodeStatistics()); + assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); + assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); + verify(consensusManager, times(1)).write(any()); + } + + @Test + public void testOrdinaryUpdatesKeepRemovingWithoutQuorumButRollbackRequiresIt() throws Exception { + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, true)); + clearInvocations(consensusManager); + doThrow(new ConsensusException("quorum unavailable")).when(consensusManager).write(any()); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, true)); + for (NodeStatus observed : + new NodeStatus[] {NodeStatus.Unknown, NodeStatus.Stopped, NodeStatus.Running}) { + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, observed, false)); + assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(NODE_ID)); + } + verify(consensusManager, never()).write(any()); + assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, true)); + assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); + assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, true)); + assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); + assertFalse(loadCache.updateNodeStatistics()); + assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); + assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(NODE_ID)); + + doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) + .when(consensusManager) + .write(any(UpdateNodeStatusPlan.class)); + assertTrue(loadCache.updateNodeStatistics()); + assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); + assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID)); + verify(consensusManager, never()) + .write(new UpdateNodeStatusPlan(NODE_ID, Operation.SET_REMOVING)); + } + + @Test + public void testConnectionFailureKeepsStoppedUntilExplicitReset() throws Exception { + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, false)); + clearInvocations(consensusManager); + + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, false)); + assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); + assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); + verify(consensusManager, never()).write(any()); + + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, true)); + assertEquals(NodeStatus.Unknown, loadCache.getNodeStatus(NODE_ID)); + assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID)); + loadCache.updateNodeStatistics(); + assertEquals(NodeStatus.Unknown, loadCache.getNodeStatus(NODE_ID)); + } + + private static void assertFailure(TSStatus status) { + assertEquals(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode(), status.getCode()); + } + + @Test + public void testCommittedClearWithLostResponseIsNotRevertedByStatistics() throws Exception { + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, true)); + clearInvocations(consensusManager); + doAnswer( + i -> { + nodeInfo.applyNodeStatusPlan(i.getArgument(0)); + throw new ConsensusException("clear response lost"); + }) + .when(consensusManager) + .write(any(UpdateNodeStatusPlan.class)); + assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, true)); + assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); + assertTrue(loadCache.updateNodeStatistics()); + assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); + assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID)); + verify(consensusManager, times(1)).write(any()); + verify(consensusManager, never()) + .write(new UpdateNodeStatusPlan(NODE_ID, Operation.SET_REMOVING)); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java new file mode 100644 index 0000000000000..1a8075a86cc83 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java @@ -0,0 +1,703 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.load.cache; + +import org.apache.iotdb.common.rpc.thrift.TAINodeConfiguration; +import org.apache.iotdb.common.rpc.thrift.TAINodeLocation; +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.cluster.NodeType; +import org.apache.iotdb.confignode.consensus.request.write.ainode.RegisterAINodePlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan.Operation; +import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; +import org.apache.iotdb.confignode.manager.IManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.load.cache.node.AINodeHeartbeatCache; +import org.apache.iotdb.confignode.manager.load.cache.node.BaseNodeCache; +import org.apache.iotdb.confignode.manager.load.cache.node.ConfigNodeHeartbeatCache; +import org.apache.iotdb.confignode.manager.load.cache.node.DataNodeHeartbeatCache; +import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample; +import org.apache.iotdb.confignode.manager.node.NodeManager; +import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; +import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class LoadCachePersistedNodeStatusTest { + + private static final int SELF_ID = ConfigNodeHeartbeatCache.CURRENT_NODE_ID; + private static final int CONFIG_NODE_ID = SELF_ID + 100; + private static final int DATA_NODE_ID = SELF_ID + 101; + private static final int REMOVING_DATA_NODE_ID = SELF_ID + 102; + private static final int AI_NODE_ID = SELF_ID + 103; + + private NodeInfo nodeInfo; + private ConsensusManager consensusManager; + private IManager configManager; + private NodeManager nodeManager; + private LoadCache loadCache; + + @Before + public void setUp() throws Exception { + nodeInfo = spy(new NodeInfo()); + configManager = mock(IManager.class); + nodeManager = mock(NodeManager.class); + consensusManager = mock(ConsensusManager.class); + ClusterSchemaManager schemaManager = mock(ClusterSchemaManager.class); + + when(configManager.getNodeManager()).thenReturn(nodeManager); + when(configManager.getConsensusManager()).thenReturn(consensusManager); + when(configManager.getClusterSchemaManager()).thenReturn(schemaManager); + when(schemaManager.getDatabaseNames(null)).thenReturn(Collections.emptyList()); + when(nodeManager.getRegisteredConfigNodes()) + .thenReturn( + Arrays.asList( + new TConfigNodeLocation( + SELF_ID, new TEndPoint("127.0.0.1", 11000), new TEndPoint("127.0.0.1", 12000)), + new TConfigNodeLocation( + CONFIG_NODE_ID, + new TEndPoint("127.0.0.1", 11001), + new TEndPoint("127.0.0.1", 12001)))); + when(nodeManager.getRegisteredDataNodes()) + .thenReturn( + Arrays.asList( + new TDataNodeConfiguration() + .setLocation(new TDataNodeLocation().setDataNodeId(DATA_NODE_ID)), + new TDataNodeConfiguration() + .setLocation(new TDataNodeLocation().setDataNodeId(REMOVING_DATA_NODE_ID)))); + when(nodeManager.getRegisteredAINodes()).thenReturn(Collections.emptyList()); + for (TConfigNodeLocation node : nodeManager.getRegisteredConfigNodes()) { + nodeInfo.applyConfigNode(new ApplyConfigNodePlan(node)); + } + for (TDataNodeConfiguration node : nodeManager.getRegisteredDataNodes()) { + nodeInfo.registerDataNode(new RegisterDataNodePlan(node)); + } + when(consensusManager.write(any(UpdateNodeStatusPlan.class))) + .thenAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))); + loadCache = new LoadCache(); + loadCache.setNodeStatusPersistence(nodeInfo::getPersistedNodeStatus, consensusManager::write); + } + + @Test + public void testInitializationRestoresStickyStatusesAndKeepsLeaderRunning() throws Exception { + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, Operation.SET_STOPPED)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + nodeInfo.applyNodeStatusPlan( + new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(SELF_ID, Operation.SET_STOPPED)); + + loadCache.initHeartbeatCache(configManager); + + Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(CONFIG_NODE_ID)); + Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(SELF_ID)); + Assert.assertTrue(loadCache.getNodeHeartbeatUnreadyReasons().isEmpty()); + verify(nodeInfo).getPersistedNodeStatus(CONFIG_NODE_ID); + verify(nodeInfo).getPersistedNodeStatus(DATA_NODE_ID); + verify(nodeInfo).getPersistedNodeStatus(REMOVING_DATA_NODE_ID); + verify(nodeInfo, never()).getPersistedNodeStatus(SELF_ID); + + loadCache.updateNodeStatistics(); + + Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(CONFIG_NODE_ID)); + Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(SELF_ID)); + Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(SELF_ID)); + } + + @Test + public void testRestoredStatusDoesNotCreateHeartbeatSamples() throws Exception { + for (NodeStatus status : Arrays.asList(NodeStatus.Stopped, NodeStatus.Removing)) { + for (BaseNodeCache cache : + Arrays.asList( + new DataNodeHeartbeatCache(DATA_NODE_ID), + new ConfigNodeHeartbeatCache(CONFIG_NODE_ID), + new AINodeHeartbeatCache(AI_NODE_ID))) { + cache.initPersistence(status, (ignoredCache, ignoredStatus) -> success()); + Assert.assertEquals(status, cache.getNodeStatus()); + Assert.assertFalse(cache.hasHeartbeatSample()); + cache.updateNodeStatistics(); + Assert.assertEquals(status, cache.getNodeStatus()); + Assert.assertFalse(cache.hasHeartbeatSample()); + } + } + } + + @Test + public void testLiveHeartbeatClearsStoppedStatusForBothNodeTypes() throws Exception { + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, Operation.SET_STOPPED)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + loadCache.initHeartbeatCache(configManager); + + loadCache.cacheConfigNodeHeartbeatSample( + CONFIG_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); + loadCache.updateNodeStatistics(); + + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(CONFIG_NODE_ID)); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(CONFIG_NODE_ID)); + Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(DATA_NODE_ID)); + } + + @Test + public void testRevivalUpdatesStatisticsWhileClearIsRetried() throws Exception { + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + loadCache.initHeartbeatCache(configManager); + doReturn(failure()) + .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) + .when(consensusManager) + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); + + Assert.assertFalse(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(DATA_NODE_ID)); + verify(consensusManager, times(2)) + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + } + + @Test + public void testClearDoesNotRecheckThePreviousPersistedStatus() throws Exception { + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + loadCache.initHeartbeatCache(configManager); + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); + // CLEAR executes the caller's decision even if the applied value has changed. + doAnswer( + invocation -> { + nodeInfo.applyNodeStatusPlan( + new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_REMOVING)); + return nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)); + }) + .when(consensusManager) + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); + } + + @Test + public void testStoppedStatisticsUpdateAfterFailedPersistenceReturns() throws Exception { + loadCache.initHeartbeatCache(configManager); + loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Running, false); + CountDownLatch persistenceStarted = new CountDownLatch(1); + CountDownLatch finishPersistence = new CountDownLatch(1); + doAnswer( + invocation -> { + persistenceStarted.countDown(); + Assert.assertTrue(finishPersistence.await(5, TimeUnit.SECONDS)); + return failure(); + }) + .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) + .when(consensusManager) + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future update = + executor.submit( + () -> loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, false)); + Assert.assertTrue(persistenceStarted.await(5, TimeUnit.SECONDS)); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(DATA_NODE_ID)); + finishPersistence.countDown(); + Assert.assertEquals(failure().getCode(), update.get(5, TimeUnit.SECONDS).getCode()); + Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + + // Even when the latest sample becomes Unknown, statistics retain the stop for retry. + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Unknown)); + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + verify(consensusManager, times(2)) + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + } finally { + finishPersistence.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void testHeartbeatAndStatisticsWaitUntilShutdownPersistenceCompletes() throws Exception { + loadCache.initHeartbeatCache(configManager); + loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Running, false); + NodeHeartbeatSample delayedRunning = new NodeHeartbeatSample(NodeStatus.Running); + CountDownLatch persistenceStarted = new CountDownLatch(1); + CountDownLatch finishPersistence = new CountDownLatch(1); + CountDownLatch updatesStarted = new CountDownLatch(2); + doAnswer( + invocation -> { + persistenceStarted.countDown(); + Assert.assertTrue(finishPersistence.await(10, TimeUnit.SECONDS)); + return nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)); + }) + .when(consensusManager) + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + ExecutorService executor = Executors.newFixedThreadPool(3); + try { + Future stopped = + executor.submit( + () -> loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, false)); + Assert.assertTrue(persistenceStarted.await(5, TimeUnit.SECONDS)); + Future heartbeat = + executor.submit( + () -> { + updatesStarted.countDown(); + loadCache.cacheDataNodeHeartbeatSample(DATA_NODE_ID, delayedRunning); + }); + Future statistics = + executor.submit( + () -> { + updatesStarted.countDown(); + return loadCache.updateNodeStatistics(); + }); + Assert.assertTrue(updatesStarted.await(5, TimeUnit.SECONDS)); + for (Future pending : Arrays.asList(heartbeat, statistics)) { + try { + pending.get(100, TimeUnit.MILLISECONDS); + Assert.fail("The node update must wait for shutdown persistence"); + } catch (TimeoutException expected) { + // Both operations must wait, while the last published status remains readable. + } + } + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + verify(consensusManager, times(1)) + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + + finishPersistence.countDown(); + Assert.assertEquals(success().getCode(), stopped.get(5, TimeUnit.SECONDS).getCode()); + heartbeat.get(5, TimeUnit.SECONDS); + Assert.assertTrue(statistics.get(5, TimeUnit.SECONDS)); + Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + } finally { + finishPersistence.countDown(); + executor.shutdownNow(); + Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + public void testHeartbeatIsNotFilteredByCacheInitializationTime() throws Exception { + long oldSampleTimestamp = System.nanoTime() - 1; + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + loadCache.initHeartbeatCache(configManager); + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, new NodeHeartbeatSample(oldSampleTimestamp, NodeStatus.Running)); + loadCache.updateNodeStatistics(); + + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + verify(consensusManager).write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + } + + @Test + public void testForcingAnotherNodeDoesNotClearRemovingStatus() throws Exception { + nodeInfo.applyNodeStatusPlan( + new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); + loadCache.initHeartbeatCache(configManager); + loadCache.cacheDataNodeHeartbeatSample( + REMOVING_DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); + + Assert.assertEquals( + success().getCode(), + loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, false).getCode()); + + Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); + Assert.assertEquals( + NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); + verify(consensusManager, never()) + .write( + argThat( + plan -> + plan instanceof UpdateNodeStatusPlan + && ((UpdateNodeStatusPlan) plan).getNodeId() == REMOVING_DATA_NODE_ID)); + + // An explicit rollback of that node still clears its durable Removing marker. + Assert.assertEquals( + success().getCode(), + loadCache.trySetNodeStatus(REMOVING_DATA_NODE_ID, NodeStatus.Running, true).getCode()); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); + Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(REMOVING_DATA_NODE_ID)); + } + + @Test + public void testExplicitRollbackRestoresOfflineAndRunningStatuses() throws Exception { + for (NodeStatus rollbackStatus : + Arrays.asList(NodeStatus.Unknown, NodeStatus.Stopped, NodeStatus.Running)) { + nodeInfo.applyNodeStatusPlan( + new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); + loadCache.initHeartbeatCache(configManager); + + Assert.assertEquals( + success().getCode(), + loadCache.trySetNodeStatus(REMOVING_DATA_NODE_ID, rollbackStatus, true).getCode()); + Assert.assertEquals(rollbackStatus, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); + Assert.assertEquals( + rollbackStatus == NodeStatus.Stopped ? NodeStatus.Stopped : null, + nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); + + // Ordinary statistics updates must preserve the successfully committed rollback. + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(rollbackStatus, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); + } + } + + @Test + public void testFailedExplicitRollbackUpdatesStatisticsAndRetriesPeriodically() throws Exception { + nodeInfo.applyNodeStatusPlan( + new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); + loadCache.initHeartbeatCache(configManager); + doReturn(failure()) + .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) + .when(consensusManager) + .write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.CLEAR)); + + Assert.assertEquals( + failure().getCode(), + loadCache.trySetNodeStatus(REMOVING_DATA_NODE_ID, NodeStatus.Running, true).getCode()); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); + Assert.assertEquals( + NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); + + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); + Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(REMOVING_DATA_NODE_ID)); + verify(consensusManager, times(2)) + .write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.CLEAR)); + verify(consensusManager, never()) + .write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); + } + + @Test + public void testShutdownReportCannotOverrideRemovingStatus() throws Exception { + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, Operation.SET_REMOVING)); + nodeInfo.applyNodeStatusPlan( + new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); + loadCache.initHeartbeatCache(configManager); + + for (int nodeId : Arrays.asList(CONFIG_NODE_ID, REMOVING_DATA_NODE_ID)) { + Assert.assertEquals( + success().getCode(), + loadCache.trySetNodeStatus(nodeId, NodeStatus.Stopped, false).getCode()); + Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(nodeId)); + Assert.assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(nodeId)); + verify(consensusManager, never()) + .write(new UpdateNodeStatusPlan(nodeId, Operation.SET_STOPPED)); + } + } + + @Test + public void testStatisticsFailureDoesNotPreventUpdatingOtherNodes() throws Exception { + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, Operation.SET_STOPPED)); + loadCache.initHeartbeatCache(configManager); + doReturn(failure()) + .when(consensusManager) + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); + loadCache.cacheConfigNodeHeartbeatSample( + CONFIG_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); + + Assert.assertFalse(loadCache.updateNodeStatistics()); + + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(CONFIG_NODE_ID)); + Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(CONFIG_NODE_ID)); + verify(consensusManager, never()).write(new UpdateNodeStatusPlan(SELF_ID, Operation.CLEAR)); + verify(consensusManager, never()) + .write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.CLEAR)); + } + + @Test + public void testRepeatedCacheCreationPreservesPendingShutdown() throws Exception { + loadCache.initHeartbeatCache(configManager); + Assert.assertEquals( + success().getCode(), + loadCache.trySetNodeStatus(CONFIG_NODE_ID, NodeStatus.Running, false).getCode()); + Assert.assertFalse(loadCache.checkAndSetHeartbeatProcessing(CONFIG_NODE_ID)); + CountDownLatch persistenceStarted = new CountDownLatch(1); + CountDownLatch finishPersistence = new CountDownLatch(1); + doAnswer( + invocation -> { + persistenceStarted.countDown(); + Assert.assertTrue(finishPersistence.await(10, TimeUnit.SECONDS)); + return nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)); + }) + .when(consensusManager) + .write(new UpdateNodeStatusPlan(CONFIG_NODE_ID, Operation.SET_STOPPED)); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future shutdown = + executor.submit( + () -> loadCache.trySetNodeStatus(CONFIG_NODE_ID, NodeStatus.Stopped, false)); + Assert.assertTrue(persistenceStarted.await(5, TimeUnit.SECONDS)); + // A recovered AddConfigNodeProcedure may repeat REGISTER_SUCCESS after registration applied. + loadCache.createNodeHeartbeatCache(NodeType.ConfigNode, CONFIG_NODE_ID); + finishPersistence.countDown(); + Assert.assertEquals(success().getCode(), shutdown.get(5, TimeUnit.SECONDS).getCode()); + Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(CONFIG_NODE_ID)); + Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(CONFIG_NODE_ID)); + Assert.assertTrue(loadCache.checkAndSetHeartbeatProcessing(CONFIG_NODE_ID)); + + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(CONFIG_NODE_ID)); + Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(CONFIG_NODE_ID)); + verify(consensusManager, never()) + .write(new UpdateNodeStatusPlan(CONFIG_NODE_ID, Operation.CLEAR)); + } finally { + finishPersistence.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void testDiscardedCacheCannotClearNewRemovingMarker() throws Exception { + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + loadCache.initHeartbeatCache(configManager); + Field field = LoadCache.class.getDeclaredField("nodeCacheMap"); + field.setAccessible(true); + Map caches = (Map) field.get(loadCache); + BaseNodeCache discarded = caches.get(DATA_NODE_ID); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_REMOVING)); + loadCache.initHeartbeatCache(configManager); + Assert.assertEquals( + failure().getCode(), discarded.trySetNodeStatus(NodeStatus.Running, true).getCode()); + Assert.assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(DATA_NODE_ID)); + verify(consensusManager, never()) + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + } + + @Test + public void testReadOnlyHeartbeatClearsStoppedButNotRemoving() throws Exception { + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + nodeInfo.applyNodeStatusPlan( + new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); + loadCache.initHeartbeatCache(configManager); + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.ReadOnly)); + loadCache.cacheDataNodeHeartbeatSample( + REMOVING_DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.ReadOnly)); + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); + Assert.assertEquals( + NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); + } + + @Test + public void testAINodeRestoresStoppedAndRemovingAcrossLeaderChange() throws Exception { + registerAINode(); + loadCache.initHeartbeatCache(configManager); + for (NodeStatus status : Arrays.asList(NodeStatus.Stopped, NodeStatus.Removing)) { + Assert.assertEquals( + success().getCode(), loadCache.trySetNodeStatus(AI_NODE_ID, status, false).getCode()); + Assert.assertEquals(status, nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); + loadCache.initHeartbeatCache(configManager); + Assert.assertEquals(status, loadCache.getNodeStatus(AI_NODE_ID)); + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(status, loadCache.getNodeStatus(AI_NODE_ID)); + } + loadCache.cacheAINodeHeartbeatSample(AI_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(AI_NODE_ID)); + Assert.assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); + } + + @Test + public void testAINodeUpdatesStatisticsOnPersistenceFailureAndRetries() throws Exception { + registerAINode(); + loadCache.initHeartbeatCache(configManager); + doReturn(failure()) + .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) + .when(consensusManager) + .write(new UpdateNodeStatusPlan(AI_NODE_ID, Operation.SET_STOPPED)); + Assert.assertEquals( + failure().getCode(), + loadCache.trySetNodeStatus(AI_NODE_ID, NodeStatus.Stopped, false).getCode()); + Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(AI_NODE_ID)); + Assert.assertNull(nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); + loadCache.cacheAINodeHeartbeatSample(AI_NODE_ID, new NodeHeartbeatSample(NodeStatus.Unknown)); + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(AI_NODE_ID)); + Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); + + doReturn(failure()) + .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) + .when(consensusManager) + .write(new UpdateNodeStatusPlan(AI_NODE_ID, Operation.CLEAR)); + loadCache.cacheAINodeHeartbeatSample(AI_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); + Assert.assertFalse(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(AI_NODE_ID)); + Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(AI_NODE_ID)); + Assert.assertNull(nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); + } + + @Test + public void testDelayedAINodeHeartbeatDoesNotRecreateRemovedCache() throws Exception { + registerAINode(); + loadCache.initHeartbeatCache(configManager); + loadCache.removeNodeCache(AI_NODE_ID); + loadCache.cacheAINodeHeartbeatSample(AI_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); + Assert.assertFalse(loadCache.getCurrentNodeStatisticsMap().containsKey(AI_NODE_ID)); + } + + private void registerAINode() { + TAINodeConfiguration node = + new TAINodeConfiguration().setLocation(new TAINodeLocation().setAiNodeId(AI_NODE_ID)); + nodeInfo.registerAINode(new RegisterAINodePlan(node)); + when(nodeManager.getRegisteredAINodes()).thenReturn(Collections.singletonList(node)); + } + + @Test + public void testOlderHeartbeatCannotUndoShutdownReport() throws Exception { + loadCache.initHeartbeatCache(configManager); + NodeHeartbeatSample delayedRunning = new NodeHeartbeatSample(NodeStatus.Running); + Assert.assertEquals( + success().getCode(), + loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, false).getCode()); + for (NodeHeartbeatSample olderHeartbeat : + Arrays.asList( + delayedRunning, + new NodeHeartbeatSample( + System.nanoTime() - TimeUnit.DAYS.toNanos(1), NodeStatus.Running))) { + loadCache.cacheDataNodeHeartbeatSample(DATA_NODE_ID, olderHeartbeat); + loadCache.updateNodeStatistics(); + Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + } + } + + @Test + public void testBlockedNodeWriteDoesNotHoldOtherNodeCacheLock() throws Exception { + loadCache.initHeartbeatCache(configManager); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + doAnswer( + i -> { + entered.countDown(); + Assert.assertTrue(release.await(10, TimeUnit.SECONDS)); + return nodeInfo.applyNodeStatusPlan(i.getArgument(0)); + }) + .when(consensusManager) + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future blocked = + executor.submit(() -> loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, true)); + Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); + Future independent = + executor.submit( + () -> loadCache.trySetNodeStatus(REMOVING_DATA_NODE_ID, NodeStatus.Removing, true)); + Assert.assertEquals(success().getCode(), independent.get(5, TimeUnit.SECONDS).getCode()); + Assert.assertEquals( + NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); + Assert.assertFalse(blocked.isDone()); + release.countDown(); + Assert.assertEquals(success().getCode(), blocked.get(5, TimeUnit.SECONDS).getCode()); + } finally { + release.countDown(); + executor.shutdownNow(); + } + } + + private static TSStatus success() { + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } + + @Test + public void testLateShutdownReportConvergesAfterRestartedNodeHeartbeat() throws Exception { + loadCache.initHeartbeatCache(configManager); + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); + loadCache.updateNodeStatistics(); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); + // Shutdown reports carry no DataNode incarnation. Document the current temporary Stopped + // result, then require a fresh heartbeat from the new process to durably clear that marker. + Assert.assertEquals( + success().getCode(), + loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, false).getCode()); + Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); + loadCache.updateNodeStatistics(); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(DATA_NODE_ID)); + } + + private static TSStatus failure() { + return new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/NodeCacheTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/NodeCacheTest.java index a400692956621..26a88b7accd1d 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/NodeCacheTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/NodeCacheTest.java @@ -19,6 +19,10 @@ package org.apache.iotdb.confignode.manager.load.cache; import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.conf.ConfigNodeConfig; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; +import org.apache.iotdb.confignode.manager.load.cache.node.AINodeHeartbeatCache; +import org.apache.iotdb.confignode.manager.load.cache.node.BaseNodeCache; import org.apache.iotdb.confignode.manager.load.cache.node.ConfigNodeHeartbeatCache; import org.apache.iotdb.confignode.manager.load.cache.node.DataNodeHeartbeatCache; import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample; @@ -35,7 +39,7 @@ public void updateStatisticsTest() { long currentTime = System.nanoTime(); dataNodeHeartbeatCache.cacheHeartbeatSample( new NodeHeartbeatSample(currentTime, NodeStatus.Running)); - dataNodeHeartbeatCache.updateCurrentStatistics(false); + dataNodeHeartbeatCache.updateNodeStatistics(); Assert.assertEquals(NodeStatus.Running, dataNodeHeartbeatCache.getNodeStatus()); Assert.assertEquals(0, dataNodeHeartbeatCache.getLoadScore()); @@ -44,8 +48,137 @@ public void updateStatisticsTest() { currentTime = System.nanoTime(); configNodeHeartbeatCache.cacheHeartbeatSample( new NodeHeartbeatSample(currentTime, NodeStatus.Running)); - configNodeHeartbeatCache.updateCurrentStatistics(false); + configNodeHeartbeatCache.updateNodeStatistics(); Assert.assertEquals(NodeStatus.Running, configNodeHeartbeatCache.getNodeStatus()); Assert.assertEquals(0, configNodeHeartbeatCache.getLoadScore()); } + + @Test + public void testRemovingAINodeStatisticsSurviveHeartbeatTimeout() { + ConfigNodeConfig config = ConfigNodeDescriptor.getInstance().getConf(); + String detector = config.getFailureDetector(); + long timeout = config.getFailureDetectorFixedThresholdInMs(); + config.setFailureDetector(IFailureDetector.FIXED_DETECTOR); + config.setFailureDetectorFixedThresholdInMs(0); + try { + AINodeHeartbeatCache cache = new AINodeHeartbeatCache(3); + cache.trySetNodeStatus(NodeStatus.Removing, false); + cache.updateNodeStatistics(); + Assert.assertEquals(NodeStatus.Removing, cache.getNodeStatus()); + Assert.assertEquals(Long.MAX_VALUE, cache.getLoadScore()); + } finally { + config.setFailureDetector(detector); + config.setFailureDetectorFixedThresholdInMs(timeout); + } + } + + @Test + public void stoppedStatusStickyAndRevivalTest() { + for (BaseNodeCache cache : + new BaseNodeCache[] { + new DataNodeHeartbeatCache(1), + new ConfigNodeHeartbeatCache(ConfigNodeHeartbeatCache.CURRENT_NODE_ID + 1), + new AINodeHeartbeatCache(3) + }) { + String cacheType = cache.getClass().getSimpleName(); + cache.cacheHeartbeatSample(new NodeHeartbeatSample(NodeStatus.Stopped)); + cache.updateNodeStatistics(); + Assert.assertEquals(cacheType, NodeStatus.Stopped, cache.getNodeStatus()); + Assert.assertEquals(cacheType, Long.MAX_VALUE, cache.getLoadScore()); + + // Unknown observations cannot undo a reported stop. + cache.cacheHeartbeatSample(new NodeHeartbeatSample(NodeStatus.Unknown)); + cache.updateNodeStatistics(); + Assert.assertEquals(cacheType, NodeStatus.Stopped, cache.getNodeStatus()); + + // A live heartbeat revives a stopped node. + cache.cacheHeartbeatSample(new NodeHeartbeatSample(NodeStatus.Running)); + cache.updateNodeStatistics(); + Assert.assertEquals(cacheType, NodeStatus.Running, cache.getNodeStatus()); + Assert.assertEquals(cacheType, 0, cache.getLoadScore()); + + // A stopped node can still enter removal. + cache.cacheHeartbeatSample(new NodeHeartbeatSample(NodeStatus.Stopped)); + cache.updateNodeStatistics(); + Assert.assertEquals(cacheType, NodeStatus.Stopped, cache.getNodeStatus()); + cache.cacheHeartbeatSample(new NodeHeartbeatSample(NodeStatus.Removing)); + cache.updateNodeStatistics(); + Assert.assertEquals(cacheType, NodeStatus.Removing, cache.getNodeStatus()); + + // Ordinary updates, including fresh Running, must not cancel removal. + for (NodeStatus observed : + new NodeStatus[] {NodeStatus.Stopped, NodeStatus.Unknown, NodeStatus.Running}) { + cache.cacheHeartbeatSample(new NodeHeartbeatSample(observed)); + cache.updateNodeStatistics(); + Assert.assertEquals(cacheType, NodeStatus.Removing, cache.getNodeStatus()); + Assert.assertEquals(cacheType, Long.MAX_VALUE, cache.getLoadScore()); + } + + // Management restores the requested state explicitly, including offline states. + for (NodeStatus restored : + new NodeStatus[] {NodeStatus.Unknown, NodeStatus.Stopped, NodeStatus.Running}) { + cache.trySetNodeStatus(NodeStatus.Removing, true); + cache.trySetNodeStatus(restored, true); + Assert.assertEquals(cacheType, restored, cache.getNodeStatus()); + cache.updateNodeStatistics(); + Assert.assertEquals(cacheType, restored, cache.getNodeStatus()); + } + } + } + + @Test + public void testConnectionFailureIsNotUndoneByPreviousRunningHeartbeat() { + for (BaseNodeCache cache : + new BaseNodeCache[] { + new DataNodeHeartbeatCache(1), + new ConfigNodeHeartbeatCache(ConfigNodeHeartbeatCache.CURRENT_NODE_ID + 1), + new AINodeHeartbeatCache(3) + }) { + cache.cacheHeartbeatSample(new NodeHeartbeatSample(NodeStatus.Running)); + cache.updateNodeStatistics(); + Assert.assertEquals(NodeStatus.Running, cache.getNodeStatus()); + + cache.trySetNodeStatus(NodeStatus.Unknown, false); + Assert.assertEquals(NodeStatus.Unknown, cache.getNodeStatus()); + Assert.assertEquals(Long.MAX_VALUE, cache.getLoadScore()); + cache.updateNodeStatistics(); + Assert.assertEquals(NodeStatus.Unknown, cache.getNodeStatus()); + } + } + + @Test + public void testExplicitStatusUpdateDoesNotRequireLiveHeartbeat() { + ConfigNodeConfig config = ConfigNodeDescriptor.getInstance().getConf(); + String detector = config.getFailureDetector(); + long timeout = config.getFailureDetectorFixedThresholdInMs(); + config.setFailureDetector(IFailureDetector.FIXED_DETECTOR); + config.setFailureDetectorFixedThresholdInMs(0); + try { + for (BaseNodeCache cache : + new BaseNodeCache[] { + new DataNodeHeartbeatCache(1), + new ConfigNodeHeartbeatCache(ConfigNodeHeartbeatCache.CURRENT_NODE_ID + 1), + new AINodeHeartbeatCache(3) + }) { + cache.cacheHeartbeatSample(new NodeHeartbeatSample(0, NodeStatus.Running)); + cache.updateNodeStatistics(); + Assert.assertEquals(NodeStatus.Unknown, cache.getNodeStatus()); + + // A shutdown report is known information even when heartbeat detection says unavailable. + cache.trySetNodeStatus(NodeStatus.Stopped, false); + Assert.assertEquals(NodeStatus.Stopped, cache.getNodeStatus()); + Assert.assertEquals(Long.MAX_VALUE, cache.getLoadScore()); + cache.updateNodeStatistics(); + Assert.assertEquals(NodeStatus.Stopped, cache.getNodeStatus()); + + cache.trySetNodeStatus(NodeStatus.Unknown, false); + Assert.assertEquals(NodeStatus.Stopped, cache.getNodeStatus()); + cache.trySetNodeStatus(NodeStatus.Unknown, true); + Assert.assertEquals(NodeStatus.Unknown, cache.getNodeStatus()); + } + } finally { + config.setFailureDetector(detector); + config.setFailureDetectorFixedThresholdInMs(timeout); + } + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinatorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinatorTest.java new file mode 100644 index 0000000000000..f3c0aab1a767c --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinatorTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.subscription.runtime; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.ProcedureManager; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.load.cache.node.NodeStatistics; +import org.apache.iotdb.confignode.manager.load.subscriber.NodeStatisticsChangeEvent; +import org.apache.iotdb.confignode.manager.subscription.SubscriptionCoordinator; +import org.apache.iotdb.confignode.manager.subscription.SubscriptionManager; +import org.apache.iotdb.confignode.persistence.subscription.SubscriptionInfo; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; + +import org.apache.tsfile.utils.Pair; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class SubscriptionRuntimeCoordinatorTest { + + private static final int DATA_NODE_ID = 1; + + private final ConfigManager configManager = mock(ConfigManager.class); + private final SubscriptionManager subscriptionManager = mock(SubscriptionManager.class); + private final SubscriptionCoordinator subscriptionCoordinator = + mock(SubscriptionCoordinator.class); + private final SubscriptionInfo subscriptionInfo = mock(SubscriptionInfo.class); + private final TopicMeta topicMeta = mock(TopicMeta.class); + private final TopicConfig topicConfig = mock(TopicConfig.class); + private final LoadManager loadManager = mock(LoadManager.class); + private final ProcedureManager procedureManager = mock(ProcedureManager.class); + + private final SubscriptionRuntimeCoordinator coordinator = + new SubscriptionRuntimeCoordinator(configManager); + + private void stubConsensusBasedTopic() { + when(configManager.getSubscriptionManager()).thenReturn(subscriptionManager); + when(subscriptionManager.getSubscriptionCoordinator()).thenReturn(subscriptionCoordinator); + when(subscriptionCoordinator.getSubscriptionInfo()).thenReturn(subscriptionInfo); + when(subscriptionInfo.getAllTopicMeta()).thenReturn(Collections.singletonList(topicMeta)); + when(topicMeta.getConfig()).thenReturn(topicConfig); + when(topicConfig.isIncrementalMode()).thenReturn(true); + + when(configManager.getLoadManager()).thenReturn(loadManager); + // A seeded DataRegion leader pair makes the refresh map non-empty. + when(loadManager.getRegionLeaderMap()) + .thenReturn( + Collections.singletonMap( + new TConsensusGroupId(TConsensusGroupType.DataRegion, 1), DATA_NODE_ID)); + when(configManager.getProcedureManager()).thenReturn(procedureManager); + } + + private NodeStatisticsChangeEvent eventOf( + final NodeStatus oldStatus, final NodeStatus newStatus) { + final Map> map = new HashMap<>(); + map.put(DATA_NODE_ID, new Pair<>(new NodeStatistics(oldStatus), new NodeStatistics(newStatus))); + return new NodeStatisticsChangeEvent(map); + } + + @Test + public void testStoppedStatusTriggersRuntimeRefresh() { + stubConsensusBasedTopic(); + + coordinator.handleNodeStatisticsChange(eventOf(NodeStatus.Running, NodeStatus.Stopped)); + + // A Stopped node is handled like Unknown/Removing: its runtime leader pairs are refreshed. + verify(procedureManager).subscriptionHandleLeaderChange(any(Map.class), anyLong()); + } + + @Test + public void testUnknownStatusTriggersRuntimeRefresh() { + stubConsensusBasedTopic(); + + coordinator.handleNodeStatisticsChange(eventOf(NodeStatus.Running, NodeStatus.Unknown)); + + verify(procedureManager).subscriptionHandleLeaderChange(any(Map.class), anyLong()); + } + + @Test + public void testReadOnlyStatusDoesNotTriggerRuntimeRefresh() { + stubConsensusBasedTopic(); + + coordinator.handleNodeStatisticsChange(eventOf(NodeStatus.Running, NodeStatus.ReadOnly)); + + // ReadOnly is not runtime-sensitive: the runtime leader pairs stay untouched. + verify(procedureManager, never()).subscriptionHandleLeaderChange(any(Map.class), anyLong()); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java index e3ced5b069428..159b4c8c306fb 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java @@ -19,15 +19,27 @@ package org.apache.iotdb.confignode.persistence; +import org.apache.iotdb.common.rpc.thrift.TAINodeConfiguration; +import org.apache.iotdb.common.rpc.thrift.TAINodeLocation; import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TNodeResource; +import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.exception.StartupException; +import org.apache.iotdb.confignode.consensus.request.write.ainode.RegisterAINodePlan; +import org.apache.iotdb.confignode.consensus.request.write.ainode.RemoveAINodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan.Operation; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateVersionInfoPlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; +import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan; import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.TException; import org.apache.tsfile.external.commons.io.FileUtils; @@ -38,6 +50,9 @@ import java.io.File; import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.Collections; +import java.util.Map; import static org.apache.iotdb.db.utils.constant.TestConstant.BASE_OUTPUT_PATH; @@ -66,11 +81,118 @@ public static void cleanup() throws IOException { public void testSnapshot() throws TException, IOException { registerConfigNodes(); registerDataNodes(); + nodeInfo.registerAINode(new RegisterAINodePlan(generateTAINodeConfiguration())); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10001, Operation.SET_STOPPED)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_REMOVING)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, Operation.SET_STOPPED)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10006, Operation.SET_REMOVING)); Assert.assertTrue(nodeInfo.processTakeSnapshot(snapshotDir)); NodeInfo nodeInfo1 = new NodeInfo(); nodeInfo1.processLoadSnapshot(snapshotDir); Assert.assertEquals(nodeInfo, nodeInfo1); + Assert.assertEquals(NodeStatus.Stopped, nodeInfo1.getPersistedNodeStatus(10001)); + Assert.assertEquals(NodeStatus.Removing, nodeInfo1.getPersistedNodeStatus(10003)); + Assert.assertEquals(NodeStatus.Stopped, nodeInfo1.getPersistedNodeStatus(10004)); + Assert.assertEquals(NodeStatus.Removing, nodeInfo1.getPersistedNodeStatus(10006)); + } + + @Test + public void testLoadSnapshotWithoutNodeStatuses() throws TException, IOException { + NodeInfo oldNodeInfo = new NodeInfo(); + oldNodeInfo.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); + oldNodeInfo.updateVersionInfo( + new UpdateVersionInfoPlan(new TNodeVersionInfo("2.0", "build"), 10003)); + File oldSnapshotDir = new File(snapshotDir, "without-node-statuses"); + Assert.assertTrue(oldSnapshotDir.mkdirs()); + Assert.assertTrue(oldNodeInfo.processTakeSnapshot(oldSnapshotDir)); + // The previous format ends immediately before the new, empty node-status map size. + try (RandomAccessFile snapshot = + new RandomAccessFile(new File(oldSnapshotDir, "node_info.bin"), "rw")) { + snapshot.setLength(snapshot.length() - Integer.BYTES); + } + + NodeInfo restored = new NodeInfo(); + restored.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); + restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_REMOVING)); + restored.processLoadSnapshot(oldSnapshotDir); + Assert.assertEquals(oldNodeInfo, restored); + Assert.assertTrue(restored.getPersistedNodeStatuses().isEmpty()); + Assert.assertEquals(new TNodeVersionInfo("2.0", "build"), restored.getVersionInfo(10003)); + } + + @Test + public void testSetOperationsReplacePersistedStatus() { + NodeInfo info = new NodeInfo(); + info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_STOPPED)); + Assert.assertEquals(NodeStatus.Stopped, info.getPersistedNodeStatus(10003)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_REMOVING)); + Assert.assertEquals(NodeStatus.Removing, info.getPersistedNodeStatus(10003)); + Map snapshot = info.getPersistedNodeStatuses(); + snapshot.clear(); + Assert.assertEquals(NodeStatus.Removing, info.getPersistedNodeStatus(10003)); + + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_STOPPED)); + Assert.assertEquals(NodeStatus.Stopped, info.getPersistedNodeStatus(10003)); + info.clear(); + Assert.assertTrue(info.getPersistedNodeStatuses().isEmpty()); + } + + @Test + public void testClearRemovesEitherPersistedStatusAndAllowsRetry() { + NodeInfo info = new NodeInfo(); + info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); + for (Operation operation : new Operation[] {Operation.SET_STOPPED, Operation.SET_REMOVING}) { + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, operation)); + Assert.assertNotNull(info.getPersistedNodeStatus(10003)); + + UpdateNodeStatusPlan clear = new UpdateNodeStatusPlan(10003, Operation.CLEAR); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), info.applyNodeStatusPlan(clear).getCode()); + Assert.assertNull(info.getPersistedNodeStatus(10003)); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), info.applyNodeStatusPlan(clear).getCode()); + Assert.assertNull(info.getPersistedNodeStatus(10003)); + } + } + + @Test + public void testNodeRemovalDiscardsStatusAndIgnoresDelayedUpdates() { + NodeInfo info = new NodeInfo(); + TConfigNodeLocation configNode = + new TConfigNodeLocation( + 10001, new TEndPoint("127.0.0.1", 22201), new TEndPoint("127.0.0.1", 22301)); + info.applyConfigNode(new ApplyConfigNodePlan(configNode)); + info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10001, Operation.SET_STOPPED)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_REMOVING)); + TAINodeConfiguration aiNode = generateTAINodeConfiguration(); + info.registerAINode(new RegisterAINodePlan(aiNode)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10006, Operation.SET_REMOVING)); + + info.removeConfigNode(new RemoveConfigNodePlan(configNode)); + info.removeDataNode( + new RemoveDataNodePlan(Collections.singletonList(generateTDataNodeLocation(3)))); + info.removeAINode(new RemoveAINodePlan(aiNode.getLocation())); + Assert.assertTrue(info.getPersistedNodeStatuses().isEmpty()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10001, Operation.SET_STOPPED)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_REMOVING)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10006, Operation.SET_STOPPED)).getCode()); + Assert.assertTrue(info.getPersistedNodeStatuses().isEmpty()); + } + + private static TAINodeConfiguration generateTAINodeConfiguration() { + return new TAINodeConfiguration( + new TAINodeLocation(10006, new TEndPoint("127.0.0.1", 10810)), new TNodeResource(8, 1024L)); } private void registerConfigNodes() { @@ -85,6 +207,60 @@ private void registerConfigNodes() { } } + @Test + public void testTruncatedStatusPayloadMustFailSnapshotLoad() throws Exception { + NodeInfo info = new NodeInfo(); + info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_REMOVING)); + File directory = new File(snapshotDir, "truncated-status-payload"); + Assert.assertTrue(directory.mkdirs()); + Assert.assertTrue(info.processTakeSnapshot(directory)); + try (RandomAccessFile file = new RandomAccessFile(new File(directory, "node_info.bin"), "rw")) { + file.setLength(file.length() - 2); + } + Assert.assertThrows(IOException.class, () -> new NodeInfo().processLoadSnapshot(directory)); + } + + @Test + public void testClearedStatusesStayClearedAfterSnapshotAndLaterPlans() throws Exception { + NodeInfo info = new NodeInfo(); + info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); + info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(4))); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_STOPPED)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, Operation.SET_REMOVING)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.CLEAR)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, Operation.CLEAR)); + File directory = new File(snapshotDir, "cleared-statuses"); + Assert.assertTrue(directory.mkdirs()); + Assert.assertTrue(info.processTakeSnapshot(directory)); + NodeInfo restored = new NodeInfo(); + restored.processLoadSnapshot(directory); + Assert.assertTrue(restored.getPersistedNodeStatuses().isEmpty()); + // Replay the entries after that snapshot in their original order. + restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_STOPPED)); + restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, Operation.SET_REMOVING)); + restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.CLEAR)); + Assert.assertNull(restored.getPersistedNodeStatus(10003)); + Assert.assertEquals(NodeStatus.Removing, restored.getPersistedNodeStatus(10004)); + } + + @Test + public void testRejoinedNodeDoesNotInheritRemovedNodeStatus() { + NodeInfo info = new NodeInfo(); + info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_REMOVING)); + info.removeDataNode( + new RemoveDataNodePlan(Collections.singletonList(generateTDataNodeLocation(3)))); + int newId = info.generateNextNodeId(); + TDataNodeConfiguration rejoined = generateTDataNodeConfiguration(3); + rejoined.getLocation().setDataNodeId(newId); + info.registerDataNode(new RegisterDataNodePlan(rejoined)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_STOPPED)); + Assert.assertNotEquals(10003, newId); + Assert.assertNull(info.getPersistedNodeStatus(newId)); + Assert.assertNull(info.getPersistedNodeStatus(10003)); + } + private void registerDataNodes() { for (int i = 3; i < 6; i++) { RegisterDataNodePlan registerDataNodePlan = diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnvTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnvTest.java new file mode 100644 index 0000000000000..7691eb45848ee --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnvTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.env; + +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ConfigNodeProcedureEnvTest { + + private static final int DATA_NODE_ID = 1; + + private final ConfigManager configManager = mock(ConfigManager.class); + private final LoadManager loadManager = mock(LoadManager.class); + private final ConfigNodeProcedureEnv env = + new ConfigNodeProcedureEnv(configManager, mock(ProcedureScheduler.class)); + + @Before + public void setUp() { + when(configManager.getLoadManager()).thenReturn(loadManager); + } + + @Test + public void testStoppedNodeIsNotRuntimeActiveWriter() { + when(loadManager.getNodeStatus(DATA_NODE_ID)).thenReturn(NodeStatus.Stopped); + + // A Stopped node is handled like Unknown: it can not serve as an active runtime writer. + Assert.assertFalse(env.isRuntimeActiveWriterNode(DATA_NODE_ID)); + } + + @Test + public void testUnknownNodeIsNotRuntimeActiveWriter() { + when(loadManager.getNodeStatus(DATA_NODE_ID)).thenReturn(NodeStatus.Unknown); + + Assert.assertFalse(env.isRuntimeActiveWriterNode(DATA_NODE_ID)); + } + + @Test + public void testRemovingNodeIsNotRuntimeActiveWriter() { + when(loadManager.getNodeStatus(DATA_NODE_ID)).thenReturn(NodeStatus.Removing); + + Assert.assertFalse(env.isRuntimeActiveWriterNode(DATA_NODE_ID)); + } + + @Test + public void testRunningNodeIsRuntimeActiveWriter() { + when(loadManager.getNodeStatus(DATA_NODE_ID)).thenReturn(NodeStatus.Running); + + Assert.assertTrue(env.isRuntimeActiveWriterNode(DATA_NODE_ID)); + } + + @Test + public void testReadOnlyNodeIsRuntimeActiveWriter() { + when(loadManager.getNodeStatus(DATA_NODE_ID)).thenReturn(NodeStatus.ReadOnly); + + // ReadOnly still responds and can serve as a runtime writer, unlike Unknown/Stopped/Removing. + Assert.assertTrue(env.isRuntimeActiveWriterNode(DATA_NODE_ID)); + } + + @Test + public void testNegativeNodeIdIsNotRuntimeActiveWriter() { + // The node id guard dominates the status check: even a Running status can not turn an + // invalid node id into an active writer. + when(loadManager.getNodeStatus(-1)).thenReturn(NodeStatus.Running); + + Assert.assertFalse(env.isRuntimeActiveWriterNode(-1)); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandlerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandlerTest.java index 8fd32922398d7..b94c96259687d 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandlerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandlerTest.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.client.sync.CnToDnSyncRequestType; import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.manager.ConfigManager; @@ -39,6 +40,7 @@ import org.apache.iotdb.confignode.manager.partition.PartitionManager; import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeReq; import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeResp; +import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.After; @@ -58,8 +60,12 @@ import static org.apache.iotdb.consensus.ConsensusFactory.RATIS_CONSENSUS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -232,4 +238,66 @@ private void assertLeaderTransfer() throws Exception { clientManagerField.set(clientPool, originalClientManager); } } + + private RegionMaintainHandler stubStatusAndSubmit( + final NodeStatus status, final TSStatus submitResult) { + RegionMaintainHandler retryHandler = spy(handler); + doReturn(status).when(retryHandler).getDataNodeStatus(original.getDataNodeId()); + doReturn(submitResult) + .when(retryHandler) + .submitDataNodeSyncRequest( + any(TEndPoint.class), + any(Object.class), + any(CnToDnSyncRequestType.class), + any(Boolean.class)); + return retryHandler; + } + + @Test + public void testDeleteOldRegionPeerUsesSingleRetryForStoppedDataNode() { + final TSStatus success = RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + final RegionMaintainHandler retryHandler = stubStatusAndSubmit(NodeStatus.Stopped, success); + + final TSStatus status = retryHandler.submitDeleteOldRegionPeerTask(1L, original, regionId); + + assertEquals(success.getCode(), status.getCode()); + // A Stopped node does not respond to requests either: like Unknown, it gets a single retry. + verify(retryHandler) + .submitDataNodeSyncRequest( + any(TEndPoint.class), + any(Object.class), + eq(CnToDnSyncRequestType.DELETE_OLD_REGION_PEER), + eq(false)); + } + + @Test + public void testDeleteOldRegionPeerUsesFullRetryForRunningDataNode() { + final TSStatus success = RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + final RegionMaintainHandler retryHandler = stubStatusAndSubmit(NodeStatus.Running, success); + + retryHandler.submitDeleteOldRegionPeerTask(1L, original, regionId); + + verify(retryHandler) + .submitDataNodeSyncRequest( + any(TEndPoint.class), + any(Object.class), + eq(CnToDnSyncRequestType.DELETE_OLD_REGION_PEER), + eq(true)); + } + + @Test + public void testDeleteOldRegionPeerUsesFullRetryForReadOnlyDataNode() { + final TSStatus success = RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + final RegionMaintainHandler retryHandler = stubStatusAndSubmit(NodeStatus.ReadOnly, success); + + retryHandler.submitDeleteOldRegionPeerTask(1L, original, regionId); + + // Only Unknown and Stopped are treated as down; ReadOnly still responds. + verify(retryHandler) + .submitDataNodeSyncRequest( + any(TEndPoint.class), + any(Object.class), + eq(CnToDnSyncRequestType.DELETE_OLD_REGION_PEER), + eq(true)); + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java new file mode 100644 index 0000000000000..2e294aae894cc --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java @@ -0,0 +1,572 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.confignode.procedure.impl.node; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.async.AsyncDataNodeInternalServiceClient; +import org.apache.iotdb.commons.client.request.AsyncRequestManager; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan.Operation; +import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; +import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.load.cache.LoadCache; +import org.apache.iotdb.confignode.manager.node.NodeManager; +import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.env.RemoveDataNodeHandler; +import org.apache.iotdb.confignode.procedure.state.RemoveDataNodeState; +import org.apache.iotdb.consensus.exception.ConsensusException; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.thrift.async.AsyncMethodCallback; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +/** Real procedure/handler/cache/NodeInfo, with faults only at the RPC and consensus boundaries. */ +public class RemoveDataNodePersistenceTest { + private static final int FIRST = 100; + private static final int SECOND = 101; + private final Map remoteStatuses = new HashMap<>(); + private final Map failRpc = new HashMap<>(); + private final Map failStop = new HashMap<>(); + private NodeInfo nodeInfo; + private LoadCache cache; + private ConfigManager manager; + private ConsensusManager consensus; + private RemoveDataNodeHandler handler; + private ConfigNodeProcedureEnv env; + private Field clientManagerField; + private Object originalClientManager; + private List nodes; + private boolean failStatusWrite; + private boolean applyRpcBeforeFailure; + private boolean failDeleteWrite; + private boolean throwDeleteWrite; + private int stopRequests; + + @Before + public void setUp() throws Exception { + nodes = Arrays.asList(location(FIRST), location(SECOND)); + nodeInfo = new NodeInfo(); + manager = mock(ConfigManager.class, RETURNS_DEEP_STUBS); + consensus = mock(ConsensusManager.class); + when(manager.getConsensusManager()).thenReturn(consensus); + when(consensus.isLeader()).thenReturn(true); + when(consensus.write(any(RemoveDataNodePlan.class))) + .thenAnswer( + i -> { + if (throwDeleteWrite) { + throw new ConsensusException("injected delete failure"); + } + return failDeleteWrite ? failure() : nodeInfo.removeDataNode(i.getArgument(0)); + }); + for (TDataNodeLocation node : nodes) { + nodeInfo.registerDataNode( + new RegisterDataNodePlan(new TDataNodeConfiguration().setLocation(node))); + remoteStatuses.put(node.getDataNodeId(), NodeStatus.Running); + } + when(manager.getNodeManager()).thenReturn(new NodeManager(manager, nodeInfo)); + when(manager.getClusterSchemaManager().getDatabaseNames(null)) + .thenReturn(Collections.emptyList()); + when(manager.getPartitionManager().getAllReplicaSets()).thenReturn(Collections.emptyList()); + when(manager.getPartitionManager().getAllReplicaSets(anyInt())) + .thenReturn(Collections.emptyList()); + cache = new LoadCache(); + cache.setNodeStatusPersistence( + nodeInfo::getPersistedNodeStatus, + plan -> failStatusWrite ? failure() : nodeInfo.applyNodeStatusPlan(plan)); + cache.initHeartbeatCache(manager); + for (TDataNodeLocation node : nodes) { + assertSuccess(cache.trySetNodeStatus(node.getDataNodeId(), NodeStatus.Running, true)); + } + LoadManager loadManager = mock(LoadManager.class); + when(manager.getLoadManager()).thenReturn(loadManager); + when(loadManager.trySetNodeStatus(anyInt(), any(), anyBoolean())) + .thenAnswer( + i -> cache.trySetNodeStatus(i.getArgument(0), i.getArgument(1), i.getArgument(2))); + when(loadManager.getNodeStatus(anyInt())) + .thenAnswer(i -> cache.getNodeStatus(i.getArgument(0))); + doAnswer( + i -> { + cache.removeNodeCache(i.getArgument(0)); + return null; + }) + .when(loadManager) + .removeNodeCache(anyInt()); + + // Keep the real request context, handler and retry loop. Restore the singleton after each test. + IClientManager transport = + mock(IClientManager.class); + for (TDataNodeLocation node : nodes) { + int id = node.getDataNodeId(); + AsyncDataNodeInternalServiceClient client = mock(AsyncDataNodeInternalServiceClient.class); + when(transport.borrowClient(node.getInternalEndPoint())).thenReturn(client); + doAnswer( + i -> { + AsyncMethodCallback callback = i.getArgument(1); + if (failRpc.getOrDefault(id, false)) { + if (applyRpcBeforeFailure) { + remoteStatuses.put(id, NodeStatus.parse(i.getArgument(0))); + } + callback.onComplete(failure()); + } else { + remoteStatuses.put(id, NodeStatus.parse(i.getArgument(0))); + callback.onComplete(success()); + } + return null; + }) + .when(client) + .setSystemStatus(anyString(), any()); + doAnswer( + i -> { + stopRequests++; + AsyncMethodCallback callback = i.getArgument(0); + callback.onComplete(failStop.getOrDefault(id, false) ? failure() : success()); + return null; + }) + .when(client) + .stopAndClearDataNode(any()); + } + clientManagerField = AsyncRequestManager.class.getDeclaredField("clientManager"); + clientManagerField.setAccessible(true); + originalClientManager = + clientManagerField.get(CnToDnInternalServiceAsyncRequestManager.getInstance()); + clientManagerField.set(CnToDnInternalServiceAsyncRequestManager.getInstance(), transport); + handler = spy(new RemoveDataNodeHandler(manager)); + // Region migration and broadcast are independent of the status persistence under test. + doReturn(Collections.emptyList()).when(handler).selectedRegionMigrationPlans(anyList()); + doNothing().when(handler).broadcastDataNodeStatusChange(anyList()); + env = mock(ConfigNodeProcedureEnv.class); + when(env.getConfigManager()).thenReturn(manager); + when(env.getRemoveDataNodeHandler()).thenReturn(handler); + } + + @After + public void tearDown() throws Exception { + if (clientManagerField != null && originalClientManager != null) { + clientManagerField.set( + CnToDnInternalServiceAsyncRequestManager.getInstance(), originalClientManager); + } + } + + @Test + public void testPreparePersistsRemoving() throws Exception { + change(FIRST, NodeStatus.Removing); + assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, NodeStatus.Removing); + } + + @Test + public void testPrepareRpcFailureStillPersistsRemoving() throws Exception { + failRpc.put(FIRST, true); + TestProcedure procedure = + procedure(NodeStatus.Running, RemoveDataNodeState.REMOVE_DATA_NODE_PREPARE); + procedure.step(env); + assertEquals(RemoveDataNodeState.BROADCAST_DISABLE_DATA_NODE, procedure.state()); + assertStates(FIRST, NodeStatus.Running, NodeStatus.Removing, NodeStatus.Removing); + } + + @Test + public void testOfflineRemovalAndRetryDoNotRequireRpcSuccess() throws Exception { + for (NodeStatus offline : Arrays.asList(NodeStatus.Unknown, NodeStatus.Stopped)) { + assertSuccess(cache.trySetNodeStatus(FIRST, offline, true)); + remoteStatuses.put(FIRST, offline); + failRpc.put(FIRST, true); + change(FIRST, NodeStatus.Removing); + assertStates(FIRST, offline, NodeStatus.Removing, NodeStatus.Removing); + change(FIRST, NodeStatus.Removing); + assertStates(FIRST, offline, NodeStatus.Removing, NodeStatus.Removing); + } + } + + @Test + public void testOfflineRemovalStillRequiresPersistence() throws Exception { + assertSuccess(cache.trySetNodeStatus(FIRST, NodeStatus.Stopped, true)); + failRpc.put(FIRST, true); + failStatusWrite = true; + expectChangeFailure(NodeStatus.Removing); + assertEquals(NodeStatus.Removing, cache.getNodeStatus(FIRST)); + assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(FIRST)); + failStatusWrite = false; + change(FIRST, NodeStatus.Removing); + assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(FIRST)); + } + + @Test + public void testOfflineCompensationDoesNotRequireRpcSuccess() throws Exception { + retainRegionOn(FIRST); + for (NodeStatus offline : Arrays.asList(NodeStatus.Unknown, NodeStatus.Stopped)) { + failRpc.put(FIRST, false); + change(FIRST, NodeStatus.Removing); + failRpc.put(FIRST, true); + assertTrue(procedure(offline, RemoveDataNodeState.STOP_DATA_NODE).finish(env)); + assertStates( + FIRST, NodeStatus.Removing, offline, offline == NodeStatus.Stopped ? offline : null); + } + } + + @Test + public void testInitialPrepareCommitFailureRetriesWithoutRecheckingCapacity() throws Exception { + for (int index = nodes.size(); index <= NodeInfo.getMinimumDataNode(); index++) { + nodeInfo.registerDataNode( + new RegisterDataNodePlan( + new TDataNodeConfiguration().setLocation(location(FIRST + index)))); + } + cache.initHeartbeatCache(manager); + for (TDataNodeConfiguration node : nodeInfo.getRegisteredDataNodes()) { + assertSuccess( + cache.trySetNodeStatus(node.getLocation().getDataNodeId(), NodeStatus.Running, true)); + } + when(manager + .getLoadManager() + .filterDataNodeThroughStatus(NodeStatus.Running, NodeStatus.ReadOnly)) + .thenAnswer( + i -> cache.filterDataNodeThroughStatus(NodeStatus.Running, NodeStatus.ReadOnly)); + + TestProcedure procedure = + procedure(NodeStatus.Running, RemoveDataNodeState.REGION_REPLICA_CHECK); + failStatusWrite = true; + procedure.step(env); + procedure.step(env); + assertFalse(procedure.isFailed()); + assertEquals(RemoveDataNodeState.REMOVE_DATA_NODE_PREPARE, procedure.state()); + assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, null); + + failStatusWrite = false; + procedure.step(env); + assertEquals(RemoveDataNodeState.BROADCAST_DISABLE_DATA_NODE, procedure.state()); + assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, NodeStatus.Removing); + } + + @Test + public void testPrepareCommitFailureUpdatesCacheAndRetries() throws Exception { + failStatusWrite = true; + TestProcedure procedure = + procedure(NodeStatus.Running, RemoveDataNodeState.REMOVE_DATA_NODE_PREPARE); + procedure.step(env); + assertFalse(procedure.isFailed()); + assertEquals(RemoveDataNodeState.REMOVE_DATA_NODE_PREPARE, procedure.state()); + assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, null); + failStatusWrite = false; + procedure.step(env); + assertEquals(RemoveDataNodeState.BROADCAST_DISABLE_DATA_NODE, procedure.state()); + assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, NodeStatus.Removing); + } + + @Test + public void testPrepareCommitFailureStopsAfterRetryThreshold() throws Exception { + failStatusWrite = true; + TestProcedure procedure = + procedure(NodeStatus.Running, RemoveDataNodeState.REMOVE_DATA_NODE_PREPARE); + for (int attempt = 0; attempt < 5; attempt++) { + procedure.step(env); + assertFalse(procedure.isFailed()); + assertEquals(RemoveDataNodeState.REMOVE_DATA_NODE_PREPARE, procedure.state()); + } + procedure.step(env); + assertTrue(procedure.isFailed()); + assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, null); + } + + @Test + public void testAppliedRpcWithLostResponseMustStillPersistIntent() throws Exception { + applyRpcBeforeFailure = true; + failRpc.put(FIRST, true); + TestProcedure procedure = + procedure(NodeStatus.Running, RemoveDataNodeState.REMOVE_DATA_NODE_PREPARE); + procedure.step(env); + assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, NodeStatus.Removing); + assertEquals(RemoveDataNodeState.BROADCAST_DISABLE_DATA_NODE, procedure.state()); + } + + @Test + public void testPreparePartialRpcFailurePreservesPerNodeResults() throws Exception { + failRpc.put(SECOND, true); + Map desired = new HashMap<>(); + desired.put(FIRST, NodeStatus.Removing); + desired.put(SECOND, NodeStatus.Removing); + handler.changeDataNodeStatus(nodes, desired); + assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, NodeStatus.Removing); + assertStates(SECOND, NodeStatus.Running, NodeStatus.Removing, NodeStatus.Removing); + } + + @Test + public void testRpcFailureOnRetryStillRequiresPersistence() throws Exception { + failStatusWrite = true; + expectChangeFailure(NodeStatus.Removing); + failRpc.put(FIRST, true); + expectChangeFailure(NodeStatus.Removing); + assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, null); + failStatusWrite = false; + change(FIRST, NodeStatus.Removing); + assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, NodeStatus.Removing); + } + + @Test + public void testCompensationRestoresRunning() throws Exception { + assertCompensation(NodeStatus.Running); + } + + @Test + public void testCompensationRestoresReadOnly() throws Exception { + assertCompensation(NodeStatus.ReadOnly); + } + + @Test + public void testCompensationRestoresUnknown() throws Exception { + assertCompensation(NodeStatus.Unknown); + } + + @Test + public void testCompensationRestoresStopped() throws Exception { + assertCompensation(NodeStatus.Stopped); + } + + @Test + public void testCompensationRpcFailureMustNotFinishProcedure() throws Exception { + change(FIRST, NodeStatus.Removing); + retainRegionOn(FIRST); + failRpc.put(FIRST, true); + for (NodeStatus original : Arrays.asList(NodeStatus.Running, NodeStatus.ReadOnly)) { + assertFalse( + "Failed compensation must not finish", + procedure(original, RemoveDataNodeState.STOP_DATA_NODE).finish(env)); + assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, NodeStatus.Removing); + } + } + + @Test + public void testCompensationCommitFailureRemainsRetryable() throws Exception { + change(FIRST, NodeStatus.Removing); + retainRegionOn(FIRST); + failStatusWrite = true; + TestProcedure procedure = procedure(NodeStatus.Running, RemoveDataNodeState.STOP_DATA_NODE); + assertFalse(procedure.finish(env)); + assertStates(FIRST, NodeStatus.Running, NodeStatus.Running, NodeStatus.Removing); + failStatusWrite = false; + assertTrue(procedure.finish(env)); + assertStates(FIRST, NodeStatus.Running, NodeStatus.Running, null); + } + + @Test + public void testDeleteErrorMustNotStopOrForgetRegisteredNode() throws Exception { + failDeleteWrite = true; + assertDeleteFailure(); + } + + @Test + public void testProcedureExecutorCanActuallyRetryCompensationCommitFailure() throws Exception { + change(FIRST, NodeStatus.Removing); + retainRegionOn(FIRST); + failStatusWrite = true; + TestProcedure procedure = procedure(NodeStatus.Running, RemoveDataNodeState.STOP_DATA_NODE); + procedure.step(env); + assertEquals( + "The state-machine executor must retain the failed compensation step", + RemoveDataNodeState.STOP_DATA_NODE, + procedure.state()); + failStatusWrite = false; + procedure.step(env); + assertStates(FIRST, NodeStatus.Running, NodeStatus.Running, null); + } + + @Test + public void testDeleteExceptionMustNotStopOrForgetRegisteredNode() throws Exception { + throwDeleteWrite = true; + assertDeleteFailure(); + } + + @Test + public void testSuccessfulDeleteCannotBeResurrectedByLateStatus() throws Exception { + change(FIRST, NodeStatus.Removing); + assertTrue(procedure(NodeStatus.Running, RemoveDataNodeState.STOP_DATA_NODE).finish(env)); + assertFalse( + nodeInfo.getRegisteredDataNodes().stream() + .anyMatch(n -> n.getLocation().getDataNodeId() == FIRST)); + assertNull(nodeInfo.getPersistedNodeStatus(FIRST)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(FIRST, Operation.SET_REMOVING)); + assertNull(nodeInfo.getPersistedNodeStatus(FIRST)); + assertTrue(stopRequests > 0); + } + + @Test + public void testFailedRemoteStopCannotResurrectDeletedNode() throws Exception { + failStop.put(FIRST, true); + testSuccessfulDeleteCannotBeResurrectedByLateStatus(); + } + + @Test + public void testPartialRemovalKeepsSuccessfulAndCompensatedResultsSeparate() throws Exception { + change(FIRST, NodeStatus.Removing); + change(SECOND, NodeStatus.Removing); + retainRegionOn(SECOND); + Map originals = new HashMap<>(); + originals.put(FIRST, NodeStatus.Running); + originals.put(SECOND, NodeStatus.ReadOnly); + TestProcedure procedure = + new TestProcedure(nodes, originals, RemoveDataNodeState.STOP_DATA_NODE); + assertTrue(procedure.finish(env)); + assertNull(nodeInfo.getPersistedNodeStatus(FIRST)); + assertEquals(1, nodeInfo.getRegisteredDataNodes().size()); + assertStates(SECOND, NodeStatus.ReadOnly, NodeStatus.ReadOnly, null); + } + + private void assertDeleteFailure() throws Exception { + change(FIRST, NodeStatus.Removing); + boolean finished = + procedure(NodeStatus.Running, RemoveDataNodeState.STOP_DATA_NODE).finish(env); + assertEquals("No remote stop before durable unregister", 0, stopRequests); + assertFalse("Failed unregister must remain retryable", finished); + assertEquals(NodeStatus.Removing, cache.getNodeStatus(FIRST)); + assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(FIRST)); + assertEquals(2, nodeInfo.getRegisteredDataNodes().size()); + } + + private void assertCompensation(NodeStatus original) throws Exception { + change(FIRST, NodeStatus.Removing); + retainRegionOn(FIRST); + assertTrue( + "Compensation should complete for " + original, + procedure(original, RemoveDataNodeState.STOP_DATA_NODE).finish(env)); + assertStates(FIRST, original, original, original == NodeStatus.Stopped ? original : null); + } + + private void retainRegionOn(int id) { + when(manager.getPartitionManager().getAllReplicaSets()) + .thenReturn( + Collections.singletonList( + new TRegionReplicaSet( + new TConsensusGroupId(TConsensusGroupType.DataRegion, 1), + Collections.singletonList(location(id))))); + } + + private void change(int id, NodeStatus status) throws Exception { + handler.changeDataNodeStatus( + Collections.singletonList(location(id)), Collections.singletonMap(id, status)); + } + + private void expectChangeFailure(NodeStatus status) throws Exception { + try { + change(FIRST, status); + fail("Unconfirmed status transition must fail"); + } catch (IOException expected) { + assertNotNull(expected.getMessage()); + } + } + + private void assertStates(int id, NodeStatus remote, NodeStatus published, NodeStatus persisted) { + assertEquals("DataNode", remote, remoteStatuses.get(id)); + assertEquals("LoadCache", published, cache.getNodeStatus(id)); + assertEquals("NodeInfo", persisted, nodeInfo.getPersistedNodeStatus(id)); + } + + private TestProcedure procedure(NodeStatus original, RemoveDataNodeState initial) { + return new TestProcedure( + Collections.singletonList(location(FIRST)), + Collections.singletonMap(FIRST, original), + initial); + } + + private static TDataNodeLocation location(int id) { + return new TDataNodeLocation() + .setDataNodeId(id) + .setInternalEndPoint(new TEndPoint("127.0.0.1", 10000 + id)) + .setClientRpcEndPoint(new TEndPoint("127.0.0.1", 11000 + id)); + } + + private static TSStatus success() { + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } + + private static TSStatus failure() { + return new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + } + + private static void assertSuccess(TSStatus status) { + assertEquals(success().getCode(), status.getCode()); + } + + private static class TestProcedure extends RemoveDataNodesProcedure { + private final RemoveDataNodeState initial; + + private TestProcedure( + List nodes, + Map originals, + RemoveDataNodeState initial) { + super(nodes, originals); + this.initial = initial; + } + + @Override + protected RemoveDataNodeState getInitialState() { + return initial; + } + + private void step(ConfigNodeProcedureEnv env) throws InterruptedException { + execute(env); + } + + private RemoveDataNodeState state() { + return getCurrentState(); + } + + private boolean finish(ConfigNodeProcedureEnv env) { + return executeFromState(env, RemoveDataNodeState.STOP_DATA_NODE) == Flow.NO_MORE_STATE; + } + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtilsTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtilsTest.java new file mode 100644 index 0000000000000..005cd93211f43 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtilsTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.schema; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.node.NodeManager; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class SchemaUtilsTest { + + private static final int STOPPED_DATA_NODE_ID = 1; + private static final int UNKNOWN_DATA_NODE_ID = 2; + private static final int RUNNING_DATA_NODE_ID = 3; + + private static TDataNodeLocation locationOf(final int dataNodeId) { + return new TDataNodeLocation( + dataNodeId, + new TEndPoint("127.0.0.1", dataNodeId * 1000), + new TEndPoint("127.0.0.1", dataNodeId * 1000 + 1), + new TEndPoint("127.0.0.1", dataNodeId * 1000 + 2), + new TEndPoint("127.0.0.1", dataNodeId * 1000 + 3), + new TEndPoint("127.0.0.1", dataNodeId * 1000 + 4)); + } + + @Test + public void testFilterFencedDataNodeTreatsStoppedLikeUnknown() { + final ConfigManager configManager = mock(ConfigManager.class); + final NodeManager nodeManager = mock(NodeManager.class); + final LoadManager loadManager = mock(LoadManager.class); + when(configManager.getNodeManager()).thenReturn(nodeManager); + when(configManager.getLoadManager()).thenReturn(loadManager); + + final Map registered = new HashMap<>(); + registered.put(STOPPED_DATA_NODE_ID, locationOf(STOPPED_DATA_NODE_ID)); + registered.put(UNKNOWN_DATA_NODE_ID, locationOf(UNKNOWN_DATA_NODE_ID)); + registered.put(RUNNING_DATA_NODE_ID, locationOf(RUNNING_DATA_NODE_ID)); + when(nodeManager.getRegisteredDataNodeLocations()).thenReturn(registered); + when(loadManager.getNodeStatus(STOPPED_DATA_NODE_ID)).thenReturn(NodeStatus.Stopped); + when(loadManager.getNodeStatus(UNKNOWN_DATA_NODE_ID)).thenReturn(NodeStatus.Unknown); + when(loadManager.getNodeStatus(RUNNING_DATA_NODE_ID)).thenReturn(NodeStatus.Running); + + // A negative fence threshold makes every never-contacted DataNode read as fenced, so the + // status condition decides alone. + final long originalFenceMs = + ConfigNodeDescriptor.getInstance().getConf().getMetadataLeaseFenceMs(); + try { + ConfigNodeDescriptor.getInstance().getConf().setMetadataLeaseFenceMs(-100_000L); + + final Map filtered = + SchemaUtils.filterFencedDataNode(configManager); + + // An additionally fenced Stopped node is skipped, exactly like an additionally fenced + // Unknown node. A Running node is kept even when fenced. + Assert.assertFalse(filtered.containsKey(STOPPED_DATA_NODE_ID)); + Assert.assertFalse(filtered.containsKey(UNKNOWN_DATA_NODE_ID)); + Assert.assertTrue(filtered.containsKey(RUNNING_DATA_NODE_ID)); + } finally { + ConfigNodeDescriptor.getInstance().getConf().setMetadataLeaseFenceMs(originalFenceMs); + } + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHookTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHookTest.java new file mode 100644 index 0000000000000..3c89c01da2ac8 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHookTest.java @@ -0,0 +1,347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.service; + +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.confignode.client.CnToCnNodeRequestType; +import org.apache.iotdb.confignode.client.sync.SyncConfigNodeClientPool; +import org.apache.iotdb.confignode.conf.ConfigNodeConfig; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.node.NodeManager; +import org.apache.iotdb.consensus.IConsensus; +import org.apache.iotdb.consensus.common.Peer; +import org.apache.iotdb.consensus.exception.ConsensusException; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InOrder; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class ConfigNodeShutdownHookTest { + + private static final CnToCnNodeRequestType REPORT = + CnToCnNodeRequestType.REPORT_CONFIG_NODE_SHUTDOWN; + + private final ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf(); + private final CommonConfig commonConf = CommonDescriptor.getInstance().getConfig(); + private final ConfigNode configNode = mock(ConfigNode.class); + private final NodeManager nodeManager = mock(NodeManager.class); + private final LoadManager loadManager = mock(LoadManager.class); + private final ConsensusManager consensusManager = mock(ConsensusManager.class); + private final IConsensus consensus = mock(IConsensus.class); + private final SyncConfigNodeClientPool clientPool = mock(SyncConfigNodeClientPool.class); + private final TEndPoint seed = new TEndPoint("127.0.0.2", 10710); + private final TEndPoint peer = new TEndPoint("127.0.0.3", 10710); + private final List registeredNodes = new ArrayList<>(); + + private TConfigNodeLocation localNode; + private TEndPoint previousSeed; + private NodeStatus previousStatus; + private String previousReason; + private int previousConnectionTimeout; + + @Before + public void setUp() { + previousSeed = conf.getSeedConfigNode(); + previousStatus = commonConf.getNodeStatus(); + previousReason = commonConf.getStatusReason(); + previousConnectionTimeout = commonConf.getCnConnectionTimeoutInMS(); + commonConf.setCnConnectionTimeoutInMS(5000); + localNode = + new TConfigNodeLocation( + conf.getConfigNodeId(), + new TEndPoint(conf.getInternalAddress(), conf.getInternalPort()), + new TEndPoint(conf.getInternalAddress(), conf.getConsensusPort())); + conf.setSeedConfigNode(seed); + registeredNodes.add(localNode); + registeredNodes.add(new TConfigNodeLocation(100, peer, new TEndPoint("127.0.0.3", 10720))); + ConfigManager configManager = mock(ConfigManager.class); + when(configNode.getConfigManager()).thenReturn(configManager); + when(configManager.getNodeManager()).thenReturn(nodeManager); + when(configManager.getLoadManager()).thenReturn(loadManager); + when(configManager.getConsensusManager()).thenReturn(consensusManager); + when(consensusManager.getConsensusImpl()).thenReturn(consensus); + when(consensusManager.getConsensusGroupId()) + .thenReturn(ConsensusManager.DEFAULT_CONSENSUS_GROUP_ID); + when(nodeManager.getRegisteredConfigNodes()).thenReturn(registeredNodes); + when(loadManager.getNodeStatus(100)).thenReturn(NodeStatus.Running); + when(clientPool.sendSyncRequestToConfigNodeWithRetry(any(), any(), eq(REPORT))) + .thenReturn(new TSStatus(TSStatusCode.INTERNAL_REQUEST_RETRY_ERROR.getStatusCode())); + } + + @After + public void tearDown() { + conf.setSeedConfigNode(previousSeed); + commonConf.setNodeStatus(previousStatus); + commonConf.setStatusReason(previousReason); + commonConf.setCnConnectionTimeoutInMS(previousConnectionTimeout); + } + + @Test + public void testStoppingSeedReportsToPeerAfterDeactivation() throws Exception { + conf.setSeedConfigNode(localNode.getInternalEndPoint()); + when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT)) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + // Deactivation may make local metadata unavailable; the report must use its saved endpoints. + doAnswer( + invocation -> { + registeredNodes.clear(); + return null; + }) + .when(configNode) + .deactivate(); + + new ConfigNodeShutdownHook(configNode, clientPool).run(); + + InOrder order = inOrder(nodeManager, configNode, clientPool); + order.verify(nodeManager).getRegisteredConfigNodes(); + order.verify(configNode).deactivate(); + order.verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT); + verify(consensus, never()).transferLeader(any(), any()); + verifyNoMoreInteractions(clientPool); + } + + @Test + public void testLeaderTransfersBeforeDeactivation() throws Exception { + conf.setSeedConfigNode(localNode.getInternalEndPoint()); + when(consensusManager.isLeader()).thenReturn(true); + when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT)) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + + new ConfigNodeShutdownHook(configNode, clientPool).run(); + + InOrder order = inOrder(consensus, configNode, clientPool); + order + .verify(consensus) + .transferLeader( + ConsensusManager.DEFAULT_CONSENSUS_GROUP_ID, + new Peer( + ConsensusManager.DEFAULT_CONSENSUS_GROUP_ID, + 100, + registeredNodes.get(1).getConsensusEndPoint())); + order.verify(configNode).deactivate(); + order.verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT); + verify(nodeManager).getRegisteredConfigNodes(); + verifyNoMoreInteractions(consensus, clientPool); + } + + @Test + public void testTransferFailureStillDeactivatesAndReports() throws Exception { + conf.setSeedConfigNode(localNode.getInternalEndPoint()); + when(consensusManager.isLeader()).thenReturn(true); + doThrow(new ConsensusException("transfer timed out")) + .when(consensus) + .transferLeader(any(), any()); + when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT)) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + + new ConfigNodeShutdownHook(configNode, clientPool).run(); + + InOrder order = inOrder(consensus, configNode, clientPool); + order.verify(consensus).transferLeader(any(), any()); + order.verify(configNode).deactivate(); + order.verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT); + } + + @Test + public void testLeaderWithoutRunningPeerStillShutsDown() throws Exception { + conf.setSeedConfigNode(localNode.getInternalEndPoint()); + when(consensusManager.isLeader()).thenReturn(true); + when(loadManager.getNodeStatus(100)).thenReturn(NodeStatus.Unknown); + when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT)) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + + new ConfigNodeShutdownHook(configNode, clientPool).run(); + + verify(consensus, never()).transferLeader(any(), any()); + verify(configNode).deactivate(); + verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT); + } + + @Test + public void testReportRetriesUntilNewLeaderIsReady() { + conf.setSeedConfigNode(localNode.getInternalEndPoint()); + when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT)) + .thenReturn( + new TSStatus(TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP.getStatusCode()), + new TSStatus(TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP.getStatusCode()), + new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + + new ConfigNodeShutdownHook(configNode, clientPool).run(); + + verify(clientPool, times(3)).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT); + verifyNoMoreInteractions(clientPool); + } + + @Test + public void testUnavailableRegisteredNodeFallsBackToPeer() { + registeredNodes.add(1, new TConfigNodeLocation(101, seed, new TEndPoint("127.0.0.2", 10720))); + when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT)) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + + new ConfigNodeShutdownHook(configNode, clientPool).run(); + + verify(clientPool, times(2)).sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT); + verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT); + verifyNoMoreInteractions(clientPool); + } + + @Test + public void testFollowerReportsWithoutStatusCacheOrSeed() { + when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT)) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + + new ConfigNodeShutdownHook(configNode, clientPool).run(); + + verifyNoMoreInteractions(loadManager, consensus); + verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT); + verifyNoMoreInteractions(clientPool); + } + + @Test + public void testSeedIsUsedOnlyWhenNoRegisteredPeersExist() { + registeredNodes.remove(1); + when(clientPool.sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT)) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + + new ConfigNodeShutdownHook(configNode, clientPool).run(); + + verify(clientPool).sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT); + verifyNoMoreInteractions(clientPool); + } + + @Test + public void testSeedIsUsedBeforeConfigManagerInitialization() { + when(configNode.getConfigManager()).thenReturn(null); + when(clientPool.sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT)) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + + new ConfigNodeShutdownHook(configNode, clientPool).run(); + + verify(clientPool).sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT); + verifyNoMoreInteractions(clientPool); + } + + @Test + public void testLocalSeedWithoutPeersDoesNotReportToItself() { + registeredNodes.remove(1); + conf.setSeedConfigNode(localNode.getInternalEndPoint()); + + new ConfigNodeShutdownHook(configNode, clientPool).run(); + + verifyNoMoreInteractions(clientPool); + } + + @Test + public void testReportFollowsRedirect() { + when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT)) + .thenReturn( + new TSStatus(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()).setRedirectNode(seed)); + when(clientPool.sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT)) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + + new ConfigNodeShutdownHook(configNode, clientPool).run(); + + InOrder order = inOrder(clientPool); + order.verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT); + order.verify(clientPool).sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT); + verifyNoMoreInteractions(clientPool); + } + + @Test + public void testRedirectToStoppingNodeFallsBackToPeer() { + registeredNodes.add(1, new TConfigNodeLocation(101, seed, new TEndPoint("127.0.0.2", 10720))); + when(clientPool.sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT)) + .thenReturn( + new TSStatus(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) + .setRedirectNode(localNode.getInternalEndPoint())); + when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT)) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + + new ConfigNodeShutdownHook(configNode, clientPool).run(); + + verify(clientPool).sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT); + verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT); + verify(clientPool, never()) + .sendSyncRequestToConfigNodeWithRetry(localNode.getInternalEndPoint(), localNode, REPORT); + verifyNoMoreInteractions(clientPool); + } + + @Test + public void testReportStopsRetryingAfterDeadline() { + commonConf.setCnConnectionTimeoutInMS(100); + when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT)) + .thenAnswer( + invocation -> { + TimeUnit.MILLISECONDS.sleep(150); + return new TSStatus(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()); + }); + + new ConfigNodeShutdownHook(configNode, clientPool).run(); + + verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT); + verifyNoMoreInteractions(clientPool); + } + + @Test + public void testReportStopsRetryingWhenInterrupted() { + when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT)) + .thenAnswer( + invocation -> { + Thread.currentThread().interrupt(); + return new TSStatus(TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP.getStatusCode()); + }); + try { + new ConfigNodeShutdownHook(configNode, clientPool).run(); + + assertTrue(Thread.currentThread().isInterrupted()); + verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT); + verifyNoMoreInteractions(clientPool); + } finally { + Thread.interrupted(); + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/cluster/NodeStatusTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/cluster/NodeStatusTest.java new file mode 100644 index 0000000000000..241077eea1b93 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/cluster/NodeStatusTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.cluster; + +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.iotdb.commons.cluster.NodeStatus.ReadOnly; +import static org.apache.iotdb.commons.cluster.NodeStatus.Removing; +import static org.apache.iotdb.commons.cluster.NodeStatus.Running; +import static org.apache.iotdb.commons.cluster.NodeStatus.Stopped; +import static org.apache.iotdb.commons.cluster.NodeStatus.Unknown; + +public class NodeStatusTest { + + @Test + public void testOrdinaryTransitionTable() { + NodeStatus[] statuses = NodeStatus.values(); + + NodeStatus[][] expected = { + {Running, Unknown, Removing, ReadOnly, Stopped}, + {Running, Unknown, Removing, ReadOnly, Stopped}, + {Removing, Removing, Removing, Removing, Removing}, + {Running, Unknown, Removing, ReadOnly, Stopped}, + {Running, Stopped, Removing, ReadOnly, Stopped} + }; + + Assert.assertEquals(statuses.length, expected.length); + for (int previous = 0; previous < statuses.length; previous++) { + Assert.assertEquals(statuses.length, expected[previous].length); + for (int requested = 0; requested < statuses.length; requested++) { + Assert.assertSame( + statuses[previous] + " -> " + statuses[requested], + expected[previous][requested], + NodeStatus.transition(statuses[previous], statuses[requested], false)); + } + } + } + + @Test + public void testManagementCanRestoreEveryPreviousStatus() { + for (NodeStatus previous : NodeStatus.values()) { + for (NodeStatus requested : NodeStatus.values()) { + Assert.assertSame(requested, NodeStatus.transition(previous, requested, true)); + } + } + } + + @Test + public void testStatusClassificationTable() { + NodeStatus[] statuses = {Running, Unknown, Removing, ReadOnly, Stopped}; + // Columns: offline, possibly offline, Region candidate, readable, persistent. + boolean[][] expected = { + {false, false, true, true, false}, + {true, true, true, false, false}, + {false, true, false, true, true}, + {false, false, false, true, false}, + {true, true, true, false, true} + }; + + Assert.assertArrayEquals(NodeStatus.values(), statuses); + Assert.assertEquals(statuses.length, expected.length); + for (int i = 0; i < statuses.length; i++) { + NodeStatus status = statuses[i]; + Assert.assertEquals(status + " isOffline", expected[i][0], status.isOffline()); + Assert.assertEquals(status + " mayBeOffline", expected[i][1], status.mayBeOffline()); + Assert.assertEquals( + status + " isRegionCandidate", expected[i][2], status.isRegionCandidate()); + Assert.assertEquals(status + " isReadable", expected[i][3], NodeStatus.isReadable(status)); + Assert.assertEquals( + status + " isPersistentStatus", expected[i][4], status.isPersistentStatus()); + } + } +} From 19c558fb3a3e4d7f982fc3c86eb79ae5481fada6 Mon Sep 17 00:00:00 2001 From: Di Wang Date: Wed, 30 Sep 2026 16:48:33 +0800 Subject: [PATCH 3/5] feature: add ReadOnly status reasons Classify local ReadOnly reasons and propagate them through heartbeats and node display APIs. Keep reasons separate from status and append StatusReason to SHOW results when present. --- .../confignode/manager/ConfigManager.java | 19 ++++-- .../confignode/manager/load/LoadManager.java | 30 ++++++--- .../manager/load/cache/LoadCache.java | 27 +++----- .../load/cache/node/BaseNodeCache.java | 12 ++-- .../load/cache/node/NodeHeartbeatSample.java | 7 ++- .../confignode/manager/node/NodeManager.java | 21 ++++++- .../procedure/env/RemoveDataNodeHandler.java | 6 +- .../impl/DataNodeInternalRPCServiceImpl.java | 35 +++++++---- ...formationSchemaContentSupplierFactory.java | 31 ++++++--- .../executor/ClusterConfigTaskExecutor.java | 8 ++- .../metadata/ShowClusterDetailsTask.java | 63 +++++++++++++++++-- .../config/metadata/ShowClusterTask.java | 40 ++++++++++-- .../config/metadata/ShowConfigNodesTask.java | 28 +++++++-- .../config/metadata/ShowDataNodesTask.java | 30 ++++++--- .../db/service/DataNodeShutdownHook.java | 4 +- .../storageengine/dataregion/DataRegion.java | 6 +- .../dataregion/memtable/TsFileProcessor.java | 4 +- .../dataregion/wal/buffer/WALBuffer.java | 10 +-- .../wal/checkpoint/CheckpointManager.java | 4 +- .../iotdb/commons/conf/CommonConfig.java | 57 ++++++++++++++--- .../iotdb/commons/disk/FolderManager.java | 14 ++--- .../disk/strategy/DirectoryStrategy.java | 4 +- .../enums/HandleSystemErrorStrategy.java | 6 +- .../schema/column/ColumnHeaderConstant.java | 2 + .../schema/table/InformationSchema.java | 3 + .../src/main/thrift/confignode.thrift | 9 +++ 26 files changed, 358 insertions(+), 122 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index af55c953b777c..98c0a1bb9fccf 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -120,6 +120,7 @@ import org.apache.iotdb.confignode.manager.externalservice.ExternalServiceInfo; import org.apache.iotdb.confignode.manager.externalservice.ExternalServiceManager; import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.load.cache.node.NodeStatistics; import org.apache.iotdb.confignode.manager.node.ClusterNodeStartUtils; import org.apache.iotdb.confignode.manager.node.NodeManager; import org.apache.iotdb.confignode.manager.node.NodeMetrics; @@ -621,7 +622,17 @@ public TShowClusterResp showCluster() { .sorted(Comparator.comparingInt(TDataNodeLocation::getDataNodeId)) .collect(Collectors.toList()); Map nodeVersionInfo = getNodeManager().getNodeVersionInfo(); - Map nodeStatus = getLoadManager().getNodeStatusWithReason(); + Map nodeStatisticsSnapshot = + getLoadManager().getNodeStatisticsSnapshot(); + Map nodeStatus = new HashMap<>(); + Map nodeStatusReason = new HashMap<>(); + nodeStatisticsSnapshot.forEach( + (nodeId, statistics) -> { + nodeStatus.put(nodeId, statistics.getStatus().getStatus()); + if (statistics.getStatusReason() != null) { + nodeStatusReason.put(nodeId, statistics.getStatusReason()); + } + }); configNodeLocations.forEach( configNodeLocation -> nodeStatus.putIfAbsent( @@ -636,11 +647,9 @@ public TShowClusterResp showCluster() { .map(TAINodeConfiguration::getLocation) .sorted(Comparator.comparingInt(TAINodeLocation::getAiNodeId)) .collect(Collectors.toList()); - Map nodeStatusMap = getLoadManager().getNodeStatusWithReason(); aiNodeLocations.forEach( aiNodeLocation -> - nodeStatusMap.putIfAbsent( - aiNodeLocation.getAiNodeId(), NodeStatus.Unknown.toString())); + nodeStatus.putIfAbsent(aiNodeLocation.getAiNodeId(), NodeStatus.Unknown.toString())); return new TShowClusterResp() .setStatus(status) @@ -648,6 +657,7 @@ public TShowClusterResp showCluster() { .setDataNodeList(dataNodeLocations) .setAiNodeList(aiNodeLocations) .setNodeStatus(nodeStatus) + .setNodeStatusReason(nodeStatusReason) .setNodeVersionInfo(nodeVersionInfo); } else { return new TShowClusterResp() @@ -656,6 +666,7 @@ public TShowClusterResp showCluster() { .setDataNodeList(Collections.emptyList()) .setAiNodeList(Collections.emptyList()) .setNodeStatus(Collections.emptyMap()) + .setNodeStatusReason(Collections.emptyMap()) .setNodeVersionInfo(Collections.emptyMap()); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java index 17d0f6494dbef..d5d7085bfa823 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java @@ -40,6 +40,7 @@ import org.apache.iotdb.confignode.manager.load.balancer.RouteBalancer; import org.apache.iotdb.confignode.manager.load.cache.LoadCache; import org.apache.iotdb.confignode.manager.load.cache.consensus.ConsensusGroupHeartbeatSample; +import org.apache.iotdb.confignode.manager.load.cache.node.NodeStatistics; import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample; import org.apache.iotdb.confignode.manager.load.service.EventService; import org.apache.iotdb.confignode.manager.load.service.HeartbeatService; @@ -269,22 +270,24 @@ public NodeStatus getNodeStatus(int nodeId) { } /** - * Safely get the specified Node's current status with reason. + * Safely get the specified Node's current status reason. * * @param nodeId The specified NodeId - * @return The specified Node's current status if the nodeCache contains it, Unknown otherwise + * @return The reason why the Node is in its current status, null if the node has no reason or the + * cache doesn't exist */ - public String getNodeStatusWithReason(int nodeId) { - return loadCache.getNodeStatusWithReason(nodeId); + public String getNodeStatusReason(int nodeId) { + return loadCache.getNodeStatusReason(nodeId); } /** - * Get all Node's current status with reason. + * Get all Nodes' current statistics in a single traversal of the node cache, so that each node's + * status and reason come from the same statistics snapshot. * - * @return Map + * @return Map */ - public Map getNodeStatusWithReason() { - return loadCache.getNodeStatusWithReason(); + public Map getNodeStatisticsSnapshot() { + return loadCache.getCurrentNodeStatisticsMap(); } /** @@ -346,18 +349,25 @@ public int getLowestLoadDataNode(List dataNodeIds) { return loadCache.getLowestLoadDataNode(dataNodeIds); } + /** Try to set the specified Node's status without a reason. */ + public TSStatus trySetNodeStatus(int nodeId, NodeStatus nodeStatus, boolean force) { + return trySetNodeStatus(nodeId, nodeStatus, null, force); + } + /** * Try to set the specified Node's status according to transition rules and broadcast statistics * changes, including when persistence fails. Does not send a status-change RPC to the node. * * @param nodeId Specified NodeId * @param nodeStatus requested status + * @param statusReason reason supplied by the caller, or null when no reason is known * @param force whether an administrative update may replace Stopped or Removing * @return processing result; success does not imply that the requested status replaced the old * one */ - public TSStatus trySetNodeStatus(int nodeId, NodeStatus nodeStatus, boolean force) { - TSStatus status = loadCache.trySetNodeStatus(nodeId, nodeStatus, force); + public TSStatus trySetNodeStatus( + int nodeId, NodeStatus nodeStatus, String statusReason, boolean force) { + TSStatus status = loadCache.trySetNodeStatus(nodeId, nodeStatus, statusReason, force); eventService.checkAndBroadcastNodeStatisticsChangeEventIfNecessary(); return status; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java index 3407ec33e8b81..f1cab44e73799 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java @@ -288,11 +288,12 @@ private TSStatus persistNodeStatus(BaseNodeCache cache, NodeStatus status) { } } - public TSStatus trySetNodeStatus(int nodeId, NodeStatus status, boolean force) { + public TSStatus trySetNodeStatus( + int nodeId, NodeStatus status, String statusReason, boolean force) { BaseNodeCache cache = nodeCacheMap.get(nodeId); return cache == null ? new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode()) - : cache.trySetNodeStatus(status, force); + : cache.trySetNodeStatus(status, statusReason, force); } /** @@ -599,25 +600,15 @@ public NodeStatus getNodeStatus(int nodeId) { } /** - * Safely get the specified Node's current status with reason. + * Safely get the specified Node's current status reason. * * @param nodeId The specified NodeId - * @return The specified Node's current status if the nodeCache contains it, Unknown otherwise + * @return The reason why the Node is in its current status, null if the node has no reason or the + * cache doesn't exist */ - public String getNodeStatusWithReason(int nodeId) { - return Optional.ofNullable(nodeCacheMap.get(nodeId)) - .map(BaseNodeCache::getNodeStatusWithReason) - .orElseGet(() -> NodeStatus.Unknown.getStatus() + "(NoHeartbeat)"); - } - - /** - * Get all Node's current status with reason. - * - * @return Map - */ - public Map getNodeStatusWithReason() { - return nodeCacheMap.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getNodeStatusWithReason())); + public String getNodeStatusReason(int nodeId) { + BaseNodeCache nodeCache = nodeCacheMap.get(nodeId); + return nodeCache == null ? null : nodeCache.getNodeStatusReason(); } /** diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java index 10a3e939c6275..a2e3bbe56e50e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java @@ -72,15 +72,15 @@ public TSStatus updateNodeStatistics() { * rules may retain the previous status even when this method returns success. A persistence * failure is returned without discarding the new statistics. */ - public TSStatus trySetNodeStatus(NodeStatus status, boolean force) { + public TSStatus trySetNodeStatus(NodeStatus status, String statusReason, boolean force) { synchronized (slidingWindow) { - NodeHeartbeatSample sample = new NodeHeartbeatSample(status); + NodeHeartbeatSample sample = new NodeHeartbeatSample(status, statusReason); cacheHeartbeatSample(sample); return applyNodeStatistics( new NodeStatistics( sample.getSampleLogicalTimestamp(), status, - null, + sample.getStatusReason(), NodeStatus.isNormalStatus(status) ? 0 : Long.MAX_VALUE), force); } @@ -135,10 +135,8 @@ public NodeStatus getNodeStatus() { /** * @return The reason why lead to current NodeStatus. */ - public String getNodeStatusWithReason() { + public String getNodeStatusReason() { NodeStatistics statistics = (NodeStatistics) this.currentStatistics.get(); - return statistics.getStatusReason() == null - ? statistics.getStatus().getStatus() - : statistics.getStatus().getStatus() + "(" + statistics.getStatusReason() + ")"; + return statistics.getStatusReason(); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeHeartbeatSample.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeHeartbeatSample.java index 8217593f5d67d..6d9be6080af6f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeHeartbeatSample.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeHeartbeatSample.java @@ -37,9 +37,14 @@ public class NodeHeartbeatSample extends AbstractHeartbeatSample { /** Constructor for generating default sample with specified status */ public NodeHeartbeatSample(NodeStatus status) { + this(status, null); + } + + /** Constructor for an explicit management status with its reason. */ + public NodeHeartbeatSample(NodeStatus status, String statusReason) { super(System.nanoTime()); this.status = status; - this.statusReason = null; + this.statusReason = statusReason; this.loadSample = null; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 3a438b3866acb..c6790da284a45 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -75,6 +75,7 @@ import org.apache.iotdb.confignode.manager.externalservice.ExternalServiceManager; import org.apache.iotdb.confignode.manager.load.LoadManager; import org.apache.iotdb.confignode.manager.load.cache.node.ConfigNodeHeartbeatCache; +import org.apache.iotdb.confignode.manager.load.cache.node.NodeStatistics; import org.apache.iotdb.confignode.manager.partition.PartitionManager; import org.apache.iotdb.confignode.manager.partition.PartitionMetrics; import org.apache.iotdb.confignode.manager.pipe.coordinator.PipeManager; @@ -505,7 +506,7 @@ public List getRegisteredAINodeInfoList() { for (TAINodeConfiguration aiNodeConfiguration : getRegisteredAINodes()) { TAINodeInfo aiNodeInfo = new TAINodeInfo(); aiNodeInfo.setAiNodeId(aiNodeConfiguration.getLocation().getAiNodeId()); - aiNodeInfo.setStatus(getLoadManager().getNodeStatusWithReason(aiNodeInfo.getAiNodeId())); + aiNodeInfo.setStatus(getLoadManager().getNodeStatus(aiNodeInfo.getAiNodeId()).getStatus()); aiNodeInfo.setInternalAddress(aiNodeConfiguration.getLocation().getInternalEndPoint().ip); aiNodeInfo.setInternalPort(aiNodeConfiguration.getLocation().getInternalEndPoint().port); aiNodeInfoList.add(aiNodeInfo); @@ -718,13 +719,20 @@ public Map getRegisteredConfigNodeLocations() { public List getRegisteredDataNodeInfoList() { List dataNodeInfoList = new ArrayList<>(); List registeredDataNodes = this.getRegisteredDataNodes(); + Map nodeStatisticsSnapshot = + getLoadManager().getNodeStatisticsSnapshot(); if (registeredDataNodes != null) { registeredDataNodes.forEach( registeredDataNode -> { TDataNodeInfo dataNodeInfo = new TDataNodeInfo(); int dataNodeId = registeredDataNode.getLocation().getDataNodeId(); dataNodeInfo.setDataNodeId(dataNodeId); - dataNodeInfo.setStatus(getLoadManager().getNodeStatusWithReason(dataNodeId)); + NodeStatistics statistics = nodeStatisticsSnapshot.get(dataNodeId); + dataNodeInfo.setStatus( + statistics == null + ? NodeStatus.Unknown.getStatus() + : statistics.getStatus().getStatus()); + dataNodeInfo.setStatusReason(statistics == null ? null : statistics.getStatusReason()); dataNodeInfo.setRpcAddresss( registeredDataNode.getLocation().getClientRpcEndPoint().getIp()); dataNodeInfo.setRpcPort( @@ -854,13 +862,20 @@ public Map getNodeVersionInfo() { public List getRegisteredConfigNodeInfoList() { List configNodeInfoList = new ArrayList<>(); List registeredConfigNodes = this.getRegisteredConfigNodes(); + Map nodeStatisticsSnapshot = + getLoadManager().getNodeStatisticsSnapshot(); if (registeredConfigNodes != null) { registeredConfigNodes.forEach( configNodeLocation -> { TConfigNodeInfo info = new TConfigNodeInfo(); int configNodeId = configNodeLocation.getConfigNodeId(); info.setConfigNodeId(configNodeId); - info.setStatus(getLoadManager().getNodeStatusWithReason(configNodeId)); + NodeStatistics statistics = nodeStatisticsSnapshot.get(configNodeId); + info.setStatus( + statistics == null + ? NodeStatus.Unknown.getStatus() + : statistics.getStatus().getStatus()); + info.setStatusReason(statistics == null ? null : statistics.getStatusReason()); info.setInternalAddress(configNodeLocation.getInternalEndPoint().getIp()); info.setInternalPort(configNodeLocation.getInternalEndPoint().getPort()); info.setRoleType( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java index d57b4a7bb32b0..5404b92060dc8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java @@ -178,8 +178,12 @@ public void changeDataNodeStatus( // Force updating NodeStatus long currentTime = System.nanoTime(); + // Match the ReadOnly reason assigned by the DataNode's SET_SYSTEM_STATUS RPC. + String statusReason = nodeStatus == NodeStatus.ReadOnly ? NodeStatus.MANUAL : null; TSStatus status = - configManager.getLoadManager().trySetNodeStatus(dataNodeId, nodeStatus, true); + configManager + .getLoadManager() + .trySetNodeStatus(dataNodeId, nodeStatus, statusReason, true); if (!isSucceed(status)) { // Do not advance the persisted procedure state before its node status is durable. statusChangeFailures.append( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 8f2c1080b3f1d..adce7c280fcd6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -2435,12 +2435,13 @@ public TDataNodeHeartbeatResp getDataNodeHeartBeat(TDataNodeHeartbeatReq req) th resp.setDataRegionRawDataSize(regionRawDataSize); } resp.setHeartbeatTimestamp(req.getHeartbeatTimestamp()); - resp.setStatus(commonConfig.getNodeStatus().getStatus()); - // Advertise that this DataNode supports metadata-lease self-fencing, so the ConfigNode may - // treat - // it as safely fenced when unreachable (older DataNodes that omit this are handled strictly). - if (commonConfig.getStatusReason() != null) { - resp.setStatusReason(commonConfig.getStatusReason()); + // Read status and reason under the same monitor used by their setters. + synchronized (commonConfig) { + resp.setStatus(commonConfig.getNodeStatus().getStatus()); + final String statusReason = commonConfig.getStatusReason(); + if (statusReason != null) { + resp.setStatusReason(statusReason); + } } MetadataLeaseFencedException schemaUsageCollectionException = null; if (req.getSchemaRegionIds() != null) { @@ -2620,12 +2621,15 @@ void sampleDiskLoad(TLoadSample loadSample) { RamUsageEstimator.humanReadableUnits((long) totalDisk), freeDiskRatio, commonConfig.getDiskSpaceWarningThreshold()); - commonConfig.setNodeStatus(NodeStatus.ReadOnly); - commonConfig.setStatusReason(NodeStatus.DISK_FULL); - } else if (NodeStatus.ReadOnly.equals(commonConfig.getNodeStatus()) - && NodeStatus.DISK_FULL.equals(commonConfig.getStatusReason())) { - commonConfig.setNodeStatus(NodeStatus.Running); - commonConfig.setStatusReason(null); + commonConfig.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL); + } else { + synchronized (commonConfig) { + // A concurrent Manual/Stopping/error transition must not be cleared by disk recovery. + if (NodeStatus.ReadOnly.equals(commonConfig.getNodeStatus()) + && NodeStatus.DISK_FULL.equals(commonConfig.getStatusReason())) { + commonConfig.setNodeStatus(NodeStatus.Running); + } + } } } } @@ -2800,7 +2804,12 @@ public TShowAppliedConfigurationsResp showAppliedConfigurations() throws TExcept @Override public TSStatus setSystemStatus(String status) throws TException { try { - commonConfig.setNodeStatus(NodeStatus.parse(status)); + NodeStatus nodeStatus = NodeStatus.parse(status); + if (nodeStatus == NodeStatus.ReadOnly) { + commonConfig.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL); + } else { + commonConfig.setNodeStatus(nodeStatus); + } } catch (Exception e) { return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java index f4fb59f7ca7a5..f57fc00154e14 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java @@ -1221,6 +1221,7 @@ protected void constructLine() { location.getConfigNodeId(), NODE_TYPE_CONFIG_NODE, showClusterResp.getNodeStatus().get(location.getConfigNodeId()), + getNodeStatusReason(location.getConfigNodeId()), location.getInternalEndPoint().getIp(), location.getInternalEndPoint().getPort(), showClusterResp.getNodeVersionInfo().get(location.getConfigNodeId())); @@ -1232,6 +1233,7 @@ protected void constructLine() { location.getDataNodeId(), NODE_TYPE_DATA_NODE, showClusterResp.getNodeStatus().get(location.getDataNodeId()), + getNodeStatusReason(location.getDataNodeId()), location.getInternalEndPoint().getIp(), location.getInternalEndPoint().getPort(), showClusterResp.getNodeVersionInfo().get(location.getDataNodeId())); @@ -1243,16 +1245,26 @@ protected void constructLine() { location.getAiNodeId(), NODE_TYPE_AI_NODE, showClusterResp.getNodeStatus().get(location.getAiNodeId()), + getNodeStatusReason(location.getAiNodeId()), location.getInternalEndPoint().getIp(), location.getInternalEndPoint().getPort(), showClusterResp.getNodeVersionInfo().get(location.getAiNodeId())); } } + private String getNodeStatusReason(int nodeId) { + // The optional field is absent when the ConfigNode is an old version. + if (!showClusterResp.isSetNodeStatusReason()) { + return null; + } + return showClusterResp.getNodeStatusReason().get(nodeId); + } + private void buildNodeTsBlock( int nodeId, String nodeType, String nodeStatus, + String nodeStatusReason, String internalAddress, int internalPort, TNodeVersionInfo versionInfo) { @@ -1263,23 +1275,28 @@ private void buildNodeTsBlock( } else { columnBuilders[2].writeBinary(new Binary(nodeStatus, TSFileConfig.STRING_CHARSET)); } + if (nodeStatusReason == null) { + columnBuilders[3].appendNull(); + } else { + columnBuilders[3].writeBinary(new Binary(nodeStatusReason, TSFileConfig.STRING_CHARSET)); + } if (internalAddress == null) { - columnBuilders[3].appendNull(); + columnBuilders[4].appendNull(); } else { - columnBuilders[3].writeBinary(new Binary(internalAddress, TSFileConfig.STRING_CHARSET)); + columnBuilders[4].writeBinary(new Binary(internalAddress, TSFileConfig.STRING_CHARSET)); } - columnBuilders[4].writeInt(internalPort); + columnBuilders[5].writeInt(internalPort); if (versionInfo == null || versionInfo.getVersion() == null) { - columnBuilders[5].appendNull(); + columnBuilders[6].appendNull(); } else { - columnBuilders[5].writeBinary( + columnBuilders[6].writeBinary( new Binary(versionInfo.getVersion(), TSFileConfig.STRING_CHARSET)); } if (versionInfo == null || versionInfo.getBuildInfo() == null) { - columnBuilders[6].appendNull(); + columnBuilders[7].appendNull(); } else { - columnBuilders[6].writeBinary( + columnBuilders[7].writeBinary( new Binary(versionInfo.getBuildInfo(), TSFileConfig.STRING_CHARSET)); } resultBuilder.declarePosition(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 46c4217f17cf5..a995d643970f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -1590,7 +1590,13 @@ public SettableFuture setSystemStatus(boolean onCluster, NodeS } } else { try { - CommonDescriptor.getInstance().getConfig().setNodeStatus(status); + if (status == NodeStatus.ReadOnly) { + CommonDescriptor.getInstance() + .getConfig() + .setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL); + } else { + CommonDescriptor.getInstance().getConfig().setNodeStatus(status); + } tsStatus = RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } catch (Exception e) { tsStatus = RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTask.java index 17778269b7ea1..2140bd53fff1a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTask.java @@ -24,7 +24,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; -import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; @@ -38,6 +37,7 @@ import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.utils.Binary; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -57,6 +57,8 @@ private static void buildConfigNodesTsBlock( TsBlockBuilder builder, int nodeId, String nodeStatus, + String nodeStatusReason, + boolean hasStatusReason, String internalAddress, int internalPort, int configConsensusPort, @@ -102,6 +104,15 @@ private static void buildConfigNodesTsBlock( .getColumnBuilder(12) .writeBinary(new Binary(versionInfo.getBuildInfo(), TSFileConfig.STRING_CHARSET)); } + if (hasStatusReason) { + if (nodeStatusReason == null) { + builder.getColumnBuilder(13).appendNull(); + } else { + builder + .getColumnBuilder(13) + .writeBinary(new Binary(nodeStatusReason, TSFileConfig.STRING_CHARSET)); + } + } builder.declarePosition(); } @@ -109,6 +120,8 @@ private static void buildAINodeTsBlock( TsBlockBuilder builder, int nodeId, String nodeStatus, + String nodeStatusReason, + boolean hasStatusReason, String internalAddress, int internalPort, TNodeVersionInfo versionInfo) { @@ -123,7 +136,6 @@ private static void buildAINodeTsBlock( } else { builder.getColumnBuilder(2).writeBinary(new Binary(nodeStatus, TSFileConfig.STRING_CHARSET)); } - if (internalAddress == null) { builder.getColumnBuilder(3).appendNull(); } else { @@ -152,6 +164,15 @@ private static void buildAINodeTsBlock( .getColumnBuilder(12) .writeBinary(new Binary(versionInfo.getBuildInfo(), TSFileConfig.STRING_CHARSET)); } + if (hasStatusReason) { + if (nodeStatusReason == null) { + builder.getColumnBuilder(13).appendNull(); + } else { + builder + .getColumnBuilder(13) + .writeBinary(new Binary(nodeStatusReason, TSFileConfig.STRING_CHARSET)); + } + } builder.declarePosition(); } @@ -160,6 +181,8 @@ private static void buildDataNodesTsBlock( TsBlockBuilder builder, int nodeId, String nodeStatus, + String nodeStatusReason, + boolean hasStatusReason, String internalAddress, int internalPort, String rpcAddress, @@ -219,15 +242,31 @@ private static void buildDataNodesTsBlock( .getColumnBuilder(12) .writeBinary(new Binary(versionInfo.getBuildInfo(), TSFileConfig.STRING_CHARSET)); } + if (hasStatusReason) { + if (nodeStatusReason == null) { + builder.getColumnBuilder(13).appendNull(); + } else { + builder + .getColumnBuilder(13) + .writeBinary(new Binary(nodeStatusReason, TSFileConfig.STRING_CHARSET)); + } + } builder.declarePosition(); } public static void buildTSBlock( TShowClusterResp clusterNodeInfos, SettableFuture future) { + boolean hasStatusReason = + clusterNodeInfos.getNodeStatusReason() != null + && clusterNodeInfos.getNodeStatusReason().values().stream() + .anyMatch(reason -> reason != null && !reason.isEmpty()); + List columnHeaders = + new ArrayList<>(ColumnHeaderConstant.showClusterDetailsColumnHeaders); + if (hasStatusReason) { + columnHeaders.add(new ColumnHeader(ColumnHeaderConstant.STATUS_REASON, TSDataType.TEXT)); + } List outputDataTypes = - ColumnHeaderConstant.showClusterDetailsColumnHeaders.stream() - .map(ColumnHeader::getColumnType) - .collect(Collectors.toList()); + columnHeaders.stream().map(ColumnHeader::getColumnType).collect(Collectors.toList()); TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); clusterNodeInfos @@ -238,6 +277,10 @@ public static void buildTSBlock( builder, e.getConfigNodeId(), clusterNodeInfos.getNodeStatus().get(e.getConfigNodeId()), + hasStatusReason + ? clusterNodeInfos.getNodeStatusReason().get(e.getConfigNodeId()) + : null, + hasStatusReason, e.getInternalEndPoint().getIp(), e.getInternalEndPoint().getPort(), e.getConsensusEndPoint().getPort(), @@ -251,6 +294,10 @@ public static void buildTSBlock( builder, e.getDataNodeId(), clusterNodeInfos.getNodeStatus().get(e.getDataNodeId()), + hasStatusReason + ? clusterNodeInfos.getNodeStatusReason().get(e.getDataNodeId()) + : null, + hasStatusReason, e.getInternalEndPoint().getIp(), e.getInternalEndPoint().getPort(), e.getClientRpcEndPoint().getIp(), @@ -267,11 +314,15 @@ public static void buildTSBlock( builder, e.getAiNodeId(), clusterNodeInfos.getNodeStatus().get(e.getAiNodeId()), + hasStatusReason + ? clusterNodeInfos.getNodeStatusReason().get(e.getAiNodeId()) + : null, + hasStatusReason, e.getInternalEndPoint().getIp(), e.getInternalEndPoint().getPort(), clusterNodeInfos.getNodeVersionInfo().get(e.getAiNodeId()))); - DatasetHeader datasetHeader = DatasetHeaderFactory.getShowClusterDetailsHeader(); + DatasetHeader datasetHeader = new DatasetHeader(columnHeaders, true); future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTask.java index 8837864ab1390..f1ad0ffe8f5f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTask.java @@ -24,7 +24,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; -import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; @@ -38,6 +37,7 @@ import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.utils.Binary; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -64,6 +64,8 @@ private static void buildTsBlock( int nodeId, String nodeType, String nodeStatus, + String nodeStatusReason, + boolean hasStatusReason, String hostAddress, int port, TNodeVersionInfo versionInfo) { @@ -100,15 +102,31 @@ private static void buildTsBlock( .writeBinary(new Binary(versionInfo.getBuildInfo(), TSFileConfig.STRING_CHARSET)); } + if (hasStatusReason) { + if (nodeStatusReason == null) { + builder.getColumnBuilder(7).appendNull(); + } else { + builder + .getColumnBuilder(7) + .writeBinary(new Binary(nodeStatusReason, TSFileConfig.STRING_CHARSET)); + } + } builder.declarePosition(); } public static void buildTsBlock( TShowClusterResp clusterNodeInfos, SettableFuture future) { + boolean hasStatusReason = + clusterNodeInfos.getNodeStatusReason() != null + && clusterNodeInfos.getNodeStatusReason().values().stream() + .anyMatch(reason -> reason != null && !reason.isEmpty()); + List columnHeaders = + new ArrayList<>(ColumnHeaderConstant.showClusterColumnHeaders); + if (hasStatusReason) { + columnHeaders.add(new ColumnHeader(ColumnHeaderConstant.STATUS_REASON, TSDataType.TEXT)); + } List outputDataTypes = - ColumnHeaderConstant.showClusterColumnHeaders.stream() - .map(ColumnHeader::getColumnType) - .collect(Collectors.toList()); + columnHeaders.stream().map(ColumnHeader::getColumnType).collect(Collectors.toList()); TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); clusterNodeInfos @@ -120,6 +138,10 @@ public static void buildTsBlock( e.getConfigNodeId(), NODE_TYPE_CONFIG_NODE, clusterNodeInfos.getNodeStatus().get(e.getConfigNodeId()), + hasStatusReason + ? clusterNodeInfos.getNodeStatusReason().get(e.getConfigNodeId()) + : null, + hasStatusReason, e.getInternalEndPoint().getIp(), e.getInternalEndPoint().getPort(), clusterNodeInfos.getNodeVersionInfo().get(e.getConfigNodeId()))); @@ -133,6 +155,10 @@ public static void buildTsBlock( e.getDataNodeId(), NODE_TYPE_DATA_NODE, clusterNodeInfos.getNodeStatus().get(e.getDataNodeId()), + hasStatusReason + ? clusterNodeInfos.getNodeStatusReason().get(e.getDataNodeId()) + : null, + hasStatusReason, e.getInternalEndPoint().getIp(), e.getInternalEndPoint().getPort(), clusterNodeInfos.getNodeVersionInfo().get(e.getDataNodeId()))); @@ -147,11 +173,15 @@ public static void buildTsBlock( e.getAiNodeId(), NODE_TYPE_AI_NODE, clusterNodeInfos.getNodeStatus().get(e.getAiNodeId()), + hasStatusReason + ? clusterNodeInfos.getNodeStatusReason().get(e.getAiNodeId()) + : null, + hasStatusReason, e.getInternalEndPoint().getIp(), e.getInternalEndPoint().getPort(), clusterNodeInfos.getNodeVersionInfo().get(e.getAiNodeId()))); } - DatasetHeader datasetHeader = DatasetHeaderFactory.getShowClusterHeader(); + DatasetHeader datasetHeader = new DatasetHeader(columnHeaders, true); future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowConfigNodesTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowConfigNodesTask.java index a666a72d81539..4bcbb9a5b3eb5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowConfigNodesTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowConfigNodesTask.java @@ -24,7 +24,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowConfigNodesResp; import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; -import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; @@ -36,6 +35,7 @@ import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.utils.BytesUtils; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -53,10 +53,20 @@ public ListenableFuture execute(IConfigTaskExecutor configTask public static void buildTSBlock( TShowConfigNodesResp showConfigNodesResp, SettableFuture future) { + boolean hasStatusReason = + showConfigNodesResp.getConfigNodesInfoList() != null + && showConfigNodesResp.getConfigNodesInfoList().stream() + .anyMatch( + configNodeInfo -> + configNodeInfo.getStatusReason() != null + && !configNodeInfo.getStatusReason().isEmpty()); + List columnHeaders = + new ArrayList<>(ColumnHeaderConstant.showConfigNodesColumnHeaders); + if (hasStatusReason) { + columnHeaders.add(new ColumnHeader(ColumnHeaderConstant.STATUS_REASON, TSDataType.TEXT)); + } List outputDataTypes = - ColumnHeaderConstant.showConfigNodesColumnHeaders.stream() - .map(ColumnHeader::getColumnType) - .collect(Collectors.toList()); + columnHeaders.stream().map(ColumnHeader::getColumnType).collect(Collectors.toList()); TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); if (showConfigNodesResp.getConfigNodesInfoList() != null) { for (TConfigNodeInfo configNodeInfo : showConfigNodesResp.getConfigNodesInfoList()) { @@ -68,10 +78,18 @@ public static void buildTSBlock( .writeBinary(BytesUtils.valueOf(configNodeInfo.getInternalAddress())); builder.getColumnBuilder(3).writeInt(configNodeInfo.getInternalPort()); builder.getColumnBuilder(4).writeBinary(BytesUtils.valueOf(configNodeInfo.getRoleType())); + if (hasStatusReason) { + String statusReason = configNodeInfo.getStatusReason(); + if (statusReason == null) { + builder.getColumnBuilder(5).appendNull(); + } else { + builder.getColumnBuilder(5).writeBinary(BytesUtils.valueOf(statusReason)); + } + } builder.declarePosition(); } } - DatasetHeader datasetHeader = DatasetHeaderFactory.getShowConfigNodesHeader(); + DatasetHeader datasetHeader = new DatasetHeader(columnHeaders, true); future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTask.java index 1adb6e68cae67..5ffdd6743edc8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTask.java @@ -24,7 +24,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp; import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; -import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; @@ -37,6 +36,7 @@ import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.utils.BytesUtils; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -58,10 +58,20 @@ public ListenableFuture execute(IConfigTaskExecutor configTask public static void buildTSBlock( TShowDataNodesResp showDataNodesResp, SettableFuture future) { + boolean hasStatusReason = + showDataNodesResp.getDataNodesInfoList() != null + && showDataNodesResp.getDataNodesInfoList().stream() + .anyMatch( + dataNodeInfo -> + dataNodeInfo.getStatusReason() != null + && !dataNodeInfo.getStatusReason().isEmpty()); + List columnHeaders = + new ArrayList<>(ColumnHeaderConstant.showDataNodesColumnHeaders); + if (hasStatusReason) { + columnHeaders.add(new ColumnHeader(ColumnHeaderConstant.STATUS_REASON, TSDataType.TEXT)); + } List outputDataTypes = - ColumnHeaderConstant.showDataNodesColumnHeaders.stream() - .map(ColumnHeader::getColumnType) - .collect(Collectors.toList()); + columnHeaders.stream().map(ColumnHeader::getColumnType).collect(Collectors.toList()); TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); if (showDataNodesResp.getDataNodesInfoList() != null) { for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) { @@ -72,16 +82,22 @@ public static void buildTSBlock( .writeBinary( BytesUtils.valueOf( dataNodeInfo.getStatus() == null ? "" : dataNodeInfo.getStatus())); - builder.getColumnBuilder(2).writeBinary(BytesUtils.valueOf(dataNodeInfo.getRpcAddresss())); builder.getColumnBuilder(3).writeInt(dataNodeInfo.getRpcPort()); builder.getColumnBuilder(4).writeInt(dataNodeInfo.getDataRegionNum()); - builder.getColumnBuilder(5).writeInt(dataNodeInfo.getSchemaRegionNum()); + if (hasStatusReason) { + String statusReason = dataNodeInfo.getStatusReason(); + if (statusReason == null) { + builder.getColumnBuilder(6).appendNull(); + } else { + builder.getColumnBuilder(6).writeBinary(BytesUtils.valueOf(statusReason)); + } + } builder.declarePosition(); } } - DatasetHeader datasetHeader = DatasetHeaderFactory.getShowDataNodesHeader(); + DatasetHeader datasetHeader = new DatasetHeader(columnHeaders, true); future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java index 6dd082c0efa4b..16f17916eb0c7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java @@ -117,7 +117,9 @@ public void run() { // Reject write operations to make sure all tsfiles will be sealed CommonDescriptor.getInstance().getConfig().setStopping(true); - CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly); + CommonDescriptor.getInstance() + .getConfig() + .setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.STOPPING); // Wait all wal are flushed WALManager.getInstance().waitAllWALFlushed(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 802d440cc4450..ead75db6bac6e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -2146,7 +2146,9 @@ protected TsFileProcessor getOrCreateTsFileProcessor(long timeRangeId, boolean s StorageEngineMessages .STORAGE_LOG_DISK_SPACE_IS_INSUFFICIENT_WHEN_CREATING_TSFILE_PROCESSOR_4032BAF0, e); - CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly); + CommonDescriptor.getInstance() + .getConfig() + .setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL); throw new WriteProcessException(e.getMessage(), e.getErrorCode(), true); } catch (IOException e) { if (retryCnt < 3) { @@ -2158,7 +2160,7 @@ protected TsFileProcessor getOrCreateTsFileProcessor(long timeRangeId, boolean s .STORAGE_LOG_MEET_IOEXCEPTION_WHEN_CREATING_TSFILEPROCESSOR_CHANGE_SYSTEM_4337F729, e); DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); - CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); + CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e); throw new WriteProcessException( String.format( StorageEngineMessages diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 78c48a2702ba7..9589dcf9a402c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -1932,7 +1932,7 @@ public void flushOneMemTable() { tsFileResource.getTsFile().getAbsolutePath(), e); DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); - CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); + CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e); try { logger.error( StorageEngineMessages @@ -2080,7 +2080,7 @@ public void flushOneMemTable() { dataRegionName, tsFileResource.getTsFile().getAbsolutePath(), e); - CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); + CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e); break; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index 13930f936d51f..45432e7ab3df5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -613,7 +613,7 @@ public void run() { failListeners(syncFailure); // SET SYSTEM TO RUNNING does not repair a failed batch or an unknown record boundary. if (CommonDescriptor.getInstance().getConfig().isRunning()) { - CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); + CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(syncFailure); } if (forceFlag && retryAfterFailedBatch) { syncFailure = null; @@ -643,7 +643,7 @@ public void run() { } failListeners(e); DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); - CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); + CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e); switchSyncingBufferToIdle(); return; } @@ -676,7 +676,7 @@ public void run() { syncFailure = e instanceof Exception exception ? exception : new IOException(e); retryAfterFailedBatch = false; failListeners(syncFailure); - CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); + CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e); return; } finally { switchSyncingBufferToIdle(); @@ -710,7 +710,7 @@ public void run() { retryAfterFailedBatch = false; } DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); - CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); + CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e); } } else if (forceFlag) { // force os cache to the storage device, avoid force twice by judging // after rolling file @@ -727,7 +727,7 @@ public void run() { failListeners(e); syncFailure = e; retryAfterFailedBatch = false; - CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); + CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java index a1bba3bad402d..ec96f43c7d401 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java @@ -222,7 +222,7 @@ public void fsyncCheckpointFile() { identifier, e); DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); - CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); + CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e); } try { @@ -243,7 +243,7 @@ public void fsyncCheckpointFile() { identifier, e); DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); - CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); + CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e); } } finally { infoLock.unlock(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 8a5b7e088ce3d..d307e8bfa365f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.enums.PipeRateAverage; import org.apache.iotdb.commons.i18n.ConfigMessages; import org.apache.iotdb.commons.pipe.config.PipeConfig; +import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.commons.utils.KillPoint.KillPoint; import org.apache.iotdb.rpc.RpcUtils; @@ -174,8 +175,11 @@ public class CommonConfig { /** Status of current system. */ private volatile NodeStatus status = NodeStatus.Running; - private NodeStatus lastStatus = NodeStatus.Unknown; - private String lastStatusReason = ""; + /** + * Maximum length of the error message embedded in the UnrecoverableError status reason, keeping + * the reason bounded in heartbeats and SHOW results. + */ + private static final int MAX_STATUS_REASON_LENGTH = 256; private volatile boolean isStopping = false; @@ -783,8 +787,26 @@ void setHandleSystemErrorStrategy(HandleSystemErrorStrategy handleSystemErrorStr this.handleSystemErrorStrategy = handleSystemErrorStrategy; } - public void handleUnrecoverableError() { - handleSystemErrorStrategy.handle(); + /** + * Handles an unrecoverable error with the given exception. The ReadOnly status reason is + * assembled here so that all call sites share one format: "UnrecoverableError, , + * ". The error message is truncated to {@link #MAX_STATUS_REASON_LENGTH} + * characters so that the reason published in heartbeats and SHOW results stays bounded. + */ + public void handleUnrecoverableError(Throwable e) { + String errorMessage = + e.getMessage() == null || e.getMessage().isEmpty() + ? e.getClass().getSimpleName() + : e.getMessage(); + if (errorMessage.length() > MAX_STATUS_REASON_LENGTH) { + errorMessage = errorMessage.substring(0, MAX_STATUS_REASON_LENGTH) + "..."; + } + handleSystemErrorStrategy.handle( + NodeStatus.UNRECOVERABLE_ERROR + + ", " + + DateTimeUtils.convertLongToDate(System.currentTimeMillis(), "ms") + + ", " + + errorMessage); } public double getDiskSpaceWarningThreshold() { @@ -827,14 +849,33 @@ public NodeStatus getNodeStatus() { return status; } - public synchronized void setNodeStatus(NodeStatus newStatus) { + /** + * Sets the node status without a new reason. An existing ReadOnly reason is preserved when + * ReadOnly is requested again; requests for other statuses clear the reason. + */ + public void setNodeStatus(NodeStatus newStatus) { + setNodeStatusWithReason(newStatus, null); + } + + /** + * Sets the node status and reason together. ReadOnly-to-ReadOnly updates select the reason + * through {@link NodeStatus#transitionReadOnlyReason(String, String)}. Other statuses clear the + * reason. + */ + public synchronized void setNodeStatusWithReason(NodeStatus newStatus, String newReason) { + if (newStatus != NodeStatus.ReadOnly) { + newReason = null; + } else if (status == NodeStatus.ReadOnly) { + newReason = NodeStatus.transitionReadOnlyReason(statusReason, newReason); + } if (status == newStatus) { + this.statusReason = newReason; return; } logger.info(ConfigMessages.SET_SYSTEM_MODE, status, newStatus); this.status = newStatus; - this.statusReason = null; + this.statusReason = newReason; switch (newStatus) { case ReadOnly: @@ -852,10 +893,6 @@ public String getStatusReason() { return statusReason; } - public void setStatusReason(String statusReason) { - this.statusReason = statusReason; - } - public int getTTimePartitionSlotTransmitLimit() { return TTimePartitionSlotTransmitLimit; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java index 93ee02a2c15d2..ecb25de88979d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java @@ -95,13 +95,8 @@ public FolderManager( throw new RuntimeException(); } this.selectStrategy.setChangeSystemStatusToReadOnly(changeSystemStatusToReadOnly); - try { - this.selectStrategy.setFolders(folders); - this.selectStrategy.setFoldersStates(foldersStates); - } catch (DiskSpaceInsufficientException e) { - changeToReadOnlyIfDiskFull(e); - throw e; - } + this.selectStrategy.setFolders(folders); + this.selectStrategy.setFoldersStates(foldersStates); } public void updateFolderState(String folder, FolderState state) { @@ -142,8 +137,9 @@ private void changeToReadOnlyIfDiskFull(DiskSpaceInsufficientException e) { if (LoggerPeriodicalLogReducer.shouldLog(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY)) { logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e); } - CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly); - CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL); + CommonDescriptor.getInstance() + .getConfig() + .setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL); } else { logger.warn(UtilMessages.CANNOT_SELECT_FOLDER_BUT_DISK_HAS_SPACE, e); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/DirectoryStrategy.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/DirectoryStrategy.java index 44c4a93749e99..978e7ecbbada8 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/DirectoryStrategy.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/DirectoryStrategy.java @@ -70,7 +70,9 @@ public void setFolders(List folders) throws DiskSpaceInsufficientExcepti if (!hasSpace) { if (changeSystemStatusToReadOnly) { LOGGER.error(UtilMessages.DISK_SPACE_INSUFFICIENT_READ_ONLY); - CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly); + CommonDescriptor.getInstance() + .getConfig() + .setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL); } else { if (LoggerPeriodicalLogReducer.shouldLog( UtilMessages.MESSAGE_DISK_SPACE_INSUFFICIENT_DF6205B0)) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/enums/HandleSystemErrorStrategy.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/enums/HandleSystemErrorStrategy.java index b21dc8db89c3a..53da902872b5c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/enums/HandleSystemErrorStrategy.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/enums/HandleSystemErrorStrategy.java @@ -33,13 +33,15 @@ public enum HandleSystemErrorStrategy { private static final Logger logger = LoggerFactory.getLogger(HandleSystemErrorStrategy.class); - public void handle() { + public void handle(String errorReason) { if (this == HandleSystemErrorStrategy.CHANGE_TO_READ_ONLY) { logger.error( CommonMessages .LOG_UNRECOVERABLE_ERROR_OCCURS_CHANGE_SYSTEM_STATUS_READ_ONLY_BECAUSE_HANDLE_05C9AD1A, new RuntimeException(CommonMessages.SYSTEM_READ_ONLY)); - CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly); + CommonDescriptor.getInstance() + .getConfig() + .setNodeStatusWithReason(NodeStatus.ReadOnly, errorReason); } else if (this == HandleSystemErrorStrategy.SHUTDOWN) { logger.error( CommonMessages diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java index e4c365818e842..239478b13fe3d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java @@ -86,6 +86,7 @@ private ColumnHeaderConstant() { public static final String NODE_ID = "NodeID"; public static final String NODE_TYPE = "NodeType"; public static final String STATUS = "Status"; + public static final String STATUS_REASON = "StatusReason"; public static final String INTERNAL_ADDRESS = "InternalAddress"; public static final String INTERNAL_PORT = "InternalPort"; public static final String CONFIG_CONSENSUS_PORT = "ConfigConsensusPort"; @@ -345,6 +346,7 @@ private ColumnHeaderConstant() { public static final String NODE_ID_TABLE_MODEL = "node_id"; public static final String NODE_TYPE_TABLE_MODEL = "node_type"; + public static final String STATUS_REASON_TABLE_MODEL = "status_reason"; public static final String INTERNAL_PORT_TABLE_MODEL = "internal_port"; public static final String BUILD_INFO_TABLE_MODEL = "build_info"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java index 92f8950a386c6..00ef05c5afa15 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java @@ -360,6 +360,9 @@ public class InformationSchema { nodesTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.STATUS.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); + nodesTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.STATUS_REASON_TABLE_MODEL, TSDataType.STRING)); nodesTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.INTERNAL_ADDRESS_TABLE_MODEL, TSDataType.STRING)); diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index b78e6ed9dff7b..48921651ee210 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -643,6 +643,9 @@ struct TShowClusterResp { 4: required list aiNodeList 5: required map nodeStatus 6: required map nodeVersionInfo + // The reason why the node is in its current status, independent of nodeStatus. + // Absent when the ConfigNode is an old version. + 7: optional map nodeStatusReason } struct TGetClusterIdResp { @@ -685,6 +688,9 @@ struct TDataNodeInfo { 5: required i32 dataRegionNum 6: required i32 schemaRegionNum 7: optional i32 cpuCoreNum + // The reason why the DataNode is in its current status, independent of status. + // Absent when the ConfigNode is an old version. + 8: optional string statusReason } struct TAINodeInfo { @@ -723,6 +729,9 @@ struct TConfigNodeInfo { 3: required string internalAddress 4: required i32 internalPort 5: required string roleType + // The reason why the ConfigNode is in its current status, independent of status. + // Absent when the ConfigNode is an old version. + 6: optional string statusReason } struct TShowConfigNodesResp { From a7bb3cd17216c970dab85722050935c6212d2f81 Mon Sep 17 00:00:00 2001 From: Di Wang Date: Wed, 30 Sep 2026 16:48:46 +0800 Subject: [PATCH 4/5] test: cover ReadOnly status reasons Cover reason precedence, explicit updates, removal rollback, SHOW layouts and table node metadata. Verify reason recovery after leader changes and cleanup after node failure. --- .../cluster/IoTDBNodeStatusPersistenceIT.java | 89 +++++-- .../it/db/it/IoTDBSetSystemStatusTableIT.java | 70 +++++- .../relational/it/schema/IoTDBDatabaseIT.java | 1 + .../manager/load/LoadManagerTest.java | 35 ++- .../load/cache/LoadCacheNodeStatusTest.java | 48 ++-- .../LoadCachePersistedNodeStatusTest.java | 142 +++++++++-- .../manager/load/cache/NodeCacheTest.java | 105 ++++++++- .../node/NodeManagerStatusReasonTest.java | 133 +++++++++++ .../node/RemoveDataNodePersistenceTest.java | 16 +- .../service/ConfigNodeShutdownHookTest.java | 3 +- ...ataNodeInternalRPCServiceImplDiskTest.java | 43 +++- ...rnalRPCServiceImplSetSystemStatusTest.java | 91 +++++++ .../metadata/ShowClusterDetailsTaskTest.java | 172 ++++++++++++++ .../config/metadata/ShowClusterTaskTest.java | 168 +++++++++++++ .../metadata/ShowConfigNodesTaskTest.java | 121 ++++++++++ .../metadata/ShowDataNodesTaskTest.java | 114 +++++++++ .../wal/buffer/WALBufferRollRecoveryTest.java | 4 +- .../rescon/disk/FolderManagerTest.java | 57 ++++- .../iotdb/commons/conf/CommonConfigTest.java | 222 +++++++++++++++++- 19 files changed, 1544 insertions(+), 90 deletions(-) create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerStatusReasonTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplSetSystemStatusTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTaskTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTaskTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowConfigNodesTaskTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTaskTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java index 04bbc101a726f..5b35fcff0d790 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java @@ -29,7 +29,9 @@ import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper; @@ -136,22 +138,8 @@ public void testStoppedNodesSurviveSnapshotRecovery() throws Exception { @Test public void testRemovingNodeSurvivesLeaderFailure() throws Exception { initCluster(3, Integer.MAX_VALUE); - final IClientManager clientManager = - new IClientManager.Factory() - .createClientManager( - new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory()); - try (SyncDataNodeInternalServiceClient client = - clientManager.borrowClient( - new TEndPoint( - EnvFactory.getEnv().getDataNodeWrapper(0).getIp(), - EnvFactory.getEnv().getDataNodeWrapper(0).getInternalPort()))) { - // Use the removal procedure's DataNode RPC. The leader learns Removing through heartbeats. - assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client.setSystemStatus(NodeStatus.Removing.getStatus()).getCode()); - } finally { - clientManager.close(); - } + // Use the removal procedure's DataNode RPC. The leader learns Removing through heartbeats. + setDataNodeSystemStatus(NodeStatus.Removing); EnvFactory.getEnv() .ensureNodeStatus( Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(0)), @@ -170,6 +158,75 @@ public void testRemovingNodeSurvivesLeaderFailure() throws Exception { Collections.singletonList(NodeStatus.Removing)); } + @Test + public void testReadOnlyReasonRebuiltAfterLeaderFailureAndClearedAfterCrash() throws Exception { + initCluster(3, Integer.MAX_VALUE); + setDataNodeSystemStatus(NodeStatus.ReadOnly); + awaitDataNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL); + + // ReadOnly and its reason belong to the live DataNode and reach the new leader in heartbeats. + final int leaderIndex = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + EnvFactory.getEnv().getConfigNodeWrapper(leaderIndex).stopForcibly(); + assertFalse(EnvFactory.getEnv().getConfigNodeWrapper(leaderIndex).isAlive()); + assertNotEquals(leaderIndex, EnvFactory.getEnv().getLeaderConfigNodeIndex()); + awaitDataNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL); + + // Without a shutdown report there is no durable Stopped marker. An unreachable ReadOnly + // DataNode must become Unknown, with its old reason cleared in both node display RPCs. + EnvFactory.getEnv().getDataNodeWrapper(0).stopForcibly(); + assertFalse(EnvFactory.getEnv().getDataNodeWrapper(0).isAlive()); + awaitDataNodeStatusWithReason(NodeStatus.Unknown, null); + } + + private void setDataNodeSystemStatus(NodeStatus status) throws Exception { + final IClientManager clientManager = + new IClientManager.Factory() + .createClientManager( + new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory()); + try (SyncDataNodeInternalServiceClient client = + clientManager.borrowClient( + new TEndPoint( + EnvFactory.getEnv().getDataNodeWrapper(0).getIp(), + EnvFactory.getEnv().getDataNodeWrapper(0).getInternalPort()))) { + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.setSystemStatus(status.getStatus()).getCode()); + } finally { + clientManager.close(); + } + } + + private void awaitDataNodeStatusWithReason(NodeStatus expectedStatus, String expectedReason) + throws InterruptedException { + Throwable lastFailure = null; + for (int retry = 0; retry < 60; retry++) { + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + final TShowClusterResp cluster = client.showCluster(); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), cluster.getStatus().getCode()); + final int dataNodeId = cluster.getDataNodeList().get(0).getDataNodeId(); + assertEquals(expectedStatus.getStatus(), cluster.getNodeStatus().get(dataNodeId)); + assertEquals(expectedReason, cluster.getNodeStatusReason().get(dataNodeId)); + + final TShowDataNodesResp dataNodes = client.showDataNodes(); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), dataNodes.getStatus().getCode()); + final TDataNodeInfo dataNode = + dataNodes.getDataNodesInfoList().stream() + .filter(node -> node.getDataNodeId() == dataNodeId) + .findFirst() + .orElseThrow(AssertionError::new); + assertEquals(expectedStatus.getStatus(), dataNode.getStatus()); + assertEquals(expectedReason, dataNode.getStatusReason()); + return; + } catch (IOException | ClientManagerException | TException | AssertionError e) { + lastFailure = e; + } + TimeUnit.SECONDS.sleep(1); + } + throw new AssertionError( + "DataNode did not reach " + expectedStatus + " with reason " + expectedReason, lastFailure); + } + private void initCluster(int configNodeCount, int snapshotThreshold) throws Exception { EnvFactory.getEnv() .getConfig() diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java index efeeb888e3fe8..4dac5e9feb4ec 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java @@ -64,18 +64,45 @@ public void setSystemStatus() { () -> { ResultSet resultSet = statement.executeQuery("SHOW DATANODES"); int num = 0; + int manualReasonNum = 0; try { while (resultSet.next()) { String status = resultSet.getString("Status"); if (status.equals("ReadOnly")) { num++; } + // The ReadOnly requested through SQL is reported with the Manual reason. + // The StatusReason column only appears once at least one node has reported + // its reason, so a missing column means the condition is not satisfied yet. + if ("Manual".equals(resultSet.getString("StatusReason"))) { + manualReasonNum++; + } } - } catch (InconsistentDataException e) { + } catch (Exception e) { + // The StatusReason column (and the Manual reason behind it) may not be + // propagated yet, or the cluster may be transiently inconsistent while the + // status is changing: treat both as "not satisfied yet". return false; } - return num == EnvFactory.getEnv().getDataNodeWrapperList().size(); + return num == EnvFactory.getEnv().getDataNodeWrapperList().size() + && manualReasonNum == EnvFactory.getEnv().getDataNodeWrapperList().size(); }); + assertShowNodeColumns(true); + + // The table-model NODES view keeps the status and its reason in two separate columns + // instead of merging them into "ReadOnly(Manual)". + try (ResultSet nodesResultSet = + statement.executeQuery("select * from information_schema.nodes")) { + int dataNodeNum = 0; + while (nodesResultSet.next()) { + if ("DataNode".equals(nodesResultSet.getString("node_type"))) { + dataNodeNum++; + Assert.assertEquals("ReadOnly", nodesResultSet.getString("status")); + Assert.assertEquals("Manual", nodesResultSet.getString("status_reason")); + } + } + Assert.assertEquals(EnvFactory.getEnv().getDataNodeWrapperList().size(), dataNodeNum); + } statement.execute("SET SYSTEM TO RUNNING ON CLUSTER"); Awaitility.await() @@ -87,6 +114,9 @@ public void setSystemStatus() { int num = 0; try { + if (resultSet.getMetaData().getColumnCount() != 6) { + return false; + } while (resultSet.next()) { String status = resultSet.getString("Status"); if (status.equals("Running")) { @@ -98,8 +128,44 @@ public void setSystemStatus() { } return num == EnvFactory.getEnv().getDataNodeWrapperList().size(); }); + assertShowNodeColumns(false); } catch (Exception e) { Assert.fail(e.getMessage()); } } + + private static void assertShowNodeColumns(boolean hasReason) throws Exception { + for (String dialect : new String[] {BaseEnv.TREE_SQL_DIALECT, BaseEnv.TABLE_SQL_DIALECT}) { + try (Connection connection = EnvFactory.getEnv().getConnection(dialect); + Statement statement = connection.createStatement()) { + assertShowColumns(statement, "SHOW CLUSTER", 7, hasReason); + assertShowColumns(statement, "SHOW CLUSTER DETAILS", 13, hasReason); + assertShowColumns(statement, "SHOW DATANODES", 6, hasReason); + // SET SYSTEM changes DataNodes only; ConfigNodes keep the layout without a reason column. + assertShowColumns(statement, "SHOW CONFIGNODES", 5, false); + } + } + } + + private static void assertShowColumns( + Statement statement, String sql, int baseColumnCount, boolean hasReason) throws Exception { + try (ResultSet resultSet = statement.executeQuery(sql)) { + int columnCount = resultSet.getMetaData().getColumnCount(); + Assert.assertEquals(sql, baseColumnCount + (hasReason ? 1 : 0), columnCount); + if (hasReason) { + Assert.assertEquals( + sql, "StatusReason", resultSet.getMetaData().getColumnName(columnCount)); + } + Assert.assertTrue(sql, resultSet.next()); + do { + if (hasReason) { + String expectedReason = + sql.startsWith("SHOW CLUSTER") && !"DataNode".equals(resultSet.getString("NodeType")) + ? null + : "Manual"; + Assert.assertEquals(sql, expectedReason, resultSet.getString(columnCount)); + } + } while (resultSet.next()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java index c7f85ea500ea7..be42be9929819 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java @@ -698,6 +698,7 @@ public void testInformationSchema() throws SQLException { "node_id,INT32,TAG,", "node_type,STRING,ATTRIBUTE,", "status,STRING,ATTRIBUTE,", + "status_reason,STRING,ATTRIBUTE,", "internal_address,STRING,ATTRIBUTE,", "internal_port,INT32,ATTRIBUTE,", "version,STRING,ATTRIBUTE,", diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java index df1b7fb1449ef..d0d896c04db9e 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java @@ -140,6 +140,39 @@ public void testNodeCache() throws InterruptedException { LOAD_MANAGER.getEventService().checkAndBroadcastNodeStatisticsChangeEventIfNecessary(); Assert.assertEquals(NodeStatus.Removing, LOAD_CACHE.getNodeStatus(1)); + // An explicit rollback forwards its reason through LoadManager and the change event. + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + LOAD_MANAGER.trySetNodeStatus(1, NodeStatus.ReadOnly, NodeStatus.MANUAL, true).getCode()); + Assert.assertTrue(NODE_SEMAPHORE.tryAcquire(5, TimeUnit.SECONDS)); + Assert.assertEquals(NodeStatus.ReadOnly, LOAD_CACHE.getNodeStatus(1)); + Assert.assertEquals(NodeStatus.MANUAL, LOAD_CACHE.getNodeStatusReason(1)); + differentNodeStatisticsMap = FAKE_SUBSCRIBER.getDifferentNodeStatisticsMap(); + Assert.assertEquals(1, differentNodeStatisticsMap.size()); + Assert.assertEquals( + NodeStatus.Removing, differentNodeStatisticsMap.get(1).getLeft().getStatus()); + Assert.assertEquals( + NodeStatus.ReadOnly, differentNodeStatisticsMap.get(1).getRight().getStatus()); + Assert.assertEquals( + NodeStatus.MANUAL, differentNodeStatisticsMap.get(1).getRight().getStatusReason()); + + // The overload without a reason clears it and broadcasts the reason-only change. + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + LOAD_MANAGER.trySetNodeStatus(1, NodeStatus.ReadOnly, false).getCode()); + Assert.assertTrue(NODE_SEMAPHORE.tryAcquire(5, TimeUnit.SECONDS)); + Assert.assertEquals(NodeStatus.ReadOnly, LOAD_CACHE.getNodeStatus(1)); + Assert.assertNull(LOAD_CACHE.getNodeStatusReason(1)); + differentNodeStatisticsMap = FAKE_SUBSCRIBER.getDifferentNodeStatisticsMap(); + Assert.assertEquals(1, differentNodeStatisticsMap.size()); + Assert.assertEquals( + NodeStatus.ReadOnly, differentNodeStatisticsMap.get(1).getLeft().getStatus()); + Assert.assertEquals( + NodeStatus.MANUAL, differentNodeStatisticsMap.get(1).getLeft().getStatusReason()); + Assert.assertEquals( + NodeStatus.ReadOnly, differentNodeStatisticsMap.get(1).getRight().getStatus()); + Assert.assertNull(differentNodeStatisticsMap.get(1).getRight().getStatusReason()); + // Remove NodeCache LOAD_MANAGER.removeNodeCache(1); NODE_SEMAPHORE.acquire(); @@ -147,7 +180,7 @@ public void testNodeCache() throws InterruptedException { // Only DataNode 1 is updated Assert.assertEquals(1, differentNodeStatisticsMap.size()); Assert.assertEquals( - new Pair<>(new NodeStatistics(NodeStatus.Removing), null), + new Pair<>(new NodeStatistics(NodeStatus.ReadOnly), null), differentNodeStatisticsMap.get(1)); } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java index 40981b07a3b6e..174195281ace2 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java @@ -104,7 +104,7 @@ public void testLoadManagerSuppliesPersistenceBeforeConsensusInitialization() th doThrow(new ConsensusException("quorum unavailable")) .when(consensusManager) .write(any(UpdateNodeStatusPlan.class)); - TSStatus result = cache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, false); + TSStatus result = cache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false); assertFailure(result); assertEquals("quorum unavailable", result.getMessage()); assertEquals(NodeStatus.Stopped, cache.getNodeStatus(NODE_ID)); @@ -115,7 +115,7 @@ public void testLoadManagerSuppliesPersistenceBeforeConsensusInitialization() th .write(any(UpdateNodeStatusPlan.class)); assertTrue(cache.updateNodeStatistics()); assertEquals(NodeStatus.Stopped, info.getPersistedNodeStatus(NODE_ID)); - assertSuccess(cache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, false)); + assertSuccess(cache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false)); verify(consensusManager, times(2)) .write(new UpdateNodeStatusPlan(NODE_ID, Operation.SET_STOPPED)); } finally { @@ -125,34 +125,34 @@ public void testLoadManagerSuppliesPersistenceBeforeConsensusInitialization() th @Test public void testOnlyDurableTransitionsWriteConsensus() throws Exception { - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, false)); - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, false)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, false)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, false)); verify(consensusManager, never()).write(any()); - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, false)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false)); assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, false)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false)); verify(consensusManager, times(1)).write(any()); - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, true)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, null, true)); assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(NODE_ID)); - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, true)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, true)); assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID)); verify(consensusManager, times(3)).write(any()); } @Test public void testFailedWriteIsRetriedAndClearIsDurable() throws Exception { - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, false)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, false)); doReturn(new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())) .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) .when(consensusManager) .write(any(UpdateNodeStatusPlan.class)); - assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, false)); + assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false)); assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID)); assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); // A later connection failure must retain the stop and retry its uncommitted record. - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, false)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, false)); assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); verify(consensusManager, times(2)) @@ -162,7 +162,7 @@ public void testFailedWriteIsRetriedAndClearIsDurable() throws Exception { .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) .when(consensusManager) .write(any(UpdateNodeStatusPlan.class)); - assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, false)); + assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, false)); assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); assertTrue(loadCache.updateNodeStatistics()); @@ -183,10 +183,10 @@ public void testCommittedWriteWithLostResponseIsNotClearedByUnknown() throws Exc }) .when(consensusManager) .write(any(UpdateNodeStatusPlan.class)); - assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, false)); + assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false)); assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, false)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, false)); assertTrue(loadCache.updateNodeStatistics()); assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); @@ -195,19 +195,19 @@ public void testCommittedWriteWithLostResponseIsNotClearedByUnknown() throws Exc @Test public void testOrdinaryUpdatesKeepRemovingWithoutQuorumButRollbackRequiresIt() throws Exception { - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, true)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, null, true)); clearInvocations(consensusManager); doThrow(new ConsensusException("quorum unavailable")).when(consensusManager).write(any()); - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, true)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, null, true)); for (NodeStatus observed : new NodeStatus[] {NodeStatus.Unknown, NodeStatus.Stopped, NodeStatus.Running}) { - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, observed, false)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, observed, null, false)); assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(NODE_ID)); } verify(consensusManager, never()).write(any()); - assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, true)); + assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, true)); assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); - assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, true)); + assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, true)); assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(NODE_ID)); assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); assertFalse(loadCache.updateNodeStatistics()); @@ -226,15 +226,15 @@ public void testOrdinaryUpdatesKeepRemovingWithoutQuorumButRollbackRequiresIt() @Test public void testConnectionFailureKeepsStoppedUntilExplicitReset() throws Exception { - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, false)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false)); clearInvocations(consensusManager); - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, false)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, false)); assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); verify(consensusManager, never()).write(any()); - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, true)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, true)); assertEquals(NodeStatus.Unknown, loadCache.getNodeStatus(NODE_ID)); assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID)); loadCache.updateNodeStatistics(); @@ -247,7 +247,7 @@ private static void assertFailure(TSStatus status) { @Test public void testCommittedClearWithLostResponseIsNotRevertedByStatistics() throws Exception { - assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, true)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, null, true)); clearInvocations(consensusManager); doAnswer( i -> { @@ -256,7 +256,7 @@ public void testCommittedClearWithLostResponseIsNotRevertedByStatistics() throws }) .when(consensusManager) .write(any(UpdateNodeStatusPlan.class)); - assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, true)); + assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, true)); assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID)); assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); assertTrue(loadCache.updateNodeStatistics()); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java index 1a8075a86cc83..d111b9abe0363 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java @@ -43,6 +43,7 @@ import org.apache.iotdb.confignode.manager.node.NodeManager; import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -234,10 +235,84 @@ public void testClearDoesNotRecheckThePreviousPersistedStatus() throws Exception Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); } + @Test + public void testReadOnlyRevivalPublishesReasonWhileStoppedMarkerClearIsRetried() + throws Exception { + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + loadCache.initHeartbeatCache(configManager); + doReturn(failure()) + .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) + .when(consensusManager) + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, readOnlySample(System.nanoTime(), NodeStatus.MANUAL)); + + Assert.assertFalse(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(DATA_NODE_ID)); + Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(DATA_NODE_ID)); + + // The next leader must learn ReadOnly from its own heartbeats, not from durable node state. + loadCache.initHeartbeatCache(configManager); + Assert.assertEquals(NodeStatus.Unknown, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertNull(loadCache.getNodeStatusReason(DATA_NODE_ID)); + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, readOnlySample(System.nanoTime(), NodeStatus.MANUAL)); + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(DATA_NODE_ID)); + } + + @Test + public void testReadOnlyHeartbeatIsNotFilteredByCacheInitializationTime() throws Exception { + long previousLeaderTimestamp = System.nanoTime(); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + loadCache.initHeartbeatCache(configManager); + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, readOnlySample(previousLeaderTimestamp, NodeStatus.MANUAL)); + + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(DATA_NODE_ID)); + Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + verify(consensusManager).write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + } + + @Test + public void testStoppingReasonIsClearedByShutdownAndCannotOverrideRemoving() throws Exception { + nodeInfo.applyNodeStatusPlan( + new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); + loadCache.initHeartbeatCache(configManager); + for (int nodeId : Arrays.asList(DATA_NODE_ID, REMOVING_DATA_NODE_ID)) { + loadCache.cacheDataNodeHeartbeatSample( + nodeId, readOnlySample(System.nanoTime(), NodeStatus.STOPPING)); + } + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.STOPPING, loadCache.getNodeStatusReason(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); + Assert.assertNull(loadCache.getNodeStatusReason(REMOVING_DATA_NODE_ID)); + + for (int nodeId : Arrays.asList(DATA_NODE_ID, REMOVING_DATA_NODE_ID)) { + Assert.assertEquals( + success().getCode(), + loadCache.trySetNodeStatus(nodeId, NodeStatus.Stopped, null, false).getCode()); + Assert.assertNull(loadCache.getNodeStatusReason(nodeId)); + } + Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + Assert.assertEquals( + NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); + } + @Test public void testStoppedStatisticsUpdateAfterFailedPersistenceReturns() throws Exception { loadCache.initHeartbeatCache(configManager); - loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Running, false); + loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Running, null, false); CountDownLatch persistenceStarted = new CountDownLatch(1); CountDownLatch finishPersistence = new CountDownLatch(1); doAnswer( @@ -253,7 +328,7 @@ public void testStoppedStatisticsUpdateAfterFailedPersistenceReturns() throws Ex try { Future update = executor.submit( - () -> loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, false)); + () -> loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, null, false)); Assert.assertTrue(persistenceStarted.await(5, TimeUnit.SECONDS)); Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(DATA_NODE_ID)); @@ -279,7 +354,7 @@ public void testStoppedStatisticsUpdateAfterFailedPersistenceReturns() throws Ex @Test public void testHeartbeatAndStatisticsWaitUntilShutdownPersistenceCompletes() throws Exception { loadCache.initHeartbeatCache(configManager); - loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Running, false); + loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Running, null, false); NodeHeartbeatSample delayedRunning = new NodeHeartbeatSample(NodeStatus.Running); CountDownLatch persistenceStarted = new CountDownLatch(1); CountDownLatch finishPersistence = new CountDownLatch(1); @@ -296,7 +371,7 @@ public void testHeartbeatAndStatisticsWaitUntilShutdownPersistenceCompletes() th try { Future stopped = executor.submit( - () -> loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, false)); + () -> loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, null, false)); Assert.assertTrue(persistenceStarted.await(5, TimeUnit.SECONDS)); Future heartbeat = executor.submit( @@ -367,7 +442,7 @@ public void testForcingAnotherNodeDoesNotClearRemovingStatus() throws Exception Assert.assertEquals( success().getCode(), - loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, false).getCode()); + loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, null, false).getCode()); Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); Assert.assertEquals( @@ -382,7 +457,9 @@ public void testForcingAnotherNodeDoesNotClearRemovingStatus() throws Exception // An explicit rollback of that node still clears its durable Removing marker. Assert.assertEquals( success().getCode(), - loadCache.trySetNodeStatus(REMOVING_DATA_NODE_ID, NodeStatus.Running, true).getCode()); + loadCache + .trySetNodeStatus(REMOVING_DATA_NODE_ID, NodeStatus.Running, null, true) + .getCode()); Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(REMOVING_DATA_NODE_ID)); } @@ -390,15 +467,25 @@ public void testForcingAnotherNodeDoesNotClearRemovingStatus() throws Exception @Test public void testExplicitRollbackRestoresOfflineAndRunningStatuses() throws Exception { for (NodeStatus rollbackStatus : - Arrays.asList(NodeStatus.Unknown, NodeStatus.Stopped, NodeStatus.Running)) { + Arrays.asList( + NodeStatus.Unknown, NodeStatus.Stopped, NodeStatus.Running, NodeStatus.ReadOnly)) { nodeInfo.applyNodeStatusPlan( new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); loadCache.initHeartbeatCache(configManager); Assert.assertEquals( success().getCode(), - loadCache.trySetNodeStatus(REMOVING_DATA_NODE_ID, rollbackStatus, true).getCode()); + loadCache + .trySetNodeStatus( + REMOVING_DATA_NODE_ID, + rollbackStatus, + rollbackStatus == NodeStatus.ReadOnly ? NodeStatus.MANUAL : null, + true) + .getCode()); Assert.assertEquals(rollbackStatus, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); + Assert.assertEquals( + rollbackStatus == NodeStatus.ReadOnly ? NodeStatus.MANUAL : null, + loadCache.getNodeStatusReason(REMOVING_DATA_NODE_ID)); Assert.assertEquals( rollbackStatus == NodeStatus.Stopped ? NodeStatus.Stopped : null, nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); @@ -406,6 +493,9 @@ public void testExplicitRollbackRestoresOfflineAndRunningStatuses() throws Excep // Ordinary statistics updates must preserve the successfully committed rollback. Assert.assertTrue(loadCache.updateNodeStatistics()); Assert.assertEquals(rollbackStatus, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); + Assert.assertEquals( + rollbackStatus == NodeStatus.ReadOnly ? NodeStatus.MANUAL : null, + loadCache.getNodeStatusReason(REMOVING_DATA_NODE_ID)); } } @@ -421,7 +511,9 @@ public void testFailedExplicitRollbackUpdatesStatisticsAndRetriesPeriodically() Assert.assertEquals( failure().getCode(), - loadCache.trySetNodeStatus(REMOVING_DATA_NODE_ID, NodeStatus.Running, true).getCode()); + loadCache + .trySetNodeStatus(REMOVING_DATA_NODE_ID, NodeStatus.Running, null, true) + .getCode()); Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); Assert.assertEquals( NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); @@ -445,7 +537,7 @@ public void testShutdownReportCannotOverrideRemovingStatus() throws Exception { for (int nodeId : Arrays.asList(CONFIG_NODE_ID, REMOVING_DATA_NODE_ID)) { Assert.assertEquals( success().getCode(), - loadCache.trySetNodeStatus(nodeId, NodeStatus.Stopped, false).getCode()); + loadCache.trySetNodeStatus(nodeId, NodeStatus.Stopped, null, false).getCode()); Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(nodeId)); Assert.assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(nodeId)); verify(consensusManager, never()) @@ -482,7 +574,7 @@ public void testRepeatedCacheCreationPreservesPendingShutdown() throws Exception loadCache.initHeartbeatCache(configManager); Assert.assertEquals( success().getCode(), - loadCache.trySetNodeStatus(CONFIG_NODE_ID, NodeStatus.Running, false).getCode()); + loadCache.trySetNodeStatus(CONFIG_NODE_ID, NodeStatus.Running, null, false).getCode()); Assert.assertFalse(loadCache.checkAndSetHeartbeatProcessing(CONFIG_NODE_ID)); CountDownLatch persistenceStarted = new CountDownLatch(1); CountDownLatch finishPersistence = new CountDownLatch(1); @@ -498,7 +590,7 @@ public void testRepeatedCacheCreationPreservesPendingShutdown() throws Exception try { Future shutdown = executor.submit( - () -> loadCache.trySetNodeStatus(CONFIG_NODE_ID, NodeStatus.Stopped, false)); + () -> loadCache.trySetNodeStatus(CONFIG_NODE_ID, NodeStatus.Stopped, null, false)); Assert.assertTrue(persistenceStarted.await(5, TimeUnit.SECONDS)); // A recovered AddConfigNodeProcedure may repeat REGISTER_SUCCESS after registration applied. loadCache.createNodeHeartbeatCache(NodeType.ConfigNode, CONFIG_NODE_ID); @@ -530,7 +622,7 @@ public void testDiscardedCacheCannotClearNewRemovingMarker() throws Exception { nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_REMOVING)); loadCache.initHeartbeatCache(configManager); Assert.assertEquals( - failure().getCode(), discarded.trySetNodeStatus(NodeStatus.Running, true).getCode()); + failure().getCode(), discarded.trySetNodeStatus(NodeStatus.Running, null, true).getCode()); Assert.assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(DATA_NODE_ID)); verify(consensusManager, never()) @@ -561,7 +653,8 @@ public void testAINodeRestoresStoppedAndRemovingAcrossLeaderChange() throws Exce loadCache.initHeartbeatCache(configManager); for (NodeStatus status : Arrays.asList(NodeStatus.Stopped, NodeStatus.Removing)) { Assert.assertEquals( - success().getCode(), loadCache.trySetNodeStatus(AI_NODE_ID, status, false).getCode()); + success().getCode(), + loadCache.trySetNodeStatus(AI_NODE_ID, status, null, false).getCode()); Assert.assertEquals(status, nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); loadCache.initHeartbeatCache(configManager); Assert.assertEquals(status, loadCache.getNodeStatus(AI_NODE_ID)); @@ -584,7 +677,7 @@ public void testAINodeUpdatesStatisticsOnPersistenceFailureAndRetries() throws E .write(new UpdateNodeStatusPlan(AI_NODE_ID, Operation.SET_STOPPED)); Assert.assertEquals( failure().getCode(), - loadCache.trySetNodeStatus(AI_NODE_ID, NodeStatus.Stopped, false).getCode()); + loadCache.trySetNodeStatus(AI_NODE_ID, NodeStatus.Stopped, null, false).getCode()); Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(AI_NODE_ID)); Assert.assertNull(nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); loadCache.cacheAINodeHeartbeatSample(AI_NODE_ID, new NodeHeartbeatSample(NodeStatus.Unknown)); @@ -627,7 +720,7 @@ public void testOlderHeartbeatCannotUndoShutdownReport() throws Exception { NodeHeartbeatSample delayedRunning = new NodeHeartbeatSample(NodeStatus.Running); Assert.assertEquals( success().getCode(), - loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, false).getCode()); + loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, null, false).getCode()); for (NodeHeartbeatSample olderHeartbeat : Arrays.asList( delayedRunning, @@ -656,11 +749,14 @@ public void testBlockedNodeWriteDoesNotHoldOtherNodeCacheLock() throws Exception ExecutorService executor = Executors.newFixedThreadPool(2); try { Future blocked = - executor.submit(() -> loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, true)); + executor.submit( + () -> loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, null, true)); Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); Future independent = executor.submit( - () -> loadCache.trySetNodeStatus(REMOVING_DATA_NODE_ID, NodeStatus.Removing, true)); + () -> + loadCache.trySetNodeStatus( + REMOVING_DATA_NODE_ID, NodeStatus.Removing, null, true)); Assert.assertEquals(success().getCode(), independent.get(5, TimeUnit.SECONDS).getCode()); Assert.assertEquals( NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); @@ -673,6 +769,14 @@ public void testBlockedNodeWriteDoesNotHoldOtherNodeCacheLock() throws Exception } } + private static NodeHeartbeatSample readOnlySample(long timestamp, String reason) { + return new NodeHeartbeatSample( + new TDataNodeHeartbeatResp() + .setHeartbeatTimestamp(timestamp) + .setStatus(NodeStatus.ReadOnly.getStatus()) + .setStatusReason(reason)); + } + private static TSStatus success() { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -688,7 +792,7 @@ public void testLateShutdownReportConvergesAfterRestartedNodeHeartbeat() throws // result, then require a fresh heartbeat from the new process to durably clear that marker. Assert.assertEquals( success().getCode(), - loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, false).getCode()); + loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, null, false).getCode()); Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); loadCache.cacheDataNodeHeartbeatSample( DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/NodeCacheTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/NodeCacheTest.java index 26a88b7accd1d..05262d3955d1f 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/NodeCacheTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/NodeCacheTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.confignode.manager.load.cache; import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.confignode.conf.ConfigNodeConfig; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.manager.load.cache.node.AINodeHeartbeatCache; @@ -26,10 +27,14 @@ import org.apache.iotdb.confignode.manager.load.cache.node.ConfigNodeHeartbeatCache; import org.apache.iotdb.confignode.manager.load.cache.node.DataNodeHeartbeatCache; import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample; +import org.apache.iotdb.confignode.manager.load.cache.node.NodeStatistics; +import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp; import org.junit.Assert; import org.junit.Test; +import java.util.Map; + public class NodeCacheTest { @Test @@ -62,7 +67,7 @@ public void testRemovingAINodeStatisticsSurviveHeartbeatTimeout() { config.setFailureDetectorFixedThresholdInMs(0); try { AINodeHeartbeatCache cache = new AINodeHeartbeatCache(3); - cache.trySetNodeStatus(NodeStatus.Removing, false); + cache.trySetNodeStatus(NodeStatus.Removing, null, false); cache.updateNodeStatistics(); Assert.assertEquals(NodeStatus.Removing, cache.getNodeStatus()); Assert.assertEquals(Long.MAX_VALUE, cache.getLoadScore()); @@ -117,8 +122,8 @@ public void stoppedStatusStickyAndRevivalTest() { // Management restores the requested state explicitly, including offline states. for (NodeStatus restored : new NodeStatus[] {NodeStatus.Unknown, NodeStatus.Stopped, NodeStatus.Running}) { - cache.trySetNodeStatus(NodeStatus.Removing, true); - cache.trySetNodeStatus(restored, true); + cache.trySetNodeStatus(NodeStatus.Removing, null, true); + cache.trySetNodeStatus(restored, null, true); Assert.assertEquals(cacheType, restored, cache.getNodeStatus()); cache.updateNodeStatistics(); Assert.assertEquals(cacheType, restored, cache.getNodeStatus()); @@ -138,7 +143,7 @@ public void testConnectionFailureIsNotUndoneByPreviousRunningHeartbeat() { cache.updateNodeStatistics(); Assert.assertEquals(NodeStatus.Running, cache.getNodeStatus()); - cache.trySetNodeStatus(NodeStatus.Unknown, false); + cache.trySetNodeStatus(NodeStatus.Unknown, null, false); Assert.assertEquals(NodeStatus.Unknown, cache.getNodeStatus()); Assert.assertEquals(Long.MAX_VALUE, cache.getLoadScore()); cache.updateNodeStatistics(); @@ -165,15 +170,15 @@ public void testExplicitStatusUpdateDoesNotRequireLiveHeartbeat() { Assert.assertEquals(NodeStatus.Unknown, cache.getNodeStatus()); // A shutdown report is known information even when heartbeat detection says unavailable. - cache.trySetNodeStatus(NodeStatus.Stopped, false); + cache.trySetNodeStatus(NodeStatus.Stopped, null, false); Assert.assertEquals(NodeStatus.Stopped, cache.getNodeStatus()); Assert.assertEquals(Long.MAX_VALUE, cache.getLoadScore()); cache.updateNodeStatistics(); Assert.assertEquals(NodeStatus.Stopped, cache.getNodeStatus()); - cache.trySetNodeStatus(NodeStatus.Unknown, false); + cache.trySetNodeStatus(NodeStatus.Unknown, null, false); Assert.assertEquals(NodeStatus.Stopped, cache.getNodeStatus()); - cache.trySetNodeStatus(NodeStatus.Unknown, true); + cache.trySetNodeStatus(NodeStatus.Unknown, null, true); Assert.assertEquals(NodeStatus.Unknown, cache.getNodeStatus()); } } finally { @@ -181,4 +186,90 @@ public void testExplicitStatusUpdateDoesNotRequireLiveHeartbeat() { config.setFailureDetectorFixedThresholdInMs(timeout); } } + + @Test + public void statusReasonPropagationTest() { + DataNodeHeartbeatCache dataNodeHeartbeatCache = new DataNodeHeartbeatCache(1); + + // A heartbeat response carrying a status reason (e.g. ReadOnly + DiskFull from the DataNode) + // publishes the reason into the node statistics. + TDataNodeHeartbeatResp heartbeatResp = + new TDataNodeHeartbeatResp() + .setHeartbeatTimestamp(System.nanoTime()) + .setStatus(NodeStatus.ReadOnly.getStatus()) + .setStatusReason(NodeStatus.DISK_FULL); + dataNodeHeartbeatCache.cacheHeartbeatSample(new NodeHeartbeatSample(heartbeatResp)); + dataNodeHeartbeatCache.updateNodeStatistics(); + Assert.assertEquals(NodeStatus.ReadOnly, dataNodeHeartbeatCache.getNodeStatus()); + Assert.assertEquals(NodeStatus.DISK_FULL, dataNodeHeartbeatCache.getNodeStatusReason()); + + // A heartbeat response without a status reason clears the previous reason. + heartbeatResp = + new TDataNodeHeartbeatResp() + .setHeartbeatTimestamp(System.nanoTime()) + .setStatus(NodeStatus.Running.getStatus()); + dataNodeHeartbeatCache.cacheHeartbeatSample(new NodeHeartbeatSample(heartbeatResp)); + dataNodeHeartbeatCache.updateNodeStatistics(); + Assert.assertEquals(NodeStatus.Running, dataNodeHeartbeatCache.getNodeStatus()); + Assert.assertNull(dataNodeHeartbeatCache.getNodeStatusReason()); + + // An Unknown decided by the failure detector (a stale heartbeat) never carries a reason. The + // stale sample is accepted because the fresh cache has an empty sliding window. + DataNodeHeartbeatCache staleCache = new DataNodeHeartbeatCache(3); + staleCache.cacheHeartbeatSample( + new NodeHeartbeatSample(System.nanoTime() - 60_000_000_000L, NodeStatus.ReadOnly)); + staleCache.updateNodeStatistics(); + Assert.assertEquals(NodeStatus.Unknown, staleCache.getNodeStatus()); + Assert.assertNull(staleCache.getNodeStatusReason()); + } + + @Test + public void explicitStatusUpdatesKeepTheSuppliedReason() { + DataNodeHeartbeatCache cache = new DataNodeHeartbeatCache(1); + // ConfigNode must use the supplied reason, without inventing Manual or applying local priority. + for (String reason : new String[] {NodeStatus.MANUAL, NodeStatus.DISK_FULL, null}) { + cache.trySetNodeStatus(NodeStatus.ReadOnly, reason, false); + Assert.assertEquals(NodeStatus.ReadOnly, cache.getNodeStatus()); + Assert.assertEquals(reason, cache.getNodeStatusReason()); + Assert.assertEquals(reason, ((NodeHeartbeatSample) cache.getLastSample()).getStatusReason()); + + cache.updateNodeStatistics(); + Assert.assertEquals(NodeStatus.ReadOnly, cache.getNodeStatus()); + Assert.assertEquals(reason, cache.getNodeStatusReason()); + } + + // A rejected ReadOnly update cannot attach its reason to a node that remains Removing. + cache.trySetNodeStatus(NodeStatus.Removing, null, true); + cache.trySetNodeStatus(NodeStatus.ReadOnly, NodeStatus.DISK_FULL, false); + Assert.assertEquals(NodeStatus.Removing, cache.getNodeStatus()); + Assert.assertNull(cache.getNodeStatusReason()); + cache.updateNodeStatistics(); + Assert.assertEquals(NodeStatus.Removing, cache.getNodeStatus()); + Assert.assertNull(cache.getNodeStatusReason()); + } + + @Test + public void loadCacheSnapshotKeepsStatusAndReasonFromSameStatistics() { + LoadCache loadCache = new LoadCache(); + loadCache.createNodeHeartbeatCache(NodeType.DataNode, 1); + loadCache.cacheDataNodeHeartbeatSample( + 1, + new NodeHeartbeatSample( + new TDataNodeHeartbeatResp() + .setHeartbeatTimestamp(System.nanoTime()) + .setStatus(NodeStatus.ReadOnly.getStatus()) + .setStatusReason(NodeStatus.MANUAL))); + loadCache.updateNodeStatistics(); + + // The snapshot used by SHOW CLUSTER reads status and reason from one statistics object. + Map snapshot = loadCache.getCurrentNodeStatisticsMap(); + Assert.assertEquals(1, snapshot.size()); + Assert.assertEquals(NodeStatus.ReadOnly, snapshot.get(1).getStatus()); + Assert.assertEquals(NodeStatus.MANUAL, snapshot.get(1).getStatusReason()); + Assert.assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(1)); + + // A missing cache yields no reason and Unknown status, without a fake merged string. + Assert.assertNull(loadCache.getNodeStatusReason(2)); + Assert.assertEquals(NodeStatus.Unknown, loadCache.getNodeStatus(2)); + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerStatusReasonTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerStatusReasonTest.java new file mode 100644 index 0000000000000..f9e086eb85c9f --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerStatusReasonTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.node; + +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TNodeResource; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.confignode.manager.IManager; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.load.cache.node.NodeStatistics; +import org.apache.iotdb.confignode.manager.partition.PartitionManager; +import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfo; + +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class NodeManagerStatusReasonTest { + private static final int NODE_ID = 1; + private static final int UNREPORTED_NODE_ID = 2; + + private NodeInfo nodeInfo; + private NodeManager nodeManager; + private AtomicReference currentStatistics; + + @Before + public void setUp() { + nodeInfo = mock(NodeInfo.class); + IManager manager = mock(IManager.class); + LoadManager loadManager = mock(LoadManager.class); + PartitionManager partitionManager = mock(PartitionManager.class); + when(manager.getLoadManager()).thenReturn(loadManager); + when(manager.getPartitionManager()).thenReturn(partitionManager); + when(partitionManager.getAllReplicaSets()).thenReturn(Collections.emptyList()); + nodeManager = new NodeManager(manager, nodeInfo); + + currentStatistics = + new AtomicReference<>( + new NodeStatistics(1, NodeStatus.ReadOnly, NodeStatus.MANUAL, Long.MAX_VALUE)); + NodeStatistics running = new NodeStatistics(2, NodeStatus.Running, null, 0); + // Simulate a heartbeat arriving immediately after the first statistics read. Reading + // status and reason separately would mix ReadOnly with the new heartbeat's null reason. + when(loadManager.getNodeStatisticsSnapshot()) + .thenAnswer( + invocation -> Collections.singletonMap(NODE_ID, currentStatistics.getAndSet(running))); + when(loadManager.getNodeStatus(NODE_ID)) + .thenAnswer(invocation -> currentStatistics.getAndSet(running).getStatus()); + when(loadManager.getNodeStatusReason(NODE_ID)) + .thenAnswer(invocation -> currentStatistics.get().getStatusReason()); + when(loadManager.getNodeStatus(UNREPORTED_NODE_ID)).thenReturn(NodeStatus.Unknown); + } + + @Test + public void testDataNodeStatusAndReasonRemainPairedDuringHeartbeatUpdate() { + when(nodeInfo.getRegisteredDataNodes()) + .thenReturn( + Arrays.asList( + dataNodeConfiguration(NODE_ID), dataNodeConfiguration(UNREPORTED_NODE_ID))); + + List result = nodeManager.getRegisteredDataNodeInfoList(); + + assertEquals(2, result.size()); + assertEquals(NodeStatus.ReadOnly.getStatus(), result.get(0).getStatus()); + assertEquals(NodeStatus.MANUAL, result.get(0).getStatusReason()); + assertEquals(NodeStatus.Unknown.getStatus(), result.get(1).getStatus()); + assertNull(result.get(1).getStatusReason()); + assertEquals(NodeStatus.Running, currentStatistics.get().getStatus()); + assertNull(currentStatistics.get().getStatusReason()); + } + + @Test + public void testConfigNodeStatusAndReasonRemainPairedDuringHeartbeatUpdate() { + when(nodeInfo.getRegisteredConfigNodes()) + .thenReturn( + Arrays.asList(configNodeLocation(NODE_ID), configNodeLocation(UNREPORTED_NODE_ID))); + + List result = nodeManager.getRegisteredConfigNodeInfoList(); + + assertEquals(2, result.size()); + assertEquals(NodeStatus.ReadOnly.getStatus(), result.get(0).getStatus()); + assertEquals(NodeStatus.MANUAL, result.get(0).getStatusReason()); + assertEquals(NodeStatus.Unknown.getStatus(), result.get(1).getStatus()); + assertNull(result.get(1).getStatusReason()); + assertEquals(NodeStatus.Running, currentStatistics.get().getStatus()); + assertNull(currentStatistics.get().getStatusReason()); + } + + private static TDataNodeConfiguration dataNodeConfiguration(int nodeId) { + return new TDataNodeConfiguration() + .setLocation( + new TDataNodeLocation() + .setDataNodeId(nodeId) + .setClientRpcEndPoint(new TEndPoint("127.0.0.1", 6667 + nodeId))) + .setResource(new TNodeResource(1, 1024)); + } + + private static TConfigNodeLocation configNodeLocation(int nodeId) { + return new TConfigNodeLocation() + .setConfigNodeId(nodeId) + .setInternalEndPoint(new TEndPoint("127.0.0.1", 10710 + nodeId)); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java index 2e294aae894cc..be61da3ee5fa6 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java @@ -133,13 +133,15 @@ public void setUp() throws Exception { plan -> failStatusWrite ? failure() : nodeInfo.applyNodeStatusPlan(plan)); cache.initHeartbeatCache(manager); for (TDataNodeLocation node : nodes) { - assertSuccess(cache.trySetNodeStatus(node.getDataNodeId(), NodeStatus.Running, true)); + assertSuccess(cache.trySetNodeStatus(node.getDataNodeId(), NodeStatus.Running, null, true)); } LoadManager loadManager = mock(LoadManager.class); when(manager.getLoadManager()).thenReturn(loadManager); - when(loadManager.trySetNodeStatus(anyInt(), any(), anyBoolean())) + when(loadManager.trySetNodeStatus(anyInt(), any(), any(), anyBoolean())) .thenAnswer( - i -> cache.trySetNodeStatus(i.getArgument(0), i.getArgument(1), i.getArgument(2))); + i -> + cache.trySetNodeStatus( + i.getArgument(0), i.getArgument(1), i.getArgument(2), i.getArgument(3))); when(loadManager.getNodeStatus(anyInt())) .thenAnswer(i -> cache.getNodeStatus(i.getArgument(0))); doAnswer( @@ -224,7 +226,7 @@ public void testPrepareRpcFailureStillPersistsRemoving() throws Exception { @Test public void testOfflineRemovalAndRetryDoNotRequireRpcSuccess() throws Exception { for (NodeStatus offline : Arrays.asList(NodeStatus.Unknown, NodeStatus.Stopped)) { - assertSuccess(cache.trySetNodeStatus(FIRST, offline, true)); + assertSuccess(cache.trySetNodeStatus(FIRST, offline, null, true)); remoteStatuses.put(FIRST, offline); failRpc.put(FIRST, true); change(FIRST, NodeStatus.Removing); @@ -236,7 +238,7 @@ public void testOfflineRemovalAndRetryDoNotRequireRpcSuccess() throws Exception @Test public void testOfflineRemovalStillRequiresPersistence() throws Exception { - assertSuccess(cache.trySetNodeStatus(FIRST, NodeStatus.Stopped, true)); + assertSuccess(cache.trySetNodeStatus(FIRST, NodeStatus.Stopped, null, true)); failRpc.put(FIRST, true); failStatusWrite = true; expectChangeFailure(NodeStatus.Removing); @@ -270,7 +272,8 @@ public void testInitialPrepareCommitFailureRetriesWithoutRecheckingCapacity() th cache.initHeartbeatCache(manager); for (TDataNodeConfiguration node : nodeInfo.getRegisteredDataNodes()) { assertSuccess( - cache.trySetNodeStatus(node.getLocation().getDataNodeId(), NodeStatus.Running, true)); + cache.trySetNodeStatus( + node.getLocation().getDataNodeId(), NodeStatus.Running, null, true)); } when(manager .getLoadManager() @@ -365,6 +368,7 @@ public void testCompensationRestoresRunning() throws Exception { @Test public void testCompensationRestoresReadOnly() throws Exception { assertCompensation(NodeStatus.ReadOnly); + assertEquals(NodeStatus.MANUAL, cache.getNodeStatusReason(FIRST)); } @Test diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHookTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHookTest.java index 3c89c01da2ac8..8ea80cff17756 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHookTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHookTest.java @@ -115,8 +115,7 @@ public void setUp() { @After public void tearDown() { conf.setSeedConfigNode(previousSeed); - commonConf.setNodeStatus(previousStatus); - commonConf.setStatusReason(previousReason); + commonConf.setNodeStatusWithReason(previousStatus, previousReason); commonConf.setCnConnectionTimeoutInMS(previousConnectionTimeout); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplDiskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplDiskTest.java index 7dce3cc3c2de1..e397baabb1672 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplDiskTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplDiskTest.java @@ -53,17 +53,17 @@ public void setUp() { originalDataNodeId = dataNodeConfig.getDataNodeId(); dataNodeConfig.setDataNodeId(0); + // Go through Running first so that a ReadOnly with a higher-priority reason left behind by + // another test can always be overridden. commonConfig.setNodeStatus(NodeStatus.Running); - commonConfig.setStatusReason(null); commonConfig.setDiskSpaceWarningThreshold(0.05); - commonConfig.setNodeStatus(NodeStatus.ReadOnly); - commonConfig.setStatusReason(NodeStatus.DISK_FULL); + commonConfig.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL); } @After public void tearDown() { - commonConfig.setNodeStatus(originalStatus); - commonConfig.setStatusReason(originalStatusReason); + commonConfig.setNodeStatus(NodeStatus.Running); + commonConfig.setNodeStatusWithReason(originalStatus, originalStatusReason); commonConfig.setDiskSpaceWarningThreshold(originalDiskSpaceWarningThreshold); dataNodeConfig.setDataNodeId(originalDataNodeId); } @@ -95,7 +95,6 @@ public void testStorageEngineDiskAggregateStillEntersReadOnly() { when(systemMetrics.getSystemDiskTotalSpace()).thenReturn(100L); commonConfig.setNodeStatus(NodeStatus.Running); - commonConfig.setStatusReason(null); DataNodeContext dataNodeContext = mock(DataNodeContext.class); DataNodeInternalRPCServiceImpl service = new DataNodeInternalRPCServiceImpl(dataNodeContext, systemMetrics); @@ -105,4 +104,36 @@ public void testStorageEngineDiskAggregateStillEntersReadOnly() { Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus()); Assert.assertEquals(NodeStatus.DISK_FULL, commonConfig.getStatusReason()); } + + @Test + public void testRecoveryRequiresExactReadOnlyWithDiskFullReason() { + SystemMetrics systemMetrics = mock(SystemMetrics.class); + // The aggregate free ratio is 52%, so a ReadOnly + DiskFull state would recover. + when(systemMetrics.getSystemDiskAvailableSpace()).thenReturn(104L); + when(systemMetrics.getSystemDiskTotalSpace()).thenReturn(200L); + + DataNodeContext dataNodeContext = mock(DataNodeContext.class); + DataNodeInternalRPCServiceImpl service = + new DataNodeInternalRPCServiceImpl(dataNodeContext, systemMetrics); + + // A manually set ReadOnly must not be auto-recovered by a disk ratio that recovered. + commonConfig.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL); + service.sampleDiskLoad(new TLoadSample()); + Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus()); + Assert.assertEquals(NodeStatus.MANUAL, commonConfig.getStatusReason()); + + // Neither can a ReadOnly entered through an unrecoverable error. + // (Manual outranks UnrecoverableError, so go through Running to enter it.) + commonConfig.setNodeStatus(NodeStatus.Running); + commonConfig.setNodeStatusWithReason( + NodeStatus.ReadOnly, NodeStatus.UNRECOVERABLE_ERROR + ", 2026-09-02 10:00:00.000, broken"); + service.sampleDiskLoad(new TLoadSample()); + Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus()); + Assert.assertTrue(commonConfig.getStatusReason().startsWith(NodeStatus.UNRECOVERABLE_ERROR)); + + commonConfig.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.STOPPING); + service.sampleDiskLoad(new TLoadSample()); + Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus()); + Assert.assertEquals(NodeStatus.STOPPING, commonConfig.getStatusReason()); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplSetSystemStatusTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplSetSystemStatusTest.java new file mode 100644 index 0000000000000..c999737dae6d2 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplSetSystemStatusTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.protocol.thrift.impl; + +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.service.DataNode.DataNodeContext; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.mockito.Mockito.mock; + +public class DataNodeInternalRPCServiceImplSetSystemStatusTest { + + private final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); + private NodeStatus originalStatus; + private String originalStatusReason; + + @BeforeClass + public static void setUpClass() { + // The static initializer of DataNodeInternalRPCServiceImpl (Coordinator) requires it. + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0); + } + + @Before + public void setUp() { + originalStatus = commonConfig.getNodeStatus(); + originalStatusReason = commonConfig.getStatusReason(); + commonConfig.setNodeStatus(NodeStatus.Running); + } + + @After + public void tearDown() { + commonConfig.setNodeStatus(NodeStatus.Running); + commonConfig.setNodeStatusWithReason(originalStatus, originalStatusReason); + } + + @Test + public void testSetSystemStatusReadOnlyCarriesManualReason() throws Exception { + final DataNodeInternalRPCServiceImpl service = + new DataNodeInternalRPCServiceImpl(mock(DataNodeContext.class)); + + // A ReadOnly requested through the RPC (i.e. by the ConfigNode on SET SYSTEM STATUS) is + // recorded with the Manual reason. + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), service.setSystemStatus("ReadOnly").getCode()); + Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus()); + Assert.assertEquals(NodeStatus.MANUAL, commonConfig.getStatusReason()); + + // Switching back to Running clears the reason. + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), service.setSystemStatus("Running").getCode()); + Assert.assertEquals(NodeStatus.Running, commonConfig.getNodeStatus()); + Assert.assertNull(commonConfig.getStatusReason()); + } + + @Test + public void testSetSystemStatusInvalidStatusReturnsError() throws Exception { + final DataNodeInternalRPCServiceImpl service = + new DataNodeInternalRPCServiceImpl(mock(DataNodeContext.class)); + + Assert.assertEquals( + TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode(), + service.setSystemStatus("NotAStatus").getCode()); + Assert.assertEquals(NodeStatus.Running, commonConfig.getNodeStatus()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTaskTest.java new file mode 100644 index 0000000000000..abf570d58fc0c --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTaskTest.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata; + +import org.apache.iotdb.common.rpc.thrift.TAINodeLocation; +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.schema.column.ColumnHeader; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.rpc.TSStatusCode; + +import com.google.common.util.concurrent.SettableFuture; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class ShowClusterDetailsTaskTest { + + private static final int CONFIG_NODE_ID = 0; + private static final int DATA_NODE_ID = 1; + private static final int AI_NODE_ID = 2; + + private static TShowClusterResp buildClusterResp(final Map statusReasonMap) { + final TConfigNodeLocation configNode = + new TConfigNodeLocation( + CONFIG_NODE_ID, new TEndPoint("127.0.0.1", 1000), new TEndPoint("127.0.0.1", 1001)); + final TDataNodeLocation dataNode = + new TDataNodeLocation( + DATA_NODE_ID, + new TEndPoint("127.0.0.1", 2000), + new TEndPoint("127.0.0.1", 2001), + new TEndPoint("127.0.0.1", 2002), + new TEndPoint("127.0.0.1", 2003), + new TEndPoint("127.0.0.1", 2004)); + final TAINodeLocation aiNode = + new TAINodeLocation(AI_NODE_ID, new TEndPoint("127.0.0.1", 3000)); + + final Map nodeStatus = new HashMap<>(); + nodeStatus.put(CONFIG_NODE_ID, NodeStatus.Running.getStatus()); + nodeStatus.put(DATA_NODE_ID, NodeStatus.ReadOnly.getStatus()); + nodeStatus.put(AI_NODE_ID, NodeStatus.Running.getStatus()); + + final Map nodeVersionInfo = new HashMap<>(); + final TNodeVersionInfo versionInfo = new TNodeVersionInfo("2.0.11", "build"); + nodeVersionInfo.put(CONFIG_NODE_ID, versionInfo); + nodeVersionInfo.put(DATA_NODE_ID, versionInfo); + nodeVersionInfo.put(AI_NODE_ID, versionInfo); + + final TShowClusterResp resp = + new TShowClusterResp( + new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()), + Collections.singletonList(configNode), + Collections.singletonList(dataNode), + Collections.singletonList(aiNode), + nodeStatus, + nodeVersionInfo); + if (statusReasonMap != null) { + resp.setNodeStatusReason(statusReasonMap); + } + return resp; + } + + private static ConfigTaskResult execute(final TShowClusterResp resp) throws Exception { + final SettableFuture future = SettableFuture.create(); + ShowClusterDetailsTask.buildTSBlock(resp, future); + return future.get(); + } + + private static List columnNames(final List headers) { + return headers.stream().map(ColumnHeader::getColumnName).collect(Collectors.toList()); + } + + @Test + public void testOldLayoutWhenNoReasonFieldPresent() throws Exception { + final ConfigTaskResult result = execute(buildClusterResp(null)); + final List headers = result.getResultSetHeader().getColumnHeaders(); + + Assert.assertEquals( + columnNames(ColumnHeaderConstant.showClusterDetailsColumnHeaders), columnNames(headers)); + Assert.assertEquals( + ColumnHeaderConstant.showClusterDetailsColumnHeaders.stream() + .map(ColumnHeader::getColumnType) + .collect(Collectors.toList()), + result.getResultSetHeader().getRespDataTypes()); + final Column[] columns = result.getResultSet().getValueColumns(); + Assert.assertEquals(headers.size(), columns.length); + + Assert.assertEquals(NodeStatus.Running.getStatus(), columns[2].getBinary(0).toString()); + Assert.assertEquals(NodeStatus.ReadOnly.getStatus(), columns[2].getBinary(1).toString()); + Assert.assertEquals("2002", columns[8].getBinary(1).toString()); + Assert.assertEquals("2004", columns[9].getBinary(1).toString()); + Assert.assertEquals("2003", columns[10].getBinary(1).toString()); + Assert.assertEquals(AI_NODE_ID, columns[0].getInt(2)); + Assert.assertEquals(3000, columns[4].getInt(2)); + Assert.assertEquals("build", columns[12].getBinary(2).toString()); + } + + @Test + public void testReasonColumnAppendedWithNullForMissingReasons() throws Exception { + final Map statusReasonMap = new HashMap<>(); + statusReasonMap.put(DATA_NODE_ID, NodeStatus.DISK_FULL); + final ConfigTaskResult result = execute(buildClusterResp(statusReasonMap)); + final List headers = result.getResultSetHeader().getColumnHeaders(); + + Assert.assertEquals( + ColumnHeaderConstant.showClusterDetailsColumnHeaders.size() + 1, headers.size()); + Assert.assertEquals( + columnNames(ColumnHeaderConstant.showClusterDetailsColumnHeaders), + columnNames(headers.subList(0, 13))); + Assert.assertEquals(ColumnHeaderConstant.STATUS_REASON, headers.get(13).getColumnName()); + Assert.assertEquals(TSDataType.TEXT, headers.get(13).getColumnType()); + Assert.assertEquals( + headers.stream().map(ColumnHeader::getColumnType).collect(Collectors.toList()), + result.getResultSetHeader().getRespDataTypes()); + final Column[] columns = result.getResultSet().getValueColumns(); + Assert.assertEquals(headers.size(), columns.length); + + Assert.assertTrue(columns[13].isNull(0)); + Assert.assertFalse(columns[13].isNull(1)); + Assert.assertEquals(NodeStatus.DISK_FULL, columns[13].getBinary(1).toString()); + Assert.assertEquals(NodeStatus.ReadOnly.getStatus(), columns[2].getBinary(1).toString()); + Assert.assertEquals("127.0.0.1", columns[3].getBinary(1).toString()); + Assert.assertEquals(2001, columns[4].getInt(1)); + Assert.assertEquals("1001", columns[5].getBinary(0).toString()); + Assert.assertEquals("", columns[6].getBinary(0).toString()); + Assert.assertEquals("127.0.0.1", columns[6].getBinary(1).toString()); + Assert.assertEquals("2000", columns[7].getBinary(1).toString()); + Assert.assertEquals("2002", columns[8].getBinary(1).toString()); + Assert.assertEquals("2004", columns[9].getBinary(1).toString()); + Assert.assertEquals("2003", columns[10].getBinary(1).toString()); + Assert.assertEquals("2.0.11", columns[11].getBinary(1).toString()); + Assert.assertEquals("build", columns[12].getBinary(1).toString()); + Assert.assertEquals(AI_NODE_ID, columns[0].getInt(2)); + Assert.assertEquals("AINode", columns[1].getBinary(2).toString()); + Assert.assertEquals("127.0.0.1", columns[3].getBinary(2).toString()); + Assert.assertEquals(3000, columns[4].getInt(2)); + Assert.assertEquals("", columns[5].getBinary(2).toString()); + Assert.assertEquals("2.0.11", columns[11].getBinary(2).toString()); + Assert.assertEquals("build", columns[12].getBinary(2).toString()); + Assert.assertTrue(columns[13].isNull(2)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTaskTest.java new file mode 100644 index 0000000000000..90d90d49de8a4 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTaskTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata; + +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.schema.column.ColumnHeader; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.rpc.TSStatusCode; + +import com.google.common.util.concurrent.SettableFuture; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class ShowClusterTaskTest { + + private static final int CONFIG_NODE_ID = 0; + private static final int DATA_NODE_ID = 1; + + private static TShowClusterResp buildClusterResp(final Map statusReasonMap) { + final TConfigNodeLocation configNode = + new TConfigNodeLocation( + CONFIG_NODE_ID, new TEndPoint("127.0.0.1", 1000), new TEndPoint("127.0.0.1", 1001)); + final TDataNodeLocation dataNode = + new TDataNodeLocation( + DATA_NODE_ID, + new TEndPoint("127.0.0.1", 2000), + new TEndPoint("127.0.0.1", 2001), + new TEndPoint("127.0.0.1", 2002), + new TEndPoint("127.0.0.1", 2003), + new TEndPoint("127.0.0.1", 2004)); + + final Map nodeStatus = new HashMap<>(); + nodeStatus.put(CONFIG_NODE_ID, NodeStatus.Running.getStatus()); + nodeStatus.put(DATA_NODE_ID, NodeStatus.ReadOnly.getStatus()); + + final Map nodeVersionInfo = new HashMap<>(); + final TNodeVersionInfo versionInfo = new TNodeVersionInfo("2.0.11", "build"); + nodeVersionInfo.put(CONFIG_NODE_ID, versionInfo); + nodeVersionInfo.put(DATA_NODE_ID, versionInfo); + + final TShowClusterResp resp = + new TShowClusterResp( + new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()), + Collections.singletonList(configNode), + Collections.singletonList(dataNode), + Collections.emptyList(), + nodeStatus, + nodeVersionInfo); + // An old ConfigNode response leaves the optional nodeStatusReason field unset. + if (statusReasonMap != null) { + resp.setNodeStatusReason(statusReasonMap); + } + return resp; + } + + private static ConfigTaskResult execute(final TShowClusterResp resp) throws Exception { + final SettableFuture future = SettableFuture.create(); + ShowClusterTask.buildTsBlock(resp, future); + return future.get(); + } + + private static List columnNames(final List headers) { + return headers.stream().map(ColumnHeader::getColumnName).collect(Collectors.toList()); + } + + @Test + public void testOldLayoutWhenNoReasonFieldPresent() throws Exception { + // A response from an old ConfigNode (or a cluster without any reason) keeps the legacy layout: + // no StatusReason column at all. + final ConfigTaskResult result = execute(buildClusterResp(null)); + final List headers = result.getResultSetHeader().getColumnHeaders(); + + Assert.assertEquals( + columnNames(ColumnHeaderConstant.showClusterColumnHeaders), columnNames(headers)); + Assert.assertEquals( + ColumnHeaderConstant.showClusterColumnHeaders.stream() + .map(ColumnHeader::getColumnType) + .collect(Collectors.toList()), + result.getResultSetHeader().getRespDataTypes()); + final Column[] columns = result.getResultSet().getValueColumns(); + Assert.assertEquals(headers.size(), columns.length); + + // The Status column carries the plain status, never a merged "ReadOnly(Manual)" string. + Assert.assertEquals(NodeStatus.Running.getStatus(), columns[2].getBinary(0).toString()); + Assert.assertEquals(NodeStatus.ReadOnly.getStatus(), columns[2].getBinary(1).toString()); + } + + @Test + public void testReasonColumnAppendedWithNullForMissingReasons() throws Exception { + final Map statusReasonMap = new HashMap<>(); + statusReasonMap.put(DATA_NODE_ID, NodeStatus.MANUAL); + final ConfigTaskResult result = execute(buildClusterResp(statusReasonMap)); + final List headers = result.getResultSetHeader().getColumnHeaders(); + + // Appending StatusReason preserves the positions of all existing columns. + Assert.assertEquals(ColumnHeaderConstant.showClusterColumnHeaders.size() + 1, headers.size()); + Assert.assertEquals( + columnNames(ColumnHeaderConstant.showClusterColumnHeaders), + columnNames(headers.subList(0, 7))); + Assert.assertEquals(ColumnHeaderConstant.STATUS_REASON, headers.get(7).getColumnName()); + Assert.assertEquals(TSDataType.TEXT, headers.get(7).getColumnType()); + // The header, its data type list and the TsBlock column count stay consistent. + Assert.assertEquals( + headers.stream().map(ColumnHeader::getColumnType).collect(Collectors.toList()), + result.getResultSetHeader().getRespDataTypes()); + final Column[] columns = result.getResultSet().getValueColumns(); + Assert.assertEquals(headers.size(), columns.length); + + // The ConfigNode has no reason: its StatusReason cell is NULL at the right position. + Assert.assertTrue(columns[7].isNull(0)); + Assert.assertFalse(columns[7].isNull(1)); + Assert.assertEquals(NodeStatus.MANUAL, columns[7].getBinary(1).toString()); + Assert.assertEquals(NodeStatus.ReadOnly.getStatus(), columns[2].getBinary(1).toString()); + Assert.assertEquals("127.0.0.1", columns[3].getBinary(1).toString()); + Assert.assertEquals(1000, columns[4].getInt(0)); + Assert.assertEquals(2001, columns[4].getInt(1)); + Assert.assertEquals("2.0.11", columns[5].getBinary(1).toString()); + Assert.assertEquals("build", columns[6].getBinary(1).toString()); + } + + @Test + public void testReasonColumnAbsentWhenAllReasonsAreEmpty() throws Exception { + // An all-empty reason map behaves like a missing field: the legacy layout is kept. + final Map statusReasonMap = new HashMap<>(); + statusReasonMap.put(CONFIG_NODE_ID, ""); + statusReasonMap.put(DATA_NODE_ID, ""); + final ConfigTaskResult result = execute(buildClusterResp(statusReasonMap)); + + Assert.assertEquals( + columnNames(ColumnHeaderConstant.showClusterColumnHeaders), + columnNames(result.getResultSetHeader().getColumnHeaders())); + Assert.assertEquals( + ColumnHeaderConstant.showClusterColumnHeaders.size(), + result.getResultSet().getValueColumns().length); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowConfigNodesTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowConfigNodesTaskTest.java new file mode 100644 index 0000000000000..3c051a2078306 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowConfigNodesTaskTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.schema.column.ColumnHeader; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowConfigNodesResp; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.rpc.TSStatusCode; + +import com.google.common.util.concurrent.SettableFuture; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class ShowConfigNodesTaskTest { + + private static final int CONFIG_NODE_WITH_REASON_ID = 1; + private static final int CONFIG_NODE_WITHOUT_REASON_ID = 2; + + private static TShowConfigNodesResp buildResp(final boolean setReasonOnFirstNode) { + final TConfigNodeInfo configNodeWithReason = + new TConfigNodeInfo( + CONFIG_NODE_WITH_REASON_ID, + NodeStatus.ReadOnly.getStatus(), + "127.0.0.1", + 1000, + "Leader"); + if (setReasonOnFirstNode) { + configNodeWithReason.setStatusReason(NodeStatus.MANUAL); + } + final TConfigNodeInfo configNodeWithoutReason = + new TConfigNodeInfo( + CONFIG_NODE_WITHOUT_REASON_ID, + NodeStatus.Running.getStatus(), + "127.0.0.1", + 1001, + "Follower"); + return new TShowConfigNodesResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())) + .setConfigNodesInfoList(Arrays.asList(configNodeWithReason, configNodeWithoutReason)); + } + + private static ConfigTaskResult execute(final TShowConfigNodesResp resp) throws Exception { + final SettableFuture future = SettableFuture.create(); + ShowConfigNodesTask.buildTSBlock(resp, future); + return future.get(); + } + + private static List columnNames(final List headers) { + return headers.stream().map(ColumnHeader::getColumnName).collect(Collectors.toList()); + } + + @Test + public void testOldLayoutWhenNoReasonIsSet() throws Exception { + final ConfigTaskResult result = execute(buildResp(false)); + final List headers = result.getResultSetHeader().getColumnHeaders(); + + Assert.assertEquals( + columnNames(ColumnHeaderConstant.showConfigNodesColumnHeaders), columnNames(headers)); + final Column[] columns = result.getResultSet().getValueColumns(); + Assert.assertEquals(headers.size(), columns.length); + + // The Status column carries the plain status, never a merged "ReadOnly(Manual)" string. + Assert.assertEquals(NodeStatus.ReadOnly.getStatus(), columns[1].getBinary(0).toString()); + Assert.assertEquals(NodeStatus.Running.getStatus(), columns[1].getBinary(1).toString()); + } + + @Test + public void testReasonColumnAppendedWithNullForMissingReasons() throws Exception { + final ConfigTaskResult result = execute(buildResp(true)); + final List headers = result.getResultSetHeader().getColumnHeaders(); + + Assert.assertEquals( + ColumnHeaderConstant.showConfigNodesColumnHeaders.size() + 1, headers.size()); + Assert.assertEquals( + columnNames(ColumnHeaderConstant.showConfigNodesColumnHeaders), + columnNames(headers.subList(0, 5))); + Assert.assertEquals(ColumnHeaderConstant.STATUS_REASON, headers.get(5).getColumnName()); + Assert.assertEquals(TSDataType.TEXT, headers.get(5).getColumnType()); + Assert.assertEquals( + headers.stream().map(ColumnHeader::getColumnType).collect(Collectors.toList()), + result.getResultSetHeader().getRespDataTypes()); + final Column[] columns = result.getResultSet().getValueColumns(); + Assert.assertEquals(headers.size(), columns.length); + + Assert.assertFalse(columns[5].isNull(0)); + Assert.assertEquals(NodeStatus.MANUAL, columns[5].getBinary(0).toString()); + Assert.assertTrue(columns[5].isNull(1)); + Assert.assertEquals(NodeStatus.ReadOnly.getStatus(), columns[1].getBinary(0).toString()); + Assert.assertEquals("127.0.0.1", columns[2].getBinary(0).toString()); + Assert.assertEquals(1000, columns[3].getInt(0)); + Assert.assertEquals(1001, columns[3].getInt(1)); + Assert.assertEquals("Leader", columns[4].getBinary(0).toString()); + Assert.assertEquals("Follower", columns[4].getBinary(1).toString()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTaskTest.java new file mode 100644 index 0000000000000..68ea1d5c4bcf0 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTaskTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.schema.column.ColumnHeader; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.rpc.TSStatusCode; + +import com.google.common.util.concurrent.SettableFuture; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.enums.TSDataType; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class ShowDataNodesTaskTest { + + private static final int DATA_NODE_WITH_REASON_ID = 1; + private static final int DATA_NODE_WITHOUT_REASON_ID = 2; + + private static TShowDataNodesResp buildResp(final boolean setReasonOnFirstNode) { + final TDataNodeInfo dataNodeWithReason = + new TDataNodeInfo( + DATA_NODE_WITH_REASON_ID, NodeStatus.ReadOnly.getStatus(), "127.0.0.1", 6667, 1, 1); + if (setReasonOnFirstNode) { + dataNodeWithReason.setStatusReason(NodeStatus.MANUAL); + } + final TDataNodeInfo dataNodeWithoutReason = + new TDataNodeInfo( + DATA_NODE_WITHOUT_REASON_ID, NodeStatus.Running.getStatus(), "127.0.0.1", 6668, 1, 1); + return new TShowDataNodesResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())) + .setDataNodesInfoList(Arrays.asList(dataNodeWithReason, dataNodeWithoutReason)); + } + + private static ConfigTaskResult execute(final TShowDataNodesResp resp) throws Exception { + final SettableFuture future = SettableFuture.create(); + ShowDataNodesTask.buildTSBlock(resp, future); + return future.get(); + } + + private static List columnNames(final List headers) { + return headers.stream().map(ColumnHeader::getColumnName).collect(Collectors.toList()); + } + + @Test + public void testOldLayoutWhenNoReasonIsSet() throws Exception { + // Responses from an old ConfigNode never set the per-node statusReason field. + final ConfigTaskResult result = execute(buildResp(false)); + final List headers = result.getResultSetHeader().getColumnHeaders(); + + Assert.assertEquals( + columnNames(ColumnHeaderConstant.showDataNodesColumnHeaders), columnNames(headers)); + final Column[] columns = result.getResultSet().getValueColumns(); + Assert.assertEquals(headers.size(), columns.length); + + // The Status column carries the plain status, never a merged "ReadOnly(Manual)" string. + Assert.assertEquals(NodeStatus.ReadOnly.getStatus(), columns[1].getBinary(0).toString()); + Assert.assertEquals(NodeStatus.Running.getStatus(), columns[1].getBinary(1).toString()); + } + + @Test + public void testReasonColumnAppendedWithNullForMissingReasons() throws Exception { + final ConfigTaskResult result = execute(buildResp(true)); + final List headers = result.getResultSetHeader().getColumnHeaders(); + + Assert.assertEquals(ColumnHeaderConstant.showDataNodesColumnHeaders.size() + 1, headers.size()); + Assert.assertEquals( + columnNames(ColumnHeaderConstant.showDataNodesColumnHeaders), + columnNames(headers.subList(0, 6))); + Assert.assertEquals(ColumnHeaderConstant.STATUS_REASON, headers.get(6).getColumnName()); + Assert.assertEquals(TSDataType.TEXT, headers.get(6).getColumnType()); + Assert.assertEquals( + headers.stream().map(ColumnHeader::getColumnType).collect(Collectors.toList()), + result.getResultSetHeader().getRespDataTypes()); + final Column[] columns = result.getResultSet().getValueColumns(); + Assert.assertEquals(headers.size(), columns.length); + + // The first DataNode has the reason; the second keeps a NULL at the same position. + Assert.assertFalse(columns[6].isNull(0)); + Assert.assertEquals(NodeStatus.MANUAL, columns[6].getBinary(0).toString()); + Assert.assertTrue(columns[6].isNull(1)); + Assert.assertEquals(NodeStatus.ReadOnly.getStatus(), columns[1].getBinary(0).toString()); + Assert.assertEquals("127.0.0.1", columns[2].getBinary(0).toString()); + Assert.assertEquals(6667, columns[3].getInt(0)); + Assert.assertEquals(6668, columns[3].getInt(1)); + Assert.assertEquals(1, columns[4].getInt(0)); + Assert.assertEquals(1, columns[5].getInt(0)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBufferRollRecoveryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBufferRollRecoveryTest.java index ab081fa138759..84e61d7f00c59 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBufferRollRecoveryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBufferRollRecoveryTest.java @@ -101,8 +101,8 @@ public void tearDown() throws Exception { doCallRealMethod().when(firstWriter).close(); buffer.close(); } finally { - commonConfig.setNodeStatus(previousStatus); - commonConfig.setStatusReason(previousReason); + commonConfig.setNodeStatus(NodeStatus.Running); + commonConfig.setNodeStatusWithReason(previousStatus, previousReason); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManagerTest.java index 44d8b5816c32a..b2a1b841b4002 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManagerTest.java @@ -198,7 +198,6 @@ public void testFolderManagerCanSkipNodeStatusChangeWhenDiskFull() { NodeStatus originalStatus = commonConfig.getNodeStatus(); String originalStatusReason = commonConfig.getStatusReason(); commonConfig.setNodeStatus(NodeStatus.Running); - commonConfig.setStatusReason(null); try { new FolderManager(Collections.emptyList(), strategyType, false); @@ -207,8 +206,60 @@ public void testFolderManagerCanSkipNodeStatusChangeWhenDiskFull() { assertEquals(NodeStatus.Running, commonConfig.getNodeStatus()); assertEquals(null, commonConfig.getStatusReason()); } finally { - commonConfig.setNodeStatus(originalStatus); - commonConfig.setStatusReason(originalStatusReason); + commonConfig.setNodeStatus(NodeStatus.Running); + commonConfig.setNodeStatusWithReason(originalStatus, originalStatusReason); + } + } + + @Test + public void testFolderManagerSetsReadOnlyWithDiskFullReasonWhenDiskFull() { + CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); + NodeStatus originalStatus = commonConfig.getNodeStatus(); + String originalStatusReason = commonConfig.getStatusReason(); + commonConfig.setNodeStatus(NodeStatus.Running); + + try { + // All folders are exhausted, so the node must switch to ReadOnly with the DiskFull reason. + new FolderManager(Collections.emptyList(), strategyType, true); + fail("Expected DiskSpaceInsufficientException"); + } catch (DiskSpaceInsufficientException e) { + assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus()); + assertEquals(NodeStatus.DISK_FULL, commonConfig.getStatusReason()); + } finally { + commonConfig.setNodeStatus(NodeStatus.Running); + commonConfig.setNodeStatusWithReason(originalStatus, originalStatusReason); + } + } + + @Test + public void testFolderManagerReadOnlyReasonWhenFoldersBecomeUnavailable() { + CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); + NodeStatus originalStatus = commonConfig.getNodeStatus(); + String originalStatusReason = commonConfig.getStatusReason(); + + try { + // Exercise runtime selection failures after the manager has initialized successfully. + testFolders.forEach( + folder -> folderManager.updateFolderState(folder, FolderManager.FolderState.ABNORMAL)); + for (String reason : Arrays.asList(null, NodeStatus.MANUAL)) { + commonConfig.setNodeStatus(NodeStatus.Running); + if (reason != null) { + commonConfig.setNodeStatusWithReason(NodeStatus.ReadOnly, reason); + } + + try { + folderManager.getNextFolder(); + fail("Expected DiskSpaceInsufficientException"); + } catch (DiskSpaceInsufficientException e) { + assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus()); + // A disk failure must not replace Manual with an automatically recoverable reason. + assertEquals( + reason == null ? NodeStatus.DISK_FULL : reason, commonConfig.getStatusReason()); + } + } + } finally { + commonConfig.setNodeStatus(NodeStatus.Running); + commonConfig.setNodeStatusWithReason(originalStatus, originalStatusReason); } } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonConfigTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonConfigTest.java index ac03de4131726..86d959c6d552d 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonConfigTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonConfigTest.java @@ -53,12 +53,230 @@ public void testConsensusSubscriptionCatchUpBatchDefaults() { @Test public void testSameNodeStatusDoesNotClearStatusReason() { CommonConfig config = new CommonConfig(); - config.setNodeStatus(NodeStatus.ReadOnly); - config.setStatusReason(NodeStatus.DISK_FULL); + config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL); config.setNodeStatus(NodeStatus.ReadOnly); Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus()); Assert.assertEquals(NodeStatus.DISK_FULL, config.getStatusReason()); } + + @Test + public void testSetReadOnlyWithReasonPriority() { + CommonConfig config = new CommonConfig(); + + // DiskFull can enter from Running. + config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL); + Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus()); + Assert.assertEquals(NodeStatus.DISK_FULL, config.getStatusReason()); + + // UnrecoverableError can override DiskFull. + config.setNodeStatusWithReason( + NodeStatus.ReadOnly, "UnrecoverableError, 2026-09-02 10:00:00.000, first"); + Assert.assertEquals( + "UnrecoverableError, 2026-09-02 10:00:00.000, first", config.getStatusReason()); + + // A second UnrecoverableError keeps the first reason. + config.setNodeStatusWithReason( + NodeStatus.ReadOnly, "UnrecoverableError, 2026-09-02 11:00:00.000, second"); + Assert.assertEquals( + "UnrecoverableError, 2026-09-02 10:00:00.000, first", config.getStatusReason()); + + // Manual overrides UnrecoverableError. + config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL); + Assert.assertEquals(NodeStatus.MANUAL, config.getStatusReason()); + + // DiskFull cannot override Manual. + config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL); + Assert.assertEquals(NodeStatus.MANUAL, config.getStatusReason()); + + // UnrecoverableError cannot override Manual either. + config.setNodeStatusWithReason( + NodeStatus.ReadOnly, "UnrecoverableError, 2026-09-02 12:00:00.000, third"); + Assert.assertEquals(NodeStatus.MANUAL, config.getStatusReason()); + + // Stopping overrides Manual. + config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.STOPPING); + Assert.assertEquals(NodeStatus.STOPPING, config.getStatusReason()); + + // Manual cannot override Stopping. + config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL); + Assert.assertEquals(NodeStatus.STOPPING, config.getStatusReason()); + + // Unknown/Removing are always overridden, as in the pre-reason behavior. + config.setNodeStatus(NodeStatus.Unknown); + config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL); + Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus()); + Assert.assertEquals(NodeStatus.DISK_FULL, config.getStatusReason()); + + config.setNodeStatus(NodeStatus.Removing); + config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL); + Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus()); + Assert.assertEquals(NodeStatus.MANUAL, config.getStatusReason()); + } + + @Test + public void testStoppingOverridesUnrecoverableError() { + CommonConfig config = new CommonConfig(); + + // Stopping is the highest-priority reason: it overrides UnrecoverableError as well. + config.setNodeStatusWithReason( + NodeStatus.ReadOnly, "UnrecoverableError, 2026-09-02 10:00:00.000, broken"); + config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.STOPPING); + Assert.assertEquals(NodeStatus.STOPPING, config.getStatusReason()); + + // And an UnrecoverableError can not override Stopping afterwards. + config.setNodeStatusWithReason( + NodeStatus.ReadOnly, "UnrecoverableError, 2026-09-02 11:00:00.000, broken again"); + Assert.assertEquals(NodeStatus.STOPPING, config.getStatusReason()); + } + + @Test + public void testClassifiedReasonOverridesUnclassifiedReadOnly() { + CommonConfig config = new CommonConfig(); + + // A null-reason ReadOnly can be entered from Running... + config.setNodeStatusWithReason(NodeStatus.ReadOnly, null); + Assert.assertNull(config.getStatusReason()); + + // ...but any classified reason overrides it. + config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL); + Assert.assertEquals(NodeStatus.DISK_FULL, config.getStatusReason()); + + config.setNodeStatusWithReason(NodeStatus.ReadOnly, null); + Assert.assertEquals(NodeStatus.DISK_FULL, config.getStatusReason()); + + config.setNodeStatusWithReason( + NodeStatus.ReadOnly, "UnrecoverableError, 2026-09-02 10:00:00.000, broken"); + Assert.assertEquals( + "UnrecoverableError, 2026-09-02 10:00:00.000, broken", config.getStatusReason()); + + config.setNodeStatusWithReason(NodeStatus.ReadOnly, null); + Assert.assertEquals( + "UnrecoverableError, 2026-09-02 10:00:00.000, broken", config.getStatusReason()); + } + + @Test + public void testHandleUnrecoverableErrorBuildsStatusReason() { + // HandleSystemErrorStrategy writes the singleton config held by CommonDescriptor. + CommonConfig config = CommonDescriptor.getInstance().getConfig(); + NodeStatus originalStatus = config.getNodeStatus(); + String originalStatusReason = config.getStatusReason(); + try { + config.setNodeStatus(NodeStatus.Running); + config.handleUnrecoverableError(new IOException("disk broken")); + Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus()); + Assert.assertTrue(config.getStatusReason().startsWith(NodeStatus.UNRECOVERABLE_ERROR + ", ")); + Assert.assertTrue(config.getStatusReason().contains("disk broken")); + } finally { + config.setNodeStatus(NodeStatus.Running); + config.setNodeStatusWithReason(originalStatus, originalStatusReason); + } + } + + @Test + public void testSetNodeStatusWithReasonClearsReasonForNonReadOnly() { + CommonConfig config = new CommonConfig(); + + for (NodeStatus status : NodeStatus.values()) { + if (status == NodeStatus.ReadOnly) { + continue; + } + config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL); + config.setNodeStatusWithReason(status, NodeStatus.DISK_FULL); + Assert.assertEquals(status, config.getNodeStatus()); + Assert.assertNull(config.getStatusReason()); + + // Repeated updates cannot attach a ReadOnly reason to another status either. + config.setNodeStatusWithReason(status, NodeStatus.MANUAL); + Assert.assertNull(config.getStatusReason()); + } + } + + @Test + public void testNullReadOnlyReasonKeepsLegacySemantics() { + CommonConfig config = new CommonConfig(); + + // A null reason can only enter from a non-ReadOnly status. + config.setNodeStatusWithReason(NodeStatus.ReadOnly, null); + Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus()); + Assert.assertNull(config.getStatusReason()); + + // ...and can never override a classified reason. + config.setNodeStatus(NodeStatus.Running); + config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL); + config.setNodeStatusWithReason(NodeStatus.ReadOnly, null); + Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus()); + Assert.assertEquals(NodeStatus.MANUAL, config.getStatusReason()); + } + + @Test + public void testHandleUnrecoverableErrorFallsBackToClassName() { + CommonConfig config = CommonDescriptor.getInstance().getConfig(); + NodeStatus originalStatus = config.getNodeStatus(); + String originalStatusReason = config.getStatusReason(); + try { + config.setNodeStatus(NodeStatus.Running); + config.handleUnrecoverableError(new RuntimeException()); + Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus()); + Assert.assertTrue(config.getStatusReason().contains("RuntimeException")); + } finally { + config.setNodeStatus(NodeStatus.Running); + config.setNodeStatusWithReason(originalStatus, originalStatusReason); + } + } + + @Test + public void testHandleUnrecoverableErrorTruncatesLongMessage() { + CommonConfig config = CommonDescriptor.getInstance().getConfig(); + NodeStatus originalStatus = config.getNodeStatus(); + String originalStatusReason = config.getStatusReason(); + try { + config.setNodeStatus(NodeStatus.Running); + StringBuilder longMessage = new StringBuilder(); + for (int i = 0; i < 300; i++) { + longMessage.append('x'); + } + config.handleUnrecoverableError(new IOException(longMessage.toString())); + String statusReason = config.getStatusReason(); + Assert.assertTrue(statusReason.endsWith("...")); + Assert.assertTrue( + statusReason.substring(statusReason.lastIndexOf(", ") + 2).length() <= 256 + 3); + } finally { + config.setNodeStatus(NodeStatus.Running); + config.setNodeStatusWithReason(originalStatus, originalStatusReason); + } + } + + @Test + public void testStoppingEntersReadOnlyFromRunningAndBlocksAllOtherReasons() { + CommonConfig config = new CommonConfig(); + + // The shutdown hook performs exactly this transition: from Running into ReadOnly with the + // Stopping reason. The state is transient in the real shutdown sequence, so the entry is + // exercised here directly instead of racing it. + config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.STOPPING); + Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus()); + Assert.assertEquals(NodeStatus.STOPPING, config.getStatusReason()); + + // Stopping has the highest priority: no classified reason can override it. + config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL); + Assert.assertEquals(NodeStatus.STOPPING, config.getStatusReason()); + + config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL); + Assert.assertEquals(NodeStatus.STOPPING, config.getStatusReason()); + + config.setNodeStatusWithReason( + NodeStatus.ReadOnly, "UnrecoverableError, 2026-09-02 10:00:00.000, broken"); + Assert.assertEquals(NodeStatus.STOPPING, config.getStatusReason()); + + // ...nor can an unclassified reason. + config.setNodeStatusWithReason(NodeStatus.ReadOnly, null); + Assert.assertEquals(NodeStatus.STOPPING, config.getStatusReason()); + + // Only an explicit management change leaving ReadOnly clears the reason. + config.setNodeStatus(NodeStatus.Running); + Assert.assertEquals(NodeStatus.Running, config.getNodeStatus()); + Assert.assertNull(config.getStatusReason()); + } } From a154878d530e41b6aca1d24e7cbabecfc921886c Mon Sep 17 00:00:00 2001 From: Di Wang Date: Fri, 9 Oct 2026 14:54:30 +0800 Subject: [PATCH 5/5] feat: persist ReadOnly node status and reasons --- .../it/cluster/IoTDBNodeStatusFailoverIT.java | 78 ++++- .../cluster/IoTDBNodeStatusPersistenceIT.java | 47 ++- .../it/cluster/NodeStatusTestAgent.java | 8 +- .../pipe/it/single/IoTDBShowReceiversIT.java | 6 +- .../confignode/UpdateNodeStatusPlan.java | 51 ++-- .../manager/load/cache/LoadCache.java | 22 +- .../load/cache/node/BaseNodeCache.java | 15 +- .../confignode/persistence/node/NodeInfo.java | 40 +-- .../request/ConfigPhysicalPlanSerDeTest.java | 70 ++++- .../manager/load/LoadManagerTest.java | 12 +- .../load/cache/LoadCacheNodeStatusTest.java | 103 +++++-- .../LoadCachePersistedNodeStatusTest.java | 267 ++++++++++++------ .../confignode/persistence/NodeInfoTest.java | 189 +++++++++---- .../node/RemoveDataNodePersistenceTest.java | 21 +- .../iotdb/commons/cluster/NodeStatus.java | 2 +- .../iotdb/commons/cluster/NodeStatusTest.java | 2 +- 16 files changed, 682 insertions(+), 251 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusFailoverIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusFailoverIT.java index 474fe9148d66a..13dd39c7a3189 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusFailoverIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusFailoverIT.java @@ -35,6 +35,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import com.sun.tools.attach.VirtualMachine; +import org.apache.tsfile.utils.Pair; import org.junit.After; import org.junit.Rule; import org.junit.Test; @@ -179,6 +180,55 @@ public void testNewLeaderChangesSurviveReturnToOriginalLeader() throws Exception assertStatus(1, removing, NodeStatus.Unknown); } + @Test + public void testReadOnlyReasonPersistsAcrossLeaderTransferAndClearsAfterRestart() + throws Exception { + init(2); + TDataNodeLocation dataNode = dataLocation(cluster(), 0); + int dataNodeId = dataNode.getDataNodeId(); + IClientManager pool = + new IClientManager.Factory() + .createClientManager( + new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory()); + try (SyncDataNodeInternalServiceClient client = + pool.borrowClient(dataNode.getInternalEndPoint())) { + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.setSystemStatus(NodeStatus.ReadOnly.getStatus()).getCode()); + } finally { + pool.close(); + } + assertStatus(0, dataNodeId, NodeStatus.ReadOnly); + assertEquals(NodeStatus.MANUAL, cluster().getNodeStatusReason().get(dataNodeId)); + // Inspect every voter's applied record before election; SHOW alone could pass on heartbeats. + for (int i = 0; i < 3; i++) { + awaitPersisted(i, dataNodeId, NodeStatus.ReadOnly, NodeStatus.MANUAL); + } + + int nextLeader = (EnvFactory.getEnv().getLeaderConfigNodeIndex() + 1) % 3; + transfer(nextLeader); + assertEquals(nextLeader, EnvFactory.getEnv().getLeaderConfigNodeIndex()); + for (int i = 0; i < 3; i++) { + awaitPersisted(i, dataNodeId, NodeStatus.ReadOnly, NodeStatus.MANUAL); + } + assertStatus(0, dataNodeId, NodeStatus.ReadOnly); + assertEquals(NodeStatus.MANUAL, cluster().getNodeStatusReason().get(dataNodeId)); + + // Explicitly report after stopping: Windows process termination does not run shutdown hooks. + assertEquals(dataNodeId, stoppedFixture(0)); + assertNull(cluster().getNodeStatusReason().get(dataNodeId)); + for (int i = 0; i < 3; i++) { + awaitPersisted(i, dataNodeId, NodeStatus.Stopped, null); + } + + EnvFactory.getEnv().startDataNode(0); + assertStatus(0, dataNodeId, NodeStatus.Running); + assertNull(cluster().getNodeStatusReason().get(dataNodeId)); + for (int i = 0; i < 3; i++) { + awaitPersisted(i, dataNodeId, null, null); + } + } + @Test public void testLiveOldLeaderPartitionAndQuorumRecovery() throws Exception { init(3); @@ -376,8 +426,12 @@ private void assertSnapshot( // The snapshot may have been replaced since it was enumerated. continue; } - if (java.util.Objects.equals(first, restored.getPersistedNodeStatus(firstId)) - && java.util.Objects.equals(second, restored.getPersistedNodeStatus(secondId))) { + if (java.util.Objects.equals( + first == null ? null : new Pair<>(first, null), + restored.getPersistedNodeStatus(firstId)) + && java.util.Objects.equals( + second == null ? null : new Pair<>(second, null), + restored.getPersistedNodeStatus(secondId))) { return; } } @@ -483,6 +537,7 @@ public void testRealRemovalPersistsIntentAndDeletesRegistration() throws Excepti private int stoppedFixture(int index) throws Exception { TDataNodeLocation node = dataLocation(cluster(), index); EnvFactory.getEnv().getDataNodeWrapper(index).stopForcibly(); + assertFalse(EnvFactory.getEnv().getDataNodeWrapper(index).isAlive()); try (SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { assertEquals(200, client.reportDataNodeShutdown(node).getCode()); @@ -579,15 +634,30 @@ private void heal() throws Exception { } private void awaitPersisted(int node, int dataNode, NodeStatus expected) throws Exception { + awaitPersisted(node, dataNode, expected, null); + } + + private void awaitPersisted(int node, int dataNode, NodeStatus expected, String expectedReason) + throws Exception { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60); do { Properties state = control(node, "inspect:" + dataNode); - if (String.valueOf(expected).equals(state.getProperty("persisted." + dataNode))) { + if (String.valueOf(expected).equals(state.getProperty("persisted." + dataNode)) + && String.valueOf(expectedReason) + .equals(state.getProperty("persistedReason." + dataNode))) { return; } TimeUnit.MILLISECONDS.sleep(200); } while (System.nanoTime() < deadline); - fail("ConfigNode " + node + " did not persist " + expected + " for " + dataNode); + fail( + "ConfigNode " + + node + + " did not persist " + + expected + + " with reason " + + expectedReason + + " for " + + dataNode); } private Path submit(int node, String command) throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java index 5b35fcff0d790..6829780c2f399 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java @@ -40,6 +40,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.TException; +import org.apache.tsfile.utils.Pair; import org.junit.After; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -54,6 +55,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -129,7 +131,11 @@ public void testStoppedNodesSurviveSnapshotRecovery() throws Exception { // Verify actual completed snapshots, including their contents, before restarting the quorum. // A low snapshot threshold alone would not prove snapshot recovery was exercised. for (ConfigNodeWrapper configNode : liveConfigNodes) { - awaitStoppedNodeSnapshot(configNode, stoppedConfigNodeId, stoppedDataNodeId); + awaitNodeStatusSnapshot( + configNode, + Map.of( + stoppedConfigNodeId, new Pair<>(NodeStatus.Stopped, null), + stoppedDataNodeId, new Pair<>(NodeStatus.Stopped, null))); } restartLiveConfigNodes(); assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Stopped); @@ -164,7 +170,7 @@ public void testReadOnlyReasonRebuiltAfterLeaderFailureAndClearedAfterCrash() th setDataNodeSystemStatus(NodeStatus.ReadOnly); awaitDataNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL); - // ReadOnly and its reason belong to the live DataNode and reach the new leader in heartbeats. + // The new leader restores ReadOnly and its reason; live heartbeats continue refreshing them. final int leaderIndex = EnvFactory.getEnv().getLeaderConfigNodeIndex(); EnvFactory.getEnv().getConfigNodeWrapper(leaderIndex).stopForcibly(); assertFalse(EnvFactory.getEnv().getConfigNodeWrapper(leaderIndex).isAlive()); @@ -178,6 +184,30 @@ public void testReadOnlyReasonRebuiltAfterLeaderFailureAndClearedAfterCrash() th awaitDataNodeStatusWithReason(NodeStatus.Unknown, null); } + @Test + public void testReadOnlyReasonSurvivesSnapshotRecovery() throws Exception { + initCluster(3, 1); + setDataNodeSystemStatus(NodeStatus.ReadOnly); + awaitDataNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL); + + final int dataNodeId; + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + dataNodeId = client.showCluster().getDataNodeList().get(0).getDataNodeId(); + } + // Inspect completed snapshots so live heartbeats cannot hide a missing durable reason. + for (ConfigNodeWrapper configNode : getLiveConfigNodes()) { + awaitNodeStatusSnapshot( + configNode, + Collections.singletonMap(dataNodeId, new Pair<>(NodeStatus.ReadOnly, NodeStatus.MANUAL))); + } + restartLiveConfigNodes(); + awaitDataNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL); + + setDataNodeSystemStatus(NodeStatus.Running); + awaitDataNodeStatusWithReason(NodeStatus.Running, null); + } + private void setDataNodeSystemStatus(NodeStatus status) throws Exception { final IClientManager clientManager = new IClientManager.Factory() @@ -299,8 +329,8 @@ private void restartLiveConfigNodes() { liveConfigNodes.forEach(ConfigNodeWrapper::start); } - private void awaitStoppedNodeSnapshot( - ConfigNodeWrapper configNode, int stoppedConfigNodeId, int stoppedDataNodeId) + private void awaitNodeStatusSnapshot( + ConfigNodeWrapper configNode, Map> expectedStatuses) throws Exception { for (int retry = 0; retry < 60; retry++) { final List snapshotFiles = new ArrayList<>(); @@ -334,13 +364,16 @@ private void awaitStoppedNodeSnapshot( // A newer snapshot may have replaced this one since the directory was enumerated. continue; } - if (NodeStatus.Stopped.equals(recovered.getPersistedNodeStatus(stoppedConfigNodeId)) - && NodeStatus.Stopped.equals(recovered.getPersistedNodeStatus(stoppedDataNodeId))) { + if (expectedStatuses.entrySet().stream() + .allMatch( + entry -> + java.util.Objects.equals( + entry.getValue(), recovered.getPersistedNodeStatus(entry.getKey())))) { return; } } TimeUnit.SECONDS.sleep(1); } - fail("No completed snapshot contains both Stopped nodes on " + configNode.getId()); + fail("No completed snapshot contains " + expectedStatuses + " on " + configNode.getId()); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/NodeStatusTestAgent.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/NodeStatusTestAgent.java index a87b6722f92a4..529afd9bccc85 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/NodeStatusTestAgent.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/NodeStatusTestAgent.java @@ -35,6 +35,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.ratis.util.CodeInjectionForTesting; import org.apache.thrift.TException; +import org.apache.tsfile.utils.Pair; import java.io.OutputStream; import java.lang.instrument.Instrumentation; @@ -126,8 +127,13 @@ private static void execute(String command, Path output) { NodeInfo nodeInfo = manager.getNodeManager().getNodeInfo(); for (int i = 1; i < args.length; i++) { int nodeId = Integer.parseInt(args[i]); + Pair persistedStatus = nodeInfo.getPersistedNodeStatus(nodeId); result.setProperty( - "persisted." + nodeId, String.valueOf(nodeInfo.getPersistedNodeStatus(nodeId))); + "persisted." + nodeId, + String.valueOf(persistedStatus == null ? null : persistedStatus.left)); + result.setProperty( + "persistedReason." + nodeId, + String.valueOf(persistedStatus == null ? null : persistedStatus.right)); } break; default: diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java index 9bbdffb9fa870..b43190e96458c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV1Req; import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.cluster.EnvUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT1; @@ -115,7 +116,8 @@ public void testShowReceiversWithStoppedDataNode() throws Exception { env.shutdownDataNode(stoppedDataNodeIndex); env.ensureNodeStatus( - Collections.singletonList(stoppedDataNode), Collections.singletonList(NodeStatus.Unknown)); + Collections.singletonList(stoppedDataNode), + Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop())); assertShowReceiversWithoutDataNode( "show receivers", BaseEnv.TREE_SQL_DIALECT, queryDataNode, stoppedDataNodeId); @@ -245,7 +247,7 @@ public void testReceiverRuntimeClearedAfterDataNodeRestartAndCanReconnect() thro env.ensureNodeStatus( Collections.singletonList(restartedDataNode), - Collections.singletonList(NodeStatus.Unknown)); + Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop())); assertNoThriftReceiverRuntimeOnDataNode( "show receivers", BaseEnv.TREE_SQL_DIALECT, restartedDataNodeId); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateNodeStatusPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateNodeStatusPlan.java index 662503607029b..35140c6177124 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateNodeStatusPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateNodeStatusPlan.java @@ -32,54 +32,55 @@ public class UpdateNodeStatusPlan extends ConfigPhysicalPlan { - public enum Operation { - SET_STOPPED, - SET_REMOVING, - CLEAR; - - /** Maps the final node status to a persistence command without applying transition rules. */ - public static Operation fromNodeStatus(NodeStatus status) { - return switch (status) { - case Stopped -> SET_STOPPED; - case Removing -> SET_REMOVING; - case Running, ReadOnly, Unknown -> CLEAR; - }; - } - } - private int nodeId; - private Operation operation; + // A null status removes the persisted record, including its reason. + private NodeStatus status; + + private String statusReason; public UpdateNodeStatusPlan() { super(ConfigPhysicalPlanType.UpdateNodeStatus); } - public UpdateNodeStatusPlan(int nodeId, Operation operation) { + public UpdateNodeStatusPlan(int nodeId, NodeStatus status) { + this(nodeId, status, null); + } + + public UpdateNodeStatusPlan(int nodeId, NodeStatus status, String statusReason) { this(); this.nodeId = nodeId; - this.operation = operation; + // Resolve clearing before serialization so replay does not depend on future persistence rules. + this.status = status != null && status.isPersistentStatus() ? status : null; + this.statusReason = this.status == null ? null : statusReason; } public int getNodeId() { return nodeId; } - public Operation getOperation() { - return operation; + public NodeStatus getStatus() { + return status; + } + + public String getStatusReason() { + return statusReason; } @Override protected void serializeImpl(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(getType().getPlanType(), stream); ReadWriteIOUtils.write(nodeId, stream); - ReadWriteIOUtils.write(operation.name(), stream); + ReadWriteIOUtils.write(status == null ? null : status.getStatus(), stream); + ReadWriteIOUtils.write(statusReason, stream); } @Override protected void deserializeImpl(ByteBuffer buffer) { nodeId = ReadWriteIOUtils.readInt(buffer); - operation = Operation.valueOf(ReadWriteIOUtils.readString(buffer)); + String statusName = ReadWriteIOUtils.readString(buffer); + status = statusName == null ? null : NodeStatus.parse(statusName); + statusReason = ReadWriteIOUtils.readString(buffer); } @Override @@ -91,11 +92,13 @@ public boolean equals(Object o) { return false; } UpdateNodeStatusPlan that = (UpdateNodeStatusPlan) o; - return nodeId == that.nodeId && operation == that.operation; + return nodeId == that.nodeId + && status == that.status + && Objects.equals(statusReason, that.statusReason); } @Override public int hashCode() { - return Objects.hash(nodeId, operation); + return Objects.hash(nodeId, status, statusReason); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java index f1cab44e73799..b7cf0639f3da7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java @@ -34,7 +34,6 @@ import org.apache.iotdb.confignode.conf.ConfigNodeConfig; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; -import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan.Operation; import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.manager.IManager; @@ -58,6 +57,7 @@ import org.apache.ratis.util.function.CheckedFunction; import org.apache.thrift.annotation.Nullable; +import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -112,7 +112,7 @@ public class LoadCache { private Map> topologyGraph; private final AtomicBoolean topologyUpdated; - private IntFunction persistedNodeStatusReader; + private IntFunction> persistedNodeStatusReader; private CheckedFunction nodeStatusWriter; public LoadCache() { @@ -128,7 +128,7 @@ public LoadCache() { } public void setNodeStatusPersistence( - IntFunction persistedNodeStatusReader, + IntFunction> persistedNodeStatusReader, CheckedFunction nodeStatusWriter) { Objects.requireNonNull(persistedNodeStatusReader); Objects.requireNonNull(nodeStatusWriter); @@ -269,18 +269,19 @@ public void createNodeHeartbeatCache(NodeType nodeType, int nodeId) { heartbeatProcessingMap.putIfAbsent(nodeId, new AtomicBoolean(false)); } - private TSStatus persistNodeStatus(BaseNodeCache cache, NodeStatus status) { + private TSStatus persistNodeStatus(BaseNodeCache cache, NodeStatus status, String reason) { int nodeId = cache.getNodeId(); if (nodeCacheMap.get(nodeId) != cache) { return new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); } - Operation operation = Operation.fromNodeStatus(status); - NodeStatus persistedStatus = operation == Operation.CLEAR ? null : status; - if (persistedNodeStatusReader.apply(nodeId) == persistedStatus) { + UpdateNodeStatusPlan plan = new UpdateNodeStatusPlan(nodeId, status, reason); + Pair persistedStatus = + plan.getStatus() == null ? null : new Pair<>(plan.getStatus(), plan.getStatusReason()); + if (Objects.equals(persistedNodeStatusReader.apply(nodeId), persistedStatus)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } try { - return nodeStatusWriter.apply(new UpdateNodeStatusPlan(nodeId, operation)); + return nodeStatusWriter.apply(plan); } catch (ConsensusException e) { LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e); return new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()) @@ -569,8 +570,9 @@ public List getNodeHeartbeatUnreadyReasons() { || nodeCache instanceof DataNodeHeartbeatCache) && !nodeCache.hasHeartbeatSample() // Stopped/Removing can be restored from persisted records without waiting for a - // heartbeat. - && !nodeCache.getNodeStatus().isPersistentStatus()) { + // heartbeat. ReadOnly still needs a heartbeat to confirm that the node is reachable. + && nodeCache.getNodeStatus() != NodeStatus.Stopped + && nodeCache.getNodeStatus() != NodeStatus.Removing) { unreadyNodes.add(nodeId); } }); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java index a2e3bbe56e50e..07a4a4877f043 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java @@ -25,9 +25,11 @@ import org.apache.iotdb.confignode.manager.load.cache.AbstractLoadCache; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.external.commons.lang3.function.TriFunction; +import org.apache.tsfile.utils.Pair; + import java.util.Collections; import java.util.List; -import java.util.function.BiFunction; /** * NodeCache caches the NodeHeartbeatSamples of a Node. Update and cache the current statistics of @@ -37,7 +39,7 @@ public abstract class BaseNodeCache extends AbstractLoadCache { protected final int nodeId; - private BiFunction nodeStatusPersister; + private TriFunction nodeStatusPersister; /** Constructor for NodeCache with default NodeStatistics. */ protected BaseNodeCache(int nodeId) { @@ -52,12 +54,13 @@ public int getNodeId() { /** Initialize a new cache before adding it to LoadCache's nodeCacheMap. */ public void initPersistence( - NodeStatus persistedStatus, - BiFunction nodeStatusPersister) { + Pair persistedStatus, + TriFunction nodeStatusPersister) { this.nodeStatusPersister = nodeStatusPersister; if (persistedStatus != null) { currentStatistics.set( - new NodeStatistics(System.nanoTime(), persistedStatus, null, Long.MAX_VALUE)); + new NodeStatistics( + System.nanoTime(), persistedStatus.left, persistedStatus.right, Long.MAX_VALUE)); } } @@ -91,7 +94,7 @@ private TSStatus applyNodeStatistics(NodeStatistics newStats, boolean force) { TSStatus result = nodeStatusPersister == null ? new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()) - : nodeStatusPersister.apply(this, newStats.getStatus()); + : nodeStatusPersister.apply(this, newStats.getStatus(), newStats.getStatusReason()); currentStatistics.set(newStats); return result; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java index aa2535bf48d83..098a1545011d5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java @@ -51,6 +51,7 @@ import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.transport.TIOStreamTransport; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -110,7 +111,8 @@ public class NodeInfo implements SnapshotProcessor { private final Map nodeVersionInfo; private final ReentrantReadWriteLock versionInfoReadWriteLock; - private final Map persistedNodeStatuses; + // Map>. A null reason still belongs to a stored status. + private final Map> persistedNodeStatuses; private final ReentrantReadWriteLock persistedNodeStatusReadWriteLock; private static final String SNAPSHOT_FILENAME = "node_info.bin"; @@ -659,16 +661,10 @@ public TSStatus applyNodeStatusPlan(UpdateNodeStatusPlan plan) { if (registeredConfigNodes.containsKey(nodeId) || registeredDataNodes.containsKey(nodeId) || registeredAINodes.containsKey(nodeId)) { - switch (plan.getOperation()) { - case SET_STOPPED: - persistedNodeStatuses.put(nodeId, NodeStatus.Stopped); - break; - case SET_REMOVING: - persistedNodeStatuses.put(nodeId, NodeStatus.Removing); - break; - case CLEAR: - persistedNodeStatuses.remove(nodeId); - break; + if (plan.getStatus() == null) { + persistedNodeStatuses.remove(nodeId); + } else { + persistedNodeStatuses.put(nodeId, new Pair<>(plan.getStatus(), plan.getStatusReason())); } } @@ -678,19 +674,24 @@ public TSStatus applyNodeStatusPlan(UpdateNodeStatusPlan plan) { } } - public NodeStatus getPersistedNodeStatus(int nodeId) { + public Pair getPersistedNodeStatus(int nodeId) { persistedNodeStatusReadWriteLock.readLock().lock(); try { - return persistedNodeStatuses.get(nodeId); + Pair status = persistedNodeStatuses.get(nodeId); + // Pair is mutable, so callers must not receive the value stored in NodeInfo. + return status == null ? null : new Pair<>(status.left, status.right); } finally { persistedNodeStatusReadWriteLock.readLock().unlock(); } } - public Map getPersistedNodeStatuses() { + public Map> getPersistedNodeStatuses() { persistedNodeStatusReadWriteLock.readLock().lock(); try { - return new HashMap<>(persistedNodeStatuses); + Map> result = new HashMap<>(); + persistedNodeStatuses.forEach( + (nodeId, status) -> result.put(nodeId, new Pair<>(status.left, status.right))); + return result; } finally { persistedNodeStatusReadWriteLock.readLock().unlock(); } @@ -793,9 +794,10 @@ private void serializeVersionInfo(OutputStream outputStream) throws IOException private void serializePersistedNodeStatuses(OutputStream outputStream) throws IOException { ReadWriteIOUtils.write(persistedNodeStatuses.size(), outputStream); - for (Entry entry : persistedNodeStatuses.entrySet()) { + for (Entry> entry : persistedNodeStatuses.entrySet()) { ReadWriteIOUtils.write(entry.getKey(), outputStream); - ReadWriteIOUtils.write(entry.getValue().getStatus(), outputStream); + ReadWriteIOUtils.write(entry.getValue().left.getStatus(), outputStream); + ReadWriteIOUtils.write(entry.getValue().right, outputStream); } } @@ -916,7 +918,9 @@ private void deserializePersistedNodeStatuses(InputStream inputStream) throws IO int size = ReadWriteIOUtils.readInt(inputStream); for (int i = 0; i < size; i++) { int nodeId = ReadWriteIOUtils.readInt(inputStream); - persistedNodeStatuses.put(nodeId, NodeStatus.parse(ReadWriteIOUtils.readString(inputStream))); + NodeStatus status = NodeStatus.parse(ReadWriteIOUtils.readString(inputStream)); + String statusReason = ReadWriteIOUtils.readString(inputStream); + persistedNodeStatuses.put(nodeId, new Pair<>(status, statusReason)); } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index 602c035750556..3ec2538c6d44e 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -35,6 +35,7 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.common.rpc.thrift.TTimedQuota; import org.apache.iotdb.common.rpc.thrift.ThrottleType; +import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.consensus.index.impl.IoTProgressIndex; import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; import org.apache.iotdb.commons.exception.IllegalPathException; @@ -74,7 +75,6 @@ import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateClusterIdPlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; -import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan.Operation; import org.apache.iotdb.confignode.consensus.request.write.cq.ActiveCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.AddCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.DropCQPlan; @@ -193,10 +193,12 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.junit.Assert; import org.junit.Test; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -218,14 +220,64 @@ public class ConfigPhysicalPlanSerDeTest { @Test public void testUpdateNodeStatusPlan() throws IOException { - for (Operation operation : Operation.values()) { - UpdateNodeStatusPlan original = new UpdateNodeStatusPlan(37, operation); - UpdateNodeStatusPlan restored = - (UpdateNodeStatusPlan) - ConfigPhysicalPlan.Factory.create(original.serializeToByteBuffer()); - Assert.assertEquals(original, restored); - Assert.assertEquals(37, restored.getNodeId()); - Assert.assertEquals(operation, restored.getOperation()); + for (NodeStatus status : + new NodeStatus[] { + NodeStatus.Stopped, + NodeStatus.Removing, + NodeStatus.ReadOnly, + NodeStatus.Running, + NodeStatus.Unknown, + null + }) { + for (String reason : + new String[] { + null, + NodeStatus.MANUAL, + NodeStatus.DISK_FULL, + NodeStatus.STOPPING, + NodeStatus.UNRECOVERABLE_ERROR + ": failed to recover root.test-0" + }) { + UpdateNodeStatusPlan original = new UpdateNodeStatusPlan(37, status, reason); + UpdateNodeStatusPlan restored = + (UpdateNodeStatusPlan) + ConfigPhysicalPlan.Factory.create(original.serializeToByteBuffer()); + Assert.assertEquals(original, restored); + Assert.assertEquals(original.hashCode(), restored.hashCode()); + Assert.assertEquals(37, restored.getNodeId()); + NodeStatus persistedStatus = status != null && status.isPersistentStatus() ? status : null; + Assert.assertEquals(persistedStatus, restored.getStatus()); + Assert.assertEquals(persistedStatus == null ? null : reason, restored.getStatusReason()); + ByteBuffer serialized = original.serializeToByteBuffer(); + Assert.assertEquals( + ConfigPhysicalPlanType.UpdateNodeStatus.getPlanType(), serialized.getShort()); + Assert.assertEquals(37, ReadWriteIOUtils.readInt(serialized)); + Assert.assertEquals( + persistedStatus == null ? null : persistedStatus.getStatus(), + ReadWriteIOUtils.readString(serialized)); + Assert.assertEquals(restored.getStatusReason(), ReadWriteIOUtils.readString(serialized)); + Assert.assertFalse(serialized.hasRemaining()); + } + } + Assert.assertNotEquals( + new UpdateNodeStatusPlan(37, NodeStatus.ReadOnly, NodeStatus.MANUAL), + new UpdateNodeStatusPlan(37, NodeStatus.ReadOnly, NodeStatus.DISK_FULL)); + } + + @Test + public void testNodeStatusPlanNormalizesNonPersistentStatuses() { + for (String reason : new String[] {null, "test reason"}) { + for (NodeStatus status : + new NodeStatus[] {NodeStatus.Stopped, NodeStatus.Removing, NodeStatus.ReadOnly}) { + UpdateNodeStatusPlan plan = new UpdateNodeStatusPlan(37, status, reason); + Assert.assertEquals(status, plan.getStatus()); + Assert.assertEquals(reason, plan.getStatusReason()); + } + for (NodeStatus status : new NodeStatus[] {NodeStatus.Running, NodeStatus.Unknown, null}) { + UpdateNodeStatusPlan plan = new UpdateNodeStatusPlan(37, status, reason); + Assert.assertNull(plan.getStatus()); + Assert.assertNull(plan.getStatusReason()); + Assert.assertEquals(new UpdateNodeStatusPlan(37, null), plan); + } } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java index d0d896c04db9e..e5028f0bca9e2 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java @@ -27,7 +27,6 @@ import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.commons.cluster.RegionStatus; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; -import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan.Operation; import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.manager.IManager; @@ -404,9 +403,10 @@ public void testLoadWarmUpWaitsForLeaderStoppedMarkerToBeCleared() throws Except when(nodeManager.getRegisteredAINodes()).thenReturn(Collections.emptyList()); when(configManager.getClusterSchemaManager().getDatabaseNames(null)) .thenReturn(Collections.emptyList()); - AtomicReference persistedLeaderStatus = new AtomicReference<>(NodeStatus.Stopped); + AtomicReference> persistedLeaderStatus = + new AtomicReference<>(new Pair<>(NodeStatus.Stopped, null)); Function nodeStatusWriter = mock(Function.class); - when(nodeStatusWriter.apply(new UpdateNodeStatusPlan(leaderId, Operation.CLEAR))) + when(nodeStatusWriter.apply(new UpdateNodeStatusPlan(leaderId, null))) .thenReturn(new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())) .thenReturn(new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())) .thenAnswer( @@ -427,17 +427,17 @@ public void testLoadWarmUpWaitsForLeaderStoppedMarkerToBeCleared() throws Except System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(31)); Assert.assertFalse(loadManager.isLoadReady()); - Assert.assertEquals(NodeStatus.Stopped, persistedLeaderStatus.get()); + Assert.assertEquals(new Pair<>(NodeStatus.Stopped, null), persistedLeaderStatus.get()); Assert.assertEquals( ManagerMessages.MESSAGE_CONFIGNODE_LEADER_IS_WAITING_FOR_NODE_STATUS_PERSISTENCE_8CA96809, loadManager.getLoadReadyReason()); // The first-heartbeat timeout must not bypass a repeated persistence failure. Assert.assertFalse(loadManager.isLoadReady()); - Assert.assertEquals(NodeStatus.Stopped, persistedLeaderStatus.get()); + Assert.assertEquals(new Pair<>(NodeStatus.Stopped, null), persistedLeaderStatus.get()); Assert.assertTrue(loadManager.isLoadReady()); Assert.assertNull(persistedLeaderStatus.get()); - verify(nodeStatusWriter, times(3)).apply(new UpdateNodeStatusPlan(leaderId, Operation.CLEAR)); + verify(nodeStatusWriter, times(3)).apply(new UpdateNodeStatusPlan(leaderId, null)); } finally { loadManager.stopLoadServices(); } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java index 174195281ace2..fa4bc5fb00f63 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java @@ -24,16 +24,18 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; -import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan.Operation; import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.manager.IManager; import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample; import org.apache.iotdb.confignode.manager.node.NodeManager; import org.apache.iotdb.confignode.persistence.node.NodeInfo; import org.apache.iotdb.consensus.exception.ConsensusException; +import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.utils.Pair; import org.junit.Before; import org.junit.Test; @@ -114,10 +116,10 @@ public void testLoadManagerSuppliesPersistenceBeforeConsensusInitialization() th .when(consensusManager) .write(any(UpdateNodeStatusPlan.class)); assertTrue(cache.updateNodeStatistics()); - assertEquals(NodeStatus.Stopped, info.getPersistedNodeStatus(NODE_ID)); + assertEquals(new Pair<>(NodeStatus.Stopped, null), info.getPersistedNodeStatus(NODE_ID)); assertSuccess(cache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false)); verify(consensusManager, times(2)) - .write(new UpdateNodeStatusPlan(NODE_ID, Operation.SET_STOPPED)); + .write(new UpdateNodeStatusPlan(NODE_ID, NodeStatus.Stopped)); } finally { manager.close(); } @@ -130,17 +132,85 @@ public void testOnlyDurableTransitionsWriteConsensus() throws Exception { verify(consensusManager, never()).write(any()); assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false)); - assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertEquals(new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(NODE_ID)); assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false)); verify(consensusManager, times(1)).write(any()); assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, null, true)); - assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertEquals(new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(NODE_ID)); assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, true)); assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID)); verify(consensusManager, times(3)).write(any()); } + @Test + public void testFailedReadOnlyReasonChangePublishesStatisticsAndRetriesPersistence() + throws Exception { + assertSuccess( + loadCache.trySetNodeStatus(NODE_ID, NodeStatus.ReadOnly, NodeStatus.DISK_FULL, false)); + UpdateNodeStatusPlan reasonChange = + new UpdateNodeStatusPlan(NODE_ID, NodeStatus.ReadOnly, NodeStatus.MANUAL); + doReturn(new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())) + .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) + .when(consensusManager) + .write(reasonChange); + + assertFailure( + loadCache.trySetNodeStatus(NODE_ID, NodeStatus.ReadOnly, NodeStatus.MANUAL, false)); + assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(NODE_ID)); + assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(NODE_ID)); + assertEquals( + new Pair<>(NodeStatus.ReadOnly, NodeStatus.DISK_FULL), + nodeInfo.getPersistedNodeStatus(NODE_ID)); + + assertTrue(loadCache.updateNodeStatistics()); + assertEquals( + new Pair<>(NodeStatus.ReadOnly, NodeStatus.MANUAL), + nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertTrue(loadCache.updateNodeStatistics()); + verify(consensusManager, times(2)).write(reasonChange); + } + + @Test + public void testReadOnlyReasonIsRemovedWhenPersistentStatusChanges() throws Exception { + for (NodeStatus requested : new NodeStatus[] {NodeStatus.Stopped, NodeStatus.Removing}) { + assertSuccess( + loadCache.trySetNodeStatus(NODE_ID, NodeStatus.ReadOnly, NodeStatus.MANUAL, true)); + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, requested, null, false)); + assertEquals(requested, loadCache.getNodeStatus(NODE_ID)); + assertNull(loadCache.getNodeStatusReason(NODE_ID)); + assertEquals(new Pair<>(requested, null), nodeInfo.getPersistedNodeStatus(NODE_ID)); + } + } + + @Test + public void testReadOnlyIsClearedOnDisconnectAndPersistedAgainAfterHeartbeat() throws Exception { + Pair readOnly = new Pair<>(NodeStatus.ReadOnly, NodeStatus.DISK_FULL); + assertSuccess( + loadCache.trySetNodeStatus(NODE_ID, NodeStatus.ReadOnly, NodeStatus.DISK_FULL, false)); + assertEquals(readOnly, nodeInfo.getPersistedNodeStatus(NODE_ID)); + + assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, false)); + assertEquals(NodeStatus.Unknown, loadCache.getNodeStatus(NODE_ID)); + assertNull(loadCache.getNodeStatusReason(NODE_ID)); + assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID)); + + loadCache.cacheDataNodeHeartbeatSample( + NODE_ID, + new NodeHeartbeatSample( + new TDataNodeHeartbeatResp() + .setHeartbeatTimestamp(System.nanoTime()) + .setStatus(NodeStatus.ReadOnly.getStatus()) + .setStatusReason(NodeStatus.DISK_FULL))); + assertTrue(loadCache.updateNodeStatistics()); + assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(NODE_ID)); + assertEquals(NodeStatus.DISK_FULL, loadCache.getNodeStatusReason(NODE_ID)); + assertEquals(readOnly, nodeInfo.getPersistedNodeStatus(NODE_ID)); + verify(consensusManager, times(2)) + .write(new UpdateNodeStatusPlan(NODE_ID, NodeStatus.ReadOnly, NodeStatus.DISK_FULL)); + verify(consensusManager).write(new UpdateNodeStatusPlan(NODE_ID, null)); + } + @Test public void testFailedWriteIsRetriedAndClearIsDurable() throws Exception { assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, false)); @@ -154,16 +224,15 @@ public void testFailedWriteIsRetriedAndClearIsDurable() throws Exception { // A later connection failure must retain the stop and retry its uncommitted record. assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, false)); assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); - assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); - verify(consensusManager, times(2)) - .write(new UpdateNodeStatusPlan(NODE_ID, Operation.SET_STOPPED)); + assertEquals(new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(NODE_ID)); + verify(consensusManager, times(2)).write(new UpdateNodeStatusPlan(NODE_ID, NodeStatus.Stopped)); doThrow(new ConsensusException("test consensus failure")) .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) .when(consensusManager) .write(any(UpdateNodeStatusPlan.class)); assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, false)); - assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertEquals(new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(NODE_ID)); assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); assertTrue(loadCache.updateNodeStatistics()); assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); @@ -184,12 +253,12 @@ public void testCommittedWriteWithLostResponseIsNotClearedByUnknown() throws Exc .when(consensusManager) .write(any(UpdateNodeStatusPlan.class)); assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false)); - assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertEquals(new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(NODE_ID)); assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, false)); assertTrue(loadCache.updateNodeStatistics()); assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); - assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertEquals(new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(NODE_ID)); verify(consensusManager, times(1)).write(any()); } @@ -208,11 +277,11 @@ public void testOrdinaryUpdatesKeepRemovingWithoutQuorumButRollbackRequiresIt() assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, true)); assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, true)); - assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertEquals(new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(NODE_ID)); assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); assertFalse(loadCache.updateNodeStatistics()); assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); - assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertEquals(new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(NODE_ID)); doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) .when(consensusManager) @@ -220,8 +289,7 @@ public void testOrdinaryUpdatesKeepRemovingWithoutQuorumButRollbackRequiresIt() assertTrue(loadCache.updateNodeStatistics()); assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID)); - verify(consensusManager, never()) - .write(new UpdateNodeStatusPlan(NODE_ID, Operation.SET_REMOVING)); + verify(consensusManager, never()).write(new UpdateNodeStatusPlan(NODE_ID, NodeStatus.Removing)); } @Test @@ -231,7 +299,7 @@ public void testConnectionFailureKeepsStoppedUntilExplicitReset() throws Excepti assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, false)); assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID)); - assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(NODE_ID)); + assertEquals(new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(NODE_ID)); verify(consensusManager, never()).write(any()); assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, true)); @@ -263,7 +331,6 @@ public void testCommittedClearWithLostResponseIsNotRevertedByStatistics() throws assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID)); assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID)); verify(consensusManager, times(1)).write(any()); - verify(consensusManager, never()) - .write(new UpdateNodeStatusPlan(NODE_ID, Operation.SET_REMOVING)); + verify(consensusManager, never()).write(new UpdateNodeStatusPlan(NODE_ID, NodeStatus.Removing)); } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java index d111b9abe0363..c8b9c84134fb5 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java @@ -31,7 +31,6 @@ import org.apache.iotdb.confignode.consensus.request.write.ainode.RegisterAINodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; -import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan.Operation; import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; import org.apache.iotdb.confignode.manager.IManager; import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; @@ -46,6 +45,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.utils.Pair; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -129,11 +129,11 @@ SELF_ID, new TEndPoint("127.0.0.1", 11000), new TEndPoint("127.0.0.1", 12000)), @Test public void testInitializationRestoresStickyStatusesAndKeepsLeaderRunning() throws Exception { - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, Operation.SET_STOPPED)); - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, NodeStatus.Stopped)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped)); nodeInfo.applyNodeStatusPlan( - new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(SELF_ID, Operation.SET_STOPPED)); + new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(SELF_ID, NodeStatus.Stopped)); loadCache.initHeartbeatCache(configManager); @@ -164,7 +164,8 @@ public void testRestoredStatusDoesNotCreateHeartbeatSamples() throws Exception { new DataNodeHeartbeatCache(DATA_NODE_ID), new ConfigNodeHeartbeatCache(CONFIG_NODE_ID), new AINodeHeartbeatCache(AI_NODE_ID))) { - cache.initPersistence(status, (ignoredCache, ignoredStatus) -> success()); + cache.initPersistence( + new Pair<>(status, null), (ignoredCache, ignoredStatus, ignoredReason) -> success()); Assert.assertEquals(status, cache.getNodeStatus()); Assert.assertFalse(cache.hasHeartbeatSample()); cache.updateNodeStatistics(); @@ -174,10 +175,78 @@ public void testRestoredStatusDoesNotCreateHeartbeatSamples() throws Exception { } } + @Test + public void testRestoredReadOnlyRequiresHeartbeatAndUnknownClearsPersistence() throws Exception { + Pair persisted = new Pair<>(NodeStatus.ReadOnly, NodeStatus.DISK_FULL); + for (BaseNodeCache cache : + Arrays.asList( + new DataNodeHeartbeatCache(DATA_NODE_ID), + new ConfigNodeHeartbeatCache(CONFIG_NODE_ID), + new AINodeHeartbeatCache(AI_NODE_ID))) { + cache.initPersistence(persisted, (ignoredCache, ignoredStatus, ignoredReason) -> success()); + Assert.assertEquals(NodeStatus.ReadOnly, cache.getNodeStatus()); + Assert.assertEquals(NodeStatus.DISK_FULL, cache.getNodeStatusReason()); + Assert.assertFalse(cache.hasHeartbeatSample()); + } + + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, NodeStatus.Stopped)); + nodeInfo.applyNodeStatusPlan( + new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing)); + nodeInfo.applyNodeStatusPlan( + new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.ReadOnly, NodeStatus.DISK_FULL)); + loadCache.initHeartbeatCache(configManager); + Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.DISK_FULL, loadCache.getNodeStatusReason(DATA_NODE_ID)); + Assert.assertEquals( + Collections.singletonList("nodes=[" + DATA_NODE_ID + "]"), + loadCache.getNodeHeartbeatUnreadyReasons()); + + // Restoring durable metadata is not evidence that the node is still reachable. + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.Unknown, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertNull(loadCache.getNodeStatusReason(DATA_NODE_ID)); + Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + Assert.assertEquals( + Collections.singletonList("nodes=[" + DATA_NODE_ID + "]"), + loadCache.getNodeHeartbeatUnreadyReasons()); + verify(consensusManager).write(new UpdateNodeStatusPlan(DATA_NODE_ID, null)); + } + + @Test + public void testReadOnlyReasonChangesArePersistedAndRunningClearsRecord() throws Exception { + loadCache.initHeartbeatCache(configManager); + // Each heartbeat replaces the previous reason, including a lower-priority or absent reason. + for (String reason : + Arrays.asList(NodeStatus.STOPPING, NodeStatus.MANUAL, NodeStatus.DISK_FULL, null)) { + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, readOnlySample(System.nanoTime(), reason)); + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(reason, loadCache.getNodeStatusReason(DATA_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.ReadOnly, reason), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + + // Repeating the same status and reason must not append another consensus log. + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, readOnlySample(System.nanoTime(), reason)); + Assert.assertTrue(loadCache.updateNodeStatistics()); + verify(consensusManager, times(1)) + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.ReadOnly, reason)); + } + + loadCache.cacheDataNodeHeartbeatSample( + DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); + Assert.assertTrue(loadCache.updateNodeStatistics()); + Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertNull(loadCache.getNodeStatusReason(DATA_NODE_ID)); + Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + verify(consensusManager).write(new UpdateNodeStatusPlan(DATA_NODE_ID, null)); + } + @Test public void testLiveHeartbeatClearsStoppedStatusForBothNodeTypes() throws Exception { - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, Operation.SET_STOPPED)); - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, NodeStatus.Stopped)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped)); loadCache.initHeartbeatCache(configManager); loadCache.cacheConfigNodeHeartbeatSample( @@ -194,29 +263,29 @@ public void testLiveHeartbeatClearsStoppedStatusForBothNodeTypes() throws Except @Test public void testRevivalUpdatesStatisticsWhileClearIsRetried() throws Exception { - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped)); loadCache.initHeartbeatCache(configManager); doReturn(failure()) .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) .when(consensusManager) - .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, null)); loadCache.cacheDataNodeHeartbeatSample( DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); Assert.assertFalse(loadCache.updateNodeStatistics()); Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); - Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); Assert.assertTrue(loadCache.updateNodeStatistics()); Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(DATA_NODE_ID)); - verify(consensusManager, times(2)) - .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + verify(consensusManager, times(2)).write(new UpdateNodeStatusPlan(DATA_NODE_ID, null)); } @Test public void testClearDoesNotRecheckThePreviousPersistedStatus() throws Exception { - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped)); loadCache.initHeartbeatCache(configManager); loadCache.cacheDataNodeHeartbeatSample( DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); @@ -224,11 +293,11 @@ public void testClearDoesNotRecheckThePreviousPersistedStatus() throws Exception doAnswer( invocation -> { nodeInfo.applyNodeStatusPlan( - new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_REMOVING)); + new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Removing)); return nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)); }) .when(consensusManager) - .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, null)); Assert.assertTrue(loadCache.updateNodeStatistics()); Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); @@ -236,31 +305,35 @@ public void testClearDoesNotRecheckThePreviousPersistedStatus() throws Exception } @Test - public void testReadOnlyRevivalPublishesReasonWhileStoppedMarkerClearIsRetried() - throws Exception { - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + public void testReadOnlyRevivalPublishesReasonWhilePersistenceIsRetried() throws Exception { + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped)); loadCache.initHeartbeatCache(configManager); doReturn(failure()) .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) .when(consensusManager) - .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.ReadOnly, NodeStatus.MANUAL)); loadCache.cacheDataNodeHeartbeatSample( DATA_NODE_ID, readOnlySample(System.nanoTime(), NodeStatus.MANUAL)); Assert.assertFalse(loadCache.updateNodeStatistics()); Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID)); Assert.assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(DATA_NODE_ID)); - Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); Assert.assertTrue(loadCache.updateNodeStatistics()); Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID)); Assert.assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(DATA_NODE_ID)); - Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(DATA_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.ReadOnly, NodeStatus.MANUAL), + nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + verify(consensusManager, times(2)) + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.ReadOnly, NodeStatus.MANUAL)); - // The next leader must learn ReadOnly from its own heartbeats, not from durable node state. + // A new leader restores the committed reason before receiving its first heartbeat. loadCache.initHeartbeatCache(configManager); - Assert.assertEquals(NodeStatus.Unknown, loadCache.getNodeStatus(DATA_NODE_ID)); - Assert.assertNull(loadCache.getNodeStatusReason(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID)); + Assert.assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(DATA_NODE_ID)); loadCache.cacheDataNodeHeartbeatSample( DATA_NODE_ID, readOnlySample(System.nanoTime(), NodeStatus.MANUAL)); Assert.assertTrue(loadCache.updateNodeStatistics()); @@ -271,7 +344,7 @@ public void testReadOnlyRevivalPublishesReasonWhileStoppedMarkerClearIsRetried() @Test public void testReadOnlyHeartbeatIsNotFilteredByCacheInitializationTime() throws Exception { long previousLeaderTimestamp = System.nanoTime(); - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped)); loadCache.initHeartbeatCache(configManager); loadCache.cacheDataNodeHeartbeatSample( DATA_NODE_ID, readOnlySample(previousLeaderTimestamp, NodeStatus.MANUAL)); @@ -279,14 +352,17 @@ public void testReadOnlyHeartbeatIsNotFilteredByCacheInitializationTime() throws Assert.assertTrue(loadCache.updateNodeStatistics()); Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID)); Assert.assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(DATA_NODE_ID)); - Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); - verify(consensusManager).write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + Assert.assertEquals( + new Pair<>(NodeStatus.ReadOnly, NodeStatus.MANUAL), + nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + verify(consensusManager) + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.ReadOnly, NodeStatus.MANUAL)); } @Test public void testStoppingReasonIsClearedByShutdownAndCannotOverrideRemoving() throws Exception { nodeInfo.applyNodeStatusPlan( - new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); + new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing)); loadCache.initHeartbeatCache(configManager); for (int nodeId : Arrays.asList(DATA_NODE_ID, REMOVING_DATA_NODE_ID)) { loadCache.cacheDataNodeHeartbeatSample( @@ -304,9 +380,11 @@ public void testStoppingReasonIsClearedByShutdownAndCannotOverrideRemoving() thr loadCache.trySetNodeStatus(nodeId, NodeStatus.Stopped, null, false).getCode()); Assert.assertNull(loadCache.getNodeStatusReason(nodeId)); } - Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); Assert.assertEquals( - NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); + new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.Removing, null), + nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); } @Test @@ -323,7 +401,7 @@ public void testStoppedStatisticsUpdateAfterFailedPersistenceReturns() throws Ex }) .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) .when(consensusManager) - .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped)); ExecutorService executor = Executors.newSingleThreadExecutor(); try { Future update = @@ -342,9 +420,10 @@ public void testStoppedStatisticsUpdateAfterFailedPersistenceReturns() throws Ex DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Unknown)); Assert.assertTrue(loadCache.updateNodeStatistics()); Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(DATA_NODE_ID)); - Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); verify(consensusManager, times(2)) - .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped)); } finally { finishPersistence.countDown(); executor.shutdownNow(); @@ -366,7 +445,7 @@ public void testHeartbeatAndStatisticsWaitUntilShutdownPersistenceCompletes() th return nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)); }) .when(consensusManager) - .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped)); ExecutorService executor = Executors.newFixedThreadPool(3); try { Future stopped = @@ -397,14 +476,15 @@ public void testHeartbeatAndStatisticsWaitUntilShutdownPersistenceCompletes() th Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); verify(consensusManager, times(1)) - .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped)); finishPersistence.countDown(); Assert.assertEquals(success().getCode(), stopped.get(5, TimeUnit.SECONDS).getCode()); heartbeat.get(5, TimeUnit.SECONDS); Assert.assertTrue(statistics.get(5, TimeUnit.SECONDS)); Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(DATA_NODE_ID)); - Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); loadCache.cacheDataNodeHeartbeatSample( DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); @@ -421,7 +501,7 @@ public void testHeartbeatAndStatisticsWaitUntilShutdownPersistenceCompletes() th @Test public void testHeartbeatIsNotFilteredByCacheInitializationTime() throws Exception { long oldSampleTimestamp = System.nanoTime() - 1; - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped)); loadCache.initHeartbeatCache(configManager); loadCache.cacheDataNodeHeartbeatSample( DATA_NODE_ID, new NodeHeartbeatSample(oldSampleTimestamp, NodeStatus.Running)); @@ -429,13 +509,13 @@ public void testHeartbeatIsNotFilteredByCacheInitializationTime() throws Excepti Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); - verify(consensusManager).write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + verify(consensusManager).write(new UpdateNodeStatusPlan(DATA_NODE_ID, null)); } @Test public void testForcingAnotherNodeDoesNotClearRemovingStatus() throws Exception { nodeInfo.applyNodeStatusPlan( - new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); + new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing)); loadCache.initHeartbeatCache(configManager); loadCache.cacheDataNodeHeartbeatSample( REMOVING_DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); @@ -446,7 +526,8 @@ public void testForcingAnotherNodeDoesNotClearRemovingStatus() throws Exception Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); Assert.assertEquals( - NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); + new Pair<>(NodeStatus.Removing, null), + nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); verify(consensusManager, never()) .write( argThat( @@ -470,7 +551,7 @@ public void testExplicitRollbackRestoresOfflineAndRunningStatuses() throws Excep Arrays.asList( NodeStatus.Unknown, NodeStatus.Stopped, NodeStatus.Running, NodeStatus.ReadOnly)) { nodeInfo.applyNodeStatusPlan( - new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); + new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing)); loadCache.initHeartbeatCache(configManager); Assert.assertEquals( @@ -487,7 +568,10 @@ public void testExplicitRollbackRestoresOfflineAndRunningStatuses() throws Excep rollbackStatus == NodeStatus.ReadOnly ? NodeStatus.MANUAL : null, loadCache.getNodeStatusReason(REMOVING_DATA_NODE_ID)); Assert.assertEquals( - rollbackStatus == NodeStatus.Stopped ? NodeStatus.Stopped : null, + rollbackStatus.isPersistentStatus() + ? new Pair<>( + rollbackStatus, rollbackStatus == NodeStatus.ReadOnly ? NodeStatus.MANUAL : null) + : null, nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); // Ordinary statistics updates must preserve the successfully committed rollback. @@ -502,12 +586,12 @@ public void testExplicitRollbackRestoresOfflineAndRunningStatuses() throws Excep @Test public void testFailedExplicitRollbackUpdatesStatisticsAndRetriesPeriodically() throws Exception { nodeInfo.applyNodeStatusPlan( - new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); + new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing)); loadCache.initHeartbeatCache(configManager); doReturn(failure()) .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) .when(consensusManager) - .write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.CLEAR)); + .write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, null)); Assert.assertEquals( failure().getCode(), @@ -516,22 +600,22 @@ public void testFailedExplicitRollbackUpdatesStatisticsAndRetriesPeriodically() .getCode()); Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); Assert.assertEquals( - NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); + new Pair<>(NodeStatus.Removing, null), + nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); Assert.assertTrue(loadCache.updateNodeStatistics()); Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(REMOVING_DATA_NODE_ID)); - verify(consensusManager, times(2)) - .write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.CLEAR)); + verify(consensusManager, times(2)).write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, null)); verify(consensusManager, never()) - .write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); + .write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing)); } @Test public void testShutdownReportCannotOverrideRemovingStatus() throws Exception { - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, Operation.SET_REMOVING)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, NodeStatus.Removing)); nodeInfo.applyNodeStatusPlan( - new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); + new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing)); loadCache.initHeartbeatCache(configManager); for (int nodeId : Arrays.asList(CONFIG_NODE_ID, REMOVING_DATA_NODE_ID)) { @@ -539,20 +623,18 @@ public void testShutdownReportCannotOverrideRemovingStatus() throws Exception { success().getCode(), loadCache.trySetNodeStatus(nodeId, NodeStatus.Stopped, null, false).getCode()); Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(nodeId)); - Assert.assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(nodeId)); - verify(consensusManager, never()) - .write(new UpdateNodeStatusPlan(nodeId, Operation.SET_STOPPED)); + Assert.assertEquals( + new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(nodeId)); + verify(consensusManager, never()).write(new UpdateNodeStatusPlan(nodeId, NodeStatus.Stopped)); } } @Test public void testStatisticsFailureDoesNotPreventUpdatingOtherNodes() throws Exception { - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, Operation.SET_STOPPED)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, NodeStatus.Stopped)); loadCache.initHeartbeatCache(configManager); - doReturn(failure()) - .when(consensusManager) - .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + doReturn(failure()).when(consensusManager).write(new UpdateNodeStatusPlan(DATA_NODE_ID, null)); loadCache.cacheDataNodeHeartbeatSample( DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); loadCache.cacheConfigNodeHeartbeatSample( @@ -561,12 +643,12 @@ public void testStatisticsFailureDoesNotPreventUpdatingOtherNodes() throws Excep Assert.assertFalse(loadCache.updateNodeStatistics()); Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID)); - Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(CONFIG_NODE_ID)); Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(CONFIG_NODE_ID)); - verify(consensusManager, never()).write(new UpdateNodeStatusPlan(SELF_ID, Operation.CLEAR)); - verify(consensusManager, never()) - .write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.CLEAR)); + verify(consensusManager, never()).write(new UpdateNodeStatusPlan(SELF_ID, null)); + verify(consensusManager, never()).write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, null)); } @Test @@ -585,7 +667,7 @@ public void testRepeatedCacheCreationPreservesPendingShutdown() throws Exception return nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)); }) .when(consensusManager) - .write(new UpdateNodeStatusPlan(CONFIG_NODE_ID, Operation.SET_STOPPED)); + .write(new UpdateNodeStatusPlan(CONFIG_NODE_ID, NodeStatus.Stopped)); ExecutorService executor = Executors.newSingleThreadExecutor(); try { Future shutdown = @@ -597,14 +679,15 @@ public void testRepeatedCacheCreationPreservesPendingShutdown() throws Exception finishPersistence.countDown(); Assert.assertEquals(success().getCode(), shutdown.get(5, TimeUnit.SECONDS).getCode()); Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(CONFIG_NODE_ID)); - Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(CONFIG_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(CONFIG_NODE_ID)); Assert.assertTrue(loadCache.checkAndSetHeartbeatProcessing(CONFIG_NODE_ID)); Assert.assertTrue(loadCache.updateNodeStatistics()); Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(CONFIG_NODE_ID)); - Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(CONFIG_NODE_ID)); - verify(consensusManager, never()) - .write(new UpdateNodeStatusPlan(CONFIG_NODE_ID, Operation.CLEAR)); + Assert.assertEquals( + new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(CONFIG_NODE_ID)); + verify(consensusManager, never()).write(new UpdateNodeStatusPlan(CONFIG_NODE_ID, null)); } finally { finishPersistence.countDown(); executor.shutdownNow(); @@ -613,27 +696,27 @@ public void testRepeatedCacheCreationPreservesPendingShutdown() throws Exception @Test public void testDiscardedCacheCannotClearNewRemovingMarker() throws Exception { - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped)); loadCache.initHeartbeatCache(configManager); Field field = LoadCache.class.getDeclaredField("nodeCacheMap"); field.setAccessible(true); Map caches = (Map) field.get(loadCache); BaseNodeCache discarded = caches.get(DATA_NODE_ID); - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_REMOVING)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Removing)); loadCache.initHeartbeatCache(configManager); Assert.assertEquals( failure().getCode(), discarded.trySetNodeStatus(NodeStatus.Running, null, true).getCode()); - Assert.assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(DATA_NODE_ID)); - verify(consensusManager, never()) - .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.CLEAR)); + verify(consensusManager, never()).write(new UpdateNodeStatusPlan(DATA_NODE_ID, null)); } @Test - public void testReadOnlyHeartbeatClearsStoppedButNotRemoving() throws Exception { - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + public void testReadOnlyHeartbeatReplacesStoppedButNotRemoving() throws Exception { + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped)); nodeInfo.applyNodeStatusPlan( - new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, Operation.SET_REMOVING)); + new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing)); loadCache.initHeartbeatCache(configManager); loadCache.cacheDataNodeHeartbeatSample( DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.ReadOnly)); @@ -641,10 +724,12 @@ public void testReadOnlyHeartbeatClearsStoppedButNotRemoving() throws Exception REMOVING_DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.ReadOnly)); Assert.assertTrue(loadCache.updateNodeStatistics()); Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID)); - Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(DATA_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.ReadOnly, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID)); Assert.assertEquals( - NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); + new Pair<>(NodeStatus.Removing, null), + nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); } @Test @@ -655,7 +740,7 @@ public void testAINodeRestoresStoppedAndRemovingAcrossLeaderChange() throws Exce Assert.assertEquals( success().getCode(), loadCache.trySetNodeStatus(AI_NODE_ID, status, null, false).getCode()); - Assert.assertEquals(status, nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); + Assert.assertEquals(new Pair<>(status, null), nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); loadCache.initHeartbeatCache(configManager); Assert.assertEquals(status, loadCache.getNodeStatus(AI_NODE_ID)); Assert.assertTrue(loadCache.updateNodeStatistics()); @@ -664,7 +749,8 @@ public void testAINodeRestoresStoppedAndRemovingAcrossLeaderChange() throws Exce loadCache.cacheAINodeHeartbeatSample(AI_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); Assert.assertTrue(loadCache.updateNodeStatistics()); Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(AI_NODE_ID)); - Assert.assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); } @Test @@ -674,7 +760,7 @@ public void testAINodeUpdatesStatisticsOnPersistenceFailureAndRetries() throws E doReturn(failure()) .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) .when(consensusManager) - .write(new UpdateNodeStatusPlan(AI_NODE_ID, Operation.SET_STOPPED)); + .write(new UpdateNodeStatusPlan(AI_NODE_ID, NodeStatus.Stopped)); Assert.assertEquals( failure().getCode(), loadCache.trySetNodeStatus(AI_NODE_ID, NodeStatus.Stopped, null, false).getCode()); @@ -683,16 +769,18 @@ public void testAINodeUpdatesStatisticsOnPersistenceFailureAndRetries() throws E loadCache.cacheAINodeHeartbeatSample(AI_NODE_ID, new NodeHeartbeatSample(NodeStatus.Unknown)); Assert.assertTrue(loadCache.updateNodeStatistics()); Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(AI_NODE_ID)); - Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); doReturn(failure()) .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0))) .when(consensusManager) - .write(new UpdateNodeStatusPlan(AI_NODE_ID, Operation.CLEAR)); + .write(new UpdateNodeStatusPlan(AI_NODE_ID, null)); loadCache.cacheAINodeHeartbeatSample(AI_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); Assert.assertFalse(loadCache.updateNodeStatistics()); Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(AI_NODE_ID)); - Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); Assert.assertTrue(loadCache.updateNodeStatistics()); Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(AI_NODE_ID)); Assert.assertNull(nodeInfo.getPersistedNodeStatus(AI_NODE_ID)); @@ -729,7 +817,8 @@ public void testOlderHeartbeatCannotUndoShutdownReport() throws Exception { loadCache.cacheDataNodeHeartbeatSample(DATA_NODE_ID, olderHeartbeat); loadCache.updateNodeStatistics(); Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(DATA_NODE_ID)); - Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); } } @@ -745,7 +834,7 @@ public void testBlockedNodeWriteDoesNotHoldOtherNodeCacheLock() throws Exception return nodeInfo.applyNodeStatusPlan(i.getArgument(0)); }) .when(consensusManager) - .write(new UpdateNodeStatusPlan(DATA_NODE_ID, Operation.SET_STOPPED)); + .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped)); ExecutorService executor = Executors.newFixedThreadPool(2); try { Future blocked = @@ -759,7 +848,8 @@ public void testBlockedNodeWriteDoesNotHoldOtherNodeCacheLock() throws Exception REMOVING_DATA_NODE_ID, NodeStatus.Removing, null, true)); Assert.assertEquals(success().getCode(), independent.get(5, TimeUnit.SECONDS).getCode()); Assert.assertEquals( - NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); + new Pair<>(NodeStatus.Removing, null), + nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID)); Assert.assertFalse(blocked.isDone()); release.countDown(); Assert.assertEquals(success().getCode(), blocked.get(5, TimeUnit.SECONDS).getCode()); @@ -793,7 +883,8 @@ public void testLateShutdownReportConvergesAfterRestartedNodeHeartbeat() throws Assert.assertEquals( success().getCode(), loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, null, false).getCode()); - Assert.assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); + Assert.assertEquals( + new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID)); loadCache.cacheDataNodeHeartbeatSample( DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running)); loadCache.updateNodeStatistics(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java index 159b4c8c306fb..667cbba357ede 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java @@ -33,7 +33,6 @@ import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; -import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan.Operation; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateVersionInfoPlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan; @@ -43,6 +42,7 @@ import org.apache.thrift.TException; import org.apache.tsfile.external.commons.io.FileUtils; +import org.apache.tsfile.utils.Pair; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -82,19 +82,23 @@ public void testSnapshot() throws TException, IOException { registerConfigNodes(); registerDataNodes(); nodeInfo.registerAINode(new RegisterAINodePlan(generateTAINodeConfiguration())); - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10001, Operation.SET_STOPPED)); - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_REMOVING)); - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, Operation.SET_STOPPED)); - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10006, Operation.SET_REMOVING)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10001, NodeStatus.Stopped)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Removing)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, NodeStatus.Stopped)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10006, NodeStatus.Removing)); Assert.assertTrue(nodeInfo.processTakeSnapshot(snapshotDir)); NodeInfo nodeInfo1 = new NodeInfo(); nodeInfo1.processLoadSnapshot(snapshotDir); Assert.assertEquals(nodeInfo, nodeInfo1); - Assert.assertEquals(NodeStatus.Stopped, nodeInfo1.getPersistedNodeStatus(10001)); - Assert.assertEquals(NodeStatus.Removing, nodeInfo1.getPersistedNodeStatus(10003)); - Assert.assertEquals(NodeStatus.Stopped, nodeInfo1.getPersistedNodeStatus(10004)); - Assert.assertEquals(NodeStatus.Removing, nodeInfo1.getPersistedNodeStatus(10006)); + Assert.assertEquals( + new Pair<>(NodeStatus.Stopped, null), nodeInfo1.getPersistedNodeStatus(10001)); + Assert.assertEquals( + new Pair<>(NodeStatus.Removing, null), nodeInfo1.getPersistedNodeStatus(10003)); + Assert.assertEquals( + new Pair<>(NodeStatus.Stopped, null), nodeInfo1.getPersistedNodeStatus(10004)); + Assert.assertEquals( + new Pair<>(NodeStatus.Removing, null), nodeInfo1.getPersistedNodeStatus(10006)); } @Test @@ -114,7 +118,7 @@ public void testLoadSnapshotWithoutNodeStatuses() throws TException, IOException NodeInfo restored = new NodeInfo(); restored.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); - restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_REMOVING)); + restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Removing)); restored.processLoadSnapshot(oldSnapshotDir); Assert.assertEquals(oldNodeInfo, restored); Assert.assertTrue(restored.getPersistedNodeStatuses().isEmpty()); @@ -125,29 +129,115 @@ public void testLoadSnapshotWithoutNodeStatuses() throws TException, IOException public void testSetOperationsReplacePersistedStatus() { NodeInfo info = new NodeInfo(); info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_STOPPED)); - Assert.assertEquals(NodeStatus.Stopped, info.getPersistedNodeStatus(10003)); - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_REMOVING)); - Assert.assertEquals(NodeStatus.Removing, info.getPersistedNodeStatus(10003)); - Map snapshot = info.getPersistedNodeStatuses(); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Stopped)); + Assert.assertEquals(new Pair<>(NodeStatus.Stopped, null), info.getPersistedNodeStatus(10003)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Removing)); + Assert.assertEquals(new Pair<>(NodeStatus.Removing, null), info.getPersistedNodeStatus(10003)); + Pair status = info.getPersistedNodeStatus(10003); + status.left = NodeStatus.ReadOnly; + status.right = NodeStatus.MANUAL; + Assert.assertEquals(new Pair<>(NodeStatus.Removing, null), info.getPersistedNodeStatus(10003)); + Map> snapshot = info.getPersistedNodeStatuses(); + snapshot.get(10003).left = NodeStatus.ReadOnly; + snapshot.get(10003).right = NodeStatus.MANUAL; + Assert.assertEquals(new Pair<>(NodeStatus.Removing, null), info.getPersistedNodeStatus(10003)); snapshot.clear(); - Assert.assertEquals(NodeStatus.Removing, info.getPersistedNodeStatus(10003)); + Assert.assertEquals(new Pair<>(NodeStatus.Removing, null), info.getPersistedNodeStatus(10003)); - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_STOPPED)); - Assert.assertEquals(NodeStatus.Stopped, info.getPersistedNodeStatus(10003)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Stopped)); + Assert.assertEquals(new Pair<>(NodeStatus.Stopped, null), info.getPersistedNodeStatus(10003)); info.clear(); Assert.assertTrue(info.getPersistedNodeStatuses().isEmpty()); } @Test - public void testClearRemovesEitherPersistedStatusAndAllowsRetry() { + public void testReadOnlyReasonsSurviveSnapshot() throws TException, IOException { + NodeInfo info = new NodeInfo(); + String[] reasons = { + null, + NodeStatus.MANUAL, + NodeStatus.DISK_FULL, + NodeStatus.STOPPING, + NodeStatus.UNRECOVERABLE_ERROR + ": failed to recover root.test-0" + }; + for (int i = 0; i < reasons.length; i++) { + info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3 + i))); + info.applyNodeStatusPlan( + new UpdateNodeStatusPlan(10003 + i, NodeStatus.ReadOnly, reasons[i])); + } + File directory = new File(snapshotDir, "readonly-reasons"); + Assert.assertTrue(directory.mkdirs()); + Assert.assertTrue(info.processTakeSnapshot(directory)); + + NodeInfo restored = new NodeInfo(); + restored.processLoadSnapshot(directory); + Assert.assertEquals(info, restored); + for (int i = 0; i < reasons.length; i++) { + Assert.assertEquals( + new Pair<>(NodeStatus.ReadOnly, reasons[i]), restored.getPersistedNodeStatus(10003 + i)); + } + } + + @Test + public void testStoppedAndRemovingReasonsSurviveSnapshot() throws TException, IOException { + NodeInfo info = new NodeInfo(); + info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); + info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(4))); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Stopped, "stop reason")); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, NodeStatus.Removing, "remove reason")); + File directory = new File(snapshotDir, "stopped-removing-reasons"); + Assert.assertTrue(directory.mkdirs()); + Assert.assertTrue(info.processTakeSnapshot(directory)); + + NodeInfo restored = new NodeInfo(); + restored.processLoadSnapshot(directory); + Assert.assertEquals(info, restored); + Assert.assertEquals( + new Pair<>(NodeStatus.Stopped, "stop reason"), restored.getPersistedNodeStatus(10003)); + Assert.assertEquals( + new Pair<>(NodeStatus.Removing, "remove reason"), restored.getPersistedNodeStatus(10004)); + } + + @Test + public void testReadOnlyReasonReplacementAndStatusChange() { NodeInfo info = new NodeInfo(); info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); - for (Operation operation : new Operation[] {Operation.SET_STOPPED, Operation.SET_REMOVING}) { - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, operation)); + // Plans contain the caller's final decision, including lower-priority or absent reasons. + for (String reason : new String[] {NodeStatus.MANUAL, NodeStatus.DISK_FULL, null}) { + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.ReadOnly, reason)); + Assert.assertEquals( + new Pair<>(NodeStatus.ReadOnly, reason), info.getPersistedNodeStatus(10003)); + } + info.applyNodeStatusPlan( + new UpdateNodeStatusPlan(10003, NodeStatus.ReadOnly, NodeStatus.MANUAL)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Stopped)); + Assert.assertEquals(new Pair<>(NodeStatus.Stopped, null), info.getPersistedNodeStatus(10003)); + } + + @Test + public void testNodeRemovalDiscardsReadOnlyReason() { + NodeInfo info = new NodeInfo(); + info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); + UpdateNodeStatusPlan readOnly = + new UpdateNodeStatusPlan(10003, NodeStatus.ReadOnly, NodeStatus.MANUAL); + info.applyNodeStatusPlan(readOnly); + info.removeDataNode( + new RemoveDataNodePlan(Collections.singletonList(generateTDataNodeLocation(3)))); + Assert.assertNull(info.getPersistedNodeStatus(10003)); + info.applyNodeStatusPlan(readOnly); + Assert.assertNull(info.getPersistedNodeStatus(10003)); + } + + @Test + public void testClearRemovesAnyPersistedStatusAndAllowsRetry() { + NodeInfo info = new NodeInfo(); + info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); + for (NodeStatus status : + new NodeStatus[] {NodeStatus.Stopped, NodeStatus.Removing, NodeStatus.ReadOnly}) { + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, status, NodeStatus.MANUAL)); Assert.assertNotNull(info.getPersistedNodeStatus(10003)); - UpdateNodeStatusPlan clear = new UpdateNodeStatusPlan(10003, Operation.CLEAR); + UpdateNodeStatusPlan clear = new UpdateNodeStatusPlan(10003, null); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), info.applyNodeStatusPlan(clear).getCode()); Assert.assertNull(info.getPersistedNodeStatus(10003)); @@ -165,11 +255,11 @@ public void testNodeRemovalDiscardsStatusAndIgnoresDelayedUpdates() { 10001, new TEndPoint("127.0.0.1", 22201), new TEndPoint("127.0.0.1", 22301)); info.applyConfigNode(new ApplyConfigNodePlan(configNode)); info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10001, Operation.SET_STOPPED)); - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_REMOVING)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10001, NodeStatus.Stopped)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Removing)); TAINodeConfiguration aiNode = generateTAINodeConfiguration(); info.registerAINode(new RegisterAINodePlan(aiNode)); - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10006, Operation.SET_REMOVING)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10006, NodeStatus.Removing)); info.removeConfigNode(new RemoveConfigNodePlan(configNode)); info.removeDataNode( @@ -179,14 +269,13 @@ public void testNodeRemovalDiscardsStatusAndIgnoresDelayedUpdates() { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10001, Operation.SET_STOPPED)).getCode()); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10001, NodeStatus.Stopped)).getCode()); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_REMOVING)) - .getCode()); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Removing)).getCode()); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10006, Operation.SET_STOPPED)).getCode()); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10006, NodeStatus.Stopped)).getCode()); Assert.assertTrue(info.getPersistedNodeStatuses().isEmpty()); } @@ -209,16 +298,19 @@ private void registerConfigNodes() { @Test public void testTruncatedStatusPayloadMustFailSnapshotLoad() throws Exception { - NodeInfo info = new NodeInfo(); - info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_REMOVING)); - File directory = new File(snapshotDir, "truncated-status-payload"); - Assert.assertTrue(directory.mkdirs()); - Assert.assertTrue(info.processTakeSnapshot(directory)); - try (RandomAccessFile file = new RandomAccessFile(new File(directory, "node_info.bin"), "rw")) { - file.setLength(file.length() - 2); + for (NodeStatus status : new NodeStatus[] {NodeStatus.Removing, NodeStatus.ReadOnly}) { + NodeInfo info = new NodeInfo(); + info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, status, NodeStatus.MANUAL)); + File directory = new File(snapshotDir, "truncated-status-payload-" + status); + Assert.assertTrue(directory.mkdirs()); + Assert.assertTrue(info.processTakeSnapshot(directory)); + try (RandomAccessFile file = + new RandomAccessFile(new File(directory, "node_info.bin"), "rw")) { + file.setLength(file.length() - 2); + } + Assert.assertThrows(IOException.class, () -> new NodeInfo().processLoadSnapshot(directory)); } - Assert.assertThrows(IOException.class, () -> new NodeInfo().processLoadSnapshot(directory)); } @Test @@ -226,10 +318,10 @@ public void testClearedStatusesStayClearedAfterSnapshotAndLaterPlans() throws Ex NodeInfo info = new NodeInfo(); info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(4))); - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_STOPPED)); - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, Operation.SET_REMOVING)); - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.CLEAR)); - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, Operation.CLEAR)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Stopped)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, NodeStatus.Removing)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, null)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, null)); File directory = new File(snapshotDir, "cleared-statuses"); Assert.assertTrue(directory.mkdirs()); Assert.assertTrue(info.processTakeSnapshot(directory)); @@ -237,25 +329,26 @@ public void testClearedStatusesStayClearedAfterSnapshotAndLaterPlans() throws Ex restored.processLoadSnapshot(directory); Assert.assertTrue(restored.getPersistedNodeStatuses().isEmpty()); // Replay the entries after that snapshot in their original order. - restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_STOPPED)); - restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, Operation.SET_REMOVING)); - restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.CLEAR)); + restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Stopped)); + restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, NodeStatus.Removing)); + restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, null)); Assert.assertNull(restored.getPersistedNodeStatus(10003)); - Assert.assertEquals(NodeStatus.Removing, restored.getPersistedNodeStatus(10004)); + Assert.assertEquals( + new Pair<>(NodeStatus.Removing, null), restored.getPersistedNodeStatus(10004)); } @Test public void testRejoinedNodeDoesNotInheritRemovedNodeStatus() { NodeInfo info = new NodeInfo(); info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3))); - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_REMOVING)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Removing)); info.removeDataNode( new RemoveDataNodePlan(Collections.singletonList(generateTDataNodeLocation(3)))); int newId = info.generateNextNodeId(); TDataNodeConfiguration rejoined = generateTDataNodeConfiguration(3); rejoined.getLocation().setDataNodeId(newId); info.registerDataNode(new RegisterDataNodePlan(rejoined)); - info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, Operation.SET_STOPPED)); + info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Stopped)); Assert.assertNotEquals(10003, newId); Assert.assertNull(info.getPersistedNodeStatus(newId)); Assert.assertNull(info.getPersistedNodeStatus(10003)); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java index be61da3ee5fa6..6ec1016191748 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java @@ -31,7 +31,6 @@ import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager; import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan; -import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan.Operation; import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan; import org.apache.iotdb.confignode.manager.ConfigManager; @@ -47,6 +46,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.async.AsyncMethodCallback; +import org.apache.tsfile.utils.Pair; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -243,10 +243,10 @@ public void testOfflineRemovalStillRequiresPersistence() throws Exception { failStatusWrite = true; expectChangeFailure(NodeStatus.Removing); assertEquals(NodeStatus.Removing, cache.getNodeStatus(FIRST)); - assertEquals(NodeStatus.Stopped, nodeInfo.getPersistedNodeStatus(FIRST)); + assertEquals(new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(FIRST)); failStatusWrite = false; change(FIRST, NodeStatus.Removing); - assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(FIRST)); + assertEquals(new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(FIRST)); } @Test @@ -443,7 +443,7 @@ public void testSuccessfulDeleteCannotBeResurrectedByLateStatus() throws Excepti nodeInfo.getRegisteredDataNodes().stream() .anyMatch(n -> n.getLocation().getDataNodeId() == FIRST)); assertNull(nodeInfo.getPersistedNodeStatus(FIRST)); - nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(FIRST, Operation.SET_REMOVING)); + nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(FIRST, NodeStatus.Removing)); assertNull(nodeInfo.getPersistedNodeStatus(FIRST)); assertTrue(stopRequests > 0); } @@ -467,7 +467,7 @@ public void testPartialRemovalKeepsSuccessfulAndCompensatedResultsSeparate() thr assertTrue(procedure.finish(env)); assertNull(nodeInfo.getPersistedNodeStatus(FIRST)); assertEquals(1, nodeInfo.getRegisteredDataNodes().size()); - assertStates(SECOND, NodeStatus.ReadOnly, NodeStatus.ReadOnly, null); + assertStates(SECOND, NodeStatus.ReadOnly, NodeStatus.ReadOnly, NodeStatus.ReadOnly); } private void assertDeleteFailure() throws Exception { @@ -477,7 +477,7 @@ private void assertDeleteFailure() throws Exception { assertEquals("No remote stop before durable unregister", 0, stopRequests); assertFalse("Failed unregister must remain retryable", finished); assertEquals(NodeStatus.Removing, cache.getNodeStatus(FIRST)); - assertEquals(NodeStatus.Removing, nodeInfo.getPersistedNodeStatus(FIRST)); + assertEquals(new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(FIRST)); assertEquals(2, nodeInfo.getRegisteredDataNodes().size()); } @@ -487,7 +487,7 @@ private void assertCompensation(NodeStatus original) throws Exception { assertTrue( "Compensation should complete for " + original, procedure(original, RemoveDataNodeState.STOP_DATA_NODE).finish(env)); - assertStates(FIRST, original, original, original == NodeStatus.Stopped ? original : null); + assertStates(FIRST, original, original, original.isPersistentStatus() ? original : null); } private void retainRegionOn(int id) { @@ -516,7 +516,12 @@ private void expectChangeFailure(NodeStatus status) throws Exception { private void assertStates(int id, NodeStatus remote, NodeStatus published, NodeStatus persisted) { assertEquals("DataNode", remote, remoteStatuses.get(id)); assertEquals("LoadCache", published, cache.getNodeStatus(id)); - assertEquals("NodeInfo", persisted, nodeInfo.getPersistedNodeStatus(id)); + assertEquals( + "NodeInfo", + persisted == null + ? null + : new Pair<>(persisted, persisted == NodeStatus.ReadOnly ? NodeStatus.MANUAL : null), + nodeInfo.getPersistedNodeStatus(id)); } private TestProcedure procedure(NodeStatus original, RemoveDataNodeState initial) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java index 8df310329247a..b422c47f36195 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java @@ -127,7 +127,7 @@ public static boolean isNormalStatus(NodeStatus status) { * existing record when the node recovers. */ public boolean isPersistentStatus() { - return this == Stopped || this == Removing; + return this == Stopped || this == Removing || this == ReadOnly; } /** No available heartbeat, or an explicit shutdown report. Removing alone is not offline. */ diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/cluster/NodeStatusTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/cluster/NodeStatusTest.java index 241077eea1b93..edc84ddfb6a0a 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/cluster/NodeStatusTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/cluster/NodeStatusTest.java @@ -71,7 +71,7 @@ public void testStatusClassificationTable() { {false, false, true, true, false}, {true, true, true, false, false}, {false, true, false, true, true}, - {false, false, false, true, false}, + {false, false, false, true, true}, {true, true, true, false, true} };