From 43e32d723dfa62b99a6295430de47c807b750b49 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Fri, 18 Sep 2026 15:28:24 +0800 Subject: [PATCH 01/24] support that auto-detection for creating measurements and adding columns for related tree-to-table view --- .../org/apache/iotdb/rpc/TSStatusCode.java | 1 + .../confignode/i18n/ManagerMessages.java | 3 + .../confignode/i18n/ProcedureMessages.java | 634 ++++++++++++------ .../confignode/i18n/ManagerMessages.java | 3 + .../confignode/i18n/ProcedureMessages.java | 9 + .../confignode/manager/ConfigManager.java | 10 + .../iotdb/confignode/manager/IManager.java | 4 + .../confignode/manager/ProcedureManager.java | 196 ++++++ .../manager/schema/ClusterSchemaManager.java | 74 +- .../schema/TreeDeviceViewFieldDetector.java | 305 +++++---- .../procedure/impl/schema/SchemaUtils.java | 54 +- .../AbstractAlterOrDropTableProcedure.java | 34 +- .../table/AbstractSetPropertiesProcedure.java | 2 +- .../schema/table/AddTableColumnProcedure.java | 2 +- .../AlterTableColumnDataTypeProcedure.java | 2 +- .../schema/table/CreateTableProcedure.java | 44 +- .../impl/schema/table/DropTableProcedure.java | 4 +- .../table/RenameTableColumnProcedure.java | 2 +- .../table/view/AddViewColumnProcedure.java | 55 +- .../table/view/CreateTableViewProcedure.java | 87 ++- .../schema/table/view/DropViewProcedure.java | 15 + .../table/view/RenameViewProcedure.java | 19 + .../view/SetViewPropertiesProcedure.java | 82 +++ .../thrift/ConfigNodeRPCServiceProcessor.java | 7 + .../iotdb/db/i18n/DataNodeMiscMessages.java | 9 +- .../iotdb/db/i18n/DataNodeQueryMessages.java | 2 + .../iotdb/db/i18n/DataNodeSchemaMessages.java | 8 + .../iotdb/db/i18n/DataNodeMiscMessages.java | 10 +- .../iotdb/db/i18n/DataNodeQueryMessages.java | 2 + .../iotdb/db/i18n/DataNodeSchemaMessages.java | 8 + .../schemaregion/SchemaExecutionVisitor.java | 119 +++- .../db/protocol/client/ConfigNodeClient.java | 9 + .../impl/DataNodeInternalRPCServiceImpl.java | 97 ++- .../config/TableConfigTaskVisitor.java | 25 +- .../view/TreeToTableViewReference.java | 94 +++ .../table/DataNodeTableCache.java | 109 ++- .../db/schemaengine/table/ITableCache.java | 11 +- .../table/ITreeToTableViewCache.java | 64 ++ .../table/TreeToTableViewCache.java | 465 +++++++++++++ .../table/TriggerAddViewColumnVisitor.java | 389 +++++++++++ .../SchemaRegionSimpleRecoverTest.java | 4 +- ...nedTreeDeviceViewScanOperatorTreeTest.java | 2 +- .../relational/analyzer/AnalyzerTest.java | 10 +- .../relational/analyzer/TreeViewTest.java | 3 +- .../cache/TableDeviceSchemaCacheTest.java | 15 +- .../TopKRuntimeFilterTreeViewPlanTest.java | 3 +- .../plan/statement/InsertStatementTest.java | 6 +- .../table/DataNodeTableCacheLeaseTest.java | 4 +- .../table/DataNodeTableCacheTest.java | 16 +- .../table/TreeToTableViewCacheTest.java | 132 ++++ .../dataregion/DataRegionTest.java | 3 +- .../CompactionDataTypeAlterTableTest.java | 5 +- .../TableModelCompactionWithTTLTest.java | 2 +- .../UnsealedTsFileRecoverPerformerTest.java | 2 +- .../iotdb/commons/i18n/SchemaMessages.java | 2 + .../iotdb/commons/i18n/SchemaMessages.java | 2 + .../table/TreeToTableViewOperationType.java | 68 ++ .../commons/schema/table/TreeViewSchema.java | 1 + .../src/main/thrift/confignode.thrift | 19 + .../src/main/thrift/datanode.thrift | 9 + 60 files changed, 2904 insertions(+), 473 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/view/TreeToTableViewReference.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITreeToTableViewCache.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TriggerAddViewColumnVisitor.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCacheTest.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewOperationType.java diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index da9b4388691e8..0da39cf56047c 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -52,6 +52,7 @@ public enum TSStatusCode { DISPATCH_ERROR(306), LICENSE_ERROR(307), THRIFT_FRAME_OVERSIZE(308), + REPEATED_WITH_EXISTING_TASK(309), // Client, REDIRECTION_RECOMMEND(400), diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 183ab36c47816..482da8daad126 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -614,6 +614,9 @@ private ManagerMessages() {} public static final String LOG_GOT_IOEXCEPTION_CONSTRUCT_DATABASE_TREE_49436621 = "Got IOException when construct database Tree"; public static final String LOG_GOT_IOEXCEPTION_DESERIALIZE_TEMPLATE_INFO_49EE617E = "Got IOException when deserialize template info"; public static final String MESSAGE_MEASUREMENTS_NOT_FOUND_ARG_CANNOT_AUTO_DETECT_980D7D44 = "Measurements not found for %s, cannot auto detect"; + public static final String EXCEPTION_THE_DUPLICATED_SOURCE_MEASUREMENT_ARG_IS_UNSUPPORTED_YET_873AF566 = "The duplicated source measurement %s is unsupported yet."; + public static final String MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB = "Multiple types encountered when auto detecting type of measurement '%s', please check"; + public static final String MESSAGE_THE_MEASUREMENTS_ARG_AND_ARG_SHARE_THE_SAME_LOWER_CASE_WHEN_AUTO_DETECTING_TYPE_PLEASE_CHECK_7225D062 = "The measurements %s and %s share the same lower case when auto detecting type, please check"; public static final String LOG_FAILED_TAKE_SNAPSHOT_BECAUSE_SNAPSHOT_FILE_ARG_ALREADY_EXIST_EB2A6093 = "Failed to take snapshot, because snapshot file [{}] is already exist."; public static final String LOG_FAILED_LOAD_SNAPSHOT_SNAPSHOT_FILE_ARG_NOT_EXIST_8828CFBA = "Failed to load snapshot,snapshot file [{}] is not exist."; public static final String LOG_YOU_MAYBE_NEED_RENAME_SIMPLE_DIR_0_0_MANUALLY_2A12C5C9 = "you maybe need to rename the simple dir to 0_0 manually."; diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java index 18aca01243908..4d47759231755 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java @@ -490,7 +490,8 @@ public final class ProcedureMessages { "Failed to rollback pipe plugin [%s] on data nodes"; public static final String FAILED_TO_ROLLBACK_PRE_RELEASE_FOR_TABLE_INFO_TO_DATANODE = "Failed to rollback pre-release {} for table {}.{} info to DataNode, failure results: {}"; - public static final String FAILED_TO_PROVE_AN_UNREACHABLE_DN_IS_FENCED = "Failed to prove an unreachable DataNode is fenced"; + public static final String FAILED_TO_PROVE_AN_UNREACHABLE_DN_IS_FENCED = + "Failed to prove an unreachable DataNode is fenced"; public static final String FAILED_TO_ROLLBACK_PRE_RELEASE_TEMPLATE_INFO_OF_TEMPLATE_SET = "Failed to rollback pre release template info of template {} set on path {} on DataNode {}"; public static final String FAILED_TO_ROLLBACK_PRE_SET_TEMPLATE_ON_PATH_DUE_TO = @@ -1188,238 +1189,432 @@ public final class ProcedureMessages { "Failed in the write API executing the consensus layer due to: "; private ProcedureMessages() {} + // --------------------------------------------------------------------------- // Additional auto-collected messages // --------------------------------------------------------------------------- public static final String EXCEPTION_MS_95BA098D = " ms."; - public static final String LOG_ARG_JOIN_WAIT_GOT_INTERRUPTED_316B5E9F = "{} join wait got interrupted"; - public static final String LOG_NO_COMPLETED_PROCEDURES_CLEANUP_50434D91 = "No completed procedures to cleanup."; - public static final String LOG_ERROR_DELETING_COMPLETED_PROCEDURES_ARG_1A3A185E = "Error deleting completed procedures {}."; + public static final String LOG_ARG_JOIN_WAIT_GOT_INTERRUPTED_316B5E9F = + "{} join wait got interrupted"; + public static final String LOG_NO_COMPLETED_PROCEDURES_CLEANUP_50434D91 = + "No completed procedures to cleanup."; + public static final String LOG_ERROR_DELETING_COMPLETED_PROCEDURES_ARG_1A3A185E = + "Error deleting completed procedures {}."; public static final String LOG_EVICT_COMPLETED_ARG_A968A070 = "Evict completed {}"; - public static final String LOG_EXECUTING_PROCEDURE_SHOULD_RUNNABLE_STATE_BUT_IT_S_NOT_PROCEDURE_7CF42CE8 = "The executing procedure should in RUNNABLE state, but it's not. Procedure is {}"; - public static final String LOG_FINISHED_SUBPROCEDURE_PID_ARG_RESUME_PROCESSING_PPID_ARG_93ED990B = "Finished subprocedure pid={}, resume processing ppid={}"; - public static final String LOG_HALT_PID_ARG_ACTIVECOUNT_ARG_411F3EBF = "Halt pid={}, activeCount={}"; - public static final String LOG_EXCEPTION_HAPPENED_WORKER_ARG_EXECUTE_PROCEDURE_ARG_6E3AD27D = "Exception happened when worker {} execute procedure {}"; - public static final String LOG_FAILED_TO_COMMIT_RELEASE_ARG_FOR_TABLE_ARG_ARG_TO_DATANODE_FAILURE_RESULTS_ARG_55386572 = - "Failed to commit-release {} for table {}.{} to DataNode, failure results: {}"; - public static final String LOG_WORKER_STUCK_ARG_ARG_RUN_TIME_ARG_MS_FB612354 = "Worker stuck {}({}), run time {} ms"; - public static final String LOG_PROCEDURE_WORKERS_ARG_RUNNING_ARG_RUNNING_STUCK_1565936D = "Procedure workers: {} is running, {} is running and stuck"; - public static final String LOG_ADD_PROCEDURE_ARG_AS_ARG_TH_ROLLBACK_STEP_C71B2184 = "Add procedure {} as the {}th rollback step"; - public static final String LOG_STATEMACHINEPROCEDURE_PID_ARG_NOT_SET_NEXT_STATE_BUT_RETURN_HAS_7F93E63F = - "StateMachineProcedure pid={} not set next state, but return HAS_MORE_STATE. It is likely that" - + " there is some problem with the code. Please check the code. This procedure is about to be" - + " terminated: {}"; - public static final String LOG_STATEMACHINEPROCEDURE_PID_ARG_SET_NEXT_STATE_ARG_BUT_RETURN_NO_0CA2D56C = "StateMachineProcedure pid={} set next state to {}, but return NO_MORE_STATE"; - public static final String LOG_DON_T_ADD_SUCCESSFUL_PROCEDURE_BACK_SCHEDULER_IT_WILL_IGNORED_E015472C = "Don't add a successful procedure back to the scheduler, it will be ignored"; + public static final String + LOG_EXECUTING_PROCEDURE_SHOULD_RUNNABLE_STATE_BUT_IT_S_NOT_PROCEDURE_7CF42CE8 = + "The executing procedure should in RUNNABLE state, but it's not. Procedure is {}"; + public static final String LOG_FINISHED_SUBPROCEDURE_PID_ARG_RESUME_PROCESSING_PPID_ARG_93ED990B = + "Finished subprocedure pid={}, resume processing ppid={}"; + public static final String LOG_HALT_PID_ARG_ACTIVECOUNT_ARG_411F3EBF = + "Halt pid={}, activeCount={}"; + public static final String LOG_EXCEPTION_HAPPENED_WORKER_ARG_EXECUTE_PROCEDURE_ARG_6E3AD27D = + "Exception happened when worker {} execute procedure {}"; + public static final String + LOG_FAILED_TO_COMMIT_RELEASE_ARG_FOR_TABLE_ARG_ARG_TO_DATANODE_FAILURE_RESULTS_ARG_55386572 = + "Failed to commit-release {} for table {}.{} to DataNode, failure results: {}"; + public static final String LOG_WORKER_STUCK_ARG_ARG_RUN_TIME_ARG_MS_FB612354 = + "Worker stuck {}({}), run time {} ms"; + public static final String LOG_PROCEDURE_WORKERS_ARG_RUNNING_ARG_RUNNING_STUCK_1565936D = + "Procedure workers: {} is running, {} is running and stuck"; + public static final String LOG_ADD_PROCEDURE_ARG_AS_ARG_TH_ROLLBACK_STEP_C71B2184 = + "Add procedure {} as the {}th rollback step"; + public static final String + LOG_STATEMACHINEPROCEDURE_PID_ARG_NOT_SET_NEXT_STATE_BUT_RETURN_HAS_7F93E63F = + "StateMachineProcedure pid={} not set next state, but return HAS_MORE_STATE. It is likely that" + + " there is some problem with the code. Please check the code. This procedure is about to be" + + " terminated: {}"; + public static final String + LOG_STATEMACHINEPROCEDURE_PID_ARG_SET_NEXT_STATE_ARG_BUT_RETURN_NO_0CA2D56C = + "StateMachineProcedure pid={} set next state to {}, but return NO_MORE_STATE"; + public static final String + LOG_DON_T_ADD_SUCCESSFUL_PROCEDURE_BACK_SCHEDULER_IT_WILL_IGNORED_E015472C = + "Don't add a successful procedure back to the scheduler, it will be ignored"; public static final String LOG_SCHEDULER_NOT_RUNNING_6969C9FF = "the scheduler is not running"; - public static final String LOG_SCHEDULER_WAITING_TIME_LEFT_ARG_NANOS_D7717019 = "the scheduler waiting time left {} nanos"; - public static final String LOG_SLEEP_FAILED_CONFIGNODEPROCEDUREENV_BCD470AC = "Sleep failed in ConfigNodeProcedureEnv: "; - public static final String LOG_INVALIDATE_CACHE_FAILED_BECAUSE_DATANODE_ARG_UNKNOWN_4F2D374C = "Invalidate cache failed, because DataNode {} is Unknown"; - public static final String LOG_INVALIDATE_CACHE_FAILED_INVALIDATE_PARTITION_CACHE_STATUS_ARG_INVALIDATE_SCHEMAENGINE_BEB7A065 = - "Invalidate cache failed, invalidate partition cache status is {}, invalidate schemaengine" - + " cache status is {}"; - public static final String MESSAGE_REMOVE_CONFIGNODE_FAILED_BECAUSE_UPDATE_CONSENSUSGROUP_PEER_INFORMATION_FAILED_FCE5302B = "Remove ConfigNode failed because update ConsensusGroup peer information failed."; - public static final String MESSAGE_CAN_T_REMOVE_DATANODE_LIMIT_REPLICATION_FACTOR_D960E3A6 = "Can't remove datanode due to the limit of replication factor, "; + public static final String LOG_SCHEDULER_WAITING_TIME_LEFT_ARG_NANOS_D7717019 = + "the scheduler waiting time left {} nanos"; + public static final String LOG_SLEEP_FAILED_CONFIGNODEPROCEDUREENV_BCD470AC = + "Sleep failed in ConfigNodeProcedureEnv: "; + public static final String LOG_INVALIDATE_CACHE_FAILED_BECAUSE_DATANODE_ARG_UNKNOWN_4F2D374C = + "Invalidate cache failed, because DataNode {} is Unknown"; + public static final String + LOG_INVALIDATE_CACHE_FAILED_INVALIDATE_PARTITION_CACHE_STATUS_ARG_INVALIDATE_SCHEMAENGINE_BEB7A065 = + "Invalidate cache failed, invalidate partition cache status is {}, invalidate schemaengine" + + " cache status is {}"; + public static final String + MESSAGE_REMOVE_CONFIGNODE_FAILED_BECAUSE_UPDATE_CONSENSUSGROUP_PEER_INFORMATION_FAILED_FCE5302B = + "Remove ConfigNode failed because update ConsensusGroup peer information failed."; + public static final String MESSAGE_CAN_T_REMOVE_DATANODE_LIMIT_REPLICATION_FACTOR_D960E3A6 = + "Can't remove datanode due to the limit of replication factor, "; public static final String MESSAGE_SUBMIT_REMOVEDATANODESPROCEDURE_FAILED_BECAUSE_WHEN_THERE_ARE_OTHER_UNKNOWN_STOPPED_OR_READONLY_NODES_IN_THE_CONSENSUS_GROUP_THAT_ARE_NOT_REMOVE_NODES_THE_REMOVE_OPERATION_CANNOT_BE_PERFORMED_FOR_SECURITY_REASONS_PLEASE_CHECK_THE_STATUS_OF_THE_NODE_ARG_AND_ENSURE_IT_IS_RUNNING_5063B3F6 = "Submit RemoveDataNodesProcedure failed, because when there are other unknown, stopped or readonly nodes in the consensus group that are not remove nodes, the remove operation cannot be performed for security reasons. Please check the status of the node %s and ensure it is running."; - public static final String MESSAGE_AVAILABLEDATANODESIZE_ARG_MAXREPLICAFACTOR_ARG_MAX_ALLOWED_REMOVED_DATA_NODE_SIZE_FB8C382C = "availableDataNodeSize: %s, maxReplicaFactor: %s, max allowed removed Data Node size is: %s"; + public static final String + MESSAGE_AVAILABLEDATANODESIZE_ARG_MAXREPLICAFACTOR_ARG_MAX_ALLOWED_REMOVED_DATA_NODE_SIZE_FB8C382C = + "availableDataNodeSize: %s, maxReplicaFactor: %s, max allowed removed Data Node size is: %s"; public static final String EXCEPTION_NOT_SUPPORTED_0A83F963 = " is not supported"; - public static final String LOG_START_ADD_TRIGGER_ARG_TRIGGERTABLE_CONFIG_NODES_NEEDTOSAVEJAR_ARG_0C23D81E = "Start to add trigger [{}] in TriggerTable on Config Nodes, needToSaveJar[{}]"; - public static final String LOG_START_CREATE_TRIGGERINSTANCE_ARG_DATA_NODES_917C3313 = "Start to create triggerInstance [{}] on Data Nodes"; - public static final String LOG_START_ACTIVE_TRIGGER_ARG_DATA_NODES_A4AB8131 = "Start to active trigger [{}] on Data Nodes"; - public static final String LOG_START_ACTIVE_TRIGGER_ARG_CONFIG_NODES_153A5D40 = "Start to active trigger [{}] on Config Nodes"; - public static final String LOG_RETRIEVABLE_ERROR_TRYING_CREATE_TRIGGER_ARG_STATE_ARG_44976C4E = "Retrievable error trying to create trigger [{}], state [{}]"; - public static final String LOG_START_CONFIG_NODE_INACTIVE_ROLLBACK_TRIGGER_ARG_536929E5 = "Start to [CONFIG_NODE_INACTIVE] rollback of trigger [{}]"; - public static final String LOG_START_DATA_NODE_INACTIVE_ROLLBACK_TRIGGER_ARG_38C93D64 = "Start to [DATA_NODE_INACTIVE] rollback of trigger [{}]"; - public static final String LOG_RETRIEVABLE_ERROR_TRYING_DROP_TRIGGER_ARG_STATE_ARG_2282AC35 = "Retrievable error trying to drop trigger [{}], state [{}]"; - public static final String LOG_DELETEDATABASEPROCEDURE_PRE_DELETE_DATABASE_ARG_6A1FEACC = "[DeleteDatabaseProcedure] Pre delete database: {}"; - public static final String LOG_DELETEDATABASEPROCEDURE_INVALIDATE_CACHE_DATABASE_ARG_299FC9BC = "[DeleteDatabaseProcedure] Invalidate cache of database: {}"; - public static final String LOG_DELETEDATABASEPROCEDURE_DELETE_DATABASESCHEMA_ARG_A49A47AC = "[DeleteDatabaseProcedure] Delete DatabaseSchema: {}"; - public static final String LOG_DELETEDATABASEPROCEDURE_SUCCESSFULLY_DELETE_SCHEMAREGION_ARG_ARG_BA0535DA = "[DeleteDatabaseProcedure] Successfully delete SchemaRegion[{}] on {}"; - public static final String LOG_DELETEDATABASEPROCEDURE_FAILED_DELETE_SCHEMAREGION_ARG_ARG_SUBMIT_ASYNC_DELETION_8C3E6DE3 = "[DeleteDatabaseProcedure] Failed to delete SchemaRegion[{}] on {}. Submit to async deletion."; - public static final String LOG_DELETEDATABASEPROCEDURE_DATA_PARTITION_POLICY_TABLE_DATABASE_ARG_CLEARED_7A32E28A = "[DeleteDatabaseProcedure] The data partition policy table of database: {} is cleared."; - public static final String LOG_DELETEDATABASEPROCEDURE_DATABASE_ARG_DELETED_SUCCESSFULLY_3A4E9202 = "[DeleteDatabaseProcedure] Database: {} is deleted successfully"; - public static final String LOG_DELETEDATABASEPROCEDURE_RETRIABLE_ERROR_TRYING_DELETE_DATABASE_ARG_STATE_ARG_8167D246 = "[DeleteDatabaseProcedure] Retriable error trying to delete database {}, state {}"; - public static final String LOG_DELETEDATABASEPROCEDURE_ROLLBACK_PREDELETED_ARG_638F53DA = "[DeleteDatabaseProcedure] Rollback to preDeleted: {}"; + public static final String + LOG_START_ADD_TRIGGER_ARG_TRIGGERTABLE_CONFIG_NODES_NEEDTOSAVEJAR_ARG_0C23D81E = + "Start to add trigger [{}] in TriggerTable on Config Nodes, needToSaveJar[{}]"; + public static final String LOG_START_CREATE_TRIGGERINSTANCE_ARG_DATA_NODES_917C3313 = + "Start to create triggerInstance [{}] on Data Nodes"; + public static final String LOG_START_ACTIVE_TRIGGER_ARG_DATA_NODES_A4AB8131 = + "Start to active trigger [{}] on Data Nodes"; + public static final String LOG_START_ACTIVE_TRIGGER_ARG_CONFIG_NODES_153A5D40 = + "Start to active trigger [{}] on Config Nodes"; + public static final String LOG_RETRIEVABLE_ERROR_TRYING_CREATE_TRIGGER_ARG_STATE_ARG_44976C4E = + "Retrievable error trying to create trigger [{}], state [{}]"; + public static final String LOG_START_CONFIG_NODE_INACTIVE_ROLLBACK_TRIGGER_ARG_536929E5 = + "Start to [CONFIG_NODE_INACTIVE] rollback of trigger [{}]"; + public static final String LOG_START_DATA_NODE_INACTIVE_ROLLBACK_TRIGGER_ARG_38C93D64 = + "Start to [DATA_NODE_INACTIVE] rollback of trigger [{}]"; + public static final String LOG_RETRIEVABLE_ERROR_TRYING_DROP_TRIGGER_ARG_STATE_ARG_2282AC35 = + "Retrievable error trying to drop trigger [{}], state [{}]"; + public static final String LOG_DELETEDATABASEPROCEDURE_PRE_DELETE_DATABASE_ARG_6A1FEACC = + "[DeleteDatabaseProcedure] Pre delete database: {}"; + public static final String LOG_DELETEDATABASEPROCEDURE_INVALIDATE_CACHE_DATABASE_ARG_299FC9BC = + "[DeleteDatabaseProcedure] Invalidate cache of database: {}"; + public static final String LOG_DELETEDATABASEPROCEDURE_DELETE_DATABASESCHEMA_ARG_A49A47AC = + "[DeleteDatabaseProcedure] Delete DatabaseSchema: {}"; + public static final String + LOG_DELETEDATABASEPROCEDURE_SUCCESSFULLY_DELETE_SCHEMAREGION_ARG_ARG_BA0535DA = + "[DeleteDatabaseProcedure] Successfully delete SchemaRegion[{}] on {}"; + public static final String + LOG_DELETEDATABASEPROCEDURE_FAILED_DELETE_SCHEMAREGION_ARG_ARG_SUBMIT_ASYNC_DELETION_8C3E6DE3 = + "[DeleteDatabaseProcedure] Failed to delete SchemaRegion[{}] on {}. Submit to async deletion."; + public static final String + LOG_DELETEDATABASEPROCEDURE_DATA_PARTITION_POLICY_TABLE_DATABASE_ARG_CLEARED_7A32E28A = + "[DeleteDatabaseProcedure] The data partition policy table of database: {} is cleared."; + public static final String + LOG_DELETEDATABASEPROCEDURE_DATABASE_ARG_DELETED_SUCCESSFULLY_3A4E9202 = + "[DeleteDatabaseProcedure] Database: {} is deleted successfully"; + public static final String + LOG_DELETEDATABASEPROCEDURE_RETRIABLE_ERROR_TRYING_DELETE_DATABASE_ARG_STATE_ARG_8167D246 = + "[DeleteDatabaseProcedure] Retriable error trying to delete database {}, state {}"; + public static final String LOG_DELETEDATABASEPROCEDURE_ROLLBACK_PREDELETED_ARG_638F53DA = + "[DeleteDatabaseProcedure] Rollback to preDeleted: {}"; public static final String EXCEPTION_FAILED_DAA6EA2F = " failed "; - public static final String EXCEPTION_FAILED_CHECK_TIME_SERIES_EXISTENCE_ALL_REPLICASET_SCHEMAREGION_ARG_FAILURES_5F668154 = "failed to check time series existence in all replicaset of schemaRegion %s. Failures: %s"; - public static final String LOG_FAILED_ROLLBACK_CONFIGNODE_TTL_STATE_9666EF54 = "Failed to rollback ConfigNode ttl state."; - public static final String LOG_FAILED_ROLLBACK_DATANODE_TTL_CACHE_436C008A = "Failed to rollback DataNode ttl cache."; - public static final String EXCEPTION_ROLLBACK_CONFIGNODE_TTL_FAILED_6D4FB59A = "Rollback ConfigNode ttl failed for "; - public static final String EXCEPTION_ROLLBACK_DATANODE_TTL_CACHE_FAILED_AF9C7102 = "Rollback dataNode ttl cache failed for "; - public static final String LOG_PLEASE_VERIFY_WHETHER_LEADER_CHANGE_HAS_OCCURRED_DURING_STAGE_9FE68EE3 = "Please verify whether a leader change has occurred during this stage. "; - public static final String LOG_IF_LOG_TRIGGERED_WITHOUT_LEADER_CHANGE_IT_INDICATES_POTENTIAL_BUG_32AE71FD = - "If this log is triggered without a leader change, it indicates a potential bug in the" - + " partition table."; - public static final String LOG_SKIP_RECOVERING_SCHEDULE_TASK_CQ_ARG_BECAUSE_ITS_METADATA_UNAVAILABLE_00286802 = "Skip recovering the schedule task of CQ {} because its metadata is unavailable."; - public static final String LOG_PROCEDUREID_ARG_ACQUIRE_LOCK_3FBF9987 = "procedureId {} acquire lock."; - public static final String LOG_PROCEDUREID_ARG_ACQUIRE_LOCK_FAILED_WILL_WAIT_LOCK_AFTER_FINISHING_3B27278E = "procedureId {} acquire lock failed, will wait for lock after finishing execution."; - public static final String LOG_PROCEDUREID_ARG_RELEASE_LOCK_FF860D6B = "procedureId {} release lock."; - public static final String LOG_RETRIEVABLE_ERROR_TRYING_ADD_CONFIG_NODE_ARG_STATE_ARG_D7285810 = "Retrievable error trying to add config node {}, state {}"; - public static final String LOG_RETRIEVABLE_ERROR_TRYING_REMOVE_CONFIG_NODE_ARG_STATE_ARG_3754EBA1 = "Retrievable error trying to remove config node {}, state {}"; - public static final String LOG_PROCEDUREID_ARG_REMOVEDATANODES_SKIPS_ACQUIRING_LOCK_SINCE_UPPER_LAYER_ENSURES_C7546FF8 = - "procedureId {}-RemoveDataNodes skips acquiring lock, since upper layer ensures the serial" - + " execution."; - public static final String LOG_PROCEDUREID_ARG_REMOVEDATANODES_SKIPS_RELEASING_LOCK_SINCE_IT_HASN_T_AED8A3DA = "procedureId {}-RemoveDataNodes skips releasing lock, since it hasn't acquire any lock."; - public static final String LOG_ARG_CAN_NOT_REMOVE_DATANODE_ARG_495F9F85 = "{}, Can not remove DataNode {} "; - public static final String LOG_BECAUSE_NUMBER_DATANODES_LESS_EQUAL_THAN_REGION_REPLICA_NUMBER_DEC0CB38 = "because the number of DataNodes is less or equal than region replica number"; - public static final String LOG_ARG_DATANODE_REGIONS_REMOVED_ARG_216A7DC7 = "{}, DataNode regions to be removed is {}"; - public static final String LOG_RETRIEVABLE_ERROR_TRYING_REMOVE_DATA_NODE_ARG_STATE_ARG_4EFEB850 = "Retrievable error trying to remove data node {}, state {}"; - public static final String LOG_SUBMIT_REGIONMIGRATEPROCEDURE_REGIONID_ARG_REMOVEDDATANODE_ARG_DESTDATANODE_ARG_COORDINATORFORADDPEER_ARG_ = - "Submit RegionMigrateProcedure for regionId {}: removedDataNode={}, destDataNode={}," - + " coordinatorForAddPeer={}, coordinatorForRemovePeer={}"; - public static final String LOG_ARG_CANNOT_FIND_TARGET_DATANODE_MIGRATE_REGION_ARG_81A78E06 = "{}, Cannot find target DataNode to migrate the region: {}"; - public static final String LOG_ARG_SOME_REGIONS_MIGRATED_FAILED_DATANODE_ARG_MIGRATEDFAILEDREGIONS_ARG_11644841 = "{}, Some regions are migrated failed in DataNode: {}, migratedFailedRegions: {}."; - public static final String LOG_REGIONS_HAVE_BEEN_SUCCESSFULLY_MIGRATED_WILL_NOT_ROLL_BACK_YOU_AE904563 = - "Regions that have been successfully migrated will not roll back, you can submit the" - + " RemoveDataNodes task again later."; - public static final String LOG_ARG_DATANODES_ARG_ALL_REGIONS_MIGRATED_SUCCESSFULLY_START_STOP_THEM_32D56F28 = "{}, DataNodes: {} all regions migrated successfully, start to stop them."; - public static final String LOG_ARG_START_ROLL_BACK_DATANODES_STATUS_ARG_05C67270 = "{}, Start to roll back the DataNodes status: {}"; - public static final String LOG_ARG_ROLL_BACK_DATANODES_STATUS_SUCCESSFULLY_ARG_6773A2DF = "{}, Roll back the DataNodes status successfully: {}"; - public static final String LOG_DATAPARTITIONINTEGRITY_NO_DATANODES_REGISTERED_NO_WAY_COLLECT_EARLIEST_TIMESLOTS_WAITING_7025EB23 = - "[DataPartitionIntegrity] No DataNodes registered, no way to collect earliest timeslots," - + " waiting for them to go up"; - public static final String LOG_DATAPARTITIONINTEGRITY_FAILED_COLLECTED_EARLIEST_TIMESLOTS_DATANODE_ID_ARG_ALREADY_OUT_834B62B9 = - "[DataPartitionIntegrity] Failed to collected earliest timeslots from the DataNode[id={}]," - + " already out of max retry time"; - public static final String LOG_DATAPARTITIONINTEGRITY_FAILED_COLLECTED_EARLIEST_TIMESLOTS_DATANODE_ID_ARG_RESPONSE_STATUS_B0A31EC4 = - "[DataPartitionIntegrity] Failed to collected earliest timeslots from the DataNode[id={}]," - + " response status is {}"; - public static final String LOG_COLLECTED_EARLIEST_TIMESLOTS_DATANODE_ID_ARG_ARG_5CDF2BA6 = "Collected earliest timeslots from the DataNode[id={}]: {}"; - public static final String LOG_DATAPARTITIONINTEGRITY_FAILED_COLLECT_EARLIEST_TIMESLOTS_DATANODE_ID_ARG_ARG_A211840A = "[DataPartitionIntegrity] Failed to collect earliest timeslots from the DataNode[id={}]: {}"; - public static final String LOG_COLLECTED_EARLIEST_TIMESLOTS_ARG_DATANODES_ARG_NUMBER_SUCCESSFUL_DATANODES_ARG_1CC129EF = "Collected earliest timeslots from {} DataNodes: {}, the number of successful DataNodes is {}"; - public static final String LOG_DATAPARTITIONINTEGRITY_NO_MISSING_DATA_PARTITIONS_DETECTED_NOTHING_NEEDS_REPAIRED_TERMINATING_72F2635F = - "[DataPartitionIntegrity] No missing data partitions detected, nothing needs to be repaired," - + " terminating procedure"; - public static final String LOG_DATAPARTITIONINTEGRITY_NO_DATA_PARTITION_TABLE_RELATED_DATABASE_ARG_WAS_FOUND_B5B90613 = - "[DataPartitionIntegrity] No data partition table related to database {} was found from the" - + " ConfigNode, and this issue needs to be repaired"; - public static final String LOG_DATAPARTITIONINTEGRITY_DATABASE_ARG_HAS_LOST_TIMESLOT_ARG_ITS_DATA_TABLE_499AF395 = - "[DataPartitionIntegrity] Database {} has lost timeslot {} in its data table partition, and" - + " this issue needs to be repaired"; - public static final String LOG_DATAPARTITIONINTEGRITY_NO_DATABASES_HAVE_LOST_DATA_PARTITIONS_TERMINATING_PROCEDURE_3E718CC3 = "[DataPartitionIntegrity] No databases have lost data partitions, terminating procedure"; - public static final String LOG_DATAPARTITIONINTEGRITY_IDENTIFIED_ARG_DATABASES_HAVE_LOST_DATA_PARTITIONS_WILL_REQUEST_6DEA7502 = - "[DataPartitionIntegrity] Identified {} databases have lost data partitions, will request" - + " DataPartitionTable generation from {} DataNodes"; - public static final String LOG_REQUESTING_DATAPARTITIONTABLE_GENERATION_ARG_DATANODES_559F97E8 = "Requesting DataPartitionTable generation from {} DataNodes..."; - public static final String LOG_DATAPARTITIONINTEGRITY_NO_DATANODES_REGISTERED_NO_WAY_REQUESTED_DATAPARTITIONTABLE_GENERATION_TERMINATING_ = - "[DataPartitionIntegrity] No DataNodes registered, no way to requested DataPartitionTable" - + " generation, terminating procedure"; - public static final String LOG_DATAPARTITIONINTEGRITY_FAILED_REQUEST_DATAPARTITIONTABLE_GENERATION_DATANODE_ID_ARG_ALREADY_OUT_6B0C9351 = - "[DataPartitionIntegrity] Failed to request DataPartitionTable generation from the" - + " DataNode[id={}], already out of max retry time"; - public static final String LOG_DATAPARTITIONINTEGRITY_FAILED_REQUEST_DATAPARTITIONTABLE_GENERATION_DATANODE_ID_ARG_RESPONSE_STATUS_93012D = - "[DataPartitionIntegrity] Failed to request DataPartitionTable generation from the" - + " DataNode[id={}], response status is {}"; - public static final String LOG_DATAPARTITIONINTEGRITY_FAILED_REQUEST_DATAPARTITIONTABLE_GENERATION_DATANODE_ID_ARG_ARG_818B47B8 = - "[DataPartitionIntegrity] Failed to request DataPartitionTable generation from" - + " DataNode[id={}]: {}"; - public static final String LOG_DATAPARTITIONINTEGRITY_FAILED_REQUEST_DATAPARTITIONTABLE_GENERATION_HEART_BEAT_DATANODE_ID_ARG_2AB63F12 = - "[DataPartitionIntegrity] Failed to request DataPartitionTable generation heart beat from the" - + " DataNode[id={}], already out of max retry time"; - public static final String LOG_DATAPARTITIONINTEGRITY_FAILED_REQUEST_DATAPARTITIONTABLE_GENERATION_HEART_BEAT_DATANODE_ID_ARG_DC1702EF = - "[DataPartitionIntegrity] Failed to request DataPartitionTable generation heart beat from the" - + " DataNode[id={}], state is {}, response status is {}"; - public static final String LOG_DATAPARTITIONINTEGRITY_DATANODE_ARG_COMPLETED_DATAPARTITIONTABLE_GENERATION_TERMINATING_HEART_BEAT_59DAAD5 = - "[DataPartitionIntegrity] DataNode {} completed DataPartitionTable generation, terminating" - + " heart beat"; - public static final String LOG_DATAPARTITIONINTEGRITY_DATANODE_ARG_STILL_GENERATING_DATAPARTITIONTABLE_63F84C78 = "[DataPartitionIntegrity] DataNode {} still generating DataPartitionTable"; - public static final String LOG_DATAPARTITIONINTEGRITY_DATANODE_ARG_RETURNED_UNKNOWN_ERROR_CODE_ARG_2DA6A21E = "[DataPartitionIntegrity] DataNode {} returned unknown error code: {}"; - public static final String LOG_DATAPARTITIONINTEGRITY_ERROR_CHECKING_DATAPARTITIONTABLE_STATUS_DATANODE_ARG_ARG_TERMINATING_HEART_D6EDA91 = - "[DataPartitionIntegrity] Error checking DataPartitionTable status from DataNode {}: {}," - + " terminating heart beat"; - public static final String LOG_DATAPARTITIONINTEGRITY_NO_DATAPARTITIONTABLES_MERGE_DATAPARTITIONTABLES_EMPTY_920E3DE6 = "[DataPartitionIntegrity] No DataPartitionTables to merge, dataPartitionTables is empty"; - public static final String LOG_DATAPARTITIONINTEGRITY_NO_DATA_PARTITION_TABLE_RELATED_DATABASE_ARG_WAS_FOUND_D1698512 = - "[DataPartitionIntegrity] No data partition table related to database {} was found from the" - + " ConfigNode, use data partition table of DataNode directly"; - public static final String LOG_DATAPARTITIONINTEGRITY_DATAPARTITIONTABLE_SUCCESSFULLY_WRITTEN_CONSENSUS_LOG_2B1634A6 = "[DataPartitionIntegrity] DataPartitionTable successfully written to consensus log"; - public static final String LOG_DATAPARTITIONINTEGRITY_ARG_SERIALIZE_FAILED_DATANODEID_ARG_967B51AA = "[DataPartitionIntegrity] {} serialize failed for dataNodeId: {}"; - public static final String LOG_DATAPARTITIONINTEGRITY_ARG_SERIALIZE_FINALDATAPARTITIONTABLES_FAILED_7E44DCD8 = "[DataPartitionIntegrity] {} serialize finalDataPartitionTables failed"; - public static final String LOG_DATAPARTITIONINTEGRITY_ARG_DESERIALIZE_FAILED_DATANODEID_ARG_22388A60 = "[DataPartitionIntegrity] {} deserialize failed for dataNodeId: {}"; - public static final String LOG_DATAPARTITIONINTEGRITY_ARG_DESERIALIZE_FINALDATAPARTITIONTABLES_FAILED_7E23E4BD = "[DataPartitionIntegrity] {} deserialize finalDataPartitionTables failed"; - public static final String LOG_DATAPARTITIONINTEGRITY_FAILED_DESERIALIZE_DATABASESCOPEDDATAPARTITIONTABLE_3B6933B5 = "[DataPartitionIntegrity] Failed to deserialize DatabaseScopedDataPartitionTable"; + public static final String + EXCEPTION_FAILED_CHECK_TIME_SERIES_EXISTENCE_ALL_REPLICASET_SCHEMAREGION_ARG_FAILURES_5F668154 = + "failed to check time series existence in all replicaset of schemaRegion %s. Failures: %s"; + public static final String LOG_FAILED_ROLLBACK_CONFIGNODE_TTL_STATE_9666EF54 = + "Failed to rollback ConfigNode ttl state."; + public static final String LOG_FAILED_ROLLBACK_DATANODE_TTL_CACHE_436C008A = + "Failed to rollback DataNode ttl cache."; + public static final String EXCEPTION_ROLLBACK_CONFIGNODE_TTL_FAILED_6D4FB59A = + "Rollback ConfigNode ttl failed for "; + public static final String EXCEPTION_ROLLBACK_DATANODE_TTL_CACHE_FAILED_AF9C7102 = + "Rollback dataNode ttl cache failed for "; + public static final String + LOG_PLEASE_VERIFY_WHETHER_LEADER_CHANGE_HAS_OCCURRED_DURING_STAGE_9FE68EE3 = + "Please verify whether a leader change has occurred during this stage. "; + public static final String + LOG_IF_LOG_TRIGGERED_WITHOUT_LEADER_CHANGE_IT_INDICATES_POTENTIAL_BUG_32AE71FD = + "If this log is triggered without a leader change, it indicates a potential bug in the" + + " partition table."; + public static final String + LOG_SKIP_RECOVERING_SCHEDULE_TASK_CQ_ARG_BECAUSE_ITS_METADATA_UNAVAILABLE_00286802 = + "Skip recovering the schedule task of CQ {} because its metadata is unavailable."; + public static final String LOG_PROCEDUREID_ARG_ACQUIRE_LOCK_3FBF9987 = + "procedureId {} acquire lock."; + public static final String + LOG_PROCEDUREID_ARG_ACQUIRE_LOCK_FAILED_WILL_WAIT_LOCK_AFTER_FINISHING_3B27278E = + "procedureId {} acquire lock failed, will wait for lock after finishing execution."; + public static final String LOG_PROCEDUREID_ARG_RELEASE_LOCK_FF860D6B = + "procedureId {} release lock."; + public static final String LOG_RETRIEVABLE_ERROR_TRYING_ADD_CONFIG_NODE_ARG_STATE_ARG_D7285810 = + "Retrievable error trying to add config node {}, state {}"; + public static final String + LOG_RETRIEVABLE_ERROR_TRYING_REMOVE_CONFIG_NODE_ARG_STATE_ARG_3754EBA1 = + "Retrievable error trying to remove config node {}, state {}"; + public static final String + LOG_PROCEDUREID_ARG_REMOVEDATANODES_SKIPS_ACQUIRING_LOCK_SINCE_UPPER_LAYER_ENSURES_C7546FF8 = + "procedureId {}-RemoveDataNodes skips acquiring lock, since upper layer ensures the serial" + + " execution."; + public static final String + LOG_PROCEDUREID_ARG_REMOVEDATANODES_SKIPS_RELEASING_LOCK_SINCE_IT_HASN_T_AED8A3DA = + "procedureId {}-RemoveDataNodes skips releasing lock, since it hasn't acquire any lock."; + public static final String LOG_ARG_CAN_NOT_REMOVE_DATANODE_ARG_495F9F85 = + "{}, Can not remove DataNode {} "; + public static final String + LOG_BECAUSE_NUMBER_DATANODES_LESS_EQUAL_THAN_REGION_REPLICA_NUMBER_DEC0CB38 = + "because the number of DataNodes is less or equal than region replica number"; + public static final String LOG_ARG_DATANODE_REGIONS_REMOVED_ARG_216A7DC7 = + "{}, DataNode regions to be removed is {}"; + public static final String LOG_RETRIEVABLE_ERROR_TRYING_REMOVE_DATA_NODE_ARG_STATE_ARG_4EFEB850 = + "Retrievable error trying to remove data node {}, state {}"; + public static final String + LOG_SUBMIT_REGIONMIGRATEPROCEDURE_REGIONID_ARG_REMOVEDDATANODE_ARG_DESTDATANODE_ARG_COORDINATORFORADDPEER_ARG_ = + "Submit RegionMigrateProcedure for regionId {}: removedDataNode={}, destDataNode={}," + + " coordinatorForAddPeer={}, coordinatorForRemovePeer={}"; + public static final String LOG_ARG_CANNOT_FIND_TARGET_DATANODE_MIGRATE_REGION_ARG_81A78E06 = + "{}, Cannot find target DataNode to migrate the region: {}"; + public static final String + LOG_ARG_SOME_REGIONS_MIGRATED_FAILED_DATANODE_ARG_MIGRATEDFAILEDREGIONS_ARG_11644841 = + "{}, Some regions are migrated failed in DataNode: {}, migratedFailedRegions: {}."; + public static final String + LOG_REGIONS_HAVE_BEEN_SUCCESSFULLY_MIGRATED_WILL_NOT_ROLL_BACK_YOU_AE904563 = + "Regions that have been successfully migrated will not roll back, you can submit the" + + " RemoveDataNodes task again later."; + public static final String + LOG_ARG_DATANODES_ARG_ALL_REGIONS_MIGRATED_SUCCESSFULLY_START_STOP_THEM_32D56F28 = + "{}, DataNodes: {} all regions migrated successfully, start to stop them."; + public static final String LOG_ARG_START_ROLL_BACK_DATANODES_STATUS_ARG_05C67270 = + "{}, Start to roll back the DataNodes status: {}"; + public static final String LOG_ARG_ROLL_BACK_DATANODES_STATUS_SUCCESSFULLY_ARG_6773A2DF = + "{}, Roll back the DataNodes status successfully: {}"; + public static final String + LOG_DATAPARTITIONINTEGRITY_NO_DATANODES_REGISTERED_NO_WAY_COLLECT_EARLIEST_TIMESLOTS_WAITING_7025EB23 = + "[DataPartitionIntegrity] No DataNodes registered, no way to collect earliest timeslots," + + " waiting for them to go up"; + public static final String + LOG_DATAPARTITIONINTEGRITY_FAILED_COLLECTED_EARLIEST_TIMESLOTS_DATANODE_ID_ARG_ALREADY_OUT_834B62B9 = + "[DataPartitionIntegrity] Failed to collected earliest timeslots from the DataNode[id={}]," + + " already out of max retry time"; + public static final String + LOG_DATAPARTITIONINTEGRITY_FAILED_COLLECTED_EARLIEST_TIMESLOTS_DATANODE_ID_ARG_RESPONSE_STATUS_B0A31EC4 = + "[DataPartitionIntegrity] Failed to collected earliest timeslots from the DataNode[id={}]," + + " response status is {}"; + public static final String LOG_COLLECTED_EARLIEST_TIMESLOTS_DATANODE_ID_ARG_ARG_5CDF2BA6 = + "Collected earliest timeslots from the DataNode[id={}]: {}"; + public static final String + LOG_DATAPARTITIONINTEGRITY_FAILED_COLLECT_EARLIEST_TIMESLOTS_DATANODE_ID_ARG_ARG_A211840A = + "[DataPartitionIntegrity] Failed to collect earliest timeslots from the DataNode[id={}]: {}"; + public static final String + LOG_COLLECTED_EARLIEST_TIMESLOTS_ARG_DATANODES_ARG_NUMBER_SUCCESSFUL_DATANODES_ARG_1CC129EF = + "Collected earliest timeslots from {} DataNodes: {}, the number of successful DataNodes is {}"; + public static final String + LOG_DATAPARTITIONINTEGRITY_NO_MISSING_DATA_PARTITIONS_DETECTED_NOTHING_NEEDS_REPAIRED_TERMINATING_72F2635F = + "[DataPartitionIntegrity] No missing data partitions detected, nothing needs to be repaired," + + " terminating procedure"; + public static final String + LOG_DATAPARTITIONINTEGRITY_NO_DATA_PARTITION_TABLE_RELATED_DATABASE_ARG_WAS_FOUND_B5B90613 = + "[DataPartitionIntegrity] No data partition table related to database {} was found from the" + + " ConfigNode, and this issue needs to be repaired"; + public static final String + LOG_DATAPARTITIONINTEGRITY_DATABASE_ARG_HAS_LOST_TIMESLOT_ARG_ITS_DATA_TABLE_499AF395 = + "[DataPartitionIntegrity] Database {} has lost timeslot {} in its data table partition, and" + + " this issue needs to be repaired"; + public static final String + LOG_DATAPARTITIONINTEGRITY_NO_DATABASES_HAVE_LOST_DATA_PARTITIONS_TERMINATING_PROCEDURE_3E718CC3 = + "[DataPartitionIntegrity] No databases have lost data partitions, terminating procedure"; + public static final String + LOG_DATAPARTITIONINTEGRITY_IDENTIFIED_ARG_DATABASES_HAVE_LOST_DATA_PARTITIONS_WILL_REQUEST_6DEA7502 = + "[DataPartitionIntegrity] Identified {} databases have lost data partitions, will request" + + " DataPartitionTable generation from {} DataNodes"; + public static final String LOG_REQUESTING_DATAPARTITIONTABLE_GENERATION_ARG_DATANODES_559F97E8 = + "Requesting DataPartitionTable generation from {} DataNodes..."; + public static final String + LOG_DATAPARTITIONINTEGRITY_NO_DATANODES_REGISTERED_NO_WAY_REQUESTED_DATAPARTITIONTABLE_GENERATION_TERMINATING_ = + "[DataPartitionIntegrity] No DataNodes registered, no way to requested DataPartitionTable" + + " generation, terminating procedure"; + public static final String + LOG_DATAPARTITIONINTEGRITY_FAILED_REQUEST_DATAPARTITIONTABLE_GENERATION_DATANODE_ID_ARG_ALREADY_OUT_6B0C9351 = + "[DataPartitionIntegrity] Failed to request DataPartitionTable generation from the" + + " DataNode[id={}], already out of max retry time"; + public static final String + LOG_DATAPARTITIONINTEGRITY_FAILED_REQUEST_DATAPARTITIONTABLE_GENERATION_DATANODE_ID_ARG_RESPONSE_STATUS_93012D = + "[DataPartitionIntegrity] Failed to request DataPartitionTable generation from the" + + " DataNode[id={}], response status is {}"; + public static final String + LOG_DATAPARTITIONINTEGRITY_FAILED_REQUEST_DATAPARTITIONTABLE_GENERATION_DATANODE_ID_ARG_ARG_818B47B8 = + "[DataPartitionIntegrity] Failed to request DataPartitionTable generation from" + + " DataNode[id={}]: {}"; + public static final String + LOG_DATAPARTITIONINTEGRITY_FAILED_REQUEST_DATAPARTITIONTABLE_GENERATION_HEART_BEAT_DATANODE_ID_ARG_2AB63F12 = + "[DataPartitionIntegrity] Failed to request DataPartitionTable generation heart beat from the" + + " DataNode[id={}], already out of max retry time"; + public static final String + LOG_DATAPARTITIONINTEGRITY_FAILED_REQUEST_DATAPARTITIONTABLE_GENERATION_HEART_BEAT_DATANODE_ID_ARG_DC1702EF = + "[DataPartitionIntegrity] Failed to request DataPartitionTable generation heart beat from the" + + " DataNode[id={}], state is {}, response status is {}"; + public static final String + LOG_DATAPARTITIONINTEGRITY_DATANODE_ARG_COMPLETED_DATAPARTITIONTABLE_GENERATION_TERMINATING_HEART_BEAT_59DAAD5 = + "[DataPartitionIntegrity] DataNode {} completed DataPartitionTable generation, terminating" + + " heart beat"; + public static final String + LOG_DATAPARTITIONINTEGRITY_DATANODE_ARG_STILL_GENERATING_DATAPARTITIONTABLE_63F84C78 = + "[DataPartitionIntegrity] DataNode {} still generating DataPartitionTable"; + public static final String + LOG_DATAPARTITIONINTEGRITY_DATANODE_ARG_RETURNED_UNKNOWN_ERROR_CODE_ARG_2DA6A21E = + "[DataPartitionIntegrity] DataNode {} returned unknown error code: {}"; + public static final String + LOG_DATAPARTITIONINTEGRITY_ERROR_CHECKING_DATAPARTITIONTABLE_STATUS_DATANODE_ARG_ARG_TERMINATING_HEART_D6EDA91 = + "[DataPartitionIntegrity] Error checking DataPartitionTable status from DataNode {}: {}," + + " terminating heart beat"; + public static final String + LOG_DATAPARTITIONINTEGRITY_NO_DATAPARTITIONTABLES_MERGE_DATAPARTITIONTABLES_EMPTY_920E3DE6 = + "[DataPartitionIntegrity] No DataPartitionTables to merge, dataPartitionTables is empty"; + public static final String + LOG_DATAPARTITIONINTEGRITY_NO_DATA_PARTITION_TABLE_RELATED_DATABASE_ARG_WAS_FOUND_D1698512 = + "[DataPartitionIntegrity] No data partition table related to database {} was found from the" + + " ConfigNode, use data partition table of DataNode directly"; + public static final String + LOG_DATAPARTITIONINTEGRITY_DATAPARTITIONTABLE_SUCCESSFULLY_WRITTEN_CONSENSUS_LOG_2B1634A6 = + "[DataPartitionIntegrity] DataPartitionTable successfully written to consensus log"; + public static final String + LOG_DATAPARTITIONINTEGRITY_ARG_SERIALIZE_FAILED_DATANODEID_ARG_967B51AA = + "[DataPartitionIntegrity] {} serialize failed for dataNodeId: {}"; + public static final String + LOG_DATAPARTITIONINTEGRITY_ARG_SERIALIZE_FINALDATAPARTITIONTABLES_FAILED_7E44DCD8 = + "[DataPartitionIntegrity] {} serialize finalDataPartitionTables failed"; + public static final String + LOG_DATAPARTITIONINTEGRITY_ARG_DESERIALIZE_FAILED_DATANODEID_ARG_22388A60 = + "[DataPartitionIntegrity] {} deserialize failed for dataNodeId: {}"; + public static final String + LOG_DATAPARTITIONINTEGRITY_ARG_DESERIALIZE_FINALDATAPARTITIONTABLES_FAILED_7E23E4BD = + "[DataPartitionIntegrity] {} deserialize finalDataPartitionTables failed"; + public static final String + LOG_DATAPARTITIONINTEGRITY_FAILED_DESERIALIZE_DATABASESCOPEDDATAPARTITIONTABLE_3B6933B5 = + "[DataPartitionIntegrity] Failed to deserialize DatabaseScopedDataPartitionTable"; public static final String EXCEPTION_FAILED_C6FF154E = " failed"; - public static final String LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_EXECUTEFROMVALIDATE_97490577 = "SubscriptionHandleLeaderChangeProcedure: executeFromValidate"; - public static final String LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_EXECUTEFROMOPERATEONCONFIGNODES_D4E8BD37 = "SubscriptionHandleLeaderChangeProcedure: executeFromOperateOnConfigNodes"; - public static final String LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_FAILED_PULL_COMMIT_PROGRESS_DATANODE_ARG_STATUS_ARG_8C6DEC4E = - "SubscriptionHandleLeaderChangeProcedure: failed to pull commit progress from DataNode {}," - + " status: {}"; - public static final String LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_FAILED_WRITE_API_EXECUTING_CONSENSUS_LAYER_56B3832A = - "SubscriptionHandleLeaderChangeProcedure: failed in the write API executing the consensus" - + " layer due to: "; - public static final String LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_EXECUTEFROMOPERATEONDATANODES_0D9F7C98 = "SubscriptionHandleLeaderChangeProcedure: executeFromOperateOnDataNodes"; - public static final String LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_IGNORED_FAILED_TOPIC_META_PUSH_DATANODE_ARG_STATUS_ARG_67FC003F = - "SubscriptionHandleLeaderChangeProcedure: ignored failed topic meta push to DataNode {}," - + " status: {}"; - public static final String LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_IGNORED_FAILED_CONSUMER_GROUP_META_PUSH_DATANODE_ARG_STATUS_17C948 = - "SubscriptionHandleLeaderChangeProcedure: ignored failed consumer group meta push to DataNode" - + " {}, status: {}"; - public static final String LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_IGNORED_FAILED_SUBSCRIPTION_RUNTIME_PUSH_UNREADABLE_DATANODE_ARG_S = - "SubscriptionHandleLeaderChangeProcedure: ignored failed subscription runtime push to" - + " unreadable DataNode {}, status: {}"; - public static final String LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_ROLLBACKFROMVALIDATE_74B408B7 = "SubscriptionHandleLeaderChangeProcedure: rollbackFromValidate"; - public static final String LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_ROLLBACKFROMOPERATEONCONFIGNODES_D4C70763 = "SubscriptionHandleLeaderChangeProcedure: rollbackFromOperateOnConfigNodes"; - public static final String LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_ROLLBACKFROMOPERATEONDATANODES_0250F6E9 = "SubscriptionHandleLeaderChangeProcedure: rollbackFromOperateOnDataNodes"; - public static final String LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_FAILED_DESERIALIZE_REGION_PROGRESS_KEY_ARG_SUMMARY_ARG_F6935E59 = - "SubscriptionHandleLeaderChangeProcedure: failed to deserialize region progress, key={}," - + " summary={}"; - public static final String EXCEPTION_FAILED_PUSH_SUBSCRIPTION_RUNTIME_STATE_READABLE_DATANODES_DURING_LEADER_CHANGE_F37E6F2C = - "Failed to push subscription runtime state to readable DataNodes during leader change," - + " details: %s"; - public static final String EXCEPTION_FAILED_SERIALIZE_REGION_PROGRESS_1769D6F1 = "Failed to serialize region progress "; - public static final String EXCEPTION_NO_READABLE_DATANODE_AVAILABLE_ACCEPT_SUBSCRIPTION_METADATA_RUNTIME_UPDATES_DURING_22E61621 = - "No readable DataNode is available to accept subscription metadata/runtime updates during" - + " leader change"; - public static final String LOG_CREATESUBSCRIPTIONPROCEDURE_TOPIC_ARG_USES_CONSENSUS_SUBSCRIPTION_MODE_031CF049 = "CreateSubscriptionProcedure: topic [{}] uses consensus subscription mode "; - public static final String LOG_MODE_ARG_SKIPPING_PIPE_CREATION_5F4D1026 = "(mode={}), skipping pipe creation"; - public static final String LOG_CREATESUBSCRIPTIONPROCEDURE_CONSENSUS_BASED_TOPICS_ARG_WILL_HANDLED_DATANODE_90A9C2FD = "CreateSubscriptionProcedure: consensus-based topics {} will be handled by DataNode "; - public static final String LOG_VIA_CONSUMER_GROUP_META_PUSH_NO_PIPE_CREATION_NEEDED_D56CFE31 = "via consumer group meta push (no pipe creation needed)"; - public static final String LOG_CREATESUBSCRIPTIONPROCEDURE_SYNCHRONIZING_COMMIT_PROGRESS_AFTER_CONSUMER_GROUP_NEWLY_SUBSCRIBED_CONSENSUS_TOPICS_ARG_F5687D36 = "CreateSubscriptionProcedure: synchronizing commit progress after consumer group newly subscribed consensus topics {}"; - public static final String LOG_DROPSUBSCRIPTIONPROCEDURE_TOPIC_ARG_USES_CONSENSUS_SUBSCRIPTION_MODE_6962D13C = "DropSubscriptionProcedure: topic [{}] uses consensus subscription mode "; - public static final String LOG_MODE_ARG_SKIPPING_PIPE_REMOVAL_133B0CD6 = "(mode={}), skipping pipe removal"; - public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_ACQUIRELOCK_SKIP_PROCEDURE_LAST_EXECUTION_TIME_ARG_CE3DD247 = "CommitProgressSyncProcedure: acquireLock, skip the procedure due to the last execution time {}"; - public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_EXECUTEFROMVALIDATE_CF220E1F = "CommitProgressSyncProcedure: executeFromValidate"; - public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_EXECUTEFROMOPERATEONCONFIGNODES_0DC818CA = "CommitProgressSyncProcedure: executeFromOperateOnConfigNodes"; - public static final String LOG_FAILED_PULL_COMMIT_PROGRESS_DATANODE_ARG_STATUS_ARG_33037B29 = "Failed to pull commit progress from DataNode {}, status: {}"; - public static final String EXCEPTION_FAILED_TO_PULL_REQUIRED_COMMIT_PROGRESS_FROM_DATANODE_ARG_STATUS_ARG_E08A2450 = "Failed to pull required commit progress from DataNode %s, status: %s"; - public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_EXECUTEFROMOPERATEONDATANODES_NO_OP_34420360 = "CommitProgressSyncProcedure: executeFromOperateOnDataNodes (no-op)"; - public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_ROLLBACKFROMVALIDATE_2309D4D2 = "CommitProgressSyncProcedure: rollbackFromValidate"; - public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_ROLLBACKFROMOPERATEONCONFIGNODES_57CB907B = "CommitProgressSyncProcedure: rollbackFromOperateOnConfigNodes"; - public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_ROLLBACKFROMOPERATEONDATANODES_0D2CEB50 = "CommitProgressSyncProcedure: rollbackFromOperateOnDataNodes"; - public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_FAILED_DESERIALIZE_REGION_PROGRESS_KEY_ARG_SUMMARY_ARG_0202F658 = "CommitProgressSyncProcedure: failed to deserialize region progress, key={}, summary={}"; + public static final String + LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_EXECUTEFROMVALIDATE_97490577 = + "SubscriptionHandleLeaderChangeProcedure: executeFromValidate"; + public static final String + LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_EXECUTEFROMOPERATEONCONFIGNODES_D4E8BD37 = + "SubscriptionHandleLeaderChangeProcedure: executeFromOperateOnConfigNodes"; + public static final String + LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_FAILED_PULL_COMMIT_PROGRESS_DATANODE_ARG_STATUS_ARG_8C6DEC4E = + "SubscriptionHandleLeaderChangeProcedure: failed to pull commit progress from DataNode {}," + + " status: {}"; + public static final String + LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_FAILED_WRITE_API_EXECUTING_CONSENSUS_LAYER_56B3832A = + "SubscriptionHandleLeaderChangeProcedure: failed in the write API executing the consensus" + + " layer due to: "; + public static final String + LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_EXECUTEFROMOPERATEONDATANODES_0D9F7C98 = + "SubscriptionHandleLeaderChangeProcedure: executeFromOperateOnDataNodes"; + public static final String + LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_IGNORED_FAILED_TOPIC_META_PUSH_DATANODE_ARG_STATUS_ARG_67FC003F = + "SubscriptionHandleLeaderChangeProcedure: ignored failed topic meta push to DataNode {}," + + " status: {}"; + public static final String + LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_IGNORED_FAILED_CONSUMER_GROUP_META_PUSH_DATANODE_ARG_STATUS_17C948 = + "SubscriptionHandleLeaderChangeProcedure: ignored failed consumer group meta push to DataNode" + + " {}, status: {}"; + public static final String + LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_IGNORED_FAILED_SUBSCRIPTION_RUNTIME_PUSH_UNREADABLE_DATANODE_ARG_S = + "SubscriptionHandleLeaderChangeProcedure: ignored failed subscription runtime push to" + + " unreadable DataNode {}, status: {}"; + public static final String + LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_ROLLBACKFROMVALIDATE_74B408B7 = + "SubscriptionHandleLeaderChangeProcedure: rollbackFromValidate"; + public static final String + LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_ROLLBACKFROMOPERATEONCONFIGNODES_D4C70763 = + "SubscriptionHandleLeaderChangeProcedure: rollbackFromOperateOnConfigNodes"; + public static final String + LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_ROLLBACKFROMOPERATEONDATANODES_0250F6E9 = + "SubscriptionHandleLeaderChangeProcedure: rollbackFromOperateOnDataNodes"; + public static final String + LOG_SUBSCRIPTIONHANDLELEADERCHANGEPROCEDURE_FAILED_DESERIALIZE_REGION_PROGRESS_KEY_ARG_SUMMARY_ARG_F6935E59 = + "SubscriptionHandleLeaderChangeProcedure: failed to deserialize region progress, key={}," + + " summary={}"; + public static final String + EXCEPTION_FAILED_PUSH_SUBSCRIPTION_RUNTIME_STATE_READABLE_DATANODES_DURING_LEADER_CHANGE_F37E6F2C = + "Failed to push subscription runtime state to readable DataNodes during leader change," + + " details: %s"; + public static final String EXCEPTION_FAILED_SERIALIZE_REGION_PROGRESS_1769D6F1 = + "Failed to serialize region progress "; + public static final String + EXCEPTION_NO_READABLE_DATANODE_AVAILABLE_ACCEPT_SUBSCRIPTION_METADATA_RUNTIME_UPDATES_DURING_22E61621 = + "No readable DataNode is available to accept subscription metadata/runtime updates during" + + " leader change"; + public static final String + LOG_CREATESUBSCRIPTIONPROCEDURE_TOPIC_ARG_USES_CONSENSUS_SUBSCRIPTION_MODE_031CF049 = + "CreateSubscriptionProcedure: topic [{}] uses consensus subscription mode "; + public static final String LOG_MODE_ARG_SKIPPING_PIPE_CREATION_5F4D1026 = + "(mode={}), skipping pipe creation"; + public static final String + LOG_CREATESUBSCRIPTIONPROCEDURE_CONSENSUS_BASED_TOPICS_ARG_WILL_HANDLED_DATANODE_90A9C2FD = + "CreateSubscriptionProcedure: consensus-based topics {} will be handled by DataNode "; + public static final String LOG_VIA_CONSUMER_GROUP_META_PUSH_NO_PIPE_CREATION_NEEDED_D56CFE31 = + "via consumer group meta push (no pipe creation needed)"; + public static final String + LOG_CREATESUBSCRIPTIONPROCEDURE_SYNCHRONIZING_COMMIT_PROGRESS_AFTER_CONSUMER_GROUP_NEWLY_SUBSCRIBED_CONSENSUS_TOPICS_ARG_F5687D36 = + "CreateSubscriptionProcedure: synchronizing commit progress after consumer group newly subscribed consensus topics {}"; + public static final String + LOG_DROPSUBSCRIPTIONPROCEDURE_TOPIC_ARG_USES_CONSENSUS_SUBSCRIPTION_MODE_6962D13C = + "DropSubscriptionProcedure: topic [{}] uses consensus subscription mode "; + public static final String LOG_MODE_ARG_SKIPPING_PIPE_REMOVAL_133B0CD6 = + "(mode={}), skipping pipe removal"; + public static final String + LOG_COMMITPROGRESSSYNCPROCEDURE_ACQUIRELOCK_SKIP_PROCEDURE_LAST_EXECUTION_TIME_ARG_CE3DD247 = + "CommitProgressSyncProcedure: acquireLock, skip the procedure due to the last execution time {}"; + public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_EXECUTEFROMVALIDATE_CF220E1F = + "CommitProgressSyncProcedure: executeFromValidate"; + public static final String + LOG_COMMITPROGRESSSYNCPROCEDURE_EXECUTEFROMOPERATEONCONFIGNODES_0DC818CA = + "CommitProgressSyncProcedure: executeFromOperateOnConfigNodes"; + public static final String LOG_FAILED_PULL_COMMIT_PROGRESS_DATANODE_ARG_STATUS_ARG_33037B29 = + "Failed to pull commit progress from DataNode {}, status: {}"; + public static final String + EXCEPTION_FAILED_TO_PULL_REQUIRED_COMMIT_PROGRESS_FROM_DATANODE_ARG_STATUS_ARG_E08A2450 = + "Failed to pull required commit progress from DataNode %s, status: %s"; + public static final String + LOG_COMMITPROGRESSSYNCPROCEDURE_EXECUTEFROMOPERATEONDATANODES_NO_OP_34420360 = + "CommitProgressSyncProcedure: executeFromOperateOnDataNodes (no-op)"; + public static final String LOG_COMMITPROGRESSSYNCPROCEDURE_ROLLBACKFROMVALIDATE_2309D4D2 = + "CommitProgressSyncProcedure: rollbackFromValidate"; + public static final String + LOG_COMMITPROGRESSSYNCPROCEDURE_ROLLBACKFROMOPERATEONCONFIGNODES_57CB907B = + "CommitProgressSyncProcedure: rollbackFromOperateOnConfigNodes"; + public static final String + LOG_COMMITPROGRESSSYNCPROCEDURE_ROLLBACKFROMOPERATEONDATANODES_0D2CEB50 = + "CommitProgressSyncProcedure: rollbackFromOperateOnDataNodes"; + public static final String + LOG_COMMITPROGRESSSYNCPROCEDURE_FAILED_DESERIALIZE_REGION_PROGRESS_KEY_ARG_SUMMARY_ARG_0202F658 = + "CommitProgressSyncProcedure: failed to deserialize region progress, key={}, summary={}"; public static final String EXCEPTION_UNEXPECTED_PARENT_444B4289 = "Unexpected parent"; public static final String LOG_ARG_8393DD4A = "{}"; - public static final String MESSAGE_HALT_PID_ARG_ACTIVECOUNT_ARG_411F3EBF = "Halt pid={}, activeCount={}"; - public static final String MESSAGE_EXCEPTION_HAPPENED_WHEN_WORKER_ARG_EXECUTE_PROCEDURE_ARG_6E3AD27D = "Exception happened when worker {} execute procedure {}"; - public static final String MESSAGE_FAILED_TO_SHOW_DATAPARTITIONTABLE_INTEGRITY_CHECK_PROGRESS_5EE98694 = "Failed to show DataPartitionTable integrity check progress"; - public static final String MESSAGE_ENCOUNTERED_UNEXPECTED_DATAPARTITIONTABLEINTEGRITYCHECKPROCEDURESTATE_ARG_WHEN_SHOWING_PROGRESS_5FA2739F = - "Encountered unexpected DataPartitionTableIntegrityCheckProcedureState {} when showing progress"; - public static final String MESSAGE_UNEXPECTED_DATAPARTITIONTABLEINTEGRITYCHECKPROCEDURESTATE_ARG_WHEN_SHOWING_PROGRESS_D3C07BA1 = - "Unexpected DataPartitionTableIntegrityCheckProcedureState {} when showing progress"; + public static final String MESSAGE_HALT_PID_ARG_ACTIVECOUNT_ARG_411F3EBF = + "Halt pid={}, activeCount={}"; + public static final String + MESSAGE_EXCEPTION_HAPPENED_WHEN_WORKER_ARG_EXECUTE_PROCEDURE_ARG_6E3AD27D = + "Exception happened when worker {} execute procedure {}"; + public static final String + MESSAGE_FAILED_TO_SHOW_DATAPARTITIONTABLE_INTEGRITY_CHECK_PROGRESS_5EE98694 = + "Failed to show DataPartitionTable integrity check progress"; + public static final String + MESSAGE_ENCOUNTERED_UNEXPECTED_DATAPARTITIONTABLEINTEGRITYCHECKPROCEDURESTATE_ARG_WHEN_SHOWING_PROGRESS_5FA2739F = + "Encountered unexpected DataPartitionTableIntegrityCheckProcedureState {} when showing progress"; + public static final String + MESSAGE_UNEXPECTED_DATAPARTITIONTABLEINTEGRITYCHECKPROCEDURESTATE_ARG_WHEN_SHOWING_PROGRESS_D3C07BA1 = + "Unexpected DataPartitionTableIntegrityCheckProcedureState {} when showing progress"; - public static final String MESSAGE_NO_PROCEDURE_WORKER_IS_CURRENTLY_AVAILABLE_WORKERS_MAY_BE_BUSY_OR_BLOCKED_BY_OTHER_PROCEDURES_AB0B1595 = - "no Procedure worker is currently available; workers may be busy or blocked by other procedures."; - public static final String MESSAGE_PIPE_OPERATION_ARG_TIMED_OUT_PROCEDUREID_ARG_STUCK_AT_ARG_REASON_ARG_THE_PROCEDURE_IS_STILL_RUNNING_7EEAC50E = - "Pipe operation %s timed out (procedureId=%d). Stuck at %s. Reason: %s. The procedure is still running."; - public static final String MESSAGE_WAITING_TO_ACQUIRE_THE_PIPETASKCOORDINATOR_LOCK_BECAUSE_ANOTHER_PIPE_OPERATION_IS_HOLDING_IT_25A3B6B8 = - "waiting to acquire the PipeTaskCoordinator lock because another Pipe operation is holding it."; - public static final String MESSAGE_WAITING_TO_ACQUIRE_THE_CONFIGNODE_NODE_LOCK_BECAUSE_ANOTHER_NODE_PROCEDURE_IS_HOLDING_IT_56494E86 = - "waiting to acquire the ConfigNode node lock because another node procedure is holding it."; - public static final String MESSAGE_WAITING_TO_ACQUIRE_THE_CONFIGNODE_NODE_LOCK_HELD_BY_ARG_PROCEDUREID_ARG_3F432041 = - "waiting to acquire the ConfigNode node lock held by %s (procedureId=%d)."; - public static final String MESSAGE_PIPE_REQUEST_OR_PLUGIN_VALIDATION_HAS_NOT_COMPLETED_A_PLUGIN_CHECK_OR_METADATA_ACCESS_MAY_BE_SLOW_57C36CEF = - "Pipe request or plugin validation has not completed; a plugin check or metadata access may be slow."; - public static final String MESSAGE_PIPE_METADATA_CALCULATION_HAS_NOT_COMPLETED_METADATA_ACCESS_OR_LOCAL_CALCULATION_MAY_BE_SLOW_DEBF2504 = - "Pipe metadata calculation has not completed; metadata access or local calculation may be slow."; - public static final String MESSAGE_THE_CONFIGNODE_CONSENSUS_WRITE_HAS_NOT_RETURNED_RUN_SHOW_CLUSTER_TO_CHECK_NODE_STATUS_B0A6E1A7 = - "the ConfigNode consensus write has not returned; run SHOW CLUSTER to check node status."; - public static final String MESSAGE_DATANODES_ARG_HAVE_NOT_RESPONDED_TO_THE_PIPE_METADATA_PUSH_RUN_SHOW_CLUSTER_TO_CHECK_THEIR_STATUS_9C2F806F = - "DataNodes %s have not responded to the Pipe metadata push; run SHOW CLUSTER to check their status."; - public static final String MESSAGE_THE_PIPE_METADATA_PUSH_HAS_NOT_COMPLETED_RUN_SHOW_CLUSTER_TO_CHECK_DATANODE_STATUS_A8F3F0A0 = - "the Pipe metadata push has not completed; run SHOW CLUSTER to check DataNode status."; - public static final String MESSAGE_THE_PREVIOUS_ATTEMPT_FAILED_WITH_ARG_AND_THIS_STATE_IS_BEING_RETRIED_7A541F27 = - "the previous attempt failed with '%s' and this state is being retried."; + public static final String + MESSAGE_NO_PROCEDURE_WORKER_IS_CURRENTLY_AVAILABLE_WORKERS_MAY_BE_BUSY_OR_BLOCKED_BY_OTHER_PROCEDURES_AB0B1595 = + "no Procedure worker is currently available; workers may be busy or blocked by other procedures."; + public static final String + MESSAGE_PIPE_OPERATION_ARG_TIMED_OUT_PROCEDUREID_ARG_STUCK_AT_ARG_REASON_ARG_THE_PROCEDURE_IS_STILL_RUNNING_7EEAC50E = + "Pipe operation %s timed out (procedureId=%d). Stuck at %s. Reason: %s. The procedure is still running."; + public static final String + MESSAGE_WAITING_TO_ACQUIRE_THE_PIPETASKCOORDINATOR_LOCK_BECAUSE_ANOTHER_PIPE_OPERATION_IS_HOLDING_IT_25A3B6B8 = + "waiting to acquire the PipeTaskCoordinator lock because another Pipe operation is holding it."; + public static final String + MESSAGE_WAITING_TO_ACQUIRE_THE_CONFIGNODE_NODE_LOCK_BECAUSE_ANOTHER_NODE_PROCEDURE_IS_HOLDING_IT_56494E86 = + "waiting to acquire the ConfigNode node lock because another node procedure is holding it."; + public static final String + MESSAGE_WAITING_TO_ACQUIRE_THE_CONFIGNODE_NODE_LOCK_HELD_BY_ARG_PROCEDUREID_ARG_3F432041 = + "waiting to acquire the ConfigNode node lock held by %s (procedureId=%d)."; + public static final String + MESSAGE_PIPE_REQUEST_OR_PLUGIN_VALIDATION_HAS_NOT_COMPLETED_A_PLUGIN_CHECK_OR_METADATA_ACCESS_MAY_BE_SLOW_57C36CEF = + "Pipe request or plugin validation has not completed; a plugin check or metadata access may be slow."; + public static final String + MESSAGE_PIPE_METADATA_CALCULATION_HAS_NOT_COMPLETED_METADATA_ACCESS_OR_LOCAL_CALCULATION_MAY_BE_SLOW_DEBF2504 = + "Pipe metadata calculation has not completed; metadata access or local calculation may be slow."; + public static final String + MESSAGE_THE_CONFIGNODE_CONSENSUS_WRITE_HAS_NOT_RETURNED_RUN_SHOW_CLUSTER_TO_CHECK_NODE_STATUS_B0A6E1A7 = + "the ConfigNode consensus write has not returned; run SHOW CLUSTER to check node status."; + public static final String + MESSAGE_DATANODES_ARG_HAVE_NOT_RESPONDED_TO_THE_PIPE_METADATA_PUSH_RUN_SHOW_CLUSTER_TO_CHECK_THEIR_STATUS_9C2F806F = + "DataNodes %s have not responded to the Pipe metadata push; run SHOW CLUSTER to check their status."; + public static final String + MESSAGE_THE_PIPE_METADATA_PUSH_HAS_NOT_COMPLETED_RUN_SHOW_CLUSTER_TO_CHECK_DATANODE_STATUS_A8F3F0A0 = + "the Pipe metadata push has not completed; run SHOW CLUSTER to check DataNode status."; + public static final String + MESSAGE_THE_PREVIOUS_ATTEMPT_FAILED_WITH_ARG_AND_THIS_STATE_IS_BEING_RETRIED_7A541F27 = + "the previous attempt failed with '%s' and this state is being retried."; public static final String MESSAGE_THE_STATE_FAILED_WITH_ARG_AND_ROLLBACK_IS_PENDING_E7B43829 = "the state failed with '%s' and rollback is pending."; public static final String MESSAGE_ROLLING_BACK_AFTER_FAILURE_ARG_474DF456 = @@ -1430,4 +1625,7 @@ private ProcedureMessages() {} "DataNode setSystemStatus RPC failed: %s: %s: %s; "; public static final String EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4 = "ConfigNode trySetNodeStatus failed: %s: %s: %s; "; + public static final String + LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_9544E1D9 = + "Failed to detect the tree series under the new view %s.%s: %s"; } diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java index fa3d63dddb3d2..739ee0029c890 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -606,6 +606,9 @@ private ManagerMessages() {} public static final String LOG_GOT_IOEXCEPTION_CONSTRUCT_DATABASE_TREE_49436621 = "构建数据库树时发生 IOException"; public static final String LOG_GOT_IOEXCEPTION_DESERIALIZE_TEMPLATE_INFO_49EE617E = "反序列化模板信息时发生 IOException"; public static final String MESSAGE_MEASUREMENTS_NOT_FOUND_ARG_CANNOT_AUTO_DETECT_980D7D44 = "未找到 %s 的测点，无法自动检测"; + public static final String EXCEPTION_THE_DUPLICATED_SOURCE_MEASUREMENT_ARG_IS_UNSUPPORTED_YET_873AF566 = "暂不支持重复的源测点 %s。"; + public static final String MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB = "自动检测测点 %s 的类型时遇到多种类型，请检查"; + public static final String MESSAGE_THE_MEASUREMENTS_ARG_AND_ARG_SHARE_THE_SAME_LOWER_CASE_WHEN_AUTO_DETECTING_TYPE_PLEASE_CHECK_7225D062 = "自动检测类型时，测点 %s 和 %s 的小写形式相同，请检查"; public static final String LOG_FAILED_TAKE_SNAPSHOT_BECAUSE_SNAPSHOT_FILE_ARG_ALREADY_EXIST_EB2A6093 = "获取快照失败，原因：快照文件 [{}] 已存在。"; public static final String LOG_FAILED_LOAD_SNAPSHOT_SNAPSHOT_FILE_ARG_NOT_EXIST_8828CFBA = "无法加载快照，快照文件 [{}] 不存在。"; public static final String LOG_YOU_MAYBE_NEED_RENAME_SIMPLE_DIR_0_0_MANUALLY_2A12C5C9 = "可能需要手动将 simple 目录重命名为 0_0。"; diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java index 14e52d72b9efe..7fed5d8317de6 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java @@ -1511,8 +1511,17 @@ private ProcedureMessages() {} public static final String MESSAGE_ROLLING_BACK_AFTER_FAILURE_ARG_474DF456 = "正在回滚，失败原因：%s。"; public static final String MESSAGE_ROLLING_BACK_AFTER_AN_EARLIER_FAILURE_850D0AF5 = "正在回滚此前发生的失败。"; +<<<<<<< HEAD public static final String EXCEPTION_DATANODE_SETSYSTEMSTATUS_RPC_FAILED_ARG_ARG_ARG_7E8E5F9F = "DataNode setSystemStatus RPC 失败：%s：%s：%s； "; public static final String EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4 = "ConfigNode trySetNodeStatus 失败：%s：%s：%s； "; +======= + public static final String + LOG_FAILED_TO_COMMIT_RELEASE_ARG_FOR_TABLE_ARG_ARG_TO_DATANODE_FAILURE_RESULTS_ARG_55386572 = + "对表 {}.{} 在 DataNode 上提交释放 {} 失败，失败结果：{}"; + public static final String + LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_9544E1D9 = + "扫描新建视图 %s.%s 下的树序列失败：%s"; +>>>>>>> 9e29979e95 (support that auto-detection for creating measurements and adding columns for related tree-to-table view) } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index f82d8187fab33..2c6fc80e17ff3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -265,6 +265,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsReq; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsResp; import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.db.schemaengine.template.TemplateAlterOperationType; @@ -3378,6 +3380,14 @@ public TSStatus alterOrDropTable(final TAlterOrDropTableReq req) { } } + @Override + public TUpdateTree2TableViewsResp updateTree2TableViews(TUpdateTree2TableViewsReq req) { + final TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? procedureManager.addColumnForTree2TableViews(req) + : new TUpdateTree2TableViewsResp(status); + } + @Override public TDeleteTableDeviceResp deleteDevice(final TDeleteTableDeviceReq req) { final TSStatus status = confirmLeader(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 37cc7c72247cc..f6e5bf897decb 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -168,6 +168,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsReq; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsResp; import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.rpc.TSStatusCode; @@ -919,6 +921,8 @@ TPermissionInfoResp login( TSStatus alterOrDropTable(final TAlterOrDropTableReq req); + TUpdateTree2TableViewsResp updateTree2TableViews(TUpdateTree2TableViewsReq req); + TDeleteTableDeviceResp deleteDevice(final TDeleteTableDeviceReq req); TShowTableResp showTables(final String database, final boolean isDetails); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 18f65a98481da..5aca993c894dc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -39,6 +39,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta; import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.commons.schema.table.AlterOrDropTableOperationType; import org.apache.iotdb.commons.schema.table.TableNodeStatus; import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.TsTable; @@ -169,6 +170,9 @@ import org.apache.iotdb.confignode.rpc.thrift.TRemoveRegionReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewFailure; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsReq; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsResp; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.exception.BatchProcessException; import org.apache.iotdb.mpp.rpc.thrift.TDropTableModProgress; @@ -216,6 +220,15 @@ public class ProcedureManager { private static final String PROCEDURE_TIMEOUT_MESSAGE = "Timed out to wait for procedure return. The procedure is still running."; + /** One addColumnForTree2TableViews call waits at most this long for its procedures to finish. */ + private static final long ADD_VIEW_COLUMN_WAIT_TIME_OUT_MS = 30_000L; + + /** How many times a request which overlapped with another task is submitted again. */ + private static final int OVERLAP_RETRY_ROUNDS = 4; + + /** How long a retry round waits for the overlapping task to finish before submitting again. */ + private static final long OVERLAP_RETRY_WAIT_MS = 2_000L; + private final ConfigManager configManager; private ProcedureExecutor executor; private ProcedureScheduler scheduler; @@ -2269,6 +2282,159 @@ public TSStatus alterTableAddColumn(final TAlterOrDropTableReq req) { false)); } + public TUpdateTree2TableViewsResp addColumnForTree2TableViews(TUpdateTree2TableViewsReq req) { + if (req == null || !req.isSetViewInfos()) { + return new TUpdateTree2TableViewsResp(RpcUtils.getStatus(TSStatusCode.ILLEGAL_PARAMETER)); + } + + final List failedViews = new ArrayList<>(); + final List>> procedures = + new ArrayList<>(); + // The requests which overlapped with another task on the same table, together with the status + // of the last submission attempt, see processOverlappingTask. + final List> overlappingReqs = new ArrayList<>(); + + // Submit every view before waiting, so procedures for different views can run concurrently. + for (final TAlterOrDropTableReq addColumnReq : req.getViewInfos()) { + if (!isAddViewColumnReq(addColumnReq)) { + failedViews.add(illegalAddViewColumnFailure(addColumnReq)); + continue; + } + + final TSStatus submissionStatus = submitAddViewColumn(addColumnReq, procedures, failedViews); + if (submissionStatus.getCode() == TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode()) { + overlappingReqs.add(new Pair<>(addColumnReq, submissionStatus)); + } + } + + processOverlappingTask(overlappingReqs, procedures, failedViews); + + final long waitDeadline = System.currentTimeMillis() + ADD_VIEW_COLUMN_WAIT_TIME_OUT_MS; + for (final Pair> procedure : + procedures) { + final long remainingWaitTime = Math.max(0, waitDeadline - System.currentTimeMillis()); + final TSStatus status = waitingProcedureFinished(procedure.getRight(), remainingWaitTime); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + failedViews.add( + new TUpdateTree2TableViewFailure( + procedure.getLeft().database, procedure.getLeft().tableName, status)); + } + } + + final TSStatus status = failedViews.isEmpty() ? StatusUtils.OK : StatusUtils.INTERNAL_ERROR; + final TUpdateTree2TableViewsResp response = new TUpdateTree2TableViewsResp(status); + if (!failedViews.isEmpty()) { + response.setFailedViews(failedViews); + } + return response; + } + + /** Whether the request carries everything an add-column operation on a view needs. */ + private static boolean isAddViewColumnReq(final @Nullable TAlterOrDropTableReq addColumnReq) { + return Objects.nonNull(addColumnReq) + && addColumnReq.isSetDatabase() + && addColumnReq.isSetTableName() + && addColumnReq.isSetQueryId() + && addColumnReq.isSetOperationType() + && addColumnReq.getOperationType() + == AlterOrDropTableOperationType.ADD_COLUMN.getTypeValue() + && addColumnReq.isSetUpdateInfo() + && addColumnReq.isSetIsView() + && addColumnReq.isIsView(); + } + + private static TUpdateTree2TableViewFailure illegalAddViewColumnFailure( + final TAlterOrDropTableReq addColumnReq) { + return new TUpdateTree2TableViewFailure( + Objects.isNull(addColumnReq) ? "" : Objects.toString(addColumnReq.getDatabase(), ""), + Objects.isNull(addColumnReq) ? "" : Objects.toString(addColumnReq.getTableName(), ""), + RpcUtils.getStatus(TSStatusCode.ILLEGAL_PARAMETER)); + } + + /** + * Submits the add-column procedure of one view and settles everything but an overlap with another + * task on the same table. + */ + private TSStatus submitAddViewColumn( + final TAlterOrDropTableReq addColumnReq, + final List>> procedures, + final List failedViews) { + final AddViewColumnProcedure procedure; + procedure = + new AddViewColumnProcedure( + addColumnReq.database, + addColumnReq.tableName, + addColumnReq.queryId, + TsTableColumnSchemaUtil.deserializeColumnSchemaList(addColumnReq.updateInfo), + false); + + final Pair submissionResult = + executeWithoutDuplicateWithoutWaiting( + addColumnReq.database, + null, + addColumnReq.tableName, + null, + addColumnReq.queryId, + ProcedureType.ADD_VIEW_COLUMN_PROCEDURE, + procedure); + final TSStatus status = submissionResult.getLeft(); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + procedures.add(new Pair<>(addColumnReq, procedure)); + } else if (status.getCode() == TSStatusCode.REPEATED_WITH_EXISTING_TASK.getStatusCode()) { + final Procedure repeatedProcedure = + executor.getProcedures().get(submissionResult.getRight()); + // null means the procedure could have finished at this moment + if (Objects.isNull(repeatedProcedure)) { + failedViews.add( + new TUpdateTree2TableViewFailure( + addColumnReq.database, + addColumnReq.tableName, + RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR))); + } else { + procedures.add(new Pair<>(addColumnReq, repeatedProcedure)); + } + } else if (status.getCode() != TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode()) { + failedViews.add( + new TUpdateTree2TableViewFailure(addColumnReq.database, addColumnReq.tableName, status)); + } + return status; + } + + /** + * Retries the requests which overlapped with another task on the same table, A request which + * still overlaps after the last round is reported as a failure, with the status of its last + * attempt. + */ + private void processOverlappingTask( + final List> overlappingReqs, + final List>> procedures, + final List failedViews) { + List> pending = overlappingReqs; + for (int round = 0; round < OVERLAP_RETRY_ROUNDS && !pending.isEmpty(); round++) { + try { + Thread.sleep(OVERLAP_RETRY_WAIT_MS); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + final List> stillOverlapping = + new ArrayList<>(pending.size()); + for (final Pair request : pending) { + final TSStatus status = submitAddViewColumn(request.getLeft(), procedures, failedViews); + if (status.getCode() == TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode()) { + stillOverlapping.add(new Pair<>(request.getLeft(), status)); + } + } + pending = stillOverlapping; + } + + for (final Pair request : pending) { + failedViews.add( + new TUpdateTree2TableViewFailure( + request.getLeft().database, request.getLeft().tableName, request.getRight())); + } + } + public TSStatus alterTableSetProperties(final TAlterOrDropTableReq req) { final boolean isView = req.isSetIsView() && req.isIsView(); return executeWithoutDuplicate( @@ -2617,6 +2783,36 @@ public TSStatus executeWithoutDuplicate( return waitingProcedureFinished(procedureToWait); } + public Pair executeWithoutDuplicateWithoutWaiting( + final String database, + final TsTable table, + final String tableName, + final @Nullable String newName, + final String queryId, + final ProcedureType thisType, + final Procedure procedure) { + final long procedureId; + synchronized (this) { + final Pair procedureIdDuplicatePair = + checkDuplicateTableTask(database, table, tableName, newName, queryId, thisType); + procedureId = procedureIdDuplicatePair.getLeft(); + if (procedureId == -1) { + if (Boolean.TRUE.equals(procedureIdDuplicatePair.getRight())) { + TSStatus status = + RpcUtils.getStatus( + TSStatusCode.OVERLAP_WITH_EXISTING_TASK, + "Some other task is operating table with same name."); + return new Pair<>(status, -1L); + } + this.executor.submitProcedure(procedure); + return new Pair<>(StatusUtils.OK, procedure.getProcId()); + } else { + TSStatus status = RpcUtils.getStatus(TSStatusCode.REPEATED_WITH_EXISTING_TASK, ""); + return new Pair<>(status, procedureId); + } + } + } + public Pair checkDuplicateTableTask( final @Nonnull String database, final TsTable table, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index e7f96322a5346..47f405c9fd2d3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -129,6 +129,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -1586,6 +1587,28 @@ public synchronized Pair tableColumnCheckForColumnExtension( .collect(Collectors.joining(", "))); final AtomicBoolean hasObject = new AtomicBoolean(false); + if (originalTable.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).isEmpty()) { + checkColumnSchema(originalTable, expandedTable, columnSchemaList, hasObject); + } else { + // only for tree_to_table view + checkColumnSchemaForTree2TableView(originalTable, expandedTable, columnSchemaList, hasObject); + } + + if (columnSchemaList.isEmpty()) { + return new Pair<>(RpcUtils.getStatus(TSStatusCode.COLUMN_ALREADY_EXISTS, errorMsg), null); + } + + if (hasObject.get()) { + expandedTable.checkTableNameAndObjectNames4Object(); + } + return new Pair<>(StatusUtils.OK, expandedTable); + } + + private static void checkColumnSchema( + TsTable originalTable, + TsTable expandedTable, + List columnSchemaList, + AtomicBoolean hasObject) { columnSchemaList.removeIf( columnSchema -> { if (Objects.isNull(originalTable.getColumnSchema(columnSchema.getColumnName()))) { @@ -1597,15 +1620,54 @@ public synchronized Pair tableColumnCheckForColumnExtension( } return true; }); + } - if (columnSchemaList.isEmpty()) { - return new Pair<>(RpcUtils.getStatus(TSStatusCode.COLUMN_ALREADY_EXISTS, errorMsg), null); + private static void checkColumnSchemaForTree2TableView( + final TsTable originalView, + final TsTable expandedView, + final List columnSchemaList, + final AtomicBoolean hasObject) { + final Set existingColumnNames = new HashSet<>(); + final Set originalNames = new HashSet<>(); + + for (final TsTableColumnSchema columnSchema : originalView.getColumnList()) { + existingColumnNames.add(columnSchema.getColumnName()); + Map props = columnSchema.getProps(); + if (props == null) { + originalNames.add(columnSchema.getColumnName()); + continue; + } + String originalName = props.get(TreeViewSchema.ORIGINAL_NAME); + if (originalName == null) { + originalNames.add(columnSchema.getColumnName()); + continue; + } + originalNames.add(originalName); } - if (hasObject.get()) { - expandedTable.checkTableNameAndObjectNames4Object(); - } - return new Pair<>(StatusUtils.OK, expandedTable); + columnSchemaList.removeIf( + columnSchema -> { + if (existingColumnNames.contains(columnSchema.getColumnName())) { + // todo log + return true; + } + + // The original time series can only be referenced once. + final String originalName = TreeViewSchema.getSourceName(columnSchema); + if (originalNames.contains(originalName)) { + // todo log + return true; + } + + if (columnSchema.getDataType().equals(TSDataType.OBJECT)) { + hasObject.set(true); + } + + existingColumnNames.add(columnSchema.getColumnName()); + originalNames.add(originalName); + expandedView.addColumnSchema(columnSchema); + return false; + }); } public synchronized Pair tableColumnCheckForColumnAltering( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java index f407d4d8ecdce..8b208f25c20c1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.table.TreeViewSchema; @@ -32,6 +33,7 @@ import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.confignode.i18n.ManagerMessages; +import org.apache.iotdb.confignode.i18n.ProcedureMessages; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.procedure.impl.schema.DataNodeRegionTaskExecutor; import org.apache.iotdb.mpp.rpc.thrift.TDeviceViewReq; @@ -40,17 +42,16 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -64,28 +65,32 @@ public class TreeDeviceViewFieldDetector { private final ConfigManager configManager; private final PartialPath path; private final TsTable table; - private final Map> fields; private TDeviceViewResp result = new TDeviceViewResp(StatusUtils.OK, new ConcurrentHashMap<>()); private final Map lowerCase2OriginalMap = new HashMap<>(); - public TreeDeviceViewFieldDetector( - final ConfigManager configManager, - final TsTable table, - final Map> fields) { + public TreeDeviceViewFieldDetector(final ConfigManager configManager, final TsTable table) { this.configManager = configManager; this.path = TreeViewSchema.getPrefixPattern(table); this.table = table; - this.fields = fields; } - public TSStatus detectMissingFieldTypes() { - if (table.getFieldNum() == 0 && Objects.isNull(fields)) { + /** + * Fills the fields of the view which is being created: a view may declare no field at all, in + * which case every field under its tree path pattern becomes a column, or declare fields whose + * type is left to the detection. + */ + public TSStatus detectFieldsForCreation() { + if (table.getFieldNum() == 0) { + // for the first time creation of the table, user do not assign any field; + // just detect all the fields of prefix path new TreeDeviceViewFieldDetectionTaskExecutor( configManager, getLatestSchemaRegionMap(), table.getTagNum(), - TreeViewSchema.isRestrict(table)) + TreeViewSchema.isRestrict(table), + null, + false) .execute(); if (result.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return result.getStatus(); @@ -93,69 +98,158 @@ public TSStatus detectMissingFieldTypes() { result .getDeviewViewFieldTypeMap() .forEach( - (field, type) -> { - final FieldColumnSchema columnSchema = - new FieldColumnSchema(field, TSDataType.getTsDataType(type)); - if (!field.equals(lowerCase2OriginalMap.get(field))) { - TreeViewSchema.setOriginalName(columnSchema, lowerCase2OriginalMap.get(field)); - } - table.addColumnSchema(columnSchema); - }); - } else { - final Map> unknownFields; - if (Objects.isNull(fields)) { - unknownFields = new HashMap<>(); - for (final TsTableColumnSchema schema : table.getColumnList()) { - if (!(schema instanceof FieldColumnSchema) - || schema.getDataType() != TSDataType.UNKNOWN) { - continue; - } - final String key = TreeViewSchema.getSourceName(schema); - if (!unknownFields.containsKey(key)) { - unknownFields.put(key, new HashSet<>()); - } - unknownFields.get(key).add((FieldColumnSchema) schema); - } - } else { - unknownFields = fields; - } + (field, type) -> + table.addColumnSchema(toFieldColumnSchemaWithOriginName(field, type))); + return StatusUtils.OK; + } + + return detectFieldTypes(collectUnknownFieldColumns()); + } + + /** + * Resolves the type of the field columns which are being added to an existing view. The columns + * to resolve are given by the caller, as only it knows which ones it is adding. + */ + public TSStatus detectFieldTypesForColumnAddition(final Map fields) { + return detectFieldTypes(fields); + } + + /** + * The fields which exist under the view's tree path pattern but are not columns of the view yet, + * as the columns to add. Used after a view is created, to pick up the series which were created + * before the view was mounted on the DataNodes, and which the DataNodes therefore never reported. + */ + public Pair> detectNewFieldColumnsAfterCreation() { + new TreeDeviceViewFieldDetectionTaskExecutor( + configManager, + getLatestSchemaRegionMap(), + table.getTagNum(), + TreeViewSchema.isRestrict(table), + null, + true) + .execute(); + if (result.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return new Pair<>(result.getStatus(), Collections.emptyList()); + } + final List columns = new ArrayList<>(); + result + .getDeviewViewFieldTypeMap() + .forEach( + (field, type) -> { + if (Objects.isNull(table.getColumnSchema(field))) { + columns.add(toFieldColumnSchemaWithOriginName(field, type)); + } + }); + return new Pair<>(StatusUtils.OK, columns); + } + + /** + * The columns which are missing from a view which auto-syncs its schema: the series which already + * existed under its tree path pattern were created before the DataNodes mounted the view on their + * tree-to-table view index, so they never triggered the addition of their column. Used right + * after the view is mounted, whether it has just been created or its auto detection has just been + * turned on. + * + * @return the columns to add; empty when the view does not auto-sync its schema, or when the + * detection itself failed + */ + public static List detectMissingColumnsAfterCreation( + final ConfigManager configManager, final String database, final TsTable table) { + final boolean autoSync = + table + .getPropValue(TreeViewSchema.AUTO_SYNC_SCHEMA) + .map(Boolean::parseBoolean) + .orElse(false); + if (!autoSync) { + return Collections.emptyList(); + } - if (unknownFields.isEmpty()) { - return StatusUtils.OK; + final Pair> detected = + new TreeDeviceViewFieldDetector(configManager, table).detectNewFieldColumnsAfterCreation(); + if (detected.getLeft().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + // todo, i18N + LOGGER.warn( + ProcedureMessages + .LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_9544E1D9, + database, + table.getTableName(), + detected.getLeft()); + return Collections.emptyList(); + } + return detected.getRight(); + } + + /** The field columns of the view whose type is not known yet, keyed by their source name. */ + private Map collectUnknownFieldColumns() { + final Map unknownFields = new HashMap<>(); + for (final TsTableColumnSchema schema : table.getColumnList()) { + if (!(schema instanceof FieldColumnSchema) || schema.getDataType() != TSDataType.UNKNOWN) { + continue; } - new TreeDeviceViewFieldDetectionTaskExecutor( - configManager, - getLatestSchemaRegionMap(), - table.getTagNum(), - TreeViewSchema.isRestrict(table), - unknownFields.keySet()) - .execute(); - if (result.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return result.getStatus(); + final String key = TreeViewSchema.getSourceName(schema); + if (unknownFields.containsKey(key)) { + // A table which references the same source measurement twice is rejected when it is + // parsed, so a table in the schema must not reach this point. + throw new SemanticException( + String.format( + ManagerMessages + .EXCEPTION_THE_DUPLICATED_SOURCE_MEASUREMENT_ARG_IS_UNSUPPORTED_YET_873AF566, + key)); } - for (final Map.Entry> unknownField : - unknownFields.entrySet()) { - if (result.getDeviewViewFieldTypeMap().containsKey(unknownField.getKey())) { - unknownField - .getValue() - .forEach( - field -> - field.setDataType( - TSDataType.getTsDataType( - result.getDeviewViewFieldTypeMap().get(unknownField.getKey())))); - } else { - return new TSStatus(TSStatusCode.TYPE_NOT_FOUND.getStatusCode()) - .setMessage( - String.format( - ManagerMessages - .MESSAGE_MEASUREMENTS_NOT_FOUND_ARG_CANNOT_AUTO_DETECT_980D7D44, - unknownField.getKey())); - } + unknownFields.put(key, (FieldColumnSchema) schema); + } + return unknownFields; + } + + /** + * Resolves the type of every field column in {@code unknownFields} from the schema of the regions + * the view covers. A measurement which the detection cannot find fails the whole detection. + */ + private TSStatus detectFieldTypes(final Map unknownFields) { + if (unknownFields.isEmpty()) { + return StatusUtils.OK; + } + new TreeDeviceViewFieldDetectionTaskExecutor( + configManager, + getLatestSchemaRegionMap(), + table.getTagNum(), + TreeViewSchema.isRestrict(table), + unknownFields.keySet(), + false) + .execute(); + if (result.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return result.getStatus(); + } + for (final Map.Entry unknownField : unknownFields.entrySet()) { + // The detection keys the field types by the lower case of the measurement name, while the + // column keeps the source name as the statement spelled it. + final String sourceName = unknownField.getKey().toLowerCase(Locale.ENGLISH); + if (result.getDeviewViewFieldTypeMap().containsKey(sourceName)) { + unknownField + .getValue() + .setDataType( + TSDataType.getTsDataType(result.getDeviewViewFieldTypeMap().get(sourceName))); + } else { + return new TSStatus(TSStatusCode.TYPE_NOT_FOUND.getStatusCode()) + .setMessage( + String.format( + ManagerMessages.MESSAGE_MEASUREMENTS_NOT_FOUND_ARG_CANNOT_AUTO_DETECT_980D7D44, + unknownField.getKey())); } } return StatusUtils.OK; } + private TsTableColumnSchema toFieldColumnSchemaWithOriginName( + final String field, final byte type) { + final FieldColumnSchema columnSchema = + new FieldColumnSchema(field, TSDataType.getTsDataType(type)); + if (!field.equals(lowerCase2OriginalMap.get(field))) { + TreeViewSchema.setOriginalName(columnSchema, lowerCase2OriginalMap.get(field)); + } + return columnSchema; + } + private Map getLatestSchemaRegionMap() { final PathPatternTree tree = new PathPatternTree(); tree.appendPathPattern(path); @@ -166,59 +260,37 @@ private Map getLatestSchemaRegionMap() { private class TreeDeviceViewFieldDetectionTaskExecutor extends DataNodeRegionTaskExecutor { - protected TreeDeviceViewFieldDetectionTaskExecutor( - final ConfigManager configManager, - final Map targetRegionGroup, - final int tagNumber, - final boolean restrict) { - super( - configManager, - targetRegionGroup, - false, - CnToDnAsyncRequestType.DETECT_TREE_DEVICE_VIEW_FIELD_TYPE, - ((dataNodeLocation, consensusGroupIdList) -> - new TDeviceViewReq( - consensusGroupIdList, Arrays.asList(path.getNodes()), tagNumber, restrict))); - configManager - .getClusterSchemaManager() - .getTemplateSetInfo(Collections.singletonList(path)) - .getPatternTemplateMap() - .values() - .stream() - .flatMap(Collection::stream) - .flatMap(template -> template.getSchemaMap().values().stream()) - .forEach( - schema -> - mergeMeasurementAndType(schema.getMeasurementName(), schema.getTypeInByte())); - } + /** + * Whether several types of the same measurement are tolerated. When they are, the type the + * detection meets first is kept instead of failing the whole detection. + */ + private final boolean tolerateTypeMismatch; protected TreeDeviceViewFieldDetectionTaskExecutor( final ConfigManager configManager, final Map targetRegionGroup, final int tagNumber, final boolean restrict, - final @Nonnull Set measurements) { + final @Nullable Set measurements, + final boolean tolerateTypeMismatch) { super( configManager, targetRegionGroup, false, CnToDnAsyncRequestType.DETECT_TREE_DEVICE_VIEW_FIELD_TYPE, - ((dataNodeLocation, consensusGroupIdList) -> - new TDeviceViewReq( - consensusGroupIdList, Arrays.asList(path.getNodes()), tagNumber, restrict) - .setRequiredMeasurements(measurements))); - configManager - .getClusterSchemaManager() - .getTemplateSetInfo(Collections.singletonList(path)) - .getPatternTemplateMap() - .values() - .stream() - .flatMap(Collection::stream) - .flatMap(template -> template.getSchemaMap().values().stream()) - .filter(schema -> measurements.contains(schema.getMeasurementName())) - .forEach( - schema -> - mergeMeasurementAndType(schema.getMeasurementName(), schema.getTypeInByte())); + ((dataNodeLocation, consensusGroupIdList) -> { + final TDeviceViewReq req = + new TDeviceViewReq( + consensusGroupIdList, Arrays.asList(path.getNodes()), tagNumber, restrict); + if (Objects.nonNull(measurements)) { + req.setRequiredMeasurements(measurements); + } + if (tolerateTypeMismatch) { + req.setTolerateTypeMismatch(true); + } + return req; + })); + this.tolerateTypeMismatch = tolerateTypeMismatch; } @Override @@ -273,25 +345,30 @@ private void mergeMeasurementAndType(final String measurement, final byte type) // Field type collection if (!result.getDeviewViewFieldTypeMap().containsKey(fieldName)) { result.getDeviewViewFieldTypeMap().put(fieldName, type); - } else if (!Objects.equals(result.getDeviewViewFieldTypeMap().get(fieldName), type)) { + } else if (!Objects.equals(result.getDeviewViewFieldTypeMap().get(fieldName), type) + && !tolerateTypeMismatch) { result.setStatus( RpcUtils.getStatus( TSStatusCode.DATA_TYPE_MISMATCH, String.format( - "Multiple types encountered when auto detecting type of measurement '%s', please check", + ManagerMessages + .MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB, measurement))); } // Field name detection if (!lowerCase2OriginalMap.containsKey(fieldName)) { lowerCase2OriginalMap.put(fieldName, measurement); - } else if (!Objects.equals(lowerCase2OriginalMap.get(fieldName), measurement)) { + } else if (!Objects.equals(lowerCase2OriginalMap.get(fieldName), measurement) + && !tolerateTypeMismatch) { result.setStatus( RpcUtils.getStatus( TSStatusCode.MEASUREMENT_NAME_CONFLICT, String.format( - "The measurements %s and %s share the same lower case when auto detecting type, please check", - lowerCase2OriginalMap.get(fieldName), measurement))); + ManagerMessages + .MESSAGE_THE_MEASUREMENTS_ARG_AND_ARG_SHARE_THE_SAME_LOWER_CASE_WHEN_AUTO_DETECTING_TYPE_PLEASE_CHECK_7225D062, + lowerCase2OriginalMap.get(fieldName), + measurement))); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java index 241eea8a70201..ad01ffc399174 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java @@ -48,6 +48,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TCountPathsUsingTemplateResp; import org.apache.iotdb.mpp.rpc.thrift.TInvalidateMatchedSchemaCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -272,6 +273,21 @@ public static Map broadcastTableUpdate( return clientHandler.getResponseMap(); } + /** + * Broadcasts a table change at commit-release through {@link ClusterCachePropagator}: proceed + * once every unreachable DataNode is provably self-fenced, instead of failing on the first one. A + * fenced DataNode fails closed and resyncs on lease recovery, which rebuilds the tree-to-table + * view index it missed; a DataNode which is not provably fenced may still serve clients, so the + * commit is reported as unproven instead. + * + * @return whether it is safe to proceed + */ + public static boolean commitReleaseTableWithHA( + final TUpdateTableReq req, final ConfigManager configManager) { + return new ClusterCachePropagator(filterFencedDataNode(configManager)) + .propagate(targets -> broadcastTableUpdate(req, targets)); + } + private static Map failedOnly(final Map responses) { return responses.entrySet().stream() .filter(entry -> entry.getValue().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) @@ -308,24 +324,22 @@ public static TUpdateTableReq buildCommitUpdateTableReq( return req; } - public static Map commitReleaseTable( + /** + * Builds the COMMIT_UPDATE_TABLE request of a change which also affects the tree-to-table view + * index of the DataNodes. + * + *

The caller declares the view attributes the DataNode needs to apply the change, so that a + * DataNode never reads them from the table it caches: that cache may hold a placeholder, in which + * case the DataNode could not compute the entry at all. + */ + public static TUpdateTableReq buildCommitUpdateTableReq( final String database, final String tableName, - final ConfigManager configManager, - final @Nullable String oldName) { + final @Nullable String oldName, + final TUpdateTree2TableView viewInfo) { final TUpdateTableReq req = buildCommitUpdateTableReq(database, tableName, oldName); - - final DataNodeAsyncRequestContext clientHandler = - new DataNodeAsyncRequestContext<>( - CnToDnAsyncRequestType.UPDATE_TABLE, req, filterFencedDataNode(configManager)); - CnToDnInternalServiceAsyncRequestManager.getInstance() - .sendAsyncRequest( - clientHandler, - ClusterCachePropagator.BROADCAST_RPC_RETRY, - ClusterCachePropagator.BROADCAST_RPC_TIMEOUT_MS); - return clientHandler.getResponseMap().entrySet().stream() - .filter(entry -> entry.getValue().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + req.setUpdateTree2TableViewInfo(viewInfo); + return req; } /** Build the ROLLBACK_UPDATE_TABLE request used to roll back a pre-released table change. */ @@ -345,6 +359,16 @@ public static TUpdateTableReq rollbackUpdateTableReq( return req; } + public static TUpdateTableReq rollbackUpdateTableReq( + final String database, + final String tableName, + final String oldName, + final TUpdateTree2TableView viewInfo) { + final TUpdateTableReq req = rollbackUpdateTableReq(database, tableName, oldName); + req.setUpdateTree2TableViewInfo(viewInfo); + return req; + } + /** * Broadcast an INVALIDATE_MATCHED_SCHEMA_CACHE to all DataNodes through {@link * ClusterCachePropagator}: proceed once every unreachable DataNode is provably self-fenced (it diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java index b4aab9949c28e..510519110f972 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java @@ -22,7 +22,6 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; -import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; @@ -118,31 +117,22 @@ protected void preRelease(final ConfigNodeProcedureEnv env, final @Nullable Stri } } - protected void commitRelease(final ConfigNodeProcedureEnv env) { - commitRelease(env, null); + /** + * The request broadcast at commit-release. A subclass whose change also affects the tree-to-table + * view index of the DataNodes overrides this to declare the effect. + */ + protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldName) { + return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName); } - protected void commitRelease(final ConfigNodeProcedureEnv env, final @Nullable String oldName) { - final Map failedResults = - SchemaUtils.commitReleaseTable( - database, table.getTableName(), env.getConfigManager(), oldName); - if (!failedResults.isEmpty()) { - LOGGER.warn( - ProcedureMessages.FAILED_TO_FOR_TABLE_TO_DATANODE_FAILURE_RESULTS, - getActionMessage(), - database, - table.getTableName(), - failedResults); - } + protected void commitReleaseWithHA(final ConfigNodeProcedureEnv env) { + commitReleaseWithHA(env, null); } protected void commitReleaseWithHA( final ConfigNodeProcedureEnv env, final @Nullable String oldName) { - final TUpdateTableReq req = - SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName); - final boolean proceeded = - new ClusterCachePropagator(SchemaUtils.filterFencedDataNode(env.getConfigManager())) - .propagate(targets -> SchemaUtils.broadcastTableUpdate(req, targets)); + final TUpdateTableReq req = buildCommitReleaseTableReq(oldName); + final boolean proceeded = SchemaUtils.commitReleaseTableWithHA(req, env.getConfigManager()); if (!proceeded) { LOGGER.warn( @@ -165,11 +155,11 @@ protected void rollbackPreRelease(final ConfigNodeProcedureEnv env) { } protected void rollbackPreRelease( - final ConfigNodeProcedureEnv env, final @Nullable String tableName) { + final ConfigNodeProcedureEnv env, final @Nullable String oldName) { // A down DataNode must not block rollback either: proceed past provably-fenced DataNodes (which // resync on recovery) and only fail on an unacked DataNode that is not provably fenced. final TUpdateTableReq req = - SchemaUtils.rollbackUpdateTableReq(database, table.getTableName(), tableName); + SchemaUtils.rollbackUpdateTableReq(database, table.getTableName(), oldName); final boolean proceeded = new ClusterCachePropagator(SchemaUtils.filterFencedDataNode(env.getConfigManager())) .propagate(targets -> SchemaUtils.broadcastTableUpdate(req, targets)); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractSetPropertiesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractSetPropertiesProcedure.java index be6f67007a3db..650b37b49a13c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractSetPropertiesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractSetPropertiesProcedure.java @@ -101,7 +101,7 @@ protected Flow executeFromState( LOGGER.info(ProcedureMessages.SET_PROPERTIES_TO_TABLE, database, tableName); break; case COMMIT_RELEASE: - commitRelease(env); + commitReleaseWithHA(env); LOGGER.info( ProcedureMessages.COMMIT_RELEASE_INFO_OF_TABLE_WHEN_SETTING_PROPERTIES, database, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java index 44f9e2f07f9fe..b6fb00cd2076d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -90,7 +90,7 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final AddTable ProcedureMessages.COMMIT_RELEASE_INFO_OF_TABLE_WHEN_ADDING_COLUMN, database, tableName); - commitRelease(env); + commitReleaseWithHA(env); return Flow.NO_MORE_STATE; default: setFailure( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AlterTableColumnDataTypeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AlterTableColumnDataTypeProcedure.java index 12491f24a4867..916426d9cc292 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AlterTableColumnDataTypeProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AlterTableColumnDataTypeProcedure.java @@ -103,7 +103,7 @@ protected Flow executeFromState( ProcedureMessages.COMMIT_RELEASE_INFO_OF_TABLE_WHEN_ALTERING_COLUMN, database, tableName); - commitRelease(env); + commitReleaseWithHA(env); return Flow.NO_MORE_STATE; default: setFailure( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java index f2379ae1256eb..a7a2af5bd70b6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java @@ -47,7 +47,6 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Map; import java.util.Objects; import static org.apache.iotdb.rpc.TSStatusCode.TABLE_ALREADY_EXISTS; @@ -98,6 +97,7 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final CreateTa case COMMIT_RELEASE: LOGGER.info(ProcedureMessages.COMMIT_RELEASE_TABLE, database, table.getTableName()); commitReleaseTable(env); + reconcileTree2TableViewColumns(env); return Flow.NO_MORE_STATE; default: setFailure( @@ -156,13 +156,13 @@ protected void preCreateTable(final ConfigNodeProcedureEnv env) { } } - private void preReleaseTable(final ConfigNodeProcedureEnv env) { + protected void preReleaseTable(final ConfigNodeProcedureEnv env) { // Broadcast the pre-update to all DataNodes. Instead of failing whenever any DataNode is // unreachable, proceed once every unacked DataNode is provably self-fenced: such a DataNode // fails closed on its (now-stale) table cache and resyncs on lease recovery, so it cannot serve // dirty schema. Only fail if an unacked DataNode is not provably fenced (it may still be // serving clients). - final TUpdateTableReq req = SchemaUtils.buildPreUpdateTableReq(database, table, null); + final TUpdateTableReq req = buildPreReleaseTableReq(); final boolean proceeded = new ClusterCachePropagator(SchemaUtils.filterFencedDataNode(env.getConfigManager())) .propagate(targets -> SchemaUtils.broadcastTableUpdate(req, targets)); @@ -181,6 +181,10 @@ private void preReleaseTable(final ConfigNodeProcedureEnv env) { setNextState(CreateTableState.COMMIT_CREATE); } + protected TUpdateTableReq buildPreReleaseTableReq() { + return SchemaUtils.buildPreUpdateTableReq(database, table, null); + } + private void commitCreateTable(final ConfigNodeProcedureEnv env) { final TSStatus status = SchemaUtils.executeInConsensusLayer( @@ -196,20 +200,35 @@ private void commitCreateTable(final ConfigNodeProcedureEnv env) { } } + /** + * The request broadcast at commit-release. A subclass whose change also affects the tree-to-table + * view index of the DataNodes overrides this to declare the effect. + */ + protected TUpdateTableReq buildCommitReleaseTableReq() { + return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), null); + } + private void commitReleaseTable(final ConfigNodeProcedureEnv env) { - final Map failedResults = - SchemaUtils.commitReleaseTable( - database, table.getTableName(), env.getConfigManager(), null); + final TUpdateTableReq req = buildCommitReleaseTableReq(); + final boolean proceeded = SchemaUtils.commitReleaseTableWithHA(req, env.getConfigManager()); - if (!failedResults.isEmpty()) { + if (!proceeded) { LOGGER.warn( ProcedureMessages.FAILED_TO_SYNC_TABLE_COMMIT_CREATE_INFO_TO_DATANODE_FAILURE, database, table.getTableName(), - failedResults); + ProcedureMessages.FAILED_TO_PROVE_AN_UNREACHABLE_DN_IS_FENCED); } } + /** + * Adds the columns of the tree series which already exist under a newly created tree-to-table + * view's path. A plain table has no such path, so the base procedure does nothing. + */ + protected void reconcileTree2TableViewColumns(final ConfigNodeProcedureEnv env) { + // do nothing here + } + @Override protected boolean isRollbackSupported(final CreateTableState state) { return true; @@ -251,10 +270,9 @@ protected void rollbackCreate(final ConfigNodeProcedureEnv env) { } } - private void rollbackPreRelease(final ConfigNodeProcedureEnv env) { + protected void rollbackPreRelease(final ConfigNodeProcedureEnv env) { // A down DataNode must not block rollback if it is already provably self-fenced. - final TUpdateTableReq req = - SchemaUtils.rollbackUpdateTableReq(database, table.getTableName(), null); + final TUpdateTableReq req = buildRollbackTableReq(); final boolean proceeded = new ClusterCachePropagator(SchemaUtils.filterFencedDataNode(env.getConfigManager())) .propagate(targets -> SchemaUtils.broadcastTableUpdate(req, targets)); @@ -271,6 +289,10 @@ private void rollbackPreRelease(final ConfigNodeProcedureEnv env) { } } + protected TUpdateTableReq buildRollbackTableReq() { + return SchemaUtils.rollbackUpdateTableReq(database, table.getTableName(), null); + } + @Override protected CreateTableState getState(final int stateId) { return CreateTableState.values()[stateId]; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index cdb4d4b71c81d..b1694f3d25262 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -254,7 +254,9 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final DropTabl ProcedureMessages.COMMIT_RELEASE_DELETE_TABLE_WHEN_DROPPING_TABLE, database, tableName); - commitRelease(env); + // A DataNode that misses this commit keeps the table in its pre-delete state, which only + // a later read of that table repairs. Retry until every reachable DataNode acks. + commitReleaseWithHA(env); return Flow.NO_MORE_STATE; default: setFailure(new ProcedureException(ProcedureMessages.UNRECOGNIZED_DROPTABLESTATE + state)); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java index 11d10901a352a..a079a04de51ef 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java @@ -94,7 +94,7 @@ protected Flow executeFromState( ProcedureMessages.COMMIT_RELEASE_INFO_OF_TABLE_WHEN_RENAMING_COLUMN, database, tableName); - commitRelease(env); + commitReleaseWithHA(env); return Flow.NO_MORE_STATE; default: setFailure( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java index 2bce5180505c2..4975b1b8f3be8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java @@ -21,26 +21,37 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.schema.table.TreeToTableViewOperationType; import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.persistence.schema.TreeDeviceViewFieldDetector; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; import org.apache.iotdb.confignode.procedure.state.ProcedureState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; +import javax.annotation.Nullable; + import java.io.DataOutputStream; import java.io.IOException; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.Optional; + +import static org.apache.iotdb.commons.schema.table.TreeToTableViewOperationType.ALTER_TAG; +import static org.apache.iotdb.commons.schema.table.TreeViewSchema.AUTO_SYNC_SCHEMA; public class AddViewColumnProcedure extends AddTableColumnProcedure { public AddViewColumnProcedure(final boolean isGeneratedByPipe) { @@ -64,28 +75,56 @@ protected void columnCheck(final ConfigNodeProcedureEnv env) { return; } - final Map> fields2Detect = new HashMap<>(); + final Map fields2Detect = new HashMap<>(); for (final TsTableColumnSchema schema : addedColumnList) { if (!(schema instanceof FieldColumnSchema) || schema.getDataType() != TSDataType.UNKNOWN) { continue; } final String key = TreeViewSchema.getSourceName(schema); - if (!fields2Detect.containsKey(key)) { - fields2Detect.put(key, new HashSet<>()); + if (fields2Detect.containsKey(key)) { + // The column check has already dropped the columns which refer to an already referenced + // source measurement, so the added columns must not share one. + throw new SemanticException( + String.format( + ManagerMessages + .EXCEPTION_THE_DUPLICATED_SOURCE_MEASUREMENT_ARG_IS_UNSUPPORTED_YET_873AF566, + key)); } - fields2Detect.get(key).add((FieldColumnSchema) schema); + fields2Detect.put(key, (FieldColumnSchema) schema); } if (!fields2Detect.isEmpty()) { final TSStatus status = - new TreeDeviceViewFieldDetector(env.getConfigManager(), table, fields2Detect) - .detectMissingFieldTypes(); + new TreeDeviceViewFieldDetector(env.getConfigManager(), table) + .detectFieldTypesForColumnAddition(fields2Detect); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { setFailure(new ProcedureException(new IoTDBException(status))); } } } + @Override + protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldName) { + final Optional type = tree2TableViewOperationType(); + if (type.isEmpty()) { + return super.buildCommitReleaseTableReq(oldName); + } + // Adding a tag column changes the tag count a device path is matched with. + final TUpdateTree2TableView viewInfo = new TUpdateTree2TableView(type.get().getOperationType()); + viewInfo.setTagNum(table.getTagNum()); + return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName, viewInfo); + } + + private Optional tree2TableViewOperationType() { + final Optional propValue = table.getPropValue(AUTO_SYNC_SCHEMA); + if (propValue.isPresent() && Boolean.parseBoolean(propValue.get())) { + final boolean hasTagColumn = + addedColumnList.stream().anyMatch(c -> c instanceof TagColumnSchema); + return hasTagColumn ? Optional.of(ALTER_TAG) : Optional.empty(); + } + return Optional.empty(); + } + @Override protected String getActionMessage() { return "add view column"; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java index f699a88a6edf2..082a18fd4497a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java @@ -23,8 +23,10 @@ import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.schema.table.TableNodeStatus; +import org.apache.iotdb.commons.schema.table.TreeToTableViewOperationType; import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.confignode.consensus.request.write.table.view.PreCreateTableViewPlan; import org.apache.iotdb.confignode.exception.DatabaseNotExistsException; import org.apache.iotdb.confignode.i18n.ProcedureMessages; @@ -36,6 +38,8 @@ import org.apache.iotdb.confignode.procedure.state.schema.CreateTableState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.Pair; @@ -46,6 +50,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.List; import java.util.Objects; import java.util.Optional; @@ -116,13 +121,91 @@ protected void checkTableExistence(final ConfigNodeProcedureEnv env) { } } final TSStatus status = - new TreeDeviceViewFieldDetector(env.getConfigManager(), table, null) - .detectMissingFieldTypes(); + new TreeDeviceViewFieldDetector(env.getConfigManager(), table).detectFieldsForCreation(); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { setFailure(new ProcedureException(new IoTDBException(status))); } } + @Override + protected TUpdateTableReq buildCommitReleaseTableReq() { + final boolean newAutoSync = + table + .getPropValue(TreeViewSchema.AUTO_SYNC_SCHEMA) + .map(Boolean::parseBoolean) + .orElse(false); + if (newAutoSync) { + // Whether the view is a new one, or replace the old table, just add view + final TUpdateTree2TableView viewInfo = + new TUpdateTree2TableView(TreeToTableViewOperationType.ADD.getOperationType()); + table.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).ifPresent(viewInfo::setTreePath); + viewInfo.setIsRestrict(TreeViewSchema.isRestrict(table)); + viewInfo.setTagNum(table.getTagNum()); + return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), null, viewInfo); + } + // Creating, or replacing with the auto detection turned off: the view the old definition had + // registered has to go. drop a non-existing view is allowed. + final boolean oldAutoSync = + Objects.nonNull(oldView) + && oldView + .getPropValue(TreeViewSchema.AUTO_SYNC_SCHEMA) + .map(Boolean::parseBoolean) + .orElse(false); + if (oldAutoSync) { + return SchemaUtils.buildCommitUpdateTableReq( + database, + table.getTableName(), + null, + new TUpdateTree2TableView(TreeToTableViewOperationType.REMOVE.getOperationType())); + } + return super.buildCommitReleaseTableReq(); + } + + /** + * A rolled-back replace restores the view the old definition had. A DataNode which restarted + * while the replace was running does not hold it: the table such a DataNode pulls is only + * pre-created, and a pre-created table does not enter the index. Registering the old view again + * is idempotent, so a DataNode which never lost it is not affected. + */ + @Override + protected TUpdateTableReq buildRollbackTableReq() { + final boolean oldAutoSync = + Objects.nonNull(oldView) + && oldView + .getPropValue(TreeViewSchema.AUTO_SYNC_SCHEMA) + .map(Boolean::parseBoolean) + .orElse(false); + if (!oldAutoSync) { + return super.buildRollbackTableReq(); + } + final TUpdateTree2TableView viewInfo = + new TUpdateTree2TableView(TreeToTableViewOperationType.ADD.getOperationType()); + oldView.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).ifPresent(viewInfo::setTreePath); + viewInfo.setIsRestrict(TreeViewSchema.isRestrict(oldView)); + viewInfo.setTagNum(oldView.getTagNum()); + return SchemaUtils.rollbackUpdateTableReq(database, table.getTableName(), null, viewInfo); + } + + @Override + protected void reconcileTree2TableViewColumns(final ConfigNodeProcedureEnv env) { + // The series which already existed under the path pattern were created before the DataNodes + // mounted the view, so they never triggered the addition of their column. + final List missingColumns = + TreeDeviceViewFieldDetector.detectMissingColumnsAfterCreation( + env.getConfigManager(), database, table); + if (missingColumns.isEmpty()) { + return; + } + + addChildProcedure( + new AddViewColumnProcedure( + database, + table.getTableName(), + String.format("create_view_reconcile_%s.%s", database, table.getTableName()), + missingColumns, + false)); + } + @Override protected void preCreateTable(final ConfigNodeProcedureEnv env) { final TSStatus status = diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java index 73a7732fc87ea..1e141e6f69850 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java @@ -19,8 +19,12 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table.view; +import org.apache.iotdb.commons.schema.table.TreeToTableViewOperationType; +import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure; import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; import java.io.DataOutputStream; import java.io.IOException; @@ -38,6 +42,17 @@ public DropViewProcedure( super(database, tableName, queryId, isGeneratedByPipe); } + /** A dropped view has to leave the tree-to-table view index of every DataNode. */ + @Override + protected TUpdateTableReq buildCommitReleaseTableReq(final String oldName) { + // Unregistering a view needs nothing but its name, which the request already carries. + return SchemaUtils.buildCommitUpdateTableReq( + database, + table.getTableName(), + null, + new TUpdateTree2TableView(TreeToTableViewOperationType.REMOVE.getOperationType())); + } + @Override public void serialize(final DataOutputStream stream) throws IOException { stream.writeShort( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java index 62f77fb3fd0de..407463c5ff6e9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java @@ -19,12 +19,20 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table.view; +import org.apache.iotdb.commons.schema.table.TreeToTableViewOperationType; +import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableProcedure; import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; + +import javax.annotation.Nullable; import java.io.DataOutputStream; import java.io.IOException; +import static org.apache.iotdb.commons.schema.table.TreeViewSchema.AUTO_SYNC_SCHEMA; + public class RenameViewProcedure extends RenameTableProcedure { public RenameViewProcedure(final boolean isGeneratedByPipe) { super(isGeneratedByPipe); @@ -39,6 +47,17 @@ public RenameViewProcedure( super(database, tableName, queryId, newName, isGeneratedByPipe); } + @Override + protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldName) { + if (!Boolean.parseBoolean(table.getPropValue(AUTO_SYNC_SCHEMA).orElse(null))) { + return super.buildCommitReleaseTableReq(oldName); + } + // A rename does not move the view, so the old name is all the DataNode needs. + final TUpdateTree2TableView viewInfo = + new TUpdateTree2TableView(TreeToTableViewOperationType.ALTER_RENAME.getOperationType()); + return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName, viewInfo); + } + @Override protected String getActionMessage() { return "rename view"; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java index ce5b223cbb181..5f1b5c2619551 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java @@ -19,12 +19,26 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table.view; +import org.apache.iotdb.commons.schema.table.TreeToTableViewOperationType; +import org.apache.iotdb.commons.schema.table.TreeViewSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.confignode.persistence.schema.TreeDeviceViewFieldDetector; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.impl.schema.table.SetTablePropertiesProcedure; import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; + +import javax.annotation.Nullable; import java.io.DataOutputStream; import java.io.IOException; +import java.util.List; import java.util.Map; +import java.util.Optional; + +import static org.apache.iotdb.commons.schema.table.TreeViewSchema.AUTO_SYNC_SCHEMA; public class SetViewPropertiesProcedure extends SetTablePropertiesProcedure { public SetViewPropertiesProcedure(final boolean isGeneratedByPipe) { @@ -40,6 +54,74 @@ public SetViewPropertiesProcedure( super(database, tableName, queryId, properties, isGeneratedByPipe); } + @Override + protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldName) { + final Optional type = tree2TableViewOperationType(); + if (type.isEmpty()) { + return super.buildCommitReleaseTableReq(oldName); + } + final TUpdateTree2TableView viewInfo = new TUpdateTree2TableView(type.get().getOperationType()); + if (TreeToTableViewOperationType.ALTER_AUTO_SYNC == type.get()) { + // Turning the auto detection on mounts the view: the DataNode needs its whole definition. + table.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).ifPresent(viewInfo::setTreePath); + viewInfo.setIsRestrict(TreeViewSchema.isRestrict(table)); + viewInfo.setTagNum(table.getTagNum()); + } + // turn down the auto_sync_schema + return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName, viewInfo); + } + + @Override + protected void commitReleaseWithHA(final ConfigNodeProcedureEnv env) { + super.commitReleaseWithHA(env); + // Turning the auto detection on mounts the view, so the series which already exist under its + // path pattern need the same second detection as the ones of a newly created view. + if (TreeToTableViewOperationType.ALTER_AUTO_SYNC + == tree2TableViewOperationType().orElse(null)) { + reconcileTree2TableViewColumns(env); + } + } + + /** Adds the columns which the view, just mounted by the commit, is still missing. */ + private void reconcileTree2TableViewColumns(final ConfigNodeProcedureEnv env) { + final List missingColumns = + TreeDeviceViewFieldDetector.detectMissingColumnsAfterCreation( + env.getConfigManager(), database, table); + if (missingColumns.isEmpty()) { + return; + } + addChildProcedure( + new AddViewColumnProcedure( + database, + table.getTableName(), + String.format("set_view_properties_reconcile_%s.%s", database, table.getTableName()), + missingColumns, + false)); + } + + /** + * The effect of the new properties on the tree-to-table view index, or empty when they do not + * change it. Turning auto_sync_schema on registers the view, turning it off unregisters it, and + * any other property leaves the index alone. + */ + private Optional tree2TableViewOperationType() { + if (!updatedProperties.containsKey(AUTO_SYNC_SCHEMA)) { + return Optional.empty(); + } + final TreeToTableViewOperationType type = + indexEffectOf(updatedProperties.get(AUTO_SYNC_SCHEMA)); + // An absent auto_sync_schema means false, so setting it to false is not a change. + return type == indexEffectOf(originalProperties.get(AUTO_SYNC_SCHEMA)) + ? Optional.empty() + : Optional.of(type); + } + + private static TreeToTableViewOperationType indexEffectOf(final @Nullable String autoSyncSchema) { + return Boolean.parseBoolean(autoSyncSchema) + ? TreeToTableViewOperationType.ALTER_AUTO_SYNC + : TreeToTableViewOperationType.REMOVE; + } + @Override protected String getActionMessage() { return "set view properties"; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index 1f89e39d44883..f62837bd428ef 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -235,6 +235,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp; import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsReq; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsResp; import org.apache.iotdb.confignode.service.ConfigNode; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.consensus.exception.ConsensusGroupNotExistException; @@ -1531,6 +1533,11 @@ public TSStatus createTableView(final TCreateTableViewReq req) { return configManager.createTableView(req); } + @Override + public TUpdateTree2TableViewsResp updateTree2TableViews(TUpdateTree2TableViewsReq req) { + return configManager.updateTree2TableViews(req); + } + @Override public TSStatus pushHeartbeat(final int dataNodeId, final TPipeHeartbeatResp resp) { return configManager.pushHeartbeat(dataNodeId, resp); diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index 43cd5114c4b9d..5481e61647bfb 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1456,7 +1456,8 @@ private DataNodeMiscMessages() {} public static final String MESSAGE_CHANGEREGIONLEADER_SUCCESSFULLY_CHANGE_LEADER_REGIONGROUP_462797B4 = "[ChangeRegionLeader] Successfully change the leader of RegionGroup: "; public static final String MESSAGE_MESSAGE_3501C7E6 = " to "; public static final String MESSAGE_EXECUTEFASTLASTDATAQUERYFORONEPREFIXPATH_DOS_NOT_SUPPORT_WILDCARDS_8E8F44F5 = "The \"executeFastLastDataQueryForOnePrefixPath\" dos not support wildcards."; - public static final String MESSAGE_TEMPLATE_NULL_TRYING_ACTIVATE_TEMPLATE_MAY_TEMPLATE_BEING_UNSET_1CE92779 = "The template is null when trying to activate template, may be the template is being unset."; + public static final String MESSAGE_THE_TEMPLATE_IS_NULL_WHEN_TRYING_TO_ACTIVATE_TEMPLATE_MAY_BE_THE_TEMPLATE_IS_BEING_UNSET_THE_TEMPLATE_IDS_ARE_ARG_7B5F4F72 = + "The template is null when trying to activate template, may be the template is being unset. The template ids are %s."; public static final String MESSAGE_DOT_9D9B854A = "."; public static final String MESSAGE_ERROR_OCCURRED_WHILE_PARSING_SQL_TO_PHYSICAL_PLAN_COLON_5C9F2C59 = "Error occurred while parsing SQL to physical plan: "; public static final String MESSAGE_ERROR_OCCURRED_IN_READ_PROCESS_COLON_CD184195 = "Error occurred in read process: "; @@ -1510,4 +1511,10 @@ private DataNodeMiscMessages() {} LOG_MPP_DATA_EXCHANGE_MAX_PAYLOAD_SIZE_ARG_IS_BELOW_MINIMUM_ALLOWED_VALUE_ARG_USING_ARG_794ABC76 = "mpp_data_exchange_max_payload_size_in_bytes (%d) is below the minimum allowed value %d, using %d"; + public static final String MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB = + "Multiple types encountered when auto detecting type of measurement '%s', please check"; + + public static final String LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_95856581 = + "Failed to collect the devices using the extended template %s"; + } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index c89034e8831df..f4e7927934713 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -4036,6 +4036,8 @@ private DataNodeQueryMessages() {} "Only InMemoryDeviceEntryDataSet supports get inline device entries"; public static final String EXCEPTION_LOAD_CONSENSUS_INVALID_PIECE_REF_F3498507 = "Invalid LOAD consensus piece ref: path %s, offset %d, size %d"; + public static final String EXCEPTION_AUTO_SYNC_SCHEMA_IS_ONLY_SUPPORTED_BY_TREE_TO_TABLE_VIEWS_C6586215 = + "auto_sync_schema is only supported by tree to table views"; public static final String EXCEPTION_CHUNKDATA_CONTAINS_NO_PHYSICAL_CHUNK_607BCFC8 = "ChunkData contains no physical chunk"; diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java index c0e0d67346b63..8de068cbbf7e6 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java @@ -564,6 +564,7 @@ public final class DataNodeSchemaMessages { // ======================== DataNodeTableCache ======================== public static final String INIT_TABLE_CACHE_SUCCESS = "Init DataNodeTableCache successfully"; + public static final String LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH_E9D65F63 = "Failed to register the tree to table view %s.%s during init: illegal path"; public static final String PRE_UPDATE_TABLE_SUCCESS = "Pre-update table {}.{} successfully"; public static final String PRE_RENAME_OLD_TABLE_SUCCESS = "Pre-rename old table {}.{} successfully"; @@ -740,5 +741,12 @@ private DataNodeSchemaMessages() {} public static final String EXCEPTION_COMMA_50AD1C01 = ", "; public static final String MESSAGE_SKIP_ROLLBACK_RENAMING_OLD_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_664F2456 = "Skip rollback renaming old table {}.{} because it has been handled."; public static final String MESSAGE_SKIP_COMMIT_UPDATE_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_31362A1C = "Skip commit-update table {}.{} because it has been handled."; + public static final String LOG_ADD_VIEW_COLUMN_CN_FAILED = "Failed to add columns to the tree to table views for operation %s, ConfigNode returned %s."; + public static final String LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW = "Failed to add a column to the view %s.%s for operation %s, ConfigNode returned %s."; + public static final String LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE = "Failed to add columns to the tree to table views for operation %s, ConfigNode returned an empty response."; + public static final String LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE = "Failed to add columns to the tree to table views for operation %s, because ConfigNode is not reachable."; + public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED = "Failed to trigger the view column addition of the tree to table views for plan node %s, the schema write is not affected."; + public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_B3449FF3 = "Tree to table view %s.%s is not registered"; + public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE = "Failed to update the tree-to-table view index of view %s.%s"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index 743d1aa19446a..40bc72108a093 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1434,8 +1434,8 @@ private DataNodeMiscMessages() {} "[ChangeRegionLeader] 成功切换 RegionGroup 的 leader："; public static final String MESSAGE_MESSAGE_3501C7E6 = " 到 "; public static final String MESSAGE_EXECUTEFASTLASTDATAQUERYFORONEPREFIXPATH_DOS_NOT_SUPPORT_WILDCARDS_8E8F44F5 = "\"executeFastLastDataQueryForOnePrefixPath\" 不支持通配符。"; - public static final String MESSAGE_TEMPLATE_NULL_TRYING_ACTIVATE_TEMPLATE_MAY_TEMPLATE_BEING_UNSET_1CE92779 = - "尝试激活模板时模板为空，可能正在取消设置该模板。"; + public static final String MESSAGE_THE_TEMPLATE_IS_NULL_WHEN_TRYING_TO_ACTIVATE_TEMPLATE_MAY_BE_THE_TEMPLATE_IS_BEING_UNSET_THE_TEMPLATE_IDS_ARE_ARG_7B5F4F72 = + "尝试激活模板时模板为空，可能正在取消设置该模板。模板 ID 为 %s。"; public static final String MESSAGE_DOT_9D9B854A = "."; public static final String MESSAGE_ERROR_OCCURRED_WHILE_PARSING_SQL_TO_PHYSICAL_PLAN_COLON_5C9F2C59 = "将 SQL 解析为物理计划时发生错误："; @@ -1491,4 +1491,10 @@ private DataNodeMiscMessages() {} LOG_MPP_DATA_EXCHANGE_MAX_PAYLOAD_SIZE_ARG_IS_BELOW_MINIMUM_ALLOWED_VALUE_ARG_USING_ARG_794ABC76 = "mpp_data_exchange_max_payload_size_in_bytes（%d）低于允许的最小值 %d，将使用 %d"; + public static final String MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB = + "自动检测测点 %s 的类型时遇到多种类型，请检查"; + + public static final String LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_95856581 = + "收集使用扩展模板 %s 的设备失败"; + } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index a3307bd07cb42..bc814d3109456 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -4792,6 +4792,8 @@ private DataNodeQueryMessages() {} "只有 InMemoryDeviceEntryDataSet 支持获取内存中的设备条目"; public static final String EXCEPTION_LOAD_CONSENSUS_INVALID_PIECE_REF_F3498507 = "无效的 LOAD 共识分片引用：路径 %s，偏移 %d，大小 %d"; + public static final String EXCEPTION_AUTO_SYNC_SCHEMA_IS_ONLY_SUPPORTED_BY_TREE_TO_TABLE_VIEWS_C6586215 = + "auto_sync_schema 仅支持树转表视图"; public static final String EXCEPTION_CHUNKDATA_CONTAINS_NO_PHYSICAL_CHUNK_607BCFC8 = "ChunkData 中不包含任何物理 chunk"; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java index 417f547573a01..12dfd0e43b57e 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java @@ -565,6 +565,7 @@ public final class DataNodeSchemaMessages { // ======================== DataNodeTableCache 相关消息 ======================== public static final String INIT_TABLE_CACHE_SUCCESS = "DataNodeTableCache 初始化成功"; + public static final String LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH_E9D65F63 = "初始化时注册树转表视图 %s.%s 失败：路径非法"; public static final String PRE_UPDATE_TABLE_SUCCESS = "预更新表 {}.{} 成功"; public static final String PRE_RENAME_OLD_TABLE_SUCCESS = "预重命名旧表 {}.{} 成功"; public static final String ROLLBACK_UPDATE_TABLE_SUCCESS = "回滚更新表 {}.{} 成功"; @@ -728,5 +729,12 @@ private DataNodeSchemaMessages() {} public static final String EXCEPTION_COMMA_50AD1C01 = ", "; public static final String MESSAGE_SKIP_ROLLBACK_RENAMING_OLD_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_664F2456 = "跳过回滚重命名旧表 {}.{}，因为该操作已被处理。"; public static final String MESSAGE_SKIP_COMMIT_UPDATE_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_31362A1C = "跳过提交更新表 {}.{}，因为该操作已被处理。"; + public static final String LOG_ADD_VIEW_COLUMN_CN_FAILED = "为操作 %s 的树转表视图加列失败，ConfigNode 返回 %s。"; + public static final String LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW = "为操作 %s 的视图 %s.%s 加列失败，ConfigNode 返回 %s。"; + public static final String LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE = "为操作 %s 的树转表视图加列失败，ConfigNode 返回了空响应。"; + public static final String LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE = "为操作 %s 的树转表视图加列失败，因为无法访问 ConfigNode。"; + public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED = "计划节点 %s 触发树转表视图加列失败，本次 schema 写入不受影响。"; + public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_B3449FF3 = "树转表视图 %s.%s 未注册"; + public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE = "更新树转表视图 %s.%s 的索引失败"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index a77b0895e4c0b..723662df783cb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -79,6 +79,8 @@ import org.apache.iotdb.db.schemaengine.schemaregion.write.req.SchemaRegionWritePlanFactory; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateAlignedTimeSeriesPlanImpl; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateTimeSeriesPlanImpl; +import org.apache.iotdb.db.schemaengine.table.TreeToTableViewCache; +import org.apache.iotdb.db.schemaengine.table.TriggerAddViewColumnVisitor; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -94,9 +96,14 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; /** Schema write {@link PlanNode} visitor */ public class SchemaExecutionVisitor implements PlanVisitor { @@ -105,6 +112,8 @@ public class SchemaExecutionVisitor implements PlanVisitor measurementGroupMap = node.getMeasurementGroupMap(); final List failingStatus = new ArrayList<>(); PartialPath devicePath; @@ -194,6 +207,8 @@ private ICreateTimeSeriesPlan transformToCreateTimeSeriesPlan( @Override public TSStatus visitInternalCreateTimeSeries( final InternalCreateTimeSeriesNode node, final ISchemaRegion schemaRegion) { + triggerAddViewColumn(node, null); + final PartialPath devicePath = node.getDevicePath(); final MeasurementGroup measurementGroup = node.getMeasurementGroup(); @@ -232,6 +247,8 @@ public TSStatus visitInternalCreateTimeSeries( @Override public TSStatus visitInternalCreateMultiTimeSeries( final InternalCreateMultiTimeSeriesNode node, final ISchemaRegion schemaRegion) { + triggerAddViewColumn(node, null); + PartialPath devicePath; MeasurementGroup measurementGroup; @@ -411,6 +428,23 @@ private void executeInternalCreateAlignedTimeSeries( } } + /** + * @param context the templates the node needs, which this visitor has already resolved for its + * own use; null for the nodes which involve no template + */ + private TSStatus triggerAddViewColumn( + final WritePlanNode node, final TriggerAddViewColumnVisitor.Context context) { + try { + if (TreeToTableViewCache.getInstance().hasView()) { + node.accept(new TriggerAddViewColumnVisitor(), context); + } + } catch (final Throwable e) { + logger.error( + DataNodeSchemaMessages.LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED, node.getPlanNodeId(), e); + } + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } + @Override public TSStatus visitAlterTimeSeries( final AlterTimeSeriesNode node, final ISchemaRegion schemaRegion) { @@ -465,15 +499,17 @@ public TSStatus visitAlterTimeSeries( @Override public TSStatus visitActivateTemplate( final ActivateTemplateNode node, final ISchemaRegion schemaRegion) { + // The template is resolved here, so that the view column trigger and the activation below share + // one instance instead of looking it up twice. + final Template template = + ClusterTemplateManager.getInstance().getTemplate(node.getTemplateId()); + final Map templateMap = new HashMap<>(1); + if (Objects.isNull(template)) { + return getUndefinedTemplateStatus(Collections.singleton(node.getTemplateId())); + } + templateMap.put(node.getTemplateId(), template); try { - final Template template = - ClusterTemplateManager.getInstance().getTemplate(node.getTemplateId()); - if (Objects.isNull(template)) { - return new TSStatus(TSStatusCode.UNDEFINED_TEMPLATE.getStatusCode()) - .setMessage( - DataNodeMiscMessages - .MESSAGE_TEMPLATE_NULL_TRYING_ACTIVATE_TEMPLATE_MAY_TEMPLATE_BEING_UNSET_1CE92779); - } + triggerAddViewColumn(node, new TriggerAddViewColumnVisitor.Context(templateMap)); schemaRegion.activateSchemaTemplate(node, template); return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } catch (final MetadataException e) { @@ -485,17 +521,20 @@ public TSStatus visitActivateTemplate( @Override public TSStatus visitBatchActivateTemplate( final BatchActivateTemplateNode node, final ISchemaRegion schemaRegion) { + final Map templateMap = new HashMap<>(); + final Set missingTemplateIds = new HashSet<>(); + collectTemplates(node.getTemplateActivationMap(), templateMap, missingTemplateIds); + + triggerAddViewColumn(node, new TriggerAddViewColumnVisitor.Context(templateMap)); + final List statusList = new ArrayList<>(); final List alreadyActivatedDeviceList = new ArrayList<>(); for (final Map.Entry> entry : node.getTemplateActivationMap().entrySet()) { - final Template template = - ClusterTemplateManager.getInstance().getTemplate(entry.getValue().left); + final Template template = templateMap.get(entry.getValue().left); if (Objects.isNull(template)) { - return new TSStatus(TSStatusCode.UNDEFINED_TEMPLATE.getStatusCode()) - .setMessage( - DataNodeMiscMessages - .MESSAGE_TEMPLATE_NULL_TRYING_ACTIVATE_TEMPLATE_MAY_TEMPLATE_BEING_UNSET_1CE92779); + // its id is in missingTemplateIds, which is reported at the end + continue; } try { schemaRegion.activateSchemaTemplate( @@ -517,21 +556,27 @@ public TSStatus visitBatchActivateTemplate( logMetaDataException(e); statusList.add(RpcUtils.getStatus(e.getErrorCode(), e.getMessage())); } + if (!missingTemplateIds.isEmpty()) { + statusList.add(getUndefinedTemplateStatus(missingTemplateIds)); + } return statusList.isEmpty() ? RpcUtils.SUCCESS_STATUS : RpcUtils.getStatus(statusList); } @Override public TSStatus visitInternalBatchActivateTemplate( final InternalBatchActivateTemplateNode node, final ISchemaRegion schemaRegion) { + final Map templateMap = new HashMap<>(); + final Set missingTemplateIds = new HashSet<>(); + collectTemplates(node.getTemplateActivationMap(), templateMap, missingTemplateIds); + + triggerAddViewColumn(node, new TriggerAddViewColumnVisitor.Context(templateMap)); + for (final Map.Entry> entry : node.getTemplateActivationMap().entrySet()) { - final Template template = - ClusterTemplateManager.getInstance().getTemplate(entry.getValue().left); + final Template template = templateMap.get(entry.getValue().left); if (Objects.isNull(template)) { - return new TSStatus(TSStatusCode.UNDEFINED_TEMPLATE.getStatusCode()) - .setMessage( - DataNodeMiscMessages - .MESSAGE_TEMPLATE_NULL_TRYING_ACTIVATE_TEMPLATE_MAY_TEMPLATE_BEING_UNSET_1CE92779); + // its id is in missingTemplateIds, which is reported at the end + continue; } try { schemaRegion.activateSchemaTemplate( @@ -548,9 +593,43 @@ public TSStatus visitInternalBatchActivateTemplate( return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); } } + if (!missingTemplateIds.isEmpty()) { + return getUndefinedTemplateStatus(missingTemplateIds); + } return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } + private static void collectTemplates( + final Map> templateActivationMap, + final Map templateMap, + final Set missingTemplateIds) { + final ClusterTemplateManager templateManager = ClusterTemplateManager.getInstance(); + for (final Pair activation : templateActivationMap.values()) { + final int templateId = activation.left; + if (templateMap.containsKey(templateId) || missingTemplateIds.contains(templateId)) { + // already resolved for another path of this plan + continue; + } + final Template template = templateManager.getTemplate(templateId); + if (Objects.isNull(template)) { + missingTemplateIds.add(templateId); + } else { + templateMap.put(templateId, template); + } + } + } + + private static TSStatus getUndefinedTemplateStatus(final Set missingTemplateIds) { + return new TSStatus(TSStatusCode.UNDEFINED_TEMPLATE.getStatusCode()) + .setMessage( + String.format( + DataNodeMiscMessages + .MESSAGE_THE_TEMPLATE_IS_NULL_WHEN_TRYING_TO_ACTIVATE_TEMPLATE_MAY_BE_THE_TEMPLATE_IS_BEING_UNSET_THE_TEMPLATE_IDS_ARE_ARG_7B5F4F72, + missingTemplateIds.stream() + .map(String::valueOf) + .collect(Collectors.joining(", ")))); + } + @Override public TSStatus visitConstructSchemaBlackList( final ConstructSchemaBlackListNode node, final ISchemaRegion schemaRegion) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 9f1e90fda10af..72aeed07483de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -188,6 +188,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp; import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsReq; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsResp; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; @@ -1315,6 +1317,13 @@ public TSStatus createTableView(TCreateTableViewReq req) throws TException { () -> client.createTableView(req), status -> !updateConfigNodeLeader(status)); } + @Override + public TUpdateTree2TableViewsResp updateTree2TableViews(TUpdateTree2TableViewsReq req) + throws TException { + return executeRemoteCallWithRetry( + () -> client.updateTree2TableViews(req), status -> !updateConfigNodeLeader(status.status)); + } + @Override public TSStatus pushHeartbeat(final int dataNodeId, final TPipeHeartbeatResp resp) throws TException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index e9b26334f66b2..a66a12e9e56b7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -58,6 +58,7 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.ConfigurationFileUtils; +import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.conf.IoTDBConstant.ClientVersion; import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.consensus.DataRegionId; @@ -192,9 +193,13 @@ import org.apache.iotdb.db.schemaengine.SchemaEngine; import org.apache.iotdb.db.schemaengine.lease.MetadataLeaseManager; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.schemaengine.schemaregion.read.req.SchemaRegionReadPlanFactory; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.db.schemaengine.table.TreeToTableViewCache; +import org.apache.iotdb.db.schemaengine.table.TriggerAddViewColumnVisitor; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUpdateType; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUtil; @@ -406,6 +411,7 @@ import static org.apache.iotdb.commons.client.request.TestConnectionUtils.testConnectionsImpl; import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; import static org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement.AlterType.SET_DATA_TYPE; +import static org.apache.iotdb.db.schemaengine.table.TriggerAddViewColumnVisitor.ALTER_DEVICE_TEMPLATE_ID_PREFIX; import static org.apache.iotdb.db.service.RegionMigrateService.REGION_MIGRATE_PROCESS; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onIoTDBException; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onQueryException; @@ -1959,14 +1965,16 @@ public TSStatus updateTable(final TUpdateTableReq req) { .rollbackUpdateTable( ReadWriteIOUtils.readString(req.tableInfo), ReadWriteIOUtils.readString(req.tableInfo), - req.oldName); + req.oldName, + req.updateTree2TableViewInfo); break; case COMMIT_UPDATE_TABLE: DataNodeTableCache.getInstance() .commitUpdateTable( ReadWriteIOUtils.readString(req.tableInfo), ReadWriteIOUtils.readString(req.tableInfo), - req.oldName); + req.oldName, + req.updateTree2TableViewInfo); break; default: LOGGER.warn(DataNodeMiscMessages.UNSUPPORTED_TYPE_UPDATING_TABLE, req.type); @@ -2149,13 +2157,15 @@ public TDeviceViewResp detectTreeDeviceViewFieldType(final TDeviceViewReq req) { for (int i = 0; i < req.getPrefixPattern().size() - 1; ++i) { nodes[i] = req.getPrefixPattern().get(i); } + final ExtendedPartialPath pathPattern = + new ExtendedPartialPath(nodes, req.isRestrict()); final ISchemaSource schemaSource = SchemaSourceFactory.getTimeSeriesSchemaCountSource( - new ExtendedPartialPath(nodes, req.isRestrict()), + pathPattern, false, // Does not support logical view currently SchemaFilterFactory.createViewTypeFilter(ViewType.BASE), - null, + ClusterTemplateManager.getInstance().checkAllRelatedTemplate(pathPattern), SchemaConstant.ALL_MATCH_SCOPE, true, true); @@ -2173,11 +2183,13 @@ public TDeviceViewResp detectTreeDeviceViewFieldType(final TDeviceViewReq req) { } if (!updateMap.containsKey(measurementName)) { updateMap.put(measurementName, schema.getTypeInByte()); - } else if (schema.getTypeInByte() != updateMap.get(measurementName)) { + } else if (schema.getTypeInByte() != updateMap.get(measurementName) + && !req.isTolerateTypeMismatch()) { return RpcUtils.getStatus( TSStatusCode.DATA_TYPE_MISMATCH, String.format( - "Multiple types encountered when auto detecting type of measurement '%s', please check", + DataNodeMiscMessages + .MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB, measurementName)); } } @@ -2902,6 +2914,24 @@ public TSStatus updateTemplate(final TUpdateTemplateReq req) { SchemaEngine.getInstance() .updateSubtreeMeasurementCountForTemplate(newTemplate.getId(), delta); } + if (TreeToTableViewCache.getInstance().hasView()) { + // addViewColumnsIfNeeded, which collects the views from the index and asks CN to add the + // columns. + try { + final Map>> device2Measurements = + collectDevicesUsingExtendedTemplate(oldTemplate, newTemplate); + TriggerAddViewColumnVisitor.addViewColumnsIfNeeded( + device2Measurements, ALTER_DEVICE_TEMPLATE_ID_PREFIX + newTemplate.getId()); + } catch (final Exception e) { + // todo + LOGGER.warn( + DataNodeMiscMessages + .LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_95856581, + newTemplate.getName(), + e); + } + } + break; default: LOGGER.warn(DataNodeMiscMessages.UNSUPPORTED_TYPE_UPDATING_TEMPLATE, req.type); @@ -2910,6 +2940,61 @@ public TSStatus updateTemplate(final TUpdateTemplateReq req) { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } + /** + * The devices which activated the template, each with the measurements the extension added to it: + * the tree-to-table views covering these devices need them as columns. + * + *

The measurements come from comparing the new template with the old one. The devices come + * from the schema regions, where every device which uses a template records its id, see {@code + * EntityTraverser}; the template manager only knows where a template is mounted, so it cannot + * tell which devices use it. + */ + private Map>> collectDevicesUsingExtendedTemplate( + final Template oldTemplate, final Template newTemplate) throws MetadataException { + final List> extendedMeasurements = + newTemplate.getSchemaMap().values().stream() + .filter( + schema -> + Objects.isNull(oldTemplate) + || !oldTemplate.getSchemaMap().containsKey(schema.getMeasurementName())) + .map(schema -> new Pair<>(schema.getMeasurementName(), schema.getType())) + .toList(); + final Pair> templateSetInfo = + ClusterTemplateManager.getInstance().getAllPathsSetTemplate(newTemplate.getName()); + final List setPaths = + Objects.isNull(templateSetInfo) ? null : templateSetInfo.getRight(); + if (extendedMeasurements.isEmpty() || Objects.isNull(setPaths)) { + return Collections.emptyMap(); + } + + final Map>> device2Measurements = new HashMap<>(); + Collection allSchemaRegions = SchemaEngine.getInstance().getAllSchemaRegions(); + for (final PartialPath setPath : setPaths) { + final String setPathString = setPath.getFullPath(); + for (final ISchemaRegion schemaRegion : allSchemaRegions) { + // A device under this path can only live in the region of the database the path belongs to. + if (!setPathString.startsWith(schemaRegion.getDatabaseFullPath())) { + continue; + } + final ISchemaReader deviceReader = + schemaRegion.getDeviceReader( + SchemaRegionReadPlanFactory.getShowDevicesPlan( + setPath.concatNode(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD), + 0, + 0, + false, + newTemplate.getId(), + SchemaConstant.ALL_MATCH_SCOPE)); + while (deviceReader.hasNext()) { + device2Measurements + .computeIfAbsent(deviceReader.next().getPartialPath(), device -> new ArrayList<>()) + .addAll(extendedMeasurements); + } + } + } + return device2Measurements; + } + @Override public TSStatus deleteRegion(TConsensusGroupId tconsensusGroupId) { TSStatus consensusStatus = waitForConsensusStarted(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 2af8eccc17388..2f11a37e74930 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -303,6 +303,7 @@ import static org.apache.iotdb.commons.schema.table.TsTable.TABLE_ALLOWED_PROPERTIES; import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME; import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY; +import static org.apache.iotdb.db.i18n.DataNodeQueryMessages.EXCEPTION_AUTO_SYNC_SCHEMA_IS_ONLY_SUPPORTED_BY_TREE_TO_TABLE_VIEWS_C6586215; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AbstractDatabaseTask.NEED_LAST_CACHE_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.MAX_DATA_REGION_GROUP_NUM_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.MAX_SCHEMA_REGION_GROUP_NUM_KEY; @@ -631,8 +632,7 @@ private Pair parseTable4CreateTableOrView( context.getSession().getUserName(), new QualifiedObjectName(database, tableName), context); final TsTable table = new TsTable(tableName); - - table.setProps(convertPropertiesToMap(node.getProperties(), false)); + table.setProps(convertPropertiesToMap(node.getProperties(), false, node instanceof CreateView)); if (Objects.nonNull(node.getComment())) { table.addProp(TsTable.COMMENT_KEY, node.getComment()); } @@ -900,7 +900,7 @@ public IConfigTask visitSetProperties(final SetProperties node, final MPPQueryCo return new AlterTableSetPropertiesTask( database, tableName, - convertPropertiesToMap(node.getProperties(), true), + convertPropertiesToMap(node.getProperties(), true, isTreeView), context.getQueryId().getId(), node.ifExists(), isTreeView); @@ -983,8 +983,20 @@ private void validateTreeViewProperties(final List propertyList) { } } + /** + * Converts the property assignments of a CREATE TABLE/VIEW or a SET PROPERTIES statement into the + * props of the TsTable. + * + * @param propertyList the property assignments to convert + * @param serializeDefault whether a property assigned {@code DEFAULT} shall be kept in the result + * as a null value, which resets the property, instead of being dropped + * @param allowAutoSyncSchema whether the statement targets a tree to table view, the only kind of + * table whose schema can be synchronized from a tree path prefix + */ private Map convertPropertiesToMap( - final List propertyList, final boolean serializeDefault) { + final List propertyList, + final boolean serializeDefault, + final boolean allowAutoSyncSchema) { final Map map = new HashMap<>(); final Set deduplicate = new HashSet<>(); for (final Property property : propertyList) { @@ -993,6 +1005,11 @@ private Map convertPropertiesToMap( throw new SemanticException(DataNodeQueryMessages.DUPLICATED_PROPERTY + key); } if (TABLE_ALLOWED_PROPERTIES.contains(key)) { + if (TreeViewSchema.AUTO_SYNC_SCHEMA.equals(key) && !allowAutoSyncSchema) { + // A property assigned DEFAULT is dropped or reset as well, so the check covers it too + throw new SemanticException( + EXCEPTION_AUTO_SYNC_SCHEMA_IS_ONLY_SUPPORTED_BY_TREE_TO_TABLE_VIEWS_C6586215); + } if (!property.isSetToDefault()) { final Expression value = property.getNonDefaultValue(); switch (key) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/view/TreeToTableViewReference.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/view/TreeToTableViewReference.java new file mode 100644 index 0000000000000..e9eb73ae8022a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/view/TreeToTableViewReference.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.view; + +import java.util.Objects; + +/** A view identity and its matching metadata. Equality depends only on database and table names. */ +public final class TreeToTableViewReference { + + public static final String NUM_OF_TAGS = "num_of_tags"; + + private final String databaseName; + + private final String tableName; + + private final boolean restrict; + + private final int numOfTags; + + public TreeToTableViewReference( + String databaseName, String tableName, boolean restrict, int numOfTags) { + this.databaseName = databaseName; + this.tableName = tableName; + this.restrict = restrict; + this.numOfTags = numOfTags; + } + + public String getDatabaseName() { + return databaseName; + } + + public String getTableName() { + return tableName; + } + + public boolean isRestrict() { + return restrict; + } + + public int getNumOfTags() { + return numOfTags; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof TreeToTableViewReference)) { + return false; + } + TreeToTableViewReference that = (TreeToTableViewReference) obj; + return Objects.equals(databaseName, that.databaseName) + && Objects.equals(tableName, that.tableName); + } + + @Override + public int hashCode() { + return Objects.hash(databaseName, tableName); + } + + @Override + public String toString() { + return "TreeToTableViewReference{" + + "databaseName='" + + databaseName + + '\'' + + ", tableName='" + + tableName + + '\'' + + ", restrict=" + + restrict + + ", numOfTags=" + + numOfTags + + '}'; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index 525bf511f4e4b..d8c9b28721372 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -23,13 +23,17 @@ import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.consensus.ConfigRegionId; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.iotdb.commons.exception.MetadataLeaseFencedException.LeaseFencedRetryPolicy; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.i18n.CommonMessages; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.table.NonCommittableTsTable; import org.apache.iotdb.commons.schema.table.PreDeleteTsTable; import org.apache.iotdb.commons.schema.table.TableNodeStatus; +import org.apache.iotdb.commons.schema.table.TreeToTableViewOperationType; +import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; @@ -44,6 +48,8 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.executor.ClusterConfigTaskExecutor; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.schemaengine.lease.MetadataLeaseManager; +import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.TException; @@ -55,11 +61,13 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; @@ -69,6 +77,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.apache.iotdb.commons.schema.table.TreeViewSchema.TREE_PATH_PATTERN; import static org.apache.iotdb.db.i18n.DataNodeSchemaMessages.FAILED_TO_REFRESH_CACHE_FROM_CN; /** It contains all tables' latest column schema */ @@ -77,6 +86,7 @@ public class DataNodeTableCache implements ITableCache { private static final Logger LOGGER = LoggerFactory.getLogger(DataNodeTableCache.class); private static final IClientManager CONFIG_NODE_CLIENT_MANAGER = ConfigNodeClientManager.getInstance(); + private final ITreeToTableViewCache treeToTableViewCache = TreeToTableViewCache.getInstance(); /** Instance-specific version counter for optimistic locking mechanisms. */ private final AtomicLong instanceVersion = new AtomicLong(0); @@ -142,12 +152,40 @@ public void init(final byte[] tableInitializationBytes) { table -> new Pair<>(table, 0L), (v1, v2) -> v2, ConcurrentHashMap::new)))); + treeToTableViewCache.init(collectInitialViews(usingMap)); LOGGER.info(DataNodeSchemaMessages.INIT_TABLE_CACHE_SUCCESS); } finally { readWriteLock.writeLock().unlock(); } } + private Map> collectInitialViews( + final Map> usingMap) { + final Map> initialViews = new HashMap<>(); + usingMap.forEach( + (key, value) -> { + final List views = + value.stream().filter(DataNodeTableCache::isAutoSyncView).toList(); + if (!views.isEmpty()) { + initialViews.put(PathUtils.unQualifyDatabaseName(key), new ArrayList<>(views)); + } + }); + return initialViews; + } + + /** + * Whether the table belongs in the tree-to-table view index: it is a tree-to-table view which + * auto-syncs its schema. A NonCommittableTsTable and a PreDeleteTsTable are placeholders which + * carry no definition at all, so they are never one. + */ + private static boolean isAutoSyncView(final TsTable table) { + final Optional autoSyncSchemaValue = + table.getPropValue(TreeViewSchema.AUTO_SYNC_SCHEMA); + return autoSyncSchemaValue.isPresent() + && Boolean.parseBoolean(autoSyncSchemaValue.get()) + && table.getPropValue(TREE_PATH_PATTERN).isPresent(); + } + // No need to acquire a lock here; reloadTableCacheAfterLeaseRecovery is within a critical section // protected by metadataLeaseManager @Override @@ -203,12 +241,12 @@ public void preUpdateTable(String database, final TsTable table, final String ol if (!table.getCachedNeedLastCache()) { TableDeviceSchemaCache.getInstance().invalidateLastCache(database, table.getTableName()); } - if (table instanceof PreDeleteTsTable) { if (databaseTableMap.containsKey(database)) { databaseTableMap.get(database).remove(table.getTableName()); } } + // If rename table if (Objects.nonNull(oldName)) { final TsTable oldTable = databaseTableMap.get(database).remove(oldName); @@ -233,11 +271,17 @@ public void preUpdateTable(String database, final TsTable table, final String ol } @Override - public void rollbackUpdateTable(String database, final String tableName, final String oldName) { + public void rollbackUpdateTable( + String database, + final String tableName, + final String oldName, + TUpdateTree2TableView updateTree2TableViewInfo) { database = PathUtils.unQualifyDatabaseName(database); readWriteLock.writeLock().lock(); try { failIfMetadataLeaseFenced(LeaseFencedRetryPolicy.RETRY_UNTIL_SUCCESS); + // for create or replace the view, only triggered when rollback + operateOnTree2TableViewCache(database, tableName, oldName, updateTree2TableViewInfo); // if rollback the drop table procedure, do nothing, // wait for triggering the action of pull table from CN final TsTable table = getTableFromSpecialStatusMap(database, tableName); @@ -316,11 +360,16 @@ private void removeTableFromSpecialStatusMap(final String database, final String @Override public void commitUpdateTable( - String database, final String tableName, final @Nullable String oldName) { + String database, + final String tableName, + final @Nullable String oldName, + TUpdateTree2TableView updateTree2TableViewInfo) { database = PathUtils.unQualifyDatabaseName(database); readWriteLock.writeLock().lock(); try { failIfMetadataLeaseFenced(LeaseFencedRetryPolicy.RETRY_UNTIL_SUCCESS); + operateOnTree2TableViewCache(database, tableName, oldName, updateTree2TableViewInfo); + // for update tree-to-table view final TsTable newTable = getTableFromSpecialStatusMap(database, tableName); if (Objects.isNull(newTable)) { LOGGER.info( @@ -370,6 +419,60 @@ public void commitUpdateTable( } } + /** + * Applies the effect of a committed change on the tree-to-table view index. Every attribute the + * operation needs comes with the request: the table this DataNode caches may be a placeholder, so + * the entry cannot be computed from it. + */ + private void operateOnTree2TableViewCache( + String database, + String tableName, + @Nullable String oldTableName, + TUpdateTree2TableView updateTree2TableViewInfo) { + if (Objects.isNull(updateTree2TableViewInfo)) { + return; + } + final TreeToTableViewOperationType type = + TreeToTableViewOperationType.getType(updateTree2TableViewInfo.getTree2TableOperationType()); + try { + switch (type) { + case REMOVE: + // Unregistering a view needs nothing but its name. + treeToTableViewCache.removeView(database, tableName); + break; + case ALTER_RENAME: + // A rename keeps the place of the view. + treeToTableViewCache.alterViewName(database, tableName, oldTableName); + break; + case ALTER_TAG: + final Map properties = new HashMap<>(); + properties.put( + TreeToTableViewReference.NUM_OF_TAGS, + String.valueOf(updateTree2TableViewInfo.getTagNum())); + treeToTableViewCache.alterViewProperties(database, tableName, properties); + break; + case ADD: + case ALTER_AUTO_SYNC: + treeToTableViewCache.addView( + database, + tableName, + updateTree2TableViewInfo.isRestrict, + updateTree2TableViewInfo.getTagNum(), + new PartialPath(updateTree2TableViewInfo.getTreePath())); + break; + default: + break; + } + } catch (final IllegalPathException e) { + LOGGER.warn( + DataNodeSchemaMessages + .LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE, + database, + tableName, + e); + } + } + /** * Repairs the cache after this DataNode missed the pre-update of a rename procedure. The * ConfigNode has already renamed the table, so the new name can only be re-fetched, and the old diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java index 13193fbff13e0..1d66e38d4b264 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.exception.MetadataLeaseFencedException.LeaseFencedRetryPolicy; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -36,10 +37,16 @@ public interface ITableCache { void preUpdateTable(final String database, final TsTable table, final @Nullable String oldName); void rollbackUpdateTable( - final String database, final String tableName, final @Nullable String oldName); + final String database, + final String tableName, + final @Nullable String oldName, + TUpdateTree2TableView updateTree2TableViewInfo); void commitUpdateTable( - final String database, final String tableName, final @Nullable String oldName); + final String database, + final String tableName, + final @Nullable String oldName, + TUpdateTree2TableView updateTree2TableViewInfo); /** * @param database shouldn't start with `root.` diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITreeToTableViewCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITreeToTableViewCache.java new file mode 100644 index 0000000000000..a3156da6ca852 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITreeToTableViewCache.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.table; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public interface ITreeToTableViewCache { + + void init(Map> initialTables); + + void clear(); + + /** + * Registers the view at {@code path}, the fixed prefix of its tree path pattern. {@code + * isRestrict} and {@code numOfTags} are the two attributes a device path is matched with, see + * {@link TreeToTableViewReference}. + */ + boolean addView( + String database, String tableName, boolean isRestrict, int numOfTags, PartialPath path) + throws IllegalPathException; + + boolean removeView(String database, String tableName); + + boolean alterViewName(String database, String newName, String oldName); + + /** + * Replaces the fields named in {@code properties} on the registered view. Only the restrict flag + * and the tag count can be replaced today, and a field whose key is absent keeps its value. + */ + boolean alterViewProperties(String database, String tableName, Map properties); + + @Nullable + PartialPath getViewPath(String database, String tableName); + + Set getTreeToTableViews(PartialPath path); + + boolean hasView(); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java new file mode 100644 index 0000000000000..0b1de30db18db --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java @@ -0,0 +1,465 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.schemaengine.table; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternUtil; +import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; +import org.apache.iotdb.commons.schema.table.TreeViewSchema; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.db.i18n.DataNodeSchemaMessages; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader; +import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference; + +import org.apache.tsfile.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; + +/** + * The index of the tree to table views on DataNode, keyed by the fixed prefix of each view's tree + * path pattern. + * + *

Only DataNodeTableCache writes this index, and it does so while holding its own write lock, so + * every write here also holds the write lock of this cache and the lock order is always + * DataNodeTableCache's write lock and then this cache's write lock. A reader only takes this + * cache's read lock, so the two locks can never form a cycle. + */ +public class TreeToTableViewCache implements ITreeToTableViewCache { + + private static final Logger LOGGER = LoggerFactory.getLogger(TreeToTableViewCache.class); + + private final IMNodeFactory nodeFactory = + MNodeFactoryLoader.getInstance().getMemMNodeIMNodeFactory(); + // This index describes path prefixes, not the database/device roles in a SchemaRegion. clear() + // replaces it with a fresh one. + private IMemMNode rootNode = nodeFactory.createInternalMNode(null, PATH_ROOT); + + private final Map> viewReferences = + new IdentityHashMap<>(); + private final Map, PartialPath> viewPaths = new HashMap<>(); + private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(); + + /** + * Whether any view is registered. A cluster without any view answers "no view covers this path" + * without taking the lock or walking the trie. It is written only under the write lock, and read + * without any lock. + */ + private volatile boolean hasView = false; + + private TreeToTableViewCache() {} + + @Override + public void clear() { + readWriteLock.writeLock().lock(); + try { + // A node is only reachable from its parent, so a fresh root drops the whole trie at once. + rootNode = nodeFactory.createInternalMNode(null, PATH_ROOT); + viewReferences.clear(); + viewPaths.clear(); + hasView = false; + } finally { + readWriteLock.writeLock().unlock(); + } + } + + @Override + public void init(final Map> initialTables) { + readWriteLock.writeLock().lock(); + try { + // The index is rebuilt from scratch, so the entry of a view which is gone does not survive. + clear(); + initialTables.forEach( + (database, tables) -> tables.forEach(table -> registerView(database, table))); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + /** + * Registers the view at the path of its tree path pattern. The caller has already picked the + * tables which belong in the index, see DataNodeTableCache#collectInitialViews. + */ + private void registerView(final String database, final TsTable table) { + final Optional treePath = table.getPropValue(TreeViewSchema.TREE_PATH_PATTERN); + if (treePath.isEmpty()) { + return; + } + try { + addView( + database, + table.getTableName(), + TreeViewSchema.isRestrict(table), + table.getTagNum(), + new PartialPath(treePath.get())); + } catch (final IllegalPathException e) { + LOGGER.warn( + DataNodeSchemaMessages + .LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH_E9D65F63, + database, + table.getTableName(), + e); + } + } + + /** + * Register a view at a fixed prefix path. + * + * @parm Partial path, eg: root.db.dev01.**; + * @return true if a new reference was registered; false if it was already present + */ + // todo, log; + @Override + public boolean addView( + final String database, + final String tableName, + final boolean isRestrict, + final int numOfTags, + final PartialPath path) + throws IllegalPathException { + int prefixLength = checkPartialPath(path); + final String[] nodes = path.getNodes(); + readWriteLock.writeLock().lock(); + try { + final PartialPath registeredPath = viewPaths.get(new Pair<>(database, tableName)); + if (Objects.nonNull(registeredPath) && !Objects.equals(registeredPath, path)) { + // A view is registered at exactly one path, so the entry left at the previous path has to + // go, or removing this view by name later would miss it. + removeView(database, tableName); + } + viewPaths.put(new Pair<>(database, tableName), path); + + IMemMNode current = rootNode; + for (int i = 1; i < prefixLength; i++) { + IMemMNode child = current.getChild(nodes[i]); + if (child == null) { + child = current.addChild(nodes[i], nodeFactory.createInternalMNode(current, nodes[i])); + } + current = child; + } + + final boolean added = + viewReferences + .computeIfAbsent(current, node -> new HashSet<>()) + .add(new TreeToTableViewReference(database, tableName, isRestrict, numOfTags)); + // whether the reference is new or not, this path is covered by a view now + hasView = true; + return added; + } finally { + readWriteLock.writeLock().unlock(); + } + } + + /** + * Removes a view from the trie. + * + * @return true if the view was registered and has been removed; false otherwise + */ + @Override + public boolean removeView(String database, String tableName) { + readWriteLock.writeLock().lock(); + try { + final RegisteredView registered = getRegisteredView(database, tableName); + if (Objects.isNull(registered)) { + LOGGER.warn( + DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_B3449FF3, + database, + tableName); + return false; + } + + registered.references.remove(registered.reference); + if (registered.references.isEmpty()) { + viewReferences.remove(registered.node); + pruneNodesWithoutView(registered.node); + hasView = !viewReferences.isEmpty(); + } + viewPaths.remove(new Pair<>(database, tableName)); + return true; + } finally { + readWriteLock.writeLock().unlock(); + } + } + + @Override + public boolean alterViewName(String database, String newName, String oldName) { + readWriteLock.writeLock().lock(); + try { + final RegisteredView registered = getRegisteredView(database, oldName); + if (Objects.isNull(registered)) { + LOGGER.warn( + DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_B3449FF3, + database, + oldName); + return false; + } + + final TreeToTableViewReference oldReference = registered.reference; + registered.references.remove(oldReference); + registered.references.add( + new TreeToTableViewReference( + database, newName, oldReference.isRestrict(), oldReference.getNumOfTags())); + // A rename does not move the view, so the path of the old name follows the new name. + final PartialPath path = viewPaths.remove(new Pair<>(database, oldName)); + viewPaths.put(new Pair<>(database, newName), path); + return true; + } finally { + readWriteLock.writeLock().unlock(); + } + } + + @Override + public boolean alterViewProperties( + final String database, final String tableName, final Map properties) { + readWriteLock.writeLock().lock(); + try { + final RegisteredView registered = getRegisteredView(database, tableName); + if (Objects.isNull(registered)) { + LOGGER.warn( + DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_B3449FF3, + database, + tableName); + return false; + } + + final TreeToTableViewReference oldReference = registered.reference; + final String restrict = properties.get(TreeViewSchema.RESTRICT); + final String numOfTags = properties.get(TreeToTableViewReference.NUM_OF_TAGS); + registered.references.remove(oldReference); + registered.references.add( + new TreeToTableViewReference( + database, + tableName, + Objects.isNull(restrict) ? oldReference.isRestrict() : Boolean.parseBoolean(restrict), + Objects.isNull(numOfTags) + ? oldReference.getNumOfTags() + : Integer.parseInt(numOfTags))); + return true; + } finally { + readWriteLock.writeLock().unlock(); + } + } + + /** + * The place a registered view lives: the trie node it sits at, the references of that node, and + * the reference of the view itself. + */ + private static class RegisteredView { + IMemMNode node; + Set references; + TreeToTableViewReference reference; + + private RegisteredView( + final IMemMNode node, + final Set references, + final TreeToTableViewReference reference) { + this.node = node; + this.references = references; + this.reference = reference; + } + } + + /** + * The registered view of these names, or null when no view of them is registered. The node may + * also exist as an intermediate node of another view's path, without any view of its own. + */ + private RegisteredView getRegisteredView(final String database, final String tableName) { + final IMemMNode node = getViewNode(database, tableName); + final Set references = + Objects.isNull(node) ? null : viewReferences.get(node); + if (Objects.isNull(references)) { + return null; + } + final TreeToTableViewReference reference = findReference(references, database, tableName); + return Objects.isNull(reference) ? null : new RegisteredView(node, references, reference); + } + + /** The trie node the view of these names is registered at, or null if it is not registered. */ + private IMemMNode getViewNode(final String database, final String tableName) { + final PartialPath path = viewPaths.get(new Pair<>(database, tableName)); + if (Objects.isNull(path)) { + return null; + } + final String[] nodes = path.getNodes(); + return findNode(nodes, nodes.length - 1); + } + + /** The reference of the view of these names in {@code references}, or null if it holds none. */ + private static TreeToTableViewReference findReference( + final Set references, + final String database, + final String tableName) { + for (final TreeToTableViewReference reference : references) { + if (Objects.equals(database, reference.getDatabaseName()) + && Objects.equals(tableName, reference.getTableName())) { + return reference; + } + } + return null; + } + + /** + * @param path the device path, eg, root.db01.plant01.dev01 + * @return the views covering the device path, empty if there is none + */ + @Override + public Set getTreeToTableViews(PartialPath path) { + if (!hasView) { + // the answer for every path when no view is registered at all + return Collections.emptySet(); + } + + final String[] nodes = path.getNodes(); + readWriteLock.readLock().lock(); + try { + Set result = new HashSet<>(); + IMemMNode current = rootNode; + int currentLevel = 1; + while (current != null && currentLevel < (nodes.length + 1)) { + // maybe has the qualified nodes, check and collect + if (viewReferences.containsKey(current)) { + checkAndCollectView(viewReferences.get(current), nodes.length - currentLevel, result); + } + if (currentLevel == nodes.length) { + break; + } + current = current.getChild(nodes[currentLevel]); + currentLevel++; + } + return result; + } finally { + readWriteLock.readLock().unlock(); + } + } + + @Override + public boolean hasView() { + return hasView; + } + + /** + * The path a view is registered at, so that a caller which knows only the names of the view can + * still locate and remove it. + * + * @return the path the view is registered at, or null if no view of these names is registered + */ + @Override + public @Nullable PartialPath getViewPath(final String database, final String tableName) { + readWriteLock.readLock().lock(); + try { + return viewPaths.get(new Pair<>(database, tableName)); + } finally { + readWriteLock.readLock().unlock(); + } + } + + /** + * Walks the fixed prefix of a checked path, which is the path without the leading "root" and the + * trailing multi level wildcard, eg, nodes of root.db.dev01.** give the node of root.db.dev01. + * + * @return the node the prefix ends at, or null if the trie does not hold that prefix + */ + private IMemMNode findNode(final String[] nodes, final int prefixLength) { + IMemMNode current = rootNode; + for (int i = 1; i < prefixLength && current != null; i++) { + current = current.getChild(nodes[i]); + } + return current; + } + + /** + * Drops the nodes which have neither a child nor a view of their own any more, starting at {@code + * node} and walking up to the first node that is still needed. Removing a view is rare, so the + * walk is affordable. + */ + private void pruneNodesWithoutView(final IMemMNode node) { + IMemMNode current = node; + while (current != rootNode) { + final IMemMNode parent = current.getParent(); + if (parent == null + || !current.getChildren().isEmpty() + || viewReferences.containsKey(current)) { + return; + } + parent.deleteChild(current.getName()); + current = parent; + } + } + + private static final class TreeToTableViewCacheHolder { + private static final TreeToTableViewCache INSTANCE = new TreeToTableViewCache(); + + private TreeToTableViewCacheHolder() {} + } + + public static ITreeToTableViewCache getInstance() { + return TreeToTableViewCacheHolder.INSTANCE; + } + + // the path has to be like root.db.dev01.**; + private int checkPartialPath(PartialPath path) throws IllegalPathException { + final String[] nodes = path.getNodes(); + if (nodes.length == 0 || !PATH_ROOT.equals(nodes[0])) { + throw new IllegalPathException(path.getFullPath()); + } + if (!MULTI_LEVEL_PATH_WILDCARD.equals(nodes[nodes.length - 1])) { + throw new IllegalPathException(path.getFullPath()); + } + final int prefixLength = nodes.length - 1; + // Validate before modifying the trie. Wildcards inside the fixed prefix are not supported. + for (int i = 1; i < prefixLength; i++) { + if (nodes[i] == null || nodes[i].isEmpty() || PathPatternUtil.hasWildcard(nodes[i])) { + throw new IllegalPathException(path.getFullPath()); + } + } + return prefixLength; + } + + private void checkAndCollectView( + Set treeToTableViewReferences, + int remainingLevel, + Set result) { + treeToTableViewReferences.forEach( + view -> { + int numOfTags = view.getNumOfTags(); + boolean match = + view.isRestrict() ? numOfTags == remainingLevel : numOfTags >= remainingLevel; + if (match) { + result.add(view); + } + }); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TriggerAddViewColumnVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TriggerAddViewColumnVisitor.java new file mode 100644 index 0000000000000..878119533031e --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TriggerAddViewColumnVisitor.java @@ -0,0 +1,389 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.table; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.AbstractConfigNodeInfo; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.consensus.ConfigRegionId; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.schema.table.AlterOrDropTableOperationType; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; +import org.apache.iotdb.commons.schema.template.Template; +import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewFailure; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsReq; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsResp; +import org.apache.iotdb.db.i18n.DataNodeSchemaMessages; +import org.apache.iotdb.db.protocol.client.ConfigNodeClient; +import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.ActivateTemplateNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.BatchActivateTemplateNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.CreateAlignedTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.CreateMultiTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.CreateTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.InternalBatchActivateTemplateNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.InternalCreateMultiTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.InternalCreateTimeSeriesNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.MeasurementGroup; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator; +import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HexFormat; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Adds the columns which the tree-to-table views covering the created timeseries need. + * + *

The trigger is a best-effort side effect of the schema creation, see addViewColumnsIfNeeded. + */ +public class TriggerAddViewColumnVisitor + implements PlanVisitor { + + private static final Logger LOGGER = LoggerFactory.getLogger(TriggerAddViewColumnVisitor.class); + private static final IClientManager CONFIG_NODE_CLIENT_MANAGER = + ConfigNodeClientManager.getInstance(); + + public static final String QUERY_ID_PREFIX = "tree_to_table_add_column"; + public static final String ALTER_DEVICE_TEMPLATE_ID_PREFIX = "template_"; + + public static final class Context { + + private final Map templates; + + public Context(Map templates) { + this.templates = Objects.isNull(templates) ? Collections.emptyMap() : templates; + } + + public Map getTemplates() { + return templates; + } + } + + /** + * Adds the columns the views covering the given devices need. + * + *

It is a best-effort side effect of the write which triggers it: every failure is logged here + * and none of them is propagated, so that the write operation is never affected. The columns + * which are not added this time are added by a later write of the same measurements, as every + * write plan carries its own planNodeId, hence its own queryId. + * + * @param device2Measurements eg. device01 -> {s1, s2, s3}; device02 -> {s3, s4} + * @param operationId the planNodeId of the plan which triggers this operation + */ + public static void addViewColumnsIfNeeded( + Map>> device2Measurements, String operationId) { + if (device2Measurements == null || device2Measurements.isEmpty()) { + return; + } + final Map>> view2Measurements = + collectViewMeasurements(device2Measurements); + if (view2Measurements.isEmpty()) { + return; + } + + try (ConfigNodeClient configNodeClient = + CONFIG_NODE_CLIENT_MANAGER.borrowClient(AbstractConfigNodeInfo.CONFIG_REGION_ID)) { + List updateViewReqs = + view2Measurements.entrySet().stream() + .map(entry -> TriggerAddViewColumnVisitor.toAlterTableReq(entry, operationId)) + .toList(); + + TUpdateTree2TableViewsResp resp = + configNodeClient.updateTree2TableViews(new TUpdateTree2TableViewsReq(updateViewReqs)); + if (resp == null || resp.getStatus() == null) { + LOGGER.error(DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE, operationId); + return; + } + if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.error( + DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_FAILED, operationId, resp.getStatus()); + } + // the views are updated one by one in ConfigNode, so the failed ones are reported one by one + if (resp.getFailedViews() != null) { + for (TUpdateTree2TableViewFailure failedView : resp.getFailedViews()) { + LOGGER.error( + DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW, + failedView.getDatabase(), + failedView.getTableName(), + operationId, + failedView.getStatus()); + } + } + } catch (ClientManagerException | TException e) { + LOGGER.warn(DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE, operationId, e); + } + } + + /** + * @return the measurements each view covering the given devices needs; eg: view01 -> {s1, s2, + * s3}, view02 -> {s1, s2, s3, s4} + */ + private static Map>> + collectViewMeasurements( + Map>> device2Measurements) { + + // deduplicate the measurement by set, the measurement with same name and dataType + // from different device would count once, eg: dev01.s1(int32) and dev02.s1(int32) + // other deduplication would leave to ConfigMTree in CN + final Map>> view2Measurements = + new HashMap<>(); + + for (Map.Entry>> entry : + device2Measurements.entrySet()) { + PartialPath path = entry.getKey(); + Set treeToTableViews = + TreeToTableViewCache.getInstance().getTreeToTableViews(path); + + treeToTableViews.forEach( + view -> + view2Measurements + .computeIfAbsent(view, key -> new LinkedHashSet<>()) + .addAll(entry.getValue())); + } + return view2Measurements; + } + + private static TAlterOrDropTableReq toAlterTableReq( + Map.Entry>> view2Measurements, + String operationId) { + final TreeToTableViewReference view = view2Measurements.getKey(); + final List columnSchemas = + view2Measurements.getValue().stream() + .map(TriggerAddViewColumnVisitor::toColumnSchema) + .toList(); + TAlterOrDropTableReq tAlterOrDropTableReq = + new TAlterOrDropTableReq( + view.getDatabaseName(), + view.getTableName(), + buildQueryId( + operationId + "_" + columnFingerprint(columnSchemas), + view.getDatabaseName(), + view.getTableName()), + AlterOrDropTableOperationType.ADD_COLUMN.getTypeValue(), + ByteBuffer.wrap(TsTableColumnSchemaUtil.serialize(columnSchemas))); + tAlterOrDropTableReq.setIsView(true); + return tAlterOrDropTableReq; + } + + // transform the upperCase to the from-attribute + private static TsTableColumnSchema toColumnSchema(final Pair measurement) { + final String measurementName = measurement.getLeft(); + final String from = + measurementName.chars().anyMatch(Character::isUpperCase) ? measurementName : null; + return TableHeaderSchemaValidator.generateColumnSchema( + TsTableColumnCategory.FIELD, + measurementName.toLowerCase(Locale.ROOT), + measurement.getRight(), + null, + from); + } + + @Override + public TSStatus visitCreateTimeSeries( + CreateTimeSeriesNode node, TriggerAddViewColumnVisitor.Context context) { + final MeasurementPath path = node.getPath(); + final Pair measurement = + new Pair<>(path.getMeasurement(), node.getDataType()); + final Map>> device2Measurements = + Collections.singletonMap(path.getDevicePath(), Collections.singletonList(measurement)); + addViewColumnsIfNeeded(device2Measurements, node.getPlanNodeId().getId()); + return null; + } + + @Override + public TSStatus visitCreateAlignedTimeSeries( + CreateAlignedTimeSeriesNode node, TriggerAddViewColumnVisitor.Context context) { + final Map>> device2Measurements = + Collections.singletonMap( + node.getDevicePath(), toMeasurements(node.getMeasurements(), node.getDataTypes())); + addViewColumnsIfNeeded(device2Measurements, node.getPlanNodeId().getId()); + return null; + } + + @Override + public TSStatus visitCreateMultiTimeSeries( + CreateMultiTimeSeriesNode node, TriggerAddViewColumnVisitor.Context context) { + final Map>> device2Measurements = + toDevice2Measurements(node.getMeasurementGroupMap()); + addViewColumnsIfNeeded(device2Measurements, node.getPlanNodeId().getId()); + return null; + } + + @Override + public TSStatus visitInternalCreateTimeSeries( + InternalCreateTimeSeriesNode node, TriggerAddViewColumnVisitor.Context context) { + final Map>> device2Measurements = + Collections.singletonMap(node.getDevicePath(), toMeasurements(node.getMeasurementGroup())); + addViewColumnsIfNeeded(device2Measurements, node.getPlanNodeId().getId()); + return null; + } + + @Override + public TSStatus visitInternalCreateMultiTimeSeries( + InternalCreateMultiTimeSeriesNode node, TriggerAddViewColumnVisitor.Context context) { + // the alignment flag is of no use to a view, only the measurements of the devices matter + final Map>> device2Measurements = + new HashMap<>(node.getDeviceMap().size()); + for (final Map.Entry> entry : + node.getDeviceMap().entrySet()) { + device2Measurements.put(entry.getKey(), toMeasurements(entry.getValue().getRight())); + } + addViewColumnsIfNeeded(device2Measurements, node.getPlanNodeId().getId()); + return null; + } + + @Override + public TSStatus visitActivateTemplate( + ActivateTemplateNode node, TriggerAddViewColumnVisitor.Context context) { + final Template template = context.getTemplates().get(node.getTemplateId()); + final Map>> device2Measurements = + Collections.singletonMap(node.getActivatePath(), toMeasurements(template)); + addViewColumnsIfNeeded(device2Measurements, node.getPlanNodeId().getId()); + return null; + } + + @Override + public TSStatus visitBatchActivateTemplate( + BatchActivateTemplateNode node, TriggerAddViewColumnVisitor.Context context) { + addViewColumnsIfNeeded( + toActivationDevice2Measurements(node.getTemplateActivationMap(), context), + node.getPlanNodeId().getId()); + return null; + } + + @Override + public TSStatus visitInternalBatchActivateTemplate( + InternalBatchActivateTemplateNode node, TriggerAddViewColumnVisitor.Context context) { + addViewColumnsIfNeeded( + toActivationDevice2Measurements(node.getTemplateActivationMap(), context), + node.getPlanNodeId().getId()); + return null; + } + + private static Map>> toActivationDevice2Measurements( + final Map> templateActivationMap, + final TriggerAddViewColumnVisitor.Context context) { + final Map>> device2Measurements = + new HashMap<>(templateActivationMap.size()); + for (final Map.Entry> entry : + templateActivationMap.entrySet()) { + final Template template = context.getTemplates().get(entry.getValue().getLeft()); + if (Objects.isNull(template)) { + continue; + } + device2Measurements.put(entry.getKey(), toMeasurements(template)); + } + return device2Measurements; + } + + private static Map>> toDevice2Measurements( + final Map measurementGroupMap) { + final Map>> device2Measurements = + new HashMap<>(measurementGroupMap.size()); + for (final Map.Entry entry : measurementGroupMap.entrySet()) { + device2Measurements.put(entry.getKey(), toMeasurements(entry.getValue())); + } + return device2Measurements; + } + + private static List> toMeasurements(final Template template) { + return template.getSchemaMap().values().stream() + .map(schema -> new Pair<>(schema.getMeasurementName(), schema.getType())) + .toList(); + } + + private static List> toMeasurements(MeasurementGroup measurementGroup) { + return toMeasurements(measurementGroup.getMeasurements(), measurementGroup.getDataTypes()); + } + + private static List> toMeasurements( + final List measurements, final List dataTypes) { + final List> result = new ArrayList<>(measurements.size()); + for (int i = 0; i < measurements.size(); i++) { + result.add(new Pair<>(measurements.get(i), dataTypes.get(i))); + } + return result; + } + + @Override + public TSStatus visitPlan(PlanNode node, TriggerAddViewColumnVisitor.Context context) { + throw new UnsupportedOperationException(); + } + + /** + * The queryId identifies one column-adding operation in ConfigNode, so as to deduplicate the + * requests sent by the leader and the followers while applying the same schema region log entry. + * + *

The {@code operationId} is the planNodeId of the plan which triggers the operation, and it + * is identical on every replica of one log entry. The database, the table and the fingerprint of + * the columns are appended because ConfigNode only compares the queryId, instead of the table, + * while deduplicating: without them the operation of another view, or of another column set, + * would be treated as a duplicate and silently skipped. + */ + private static String buildQueryId(String operationId, String database, String table) { + return QUERY_ID_PREFIX + "_" + operationId + "_" + database + "_" + table; + } + + /** + * The columns are sorted first so that the same column set always yields the same fingerprint, no + * matter in which order the measurements were collected. + */ + private static String columnFingerprint(final List columnSchemas) { + final List sortedColumnSchemas = new ArrayList<>(columnSchemas); + sortedColumnSchemas.sort(Comparator.comparing(TsTableColumnSchema::getColumnName)); + return digestOf(TsTableColumnSchemaUtil.serialize(sortedColumnSchemas)); + } + + private static String digestOf(final byte[] content) { + try { + // 4 bytes are enough here, the number of distinct column sets of one view is rather small + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content), 0, 4); + } catch (final NoSuchAlgorithmException e) { + // SHA-256 is supported by every Java platform + throw new IllegalStateException(e); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java index d60a8f2cc3cca..98ef2dc61c012 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java @@ -290,7 +290,7 @@ public void testRecoverAfterCreateAndUpdateDevice() throws Exception { new TagColumnSchema(columnHeader.getColumnName(), columnHeader.getColumnType()))); testTable.addColumnSchema(new AttributeColumnSchema(attributeName, TSDataType.STRING)); DataNodeTableCache.getInstance().preUpdateTable(database, testTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null, null); schemaRegion.updateTableDeviceAttribute( new TableDeviceAttributeUpdateNode( @@ -337,7 +337,7 @@ public void testUpdateDeviceRejectsFieldAssignment() throws Exception { testTable.addColumnSchema(new AttributeColumnSchema("attr", TSDataType.STRING)); testTable.addColumnSchema(new FieldColumnSchema("field", TSDataType.STRING)); DataNodeTableCache.getInstance().preUpdateTable(database, testTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null, null); try { final ISchemaRegion schemaRegion = getSchemaRegion(database, 0); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java index 744bda9ce5b84..cc7f97984715f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java @@ -159,7 +159,7 @@ public void setUp() throws MetadataException, IOException, WriteProcessException TreeViewSchema.TREE_PATH_PATTERN, NON_ALIGNED_TREE_DEVICE_VIEW_SCAN_OPERATOR_TREE_TEST + ".**"); DataNodeTableCache.getInstance().preUpdateTable(tableDbName, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(tableDbName, tableViewName, null); + DataNodeTableCache.getInstance().commitUpdateTable(tableDbName, tableViewName, null, null); } @After diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java index 9c9918ddbc8c4..bc4fd02b192d7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java @@ -1149,7 +1149,7 @@ public void testPipeTransferredUpdateRejectsInvalidTarget() { tsTable.addColumnSchema(new AttributeColumnSchema("attr1", TSDataType.STRING)); tsTable.addColumnSchema(new FieldColumnSchema("s1", TSDataType.INT64)); DataNodeTableCache.getInstance().preUpdateTable(database, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, table, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, table, null, null); try { final Update statement = @@ -1211,7 +1211,7 @@ public void testSqlUpdateRejectsInvalidColumnsWithExplicitErrors() { tsTable.addColumnSchema(new AttributeColumnSchema("attr1", TSDataType.STRING)); tsTable.addColumnSchema(new FieldColumnSchema("s1", TSDataType.INT64)); DataNodeTableCache.getInstance().preUpdateTable(database, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, table, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, table, null, null); try { assertUpdateSemanticException( @@ -1276,7 +1276,7 @@ public void testInternalShowDeviceKeepsPreparedTraversalState() { tsTable.addColumnSchema(new TagColumnSchema("tag1", TSDataType.STRING)); tsTable.addColumnSchema(new AttributeColumnSchema("attr1", TSDataType.STRING)); DataNodeTableCache.getInstance().preUpdateTable(database, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, table, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, table, null, null); try { final ShowDevice statement = new ShowDevice(database, table); @@ -1316,7 +1316,7 @@ private void assertDeviceQueryCanBeAnalyzedAgain(final String sql) { tsTable.addColumnSchema(new AttributeColumnSchema("attr1", TSDataType.STRING)); tsTable.addColumnSchema(new AttributeColumnSchema("attr2", TSDataType.STRING)); DataNodeTableCache.getInstance().preUpdateTable(database, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, table, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, table, null, null); try { final SqlParser sqlParser = new SqlParser(); @@ -1370,7 +1370,7 @@ private void assertParsedDeviceStatementUnchanged( private Metadata mockMetadataForInsertion() { final TsTable tsTable = StatementTestUtils.genTsTable(); DataNodeTableCache.getInstance().preUpdateTable(database, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, table, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, table, null, null); return new TestMetadata() { @Override public Optional validateTableHeaderSchema4TsFile( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TreeViewTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TreeViewTest.java index 959bd4b9d6abb..6a7e169257a3f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TreeViewTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TreeViewTest.java @@ -69,7 +69,8 @@ public void setup() { tsTable.addProp(TsTable.TTL_PROPERTY, Long.MAX_VALUE + ""); tsTable.addProp(TreeViewSchema.TREE_PATH_PATTERN, "root.test" + ".**"); DataNodeTableCache.getInstance().preUpdateTable(TREE_VIEW_DB, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(TREE_VIEW_DB, DEVICE_VIEW_TEST_TABLE, null); + DataNodeTableCache.getInstance() + .commitUpdateTable(TREE_VIEW_DB, DEVICE_VIEW_TEST_TABLE, null, null); } @After diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java index 2b8909f38613e..a59d36839029d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java @@ -107,10 +107,10 @@ public static void prepareEnvironment() { new FieldColumnSchema( measurement6, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); DataNodeTableCache.getInstance().preUpdateTable(database1, testTable1, null); - DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null); + DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null, null); DataNodeTableCache.getInstance().preUpdateTable(database2, testTable1, null); - DataNodeTableCache.getInstance().commitUpdateTable(database2, table1, null); + DataNodeTableCache.getInstance().commitUpdateTable(database2, table1, null, null); final TsTable testTable2 = new TsTable(table2); columnHeaderList.forEach( @@ -139,7 +139,7 @@ public static void prepareEnvironment() { new FieldColumnSchema( measurement6, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); DataNodeTableCache.getInstance().preUpdateTable(database1, testTable2, null); - DataNodeTableCache.getInstance().commitUpdateTable(database1, table2, null); + DataNodeTableCache.getInstance().commitUpdateTable(database1, table2, null, null); final TsTable testTableNeedLastCacheDisabled = new TsTable(tableNeedLastCacheDisabled); columnHeaderList.forEach( @@ -173,7 +173,8 @@ public static void prepareEnvironment() { TsTable.NEED_LAST_CACHE_PROPERTY, Boolean.FALSE.toString()); DataNodeTableCache.getInstance() .preUpdateTable(database1, testTableNeedLastCacheDisabled, null); - DataNodeTableCache.getInstance().commitUpdateTable(database1, tableNeedLastCacheDisabled, null); + DataNodeTableCache.getInstance() + .commitUpdateTable(database1, tableNeedLastCacheDisabled, null, null); } @AfterClass @@ -628,7 +629,7 @@ public void testLastCacheIsInvalidatedWhenTableNeedLastCacheTurnsFalse() { new TsTable(DataNodeTableCache.getInstance().getTable(database1, table1)); disabledTable.addProp(TsTable.NEED_LAST_CACHE_PROPERTY, Boolean.FALSE.toString()); DataNodeTableCache.getInstance().preUpdateTable(database1, disabledTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null); + DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null, null); try { Assert.assertNull(cache.getLastEntry(database1, deviceID, measurement1)); @@ -636,7 +637,7 @@ public void testLastCacheIsInvalidatedWhenTableNeedLastCacheTurnsFalse() { final TsTable enabledTable = new TsTable(disabledTable); enabledTable.removeProp(TsTable.NEED_LAST_CACHE_PROPERTY); DataNodeTableCache.getInstance().preUpdateTable(database1, enabledTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null); + DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null, null); } } @@ -654,7 +655,7 @@ public void testLastCacheIsInvalidatedWhenTableNeedLastCacheSetFalseWithoutCache new TsTable(DataNodeTableCache.getInstance().getTable(database1, table1)); enabledTable.renameTable(tableName); DataNodeTableCache.getInstance().preUpdateTable(database, enabledTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null, null); final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); updateLastCache4Query(cache, database, deviceID, measurements, values); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TopKRuntimeFilterTreeViewPlanTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TopKRuntimeFilterTreeViewPlanTest.java index 91b16f17d5ee3..305f6dd706c4f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TopKRuntimeFilterTreeViewPlanTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TopKRuntimeFilterTreeViewPlanTest.java @@ -48,7 +48,8 @@ public void setup() { tsTable.addProp(TsTable.TTL_PROPERTY, Long.MAX_VALUE + ""); tsTable.addProp(TreeViewSchema.TREE_PATH_PATTERN, "root.test.**"); DataNodeTableCache.getInstance().preUpdateTable(TREE_VIEW_DB, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(TREE_VIEW_DB, DEVICE_VIEW_TEST_TABLE, null); + DataNodeTableCache.getInstance() + .commitUpdateTable(TREE_VIEW_DB, DEVICE_VIEW_TEST_TABLE, null, null); } @After diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java index bf42e64460982..e0ddc58039fa2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java @@ -99,7 +99,7 @@ public void setUp() throws Exception { TsTable tsTable = convertTableSchemaToTsTable(tableSchema); DataNodeTableCache.getInstance().preUpdateTable("test", tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null); + DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null, null); when(metadata.validateTableHeaderSchema4TsFile( any(String.class), @@ -252,7 +252,7 @@ public void testConflictCategory() { TsTable tsTable = convertTableSchemaToTsTable(tableSchema); DataNodeTableCache.getInstance().preUpdateTable("test", tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null); + DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null, null); when(metadata.validateTableHeaderSchema4TsFile( any(String.class), @@ -288,7 +288,7 @@ public void testMissingIdColumn() { TsTable tsTable = convertTableSchemaToTsTable(tableSchema); DataNodeTableCache.getInstance().preUpdateTable("test", tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null); + DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null, null); when(metadata.validateTableHeaderSchema4TsFile( any(String.class), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheLeaseTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheLeaseTest.java index 00c5388e747f0..7c39b9f8a9799 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheLeaseTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheLeaseTest.java @@ -80,8 +80,8 @@ public void fencedLeaseFailsClosedForUpdateApis() { nowNanos.addAndGet((T_FENCE_MS + 1) * 1_000_000L); assertLeaseFenced(() -> tableCache.preUpdateTable("root.db", table, null)); - assertLeaseFenced(() -> tableCache.rollbackUpdateTable("root.db", "t", null)); - assertLeaseFenced(() -> tableCache.commitUpdateTable("root.db", "t", null)); + assertLeaseFenced(() -> tableCache.rollbackUpdateTable("root.db", "t", null, null)); + assertLeaseFenced(() -> tableCache.commitUpdateTable("root.db", "t", null, null)); } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java index bec226b21ab09..1734aaa678f31 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java @@ -80,8 +80,8 @@ public void commitUpdateTableIsIdempotent() { try { cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, createTable(TABLE_NAME), null); - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null); - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null); Assert.assertEquals( TABLE_NAME, cache.getTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME).getTableName()); @@ -97,8 +97,8 @@ public void commitAfterRollbackUpdateTableIsIgnored() { try { cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, createTable(TABLE_NAME), null); - cache.rollbackUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null); - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null); + cache.rollbackUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null); Assert.assertNull(cache.getTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, false)); } finally { @@ -113,10 +113,10 @@ public void rollbackRenameTableRestoresOldName() { cache.invalid(TABLE_CACHE_TEST_DATABASE); try { cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, createTable(TABLE_NAME), null); - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null); cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, createTable(newTableName), TABLE_NAME); - cache.rollbackUpdateTable(TABLE_CACHE_TEST_DATABASE, newTableName, TABLE_NAME); + cache.rollbackUpdateTable(TABLE_CACHE_TEST_DATABASE, newTableName, TABLE_NAME, null); Assert.assertEquals( TABLE_NAME, cache.getTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME).getTableName()); @@ -144,9 +144,9 @@ public void commitRenameRepairsCacheWhenPreReleaseWasMissed() throws Exception { TsTableInternalRPCUtil.serializeTableInitializationInfo(usingTables, specialTables)); // The commit is the only sign of the rename, and the HA broadcast may send it more than once. - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, newName, oldName); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, newName, oldName, null); assertRenameRepaired(cache, oldName, newName); - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, newName, oldName); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, newName, oldName, null); assertRenameRepaired(cache, oldName, newName); } finally { cache.invalid(TABLE_CACHE_TEST_DATABASE); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCacheTest.java new file mode 100644 index 0000000000000..58ec3c7b4aa43 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCacheTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.table; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference; + +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.Set; + +public class TreeToTableViewCacheTest { + + /** + * The cache is a singleton, so every case registers its own views and removes them again, and + * every case uses its own database. + */ + private final ITreeToTableViewCache viewCache = TreeToTableViewCache.getInstance(); + + @Test + public void testAddAlterAndRemoveView() throws Exception { + final String database = "add_test"; + final PartialPath viewPath = new PartialPath("root.add_test.dev01.**"); + final PartialPath devicePath = new PartialPath("root.add_test.dev01"); + try { + Assert.assertTrue(viewCache.getTreeToTableViews(devicePath).isEmpty()); + + Assert.assertTrue(viewCache.addView(database, "view1", false, 0, viewPath)); + // the same view at the same path twice does not add a second reference + Assert.assertFalse(viewCache.addView(database, "view1", false, 0, viewPath)); + Assert.assertTrue(isCovered(devicePath, "view1")); + + Assert.assertTrue(viewCache.alterViewName(database, "view2", "view1")); + Assert.assertFalse(isCovered(devicePath, "view1")); + Assert.assertTrue(isCovered(devicePath, "view2")); + + Assert.assertTrue(viewCache.removeView(database, "view2")); + Assert.assertTrue(viewCache.getTreeToTableViews(devicePath).isEmpty()); + // the view is gone, so removing it again fails + Assert.assertFalse(viewCache.removeView(database, "view2")); + } finally { + viewCache.removeView(database, "view1"); + viewCache.removeView(database, "view2"); + } + } + + @Test + public void testRemovingOneViewKeepsTheOthers() throws Exception { + final String database = "remove_test"; + final PartialPath viewPath = new PartialPath("root.remove_test.dev01.**"); + final PartialPath devicePath = new PartialPath("root.remove_test.dev01"); + try { + Assert.assertTrue(viewCache.addView(database, "view3", false, 0, viewPath)); + Assert.assertTrue(viewCache.addView(database, "view4", false, 0, viewPath)); + + Assert.assertTrue(viewCache.removeView(database, "view3")); + + final Set views = viewCache.getTreeToTableViews(devicePath); + Assert.assertEquals(1, views.size()); + Assert.assertEquals("view4", views.iterator().next().getTableName()); + } finally { + viewCache.removeView(database, "view3"); + viewCache.removeView(database, "view4"); + } + } + + /** The nodes created for the path of the removed view have to be dropped together with it. */ + @Test + public void testRemoveViewPrunesTheTrie() throws Exception { + final String database = "prune_test"; + final PartialPath viewPath = new PartialPath("root.prune_test.dev01.**"); + final PartialPath devicePath = new PartialPath("root.prune_test.dev01"); + try { + Assert.assertTrue(viewCache.addView(database, "view5", false, 0, viewPath)); + + Assert.assertTrue(viewCache.removeView(database, "view5")); + + // the path of the removed view is not held by the trie any more + Assert.assertNull(getRootNode().getChild(database)); + Assert.assertTrue(viewCache.getTreeToTableViews(devicePath).isEmpty()); + } finally { + viewCache.removeView(database, "view5"); + } + } + + @Test + public void testRemoveViewOfAnotherDatabase() throws Exception { + final String database = "database_test"; + final PartialPath viewPath = new PartialPath("root.database_test.dev01.**"); + final PartialPath devicePath = new PartialPath("root.database_test.dev01"); + try { + Assert.assertTrue(viewCache.addView(database, "view6", false, 0, viewPath)); + + // the view of another database is another view, even if the name and the path are the same + Assert.assertFalse(viewCache.removeView("another_database", "view6")); + Assert.assertTrue(isCovered(devicePath, "view6")); + } finally { + viewCache.removeView(database, "view6"); + } + } + + private boolean isCovered(final PartialPath devicePath, final String tableName) { + return viewCache.getTreeToTableViews(devicePath).stream() + .anyMatch(view -> tableName.equals(view.getTableName())); + } + + private static IMemMNode getRootNode() throws Exception { + final Field field = TreeToTableViewCache.class.getDeclaredField("rootNode"); + field.setAccessible(true); + return (IMemMNode) field.get(TreeToTableViewCache.getInstance()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index f7a594659e771..a70315cca3491 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -177,7 +177,8 @@ public void setUp() throws Exception { DataNodeTableCache.getInstance() .preUpdateTable(dataRegion.getDatabaseName(), StatementTestUtils.genTsTable(), null); DataNodeTableCache.getInstance() - .commitUpdateTable(dataRegion.getDatabaseName(), StatementTestUtils.tableName(), null); + .commitUpdateTable( + dataRegion.getDatabaseName(), StatementTestUtils.tableName(), null, null); } @After diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java index e9ea0025250ed..9aa2143ab8ea1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java @@ -188,8 +188,11 @@ private void createTable(String tableName, TSDataType dataType) { tsTable.addColumnSchema(new TagColumnSchema("id_column", TSDataType.STRING)); tsTable.addColumnSchema( new FieldColumnSchema("s1", dataType, TSEncoding.PLAIN, CompressionType.LZ4)); + DataNodeTableCache.getInstance().preUpdateTable(this.COMPACTION_TEST_SG, tsTable, null); + DataNodeTableCache.getInstance() + .commitUpdateTable(this.COMPACTION_TEST_SG, tableName, null, null); DataNodeTableCache.getInstance().preUpdateTable(TABLE_MODEL_TEST_SG, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null); + DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null, null); } private TsFileResource generateInt32TableFile(TimeRange timeRange, boolean seq) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java index 877756b73d5b7..e372b0b43ace9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java @@ -170,6 +170,6 @@ public void createTable(String tableName, long ttl) { new FieldColumnSchema("s1", TSDataType.STRING, TSEncoding.PLAIN, CompressionType.LZ4)); tsTable.addProp(TsTable.TTL_PROPERTY, ttl + ""); DataNodeTableCache.getInstance().preUpdateTable(TABLE_MODEL_TEST_SG, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null); + DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null, null); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java index 49f22af864070..8d43c77970cd4 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java @@ -190,7 +190,7 @@ public void testRedoRelationalInsertPlan() throws Exception { testTable1.addColumnSchema( new FieldColumnSchema("m1", TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); DataNodeTableCache.getInstance().preUpdateTable(SG_NAME, testTable1, null); - DataNodeTableCache.getInstance().commitUpdateTable(SG_NAME, "table1", null); + DataNodeTableCache.getInstance().commitUpdateTable(SG_NAME, "table1", null, null); // generate crashed .tsfile File file = new File(FILE_NAME); Files.createDirectories(file.getParentFile().toPath()); diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/SchemaMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/SchemaMessages.java index 58e7f6fe22012..966512e4bfb2d 100644 --- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/SchemaMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/SchemaMessages.java @@ -38,6 +38,8 @@ public final class SchemaMessages { public static final String CANNOT_REMOVE_TAG_COLUMN = "Cannot remove an tag column: %s"; public static final String UNKNOWN_TABLE_UPDATE_OPERATION_TYPE = "Unknown table update operation type%s"; + public static final String EXCEPTION_UNKNOWN_TREE_TO_TABLE_VIEW_OPERATION_TYPE_ARG_C13CAC3F = + "Unknown tree to table view operation type: %s"; // Column messages public static final String UNSUPPORTED_COLUMN_TYPE_IN_TSFILE = diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/SchemaMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/SchemaMessages.java index 776690108a0df..dcbdec3a1b6c7 100644 --- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/SchemaMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/SchemaMessages.java @@ -38,6 +38,8 @@ public final class SchemaMessages { public static final String CANNOT_REMOVE_TAG_COLUMN = "无法移除 tag 列：%s"; public static final String UNKNOWN_TABLE_UPDATE_OPERATION_TYPE = "未知的 table 更新操作类型%s"; + public static final String EXCEPTION_UNKNOWN_TREE_TO_TABLE_VIEW_OPERATION_TYPE_ARG_C13CAC3F = + "未知的树转表视图操作类型：%s"; // Column 相关消息 public static final String UNSUPPORTED_COLUMN_TYPE_IN_TSFILE = diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewOperationType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewOperationType.java new file mode 100644 index 0000000000000..ac7deccfd918e --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewOperationType.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.schema.table; + +import org.apache.iotdb.commons.i18n.SchemaMessages; + +public enum TreeToTableViewOperationType { + /** Register the view, or replace the entry stored for the same view. */ + ADD((byte) 0), + /** Unregister the view. */ + REMOVE((byte) 1), + + /** Replace the registered entry of the view with the one computed from the new table. */ + ALTER_TAG((byte) 2), + + ALTER_RENAME((byte) 3), + + ALTER_AUTO_SYNC((byte) 4); + + private final byte operationType; + + TreeToTableViewOperationType(final byte operationType) { + this.operationType = operationType; + } + + public byte getOperationType() { + return operationType; + } + + public static TreeToTableViewOperationType getType(final byte type) { + switch (type) { + case -1: + return null; + case 0: + return ADD; + case 1: + return REMOVE; + case 2: + return ALTER_TAG; + case 3: + return ALTER_RENAME; + case 4: + return ALTER_AUTO_SYNC; + default: + throw new IllegalArgumentException( + String.format( + SchemaMessages.EXCEPTION_UNKNOWN_TREE_TO_TABLE_VIEW_OPERATION_TYPE_ARG_C13CAC3F, + type)); + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java index 2acbb993254d7..d0d2d6e9f96c1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java @@ -35,6 +35,7 @@ public class TreeViewSchema { public static final String RESTRICT = "__restrict"; public static final String UNSUPPORTED_NEED_LAST_CACHE_PROPERTY = "The tree view does not support need_last_cache property."; + public static final String AUTO_SYNC_SCHEMA = "auto_sync_schema"; public static boolean isTreeViewTable(final TsTable table) { return table.getPropValue(TREE_PATH_PATTERN).isPresent(); diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 0b733829d87b1..88ecee141dada 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1368,6 +1368,23 @@ struct TCreateTableViewReq { 2: required bool replace } + struct TUpdateTree2TableViewFailure { + 1: required string database + 2: required string tableName + 3: required common.TSStatus status + } + + +struct TUpdateTree2TableViewsResp{ + 1: required common.TSStatus status + 2: optional list failedViews +} + +struct TUpdateTree2TableViewsReq{ + 1:required list viewInfos +} + + service IConfigNodeRPCService { // ====================================================== @@ -2171,4 +2188,6 @@ service IConfigNodeRPCService { // Table view common.TSStatus createTableView(TCreateTableViewReq req) + + TUpdateTree2TableViewsResp updateTree2TableViews(TUpdateTree2TableViewsReq req) } diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 76c0860a24fa8..57afa70e72846 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -356,6 +356,14 @@ struct TUpdateTableReq { 1: required byte type 2: required binary tableInfo 3: optional string oldName + 4: optional TUpdateTree2TableView updateTree2TableViewInfo +} + +struct TUpdateTree2TableView{ + 1: required byte tree2TableOperationType + 2: optional string treePath + 3: optional bool isRestrict + 4: optional i32 tagNum } struct TInvalidateTableCacheReq { @@ -450,6 +458,7 @@ struct TDeviceViewReq { 3: required i32 tagNumber 4: required bool restrict 5: optional set requiredMeasurements + 6: optional bool tolerateTypeMismatch } struct TDeviceViewResp { From daa30b0d0177a4084c205209f75c4ea502efd72c Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Wed, 14 Oct 2026 19:02:54 +0800 Subject: [PATCH 02/24] fix the i18n and add the try-with-resources --- .../iotdb/confignode/i18n/ProcedureMessages.java | 6 ------ .../impl/DataNodeInternalRPCServiceImpl.java | 15 ++++++++------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java index 7fed5d8317de6..24ce6192841fa 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java @@ -1511,17 +1511,11 @@ private ProcedureMessages() {} public static final String MESSAGE_ROLLING_BACK_AFTER_FAILURE_ARG_474DF456 = "正在回滚，失败原因：%s。"; public static final String MESSAGE_ROLLING_BACK_AFTER_AN_EARLIER_FAILURE_850D0AF5 = "正在回滚此前发生的失败。"; -<<<<<<< HEAD public static final String EXCEPTION_DATANODE_SETSYSTEMSTATUS_RPC_FAILED_ARG_ARG_ARG_7E8E5F9F = "DataNode setSystemStatus RPC 失败：%s：%s：%s； "; public static final String EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4 = "ConfigNode trySetNodeStatus 失败：%s：%s：%s； "; -======= - public static final String - LOG_FAILED_TO_COMMIT_RELEASE_ARG_FOR_TABLE_ARG_ARG_TO_DATANODE_FAILURE_RESULTS_ARG_55386572 = - "对表 {}.{} 在 DataNode 上提交释放 {} 失败，失败结果：{}"; public static final String LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_9544E1D9 = "扫描新建视图 %s.%s 下的树序列失败：%s"; ->>>>>>> 9e29979e95 (support that auto-detection for creating measurements and adding columns for related tree-to-table view) } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index a66a12e9e56b7..ad9c03752ba13 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -2950,7 +2950,7 @@ public TSStatus updateTemplate(final TUpdateTemplateReq req) { * tell which devices use it. */ private Map>> collectDevicesUsingExtendedTemplate( - final Template oldTemplate, final Template newTemplate) throws MetadataException { + final Template oldTemplate, final Template newTemplate) throws Exception { final List> extendedMeasurements = newTemplate.getSchemaMap().values().stream() .filter( @@ -2976,7 +2976,7 @@ private Map>> collectDevicesUsingExte if (!setPathString.startsWith(schemaRegion.getDatabaseFullPath())) { continue; } - final ISchemaReader deviceReader = + try (final ISchemaReader deviceReader = schemaRegion.getDeviceReader( SchemaRegionReadPlanFactory.getShowDevicesPlan( setPath.concatNode(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD), @@ -2984,11 +2984,12 @@ private Map>> collectDevicesUsingExte 0, false, newTemplate.getId(), - SchemaConstant.ALL_MATCH_SCOPE)); - while (deviceReader.hasNext()) { - device2Measurements - .computeIfAbsent(deviceReader.next().getPartialPath(), device -> new ArrayList<>()) - .addAll(extendedMeasurements); + SchemaConstant.ALL_MATCH_SCOPE))) { + while (deviceReader.hasNext()) { + device2Measurements + .computeIfAbsent(deviceReader.next().getPartialPath(), device -> new ArrayList<>()) + .addAll(extendedMeasurements); + } } } } From 08ee0a7163fe26c2ea7abe57686ebbc99c24d3d9 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Thu, 15 Oct 2026 11:36:10 +0800 Subject: [PATCH 03/24] reorder so that the "extend column" request comes after sequence creation --- .../schemaregion/SchemaExecutionVisitor.java | 166 ++++++-- .../impl/DataNodeInternalRPCServiceImpl.java | 6 +- .../table/TriggerAddViewColumnManager.java | 223 ++++++++++ .../table/TriggerAddViewColumnVisitor.java | 389 ------------------ 4 files changed, 360 insertions(+), 424 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TriggerAddViewColumnManager.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TriggerAddViewColumnVisitor.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index 723662df783cb..742539a56874a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -80,7 +80,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateAlignedTimeSeriesPlanImpl; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateTimeSeriesPlanImpl; import org.apache.iotdb.db.schemaengine.table.TreeToTableViewCache; -import org.apache.iotdb.db.schemaengine.table.TriggerAddViewColumnVisitor; +import org.apache.iotdb.db.schemaengine.table.TriggerAddViewColumnManager; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -112,22 +112,22 @@ public class SchemaExecutionVisitor implements PlanVisitor>> createdMeasurement = + toSingleMeasurement(path.getDevicePath(), path.getMeasurement(), node.getDataType()); + syncViewColumns(node, createdMeasurement); return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Execute successfully"); } @Override public TSStatus visitCreateAlignedTimeSeries( final CreateAlignedTimeSeriesNode node, final ISchemaRegion schemaRegion) { - triggerAddViewColumn(node, null); - try { if (node.isGeneratedByPipe()) { final ICreateAlignedTimeSeriesPlan plan = @@ -149,15 +149,19 @@ public TSStatus visitCreateAlignedTimeSeries( logMetaDataException(String.format("%s: MetaData error: ", IoTDBConstant.GLOBAL_DB_NAME), e); return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); } + + final Map>> createdMeasurements = + Collections.singletonMap( + node.getDevicePath(), toMeasurements(node.getMeasurements(), node.getDataTypes())); + syncViewColumns(node, createdMeasurements); return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Execute successfully"); } @Override public TSStatus visitCreateMultiTimeSeries( final CreateMultiTimeSeriesNode node, final ISchemaRegion schemaRegion) { - triggerAddViewColumn(node, null); - final Map measurementGroupMap = node.getMeasurementGroupMap(); + final Map>> createdMeasurements = new HashMap<>(); final List failingStatus = new ArrayList<>(); PartialPath devicePath; MeasurementGroup measurementGroup; @@ -173,6 +177,11 @@ public TSStatus visitCreateMultiTimeSeries( transformToCreateTimeSeriesPlan(devicePath, measurementGroup, i); ((CreateTimeSeriesPlanImpl) createTimeSeriesPlan).setWithMerge(node.isGeneratedByPipe()); schemaRegion.createTimeSeries(createTimeSeriesPlan, -1); + recordCreatedMeasurement( + createdMeasurements, + devicePath, + measurementGroup.getMeasurements().get(i), + measurementGroup.getDataTypes().get(i)); } catch (final MetadataException e) { logMetaDataException( String.format("%s: MetaData error: ", IoTDBConstant.GLOBAL_DB_NAME), e); @@ -181,6 +190,10 @@ public TSStatus visitCreateMultiTimeSeries( } } + // The measurements which were created need their columns even when another one of the same + // statement failed: no later write of them triggers this addition again. + syncViewColumns(node, createdMeasurements); + if (!failingStatus.isEmpty()) { return RpcUtils.getStatus(failingStatus); } @@ -207,11 +220,11 @@ private ICreateTimeSeriesPlan transformToCreateTimeSeriesPlan( @Override public TSStatus visitInternalCreateTimeSeries( final InternalCreateTimeSeriesNode node, final ISchemaRegion schemaRegion) { - triggerAddViewColumn(node, null); final PartialPath devicePath = node.getDevicePath(); final MeasurementGroup measurementGroup = node.getMeasurementGroup(); + final Map>> createdMeasurements = new HashMap<>(); final List existingTimeSeriesAndAlignmentMismatch = new ArrayList<>(); final List failingStatus = new ArrayList<>(); @@ -222,7 +235,8 @@ public TSStatus visitInternalCreateTimeSeries( schemaRegion, existingTimeSeriesAndAlignmentMismatch, failingStatus, - node.isGeneratedByPipe()); + node.isGeneratedByPipe(), + createdMeasurements); } else { executeInternalCreateTimeSeries( devicePath, @@ -230,9 +244,12 @@ public TSStatus visitInternalCreateTimeSeries( schemaRegion, existingTimeSeriesAndAlignmentMismatch, failingStatus, - node.isGeneratedByPipe()); + node.isGeneratedByPipe(), + createdMeasurements); } + syncViewColumns(node, createdMeasurements); + if (!failingStatus.isEmpty()) { return RpcUtils.getStatus(failingStatus); } @@ -247,8 +264,8 @@ public TSStatus visitInternalCreateTimeSeries( @Override public TSStatus visitInternalCreateMultiTimeSeries( final InternalCreateMultiTimeSeriesNode node, final ISchemaRegion schemaRegion) { - triggerAddViewColumn(node, null); + final Map>> createdMeasurements = new HashMap<>(); PartialPath devicePath; MeasurementGroup measurementGroup; @@ -266,7 +283,8 @@ public TSStatus visitInternalCreateMultiTimeSeries( schemaRegion, existingTimeSeriesAndAlignmentMismatch, failingStatus, - node.isGeneratedByPipe()); + node.isGeneratedByPipe(), + createdMeasurements); } else { executeInternalCreateTimeSeries( devicePath, @@ -274,10 +292,13 @@ public TSStatus visitInternalCreateMultiTimeSeries( schemaRegion, existingTimeSeriesAndAlignmentMismatch, failingStatus, - node.isGeneratedByPipe()); + node.isGeneratedByPipe(), + createdMeasurements); } } + syncViewColumns(node, createdMeasurements); + if (!failingStatus.isEmpty()) { return RpcUtils.getStatus(failingStatus); } @@ -295,7 +316,8 @@ private void executeInternalCreateTimeSeries( final ISchemaRegion schemaRegion, final List existingTimeSeriesAndAlignmentMismatch, final List failingStatus, - final boolean withMerge) { + final boolean withMerge, + final Map>> createdMeasurements) { final int size = measurementGroup.getMeasurements().size(); // todo implement batch creation of one device in SchemaRegion boolean alignedIsSet = false; @@ -309,6 +331,13 @@ private void executeInternalCreateTimeSeries( // Thus the original ones are not altered ((CreateTimeSeriesPlanImpl) createTimeSeriesPlan).setWithMerge(withMerge); schemaRegion.createTimeSeries(createTimeSeriesPlan, -1); + // The alignment mismatch below does not undo the creation, so the measurement is created + // either way. + recordCreatedMeasurement( + createdMeasurements, + devicePath, + measurementGroup.getMeasurements().get(i), + measurementGroup.getDataTypes().get(i)); if (((CreateTimeSeriesPlanImpl) createTimeSeriesPlan).getAligned().get() && !alignedIsSet) { existingTimeSeriesAndAlignmentMismatch.add( new TSStatus(TSStatusCode.ALIGNED_TIMESERIES_ERROR.getStatusCode()) @@ -333,7 +362,8 @@ private void executeInternalCreateAlignedTimeSeries( final ISchemaRegion schemaRegion, final List existingTimeSeriesAndAlignmentMismatch, final List failingStatus, - final boolean withMerge) { + final boolean withMerge, + final Map>> createdMeasurements) { final List measurementList = measurementGroup.getMeasurements(); final List dataTypeList = measurementGroup.getDataTypes(); final List encodingList = measurementGroup.getEncodings(); @@ -361,8 +391,19 @@ private void executeInternalCreateAlignedTimeSeries( boolean shouldRetry = true; while (shouldRetry) { + // The retry trims the plan, so the measurements of this attempt are the ones the plan holds + // now. They are captured before the call, as the call itself trims the measurements it merges + // instead of creating. + final List attemptedMeasurements = + new ArrayList<>(createAlignedTimeSeriesPlan.getMeasurements()); + final List attemptedDataTypes = + new ArrayList<>(createAlignedTimeSeriesPlan.getDataTypes()); try { schemaRegion.createAlignedTimeSeries(createAlignedTimeSeriesPlan); + // An aligned batch is validated as a whole before it is created, so a call which returns + // without an exception has created every measurement it was given. + recordCreatedMeasurements( + createdMeasurements, devicePath, attemptedMeasurements, attemptedDataTypes); shouldRetry = false; } catch (final MeasurementAlreadyExistException e) { // The existence check will be executed before truly creation @@ -429,20 +470,75 @@ private void executeInternalCreateAlignedTimeSeries( } /** - * @param context the templates the node needs, which this visitor has already resolved for its - * own use; null for the nodes which involve no template + * Adds to the tree-to-table views the columns the measurements a schema write has created need. + * + *

It is a best-effort side effect of the write which triggers it: every failure is logged here + * and none of them is propagated, so that the write operation is never affected. + * + *

A plan which a pipe pushed is skipped: the cluster the measurements come from has already + * added their columns, and those columns reach this cluster with the view schema itself, so + * asking ConfigNode for them again can only be rejected as a duplicate. + * + * @param node the plan which triggers the addition + * @param createdMeasurements the measurements this operation really created; only they get a + * column, so that a creation which failed leaves no column behind */ - private TSStatus triggerAddViewColumn( - final WritePlanNode node, final TriggerAddViewColumnVisitor.Context context) { + private void syncViewColumns( + final WritePlanNode node, + final Map>> createdMeasurements) { + if (node.isGeneratedByPipe()) { + return; + } + final String operationId = node.getPlanNodeId().getId(); try { if (TreeToTableViewCache.getInstance().hasView()) { - node.accept(new TriggerAddViewColumnVisitor(), context); + TriggerAddViewColumnManager.addViewColumnsIfNeeded(createdMeasurements, operationId); } } catch (final Throwable e) { - logger.error( - DataNodeSchemaMessages.LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED, node.getPlanNodeId(), e); + logger.error(DataNodeSchemaMessages.LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED, operationId, e); } - return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } + + private static void recordCreatedMeasurement( + final Map>> createdMeasurements, + final PartialPath devicePath, + final String measurement, + final TSDataType dataType) { + createdMeasurements + .computeIfAbsent(devicePath, key -> new ArrayList<>()) + .add(new Pair<>(measurement, dataType)); + } + + private static void recordCreatedMeasurements( + final Map>> createdMeasurements, + final PartialPath devicePath, + final List measurements, + final List dataTypes) { + for (int i = 0; i < measurements.size(); i++) { + recordCreatedMeasurement( + createdMeasurements, devicePath, measurements.get(i), dataTypes.get(i)); + } + } + + private static Map>> toSingleMeasurement( + final PartialPath devicePath, final String measurement, final TSDataType dataType) { + return Collections.singletonMap( + devicePath, Collections.singletonList(new Pair<>(measurement, dataType))); + } + + private static List> toMeasurements(final Template template) { + return template.getSchemaMap().values().stream() + .map(schema -> new Pair<>(schema.getMeasurementName(), schema.getType())) + .toList(); + } + + private static List> toMeasurements( + final List measurements, final List dataTypes) { + final List> pairs = new ArrayList<>(measurements.size()); + for (int i = 0; i < measurements.size(); i++) { + pairs.add(new Pair<>(measurements.get(i), dataTypes.get(i))); + } + return pairs; } @Override @@ -499,23 +595,22 @@ public TSStatus visitAlterTimeSeries( @Override public TSStatus visitActivateTemplate( final ActivateTemplateNode node, final ISchemaRegion schemaRegion) { - // The template is resolved here, so that the view column trigger and the activation below share - // one instance instead of looking it up twice. final Template template = ClusterTemplateManager.getInstance().getTemplate(node.getTemplateId()); - final Map templateMap = new HashMap<>(1); if (Objects.isNull(template)) { return getUndefinedTemplateStatus(Collections.singleton(node.getTemplateId())); } - templateMap.put(node.getTemplateId(), template); try { - triggerAddViewColumn(node, new TriggerAddViewColumnVisitor.Context(templateMap)); schemaRegion.activateSchemaTemplate(node, template); - return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } catch (final MetadataException e) { logMetaDataException(e); return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); } + final PartialPath devicePath = node.getActivatePath(); + final Map>> activatedMeasurements = + Collections.singletonMap(devicePath, toMeasurements(template)); + syncViewColumns(node, activatedMeasurements); + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } @Override @@ -525,8 +620,7 @@ public TSStatus visitBatchActivateTemplate( final Set missingTemplateIds = new HashSet<>(); collectTemplates(node.getTemplateActivationMap(), templateMap, missingTemplateIds); - triggerAddViewColumn(node, new TriggerAddViewColumnVisitor.Context(templateMap)); - + final Map>> activatedMeasurements = new HashMap<>(); final List statusList = new ArrayList<>(); final List alreadyActivatedDeviceList = new ArrayList<>(); for (final Map.Entry> entry : @@ -541,6 +635,9 @@ public TSStatus visitBatchActivateTemplate( SchemaRegionWritePlanFactory.getActivateTemplateInClusterPlan( entry.getKey(), entry.getValue().right, entry.getValue().left), template); + // A device which already carries the template is reported below and gets no column here: + // its measurements were already covered when the template was activated on it. + activatedMeasurements.put(entry.getKey(), toMeasurements(template)); } catch (final MetadataException e) { if (e.getErrorCode() == TSStatusCode.TEMPLATE_IS_IN_USE.getStatusCode()) { alreadyActivatedDeviceList.add(entry.getKey()); @@ -550,6 +647,7 @@ public TSStatus visitBatchActivateTemplate( } } } + syncViewColumns(node, activatedMeasurements); if (!alreadyActivatedDeviceList.isEmpty()) { final TemplateIsInUseException e = new TemplateIsInUseException(alreadyActivatedDeviceList.toString()); @@ -569,7 +667,7 @@ public TSStatus visitInternalBatchActivateTemplate( final Set missingTemplateIds = new HashSet<>(); collectTemplates(node.getTemplateActivationMap(), templateMap, missingTemplateIds); - triggerAddViewColumn(node, new TriggerAddViewColumnVisitor.Context(templateMap)); + final Map>> activatedMeasurements = new HashMap<>(); for (final Map.Entry> entry : node.getTemplateActivationMap().entrySet()) { @@ -583,7 +681,10 @@ public TSStatus visitInternalBatchActivateTemplate( SchemaRegionWritePlanFactory.getActivateTemplateInClusterPlan( entry.getKey(), entry.getValue().right, entry.getValue().left), template); + activatedMeasurements.put(entry.getKey(), toMeasurements(template)); } catch (final TemplateIsInUseException e) { + // The measurements of a device which already carries the template were already covered when + // the template was activated on it. logger.info( String.format( DataNodeSchemaMessages.DEVICE_TEMPLATE_ALREADY_ACTIVATED_ON_PATH_FMT, @@ -593,6 +694,7 @@ public TSStatus visitInternalBatchActivateTemplate( return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); } } + syncViewColumns(node, activatedMeasurements); if (!missingTemplateIds.isEmpty()) { return getUndefinedTemplateStatus(missingTemplateIds); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index ad9c03752ba13..53a2722e16a0a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -199,7 +199,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.table.TreeToTableViewCache; -import org.apache.iotdb.db.schemaengine.table.TriggerAddViewColumnVisitor; +import org.apache.iotdb.db.schemaengine.table.TriggerAddViewColumnManager; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUpdateType; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUtil; @@ -411,7 +411,7 @@ import static org.apache.iotdb.commons.client.request.TestConnectionUtils.testConnectionsImpl; import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; import static org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement.AlterType.SET_DATA_TYPE; -import static org.apache.iotdb.db.schemaengine.table.TriggerAddViewColumnVisitor.ALTER_DEVICE_TEMPLATE_ID_PREFIX; +import static org.apache.iotdb.db.schemaengine.table.TriggerAddViewColumnManager.ALTER_DEVICE_TEMPLATE_ID_PREFIX; import static org.apache.iotdb.db.service.RegionMigrateService.REGION_MIGRATE_PROCESS; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onIoTDBException; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onQueryException; @@ -2920,7 +2920,7 @@ public TSStatus updateTemplate(final TUpdateTemplateReq req) { try { final Map>> device2Measurements = collectDevicesUsingExtendedTemplate(oldTemplate, newTemplate); - TriggerAddViewColumnVisitor.addViewColumnsIfNeeded( + TriggerAddViewColumnManager.addViewColumnsIfNeeded( device2Measurements, ALTER_DEVICE_TEMPLATE_ID_PREFIX + newTemplate.getId()); } catch (final Exception e) { // todo diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TriggerAddViewColumnManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TriggerAddViewColumnManager.java new file mode 100644 index 0000000000000..2413fe54fc46d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TriggerAddViewColumnManager.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.table; + +import org.apache.iotdb.commons.client.AbstractConfigNodeInfo; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.consensus.ConfigRegionId; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.table.AlterOrDropTableOperationType; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; +import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewFailure; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsReq; +import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsResp; +import org.apache.iotdb.db.i18n.DataNodeSchemaMessages; +import org.apache.iotdb.db.protocol.client.ConfigNodeClient; +import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator; +import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HexFormat; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Adds to the tree-to-table views the columns which the measurements a schema write has created + * need. + * + *

The addition is a best-effort side effect of the schema write, see addViewColumnsIfNeeded. + */ +public class TriggerAddViewColumnManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(TriggerAddViewColumnManager.class); + private static final IClientManager CONFIG_NODE_CLIENT_MANAGER = + ConfigNodeClientManager.getInstance(); + + public static final String QUERY_ID_PREFIX = "tree_to_table_add_column"; + public static final String ALTER_DEVICE_TEMPLATE_ID_PREFIX = "template_"; + + /** + * Adds the columns the views covering the given devices need. + * + *

It is a best-effort side effect of the write which triggers it: every failure is logged here + * and none of them is propagated, so that the write operation is never affected. The columns + * which are not added this time are only added again when a later schema write covers the same + * measurements. + * + * @param device2Measurements the measurements the schema write really created, eg. device01 -> + * {s1, s2, s3}; device02 -> {s3, s4}. Only they get a column, so that a creation which failed + * cannot leave a column behind. + * @param operationId the planNodeId of the plan which triggers this operation, for the log + * messages only: it is a per-query index, so it does not identify the operation + */ + public static void addViewColumnsIfNeeded( + final Map>> device2Measurements, + final String operationId) { + if (device2Measurements == null || device2Measurements.isEmpty()) { + return; + } + + // the views to be added columns + final Map>> view2Measurements = + collectViewMeasurements(device2Measurements); + if (view2Measurements.isEmpty()) { + return; + } + + try (ConfigNodeClient configNodeClient = + CONFIG_NODE_CLIENT_MANAGER.borrowClient(AbstractConfigNodeInfo.CONFIG_REGION_ID)) { + List updateViewReqs = + view2Measurements.entrySet().stream() + .map(TriggerAddViewColumnManager::toAlterTableReq) + .toList(); + + TUpdateTree2TableViewsResp resp = + configNodeClient.updateTree2TableViews(new TUpdateTree2TableViewsReq(updateViewReqs)); + if (resp == null || resp.getStatus() == null) { + LOGGER.error(DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE, operationId); + return; + } + if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.error( + DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_FAILED, operationId, resp.getStatus()); + } + // the views are updated one by one in ConfigNode, so the failed ones are reported one by one + if (resp.getFailedViews() != null) { + for (TUpdateTree2TableViewFailure failedView : resp.getFailedViews()) { + LOGGER.error( + DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW, + failedView.getDatabase(), + failedView.getTableName(), + operationId, + failedView.getStatus()); + } + } + } catch (ClientManagerException | TException e) { + LOGGER.warn(DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE, operationId, e); + } + } + + /** + * @return the measurements each view covering the given devices needs; eg: view01 -> {s1, s2, + * s3}, view02 -> {s1, s2, s3, s4} + */ + private static Map>> + collectViewMeasurements( + Map>> device2Measurements) { + + // deduplicate the measurement by set, the measurement with same name and dataType + // from different device would count once, eg: dev01.s1(int32) and dev02.s1(int32) + // other deduplication would leave to ConfigMTree in CN + final Map>> view2Measurements = + new HashMap<>(); + + for (Map.Entry>> entry : + device2Measurements.entrySet()) { + PartialPath path = entry.getKey(); + Set treeToTableViews = + TreeToTableViewCache.getInstance().getTreeToTableViews(path); + + treeToTableViews.forEach( + view -> + view2Measurements + .computeIfAbsent(view, key -> new LinkedHashSet<>()) + .addAll(entry.getValue())); + } + return view2Measurements; + } + + private static TAlterOrDropTableReq toAlterTableReq( + Map.Entry>> view2Measurements) { + final TreeToTableViewReference view = view2Measurements.getKey(); + final List columnSchemas = + view2Measurements.getValue().stream() + .map(TriggerAddViewColumnManager::toColumnSchema) + .toList(); + TAlterOrDropTableReq tAlterOrDropTableReq = + new TAlterOrDropTableReq( + view.getDatabaseName(), + view.getTableName(), + buildQueryId( + columnFingerprint(columnSchemas), view.getDatabaseName(), view.getTableName()), + AlterOrDropTableOperationType.ADD_COLUMN.getTypeValue(), + ByteBuffer.wrap(TsTableColumnSchemaUtil.serialize(columnSchemas))); + tAlterOrDropTableReq.setIsView(true); + return tAlterOrDropTableReq; + } + + // transform the upperCase to the from-attribute + private static TsTableColumnSchema toColumnSchema(final Pair measurement) { + final String measurementName = measurement.getLeft(); + final String from = + measurementName.chars().anyMatch(Character::isUpperCase) ? measurementName : null; + return TableHeaderSchemaValidator.generateColumnSchema( + TsTableColumnCategory.FIELD, + measurementName.toLowerCase(Locale.ROOT), + measurement.getRight(), + null, + from); + } + + // leader and follower would build the same queryId when apply the same log; so CN could recognize + // the same add view column request + private static String buildQueryId(String columnFingerprint, String database, String table) { + return QUERY_ID_PREFIX + "_" + columnFingerprint + "_" + database + "_" + table; + } + + /** + * The columns are sorted first so that the same column set always yields the same fingerprint, no + * matter in which order the measurements were collected. + */ + private static String columnFingerprint(final List columnSchemas) { + final List sortedColumnSchemas = new ArrayList<>(columnSchemas); + sortedColumnSchemas.sort(Comparator.comparing(TsTableColumnSchema::getColumnName)); + return digestOf(TsTableColumnSchemaUtil.serialize(sortedColumnSchemas)); + } + + private static String digestOf(final byte[] content) { + try { + // 4 bytes are enough here, the number of distinct column sets of one view is rather small + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content), 0, 4); + } catch (final NoSuchAlgorithmException e) { + // SHA-256 is supported by every Java platform + throw new IllegalStateException(e); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TriggerAddViewColumnVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TriggerAddViewColumnVisitor.java deleted file mode 100644 index 878119533031e..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TriggerAddViewColumnVisitor.java +++ /dev/null @@ -1,389 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.schemaengine.table; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.AbstractConfigNodeInfo; -import org.apache.iotdb.commons.client.IClientManager; -import org.apache.iotdb.commons.client.exception.ClientManagerException; -import org.apache.iotdb.commons.consensus.ConfigRegionId; -import org.apache.iotdb.commons.path.MeasurementPath; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; -import org.apache.iotdb.commons.schema.table.AlterOrDropTableOperationType; -import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; -import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; -import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; -import org.apache.iotdb.commons.schema.template.Template; -import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewFailure; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsReq; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsResp; -import org.apache.iotdb.db.i18n.DataNodeSchemaMessages; -import org.apache.iotdb.db.protocol.client.ConfigNodeClient; -import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.ActivateTemplateNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.BatchActivateTemplateNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.CreateAlignedTimeSeriesNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.CreateMultiTimeSeriesNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.CreateTimeSeriesNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.InternalBatchActivateTemplateNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.InternalCreateMultiTimeSeriesNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.InternalCreateTimeSeriesNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.MeasurementGroup; -import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator; -import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.apache.thrift.TException; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HexFormat; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -/** - * Adds the columns which the tree-to-table views covering the created timeseries need. - * - *

The trigger is a best-effort side effect of the schema creation, see addViewColumnsIfNeeded. - */ -public class TriggerAddViewColumnVisitor - implements PlanVisitor { - - private static final Logger LOGGER = LoggerFactory.getLogger(TriggerAddViewColumnVisitor.class); - private static final IClientManager CONFIG_NODE_CLIENT_MANAGER = - ConfigNodeClientManager.getInstance(); - - public static final String QUERY_ID_PREFIX = "tree_to_table_add_column"; - public static final String ALTER_DEVICE_TEMPLATE_ID_PREFIX = "template_"; - - public static final class Context { - - private final Map templates; - - public Context(Map templates) { - this.templates = Objects.isNull(templates) ? Collections.emptyMap() : templates; - } - - public Map getTemplates() { - return templates; - } - } - - /** - * Adds the columns the views covering the given devices need. - * - *

It is a best-effort side effect of the write which triggers it: every failure is logged here - * and none of them is propagated, so that the write operation is never affected. The columns - * which are not added this time are added by a later write of the same measurements, as every - * write plan carries its own planNodeId, hence its own queryId. - * - * @param device2Measurements eg. device01 -> {s1, s2, s3}; device02 -> {s3, s4} - * @param operationId the planNodeId of the plan which triggers this operation - */ - public static void addViewColumnsIfNeeded( - Map>> device2Measurements, String operationId) { - if (device2Measurements == null || device2Measurements.isEmpty()) { - return; - } - final Map>> view2Measurements = - collectViewMeasurements(device2Measurements); - if (view2Measurements.isEmpty()) { - return; - } - - try (ConfigNodeClient configNodeClient = - CONFIG_NODE_CLIENT_MANAGER.borrowClient(AbstractConfigNodeInfo.CONFIG_REGION_ID)) { - List updateViewReqs = - view2Measurements.entrySet().stream() - .map(entry -> TriggerAddViewColumnVisitor.toAlterTableReq(entry, operationId)) - .toList(); - - TUpdateTree2TableViewsResp resp = - configNodeClient.updateTree2TableViews(new TUpdateTree2TableViewsReq(updateViewReqs)); - if (resp == null || resp.getStatus() == null) { - LOGGER.error(DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE, operationId); - return; - } - if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - LOGGER.error( - DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_FAILED, operationId, resp.getStatus()); - } - // the views are updated one by one in ConfigNode, so the failed ones are reported one by one - if (resp.getFailedViews() != null) { - for (TUpdateTree2TableViewFailure failedView : resp.getFailedViews()) { - LOGGER.error( - DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW, - failedView.getDatabase(), - failedView.getTableName(), - operationId, - failedView.getStatus()); - } - } - } catch (ClientManagerException | TException e) { - LOGGER.warn(DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE, operationId, e); - } - } - - /** - * @return the measurements each view covering the given devices needs; eg: view01 -> {s1, s2, - * s3}, view02 -> {s1, s2, s3, s4} - */ - private static Map>> - collectViewMeasurements( - Map>> device2Measurements) { - - // deduplicate the measurement by set, the measurement with same name and dataType - // from different device would count once, eg: dev01.s1(int32) and dev02.s1(int32) - // other deduplication would leave to ConfigMTree in CN - final Map>> view2Measurements = - new HashMap<>(); - - for (Map.Entry>> entry : - device2Measurements.entrySet()) { - PartialPath path = entry.getKey(); - Set treeToTableViews = - TreeToTableViewCache.getInstance().getTreeToTableViews(path); - - treeToTableViews.forEach( - view -> - view2Measurements - .computeIfAbsent(view, key -> new LinkedHashSet<>()) - .addAll(entry.getValue())); - } - return view2Measurements; - } - - private static TAlterOrDropTableReq toAlterTableReq( - Map.Entry>> view2Measurements, - String operationId) { - final TreeToTableViewReference view = view2Measurements.getKey(); - final List columnSchemas = - view2Measurements.getValue().stream() - .map(TriggerAddViewColumnVisitor::toColumnSchema) - .toList(); - TAlterOrDropTableReq tAlterOrDropTableReq = - new TAlterOrDropTableReq( - view.getDatabaseName(), - view.getTableName(), - buildQueryId( - operationId + "_" + columnFingerprint(columnSchemas), - view.getDatabaseName(), - view.getTableName()), - AlterOrDropTableOperationType.ADD_COLUMN.getTypeValue(), - ByteBuffer.wrap(TsTableColumnSchemaUtil.serialize(columnSchemas))); - tAlterOrDropTableReq.setIsView(true); - return tAlterOrDropTableReq; - } - - // transform the upperCase to the from-attribute - private static TsTableColumnSchema toColumnSchema(final Pair measurement) { - final String measurementName = measurement.getLeft(); - final String from = - measurementName.chars().anyMatch(Character::isUpperCase) ? measurementName : null; - return TableHeaderSchemaValidator.generateColumnSchema( - TsTableColumnCategory.FIELD, - measurementName.toLowerCase(Locale.ROOT), - measurement.getRight(), - null, - from); - } - - @Override - public TSStatus visitCreateTimeSeries( - CreateTimeSeriesNode node, TriggerAddViewColumnVisitor.Context context) { - final MeasurementPath path = node.getPath(); - final Pair measurement = - new Pair<>(path.getMeasurement(), node.getDataType()); - final Map>> device2Measurements = - Collections.singletonMap(path.getDevicePath(), Collections.singletonList(measurement)); - addViewColumnsIfNeeded(device2Measurements, node.getPlanNodeId().getId()); - return null; - } - - @Override - public TSStatus visitCreateAlignedTimeSeries( - CreateAlignedTimeSeriesNode node, TriggerAddViewColumnVisitor.Context context) { - final Map>> device2Measurements = - Collections.singletonMap( - node.getDevicePath(), toMeasurements(node.getMeasurements(), node.getDataTypes())); - addViewColumnsIfNeeded(device2Measurements, node.getPlanNodeId().getId()); - return null; - } - - @Override - public TSStatus visitCreateMultiTimeSeries( - CreateMultiTimeSeriesNode node, TriggerAddViewColumnVisitor.Context context) { - final Map>> device2Measurements = - toDevice2Measurements(node.getMeasurementGroupMap()); - addViewColumnsIfNeeded(device2Measurements, node.getPlanNodeId().getId()); - return null; - } - - @Override - public TSStatus visitInternalCreateTimeSeries( - InternalCreateTimeSeriesNode node, TriggerAddViewColumnVisitor.Context context) { - final Map>> device2Measurements = - Collections.singletonMap(node.getDevicePath(), toMeasurements(node.getMeasurementGroup())); - addViewColumnsIfNeeded(device2Measurements, node.getPlanNodeId().getId()); - return null; - } - - @Override - public TSStatus visitInternalCreateMultiTimeSeries( - InternalCreateMultiTimeSeriesNode node, TriggerAddViewColumnVisitor.Context context) { - // the alignment flag is of no use to a view, only the measurements of the devices matter - final Map>> device2Measurements = - new HashMap<>(node.getDeviceMap().size()); - for (final Map.Entry> entry : - node.getDeviceMap().entrySet()) { - device2Measurements.put(entry.getKey(), toMeasurements(entry.getValue().getRight())); - } - addViewColumnsIfNeeded(device2Measurements, node.getPlanNodeId().getId()); - return null; - } - - @Override - public TSStatus visitActivateTemplate( - ActivateTemplateNode node, TriggerAddViewColumnVisitor.Context context) { - final Template template = context.getTemplates().get(node.getTemplateId()); - final Map>> device2Measurements = - Collections.singletonMap(node.getActivatePath(), toMeasurements(template)); - addViewColumnsIfNeeded(device2Measurements, node.getPlanNodeId().getId()); - return null; - } - - @Override - public TSStatus visitBatchActivateTemplate( - BatchActivateTemplateNode node, TriggerAddViewColumnVisitor.Context context) { - addViewColumnsIfNeeded( - toActivationDevice2Measurements(node.getTemplateActivationMap(), context), - node.getPlanNodeId().getId()); - return null; - } - - @Override - public TSStatus visitInternalBatchActivateTemplate( - InternalBatchActivateTemplateNode node, TriggerAddViewColumnVisitor.Context context) { - addViewColumnsIfNeeded( - toActivationDevice2Measurements(node.getTemplateActivationMap(), context), - node.getPlanNodeId().getId()); - return null; - } - - private static Map>> toActivationDevice2Measurements( - final Map> templateActivationMap, - final TriggerAddViewColumnVisitor.Context context) { - final Map>> device2Measurements = - new HashMap<>(templateActivationMap.size()); - for (final Map.Entry> entry : - templateActivationMap.entrySet()) { - final Template template = context.getTemplates().get(entry.getValue().getLeft()); - if (Objects.isNull(template)) { - continue; - } - device2Measurements.put(entry.getKey(), toMeasurements(template)); - } - return device2Measurements; - } - - private static Map>> toDevice2Measurements( - final Map measurementGroupMap) { - final Map>> device2Measurements = - new HashMap<>(measurementGroupMap.size()); - for (final Map.Entry entry : measurementGroupMap.entrySet()) { - device2Measurements.put(entry.getKey(), toMeasurements(entry.getValue())); - } - return device2Measurements; - } - - private static List> toMeasurements(final Template template) { - return template.getSchemaMap().values().stream() - .map(schema -> new Pair<>(schema.getMeasurementName(), schema.getType())) - .toList(); - } - - private static List> toMeasurements(MeasurementGroup measurementGroup) { - return toMeasurements(measurementGroup.getMeasurements(), measurementGroup.getDataTypes()); - } - - private static List> toMeasurements( - final List measurements, final List dataTypes) { - final List> result = new ArrayList<>(measurements.size()); - for (int i = 0; i < measurements.size(); i++) { - result.add(new Pair<>(measurements.get(i), dataTypes.get(i))); - } - return result; - } - - @Override - public TSStatus visitPlan(PlanNode node, TriggerAddViewColumnVisitor.Context context) { - throw new UnsupportedOperationException(); - } - - /** - * The queryId identifies one column-adding operation in ConfigNode, so as to deduplicate the - * requests sent by the leader and the followers while applying the same schema region log entry. - * - *

The {@code operationId} is the planNodeId of the plan which triggers the operation, and it - * is identical on every replica of one log entry. The database, the table and the fingerprint of - * the columns are appended because ConfigNode only compares the queryId, instead of the table, - * while deduplicating: without them the operation of another view, or of another column set, - * would be treated as a duplicate and silently skipped. - */ - private static String buildQueryId(String operationId, String database, String table) { - return QUERY_ID_PREFIX + "_" + operationId + "_" + database + "_" + table; - } - - /** - * The columns are sorted first so that the same column set always yields the same fingerprint, no - * matter in which order the measurements were collected. - */ - private static String columnFingerprint(final List columnSchemas) { - final List sortedColumnSchemas = new ArrayList<>(columnSchemas); - sortedColumnSchemas.sort(Comparator.comparing(TsTableColumnSchema::getColumnName)); - return digestOf(TsTableColumnSchemaUtil.serialize(sortedColumnSchemas)); - } - - private static String digestOf(final byte[] content) { - try { - // 4 bytes are enough here, the number of distinct column sets of one view is rather small - return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content), 0, 4); - } catch (final NoSuchAlgorithmException e) { - // SHA-256 is supported by every Java platform - throw new IllegalStateException(e); - } - } -} From 7ff273ddfcf3d0b2d8a3023de66bf8c13e81b88b Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Thu, 15 Oct 2026 17:47:29 +0800 Subject: [PATCH 04/24] rename the interface --- .../confignode/manager/ConfigManager.java | 10 +++--- .../iotdb/confignode/manager/IManager.java | 6 ++-- .../confignode/manager/ProcedureManager.java | 30 ++++++++--------- .../table/view/AddViewColumnProcedure.java | 8 ++--- .../table/view/CreateTableViewProcedure.java | 8 ++--- .../schema/table/view/DropViewProcedure.java | 4 +-- .../table/view/RenameViewProcedure.java | 4 +-- .../view/SetViewPropertiesProcedure.java | 19 +++++------ .../thrift/ConfigNodeRPCServiceProcessor.java | 8 ++--- .../schemaregion/SchemaExecutionVisitor.java | 4 +-- .../db/protocol/client/ConfigNodeClient.java | 8 ++--- .../impl/DataNodeInternalRPCServiceImpl.java | 10 +++--- .../table/DataNodeTableCache.java | 7 ++-- ...mnManager.java => ViewColumnAutoSync.java} | 32 +++++++++---------- ... => TreeToTableViewTrieOperationType.java} | 6 ++-- .../src/main/thrift/confignode.thrift | 10 +++--- 16 files changed, 87 insertions(+), 87 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/{TriggerAddViewColumnManager.java => ViewColumnAutoSync.java} (89%) rename iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/{TreeToTableViewOperationType.java => TreeToTableViewTrieOperationType.java} (90%) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 2c6fc80e17ff3..97fc460320888 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -158,6 +158,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterTimeSeriesReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterTopicReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TClusterParameters; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; @@ -265,8 +267,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsReq; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsResp; import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.db.schemaengine.template.TemplateAlterOperationType; @@ -3381,11 +3381,11 @@ public TSStatus alterOrDropTable(final TAlterOrDropTableReq req) { } @Override - public TUpdateTree2TableViewsResp updateTree2TableViews(TUpdateTree2TableViewsReq req) { + public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) { final TSStatus status = confirmLeader(); return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() - ? procedureManager.addColumnForTree2TableViews(req) - : new TUpdateTree2TableViewsResp(status); + ? procedureManager.autoSyncViewColumns(req) + : new TAutoSyncViewColumnsResp(status); } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index f6e5bf897decb..9d2ea4c725d03 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -72,6 +72,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterTimeSeriesReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterTopicReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp; @@ -168,8 +170,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsReq; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsResp; import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.rpc.TSStatusCode; @@ -921,7 +921,7 @@ TPermissionInfoResp login( TSStatus alterOrDropTable(final TAlterOrDropTableReq req); - TUpdateTree2TableViewsResp updateTree2TableViews(TUpdateTree2TableViewsReq req); + TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req); TDeleteTableDeviceResp deleteDevice(final TDeleteTableDeviceReq req); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 5aca993c894dc..1c51132317c73 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -152,6 +152,9 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterTopicReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsFailure; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; @@ -170,9 +173,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TRemoveRegionReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewFailure; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsReq; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsResp; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.exception.BatchProcessException; import org.apache.iotdb.mpp.rpc.thrift.TDropTableModProgress; @@ -2282,12 +2282,12 @@ public TSStatus alterTableAddColumn(final TAlterOrDropTableReq req) { false)); } - public TUpdateTree2TableViewsResp addColumnForTree2TableViews(TUpdateTree2TableViewsReq req) { + public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) { if (req == null || !req.isSetViewInfos()) { - return new TUpdateTree2TableViewsResp(RpcUtils.getStatus(TSStatusCode.ILLEGAL_PARAMETER)); + return new TAutoSyncViewColumnsResp(RpcUtils.getStatus(TSStatusCode.ILLEGAL_PARAMETER)); } - final List failedViews = new ArrayList<>(); + final List failedViews = new ArrayList<>(); final List>> procedures = new ArrayList<>(); // The requests which overlapped with another task on the same table, together with the status @@ -2316,13 +2316,13 @@ public TUpdateTree2TableViewsResp addColumnForTree2TableViews(TUpdateTree2TableV final TSStatus status = waitingProcedureFinished(procedure.getRight(), remainingWaitTime); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { failedViews.add( - new TUpdateTree2TableViewFailure( + new TAutoSyncViewColumnsFailure( procedure.getLeft().database, procedure.getLeft().tableName, status)); } } final TSStatus status = failedViews.isEmpty() ? StatusUtils.OK : StatusUtils.INTERNAL_ERROR; - final TUpdateTree2TableViewsResp response = new TUpdateTree2TableViewsResp(status); + final TAutoSyncViewColumnsResp response = new TAutoSyncViewColumnsResp(status); if (!failedViews.isEmpty()) { response.setFailedViews(failedViews); } @@ -2343,9 +2343,9 @@ private static boolean isAddViewColumnReq(final @Nullable TAlterOrDropTableReq a && addColumnReq.isIsView(); } - private static TUpdateTree2TableViewFailure illegalAddViewColumnFailure( + private static TAutoSyncViewColumnsFailure illegalAddViewColumnFailure( final TAlterOrDropTableReq addColumnReq) { - return new TUpdateTree2TableViewFailure( + return new TAutoSyncViewColumnsFailure( Objects.isNull(addColumnReq) ? "" : Objects.toString(addColumnReq.getDatabase(), ""), Objects.isNull(addColumnReq) ? "" : Objects.toString(addColumnReq.getTableName(), ""), RpcUtils.getStatus(TSStatusCode.ILLEGAL_PARAMETER)); @@ -2358,7 +2358,7 @@ private static TUpdateTree2TableViewFailure illegalAddViewColumnFailure( private TSStatus submitAddViewColumn( final TAlterOrDropTableReq addColumnReq, final List>> procedures, - final List failedViews) { + final List failedViews) { final AddViewColumnProcedure procedure; procedure = new AddViewColumnProcedure( @@ -2386,7 +2386,7 @@ private TSStatus submitAddViewColumn( // null means the procedure could have finished at this moment if (Objects.isNull(repeatedProcedure)) { failedViews.add( - new TUpdateTree2TableViewFailure( + new TAutoSyncViewColumnsFailure( addColumnReq.database, addColumnReq.tableName, RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR))); @@ -2395,7 +2395,7 @@ private TSStatus submitAddViewColumn( } } else if (status.getCode() != TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode()) { failedViews.add( - new TUpdateTree2TableViewFailure(addColumnReq.database, addColumnReq.tableName, status)); + new TAutoSyncViewColumnsFailure(addColumnReq.database, addColumnReq.tableName, status)); } return status; } @@ -2408,7 +2408,7 @@ private TSStatus submitAddViewColumn( private void processOverlappingTask( final List> overlappingReqs, final List>> procedures, - final List failedViews) { + final List failedViews) { List> pending = overlappingReqs; for (int round = 0; round < OVERLAP_RETRY_ROUNDS && !pending.isEmpty(); round++) { try { @@ -2430,7 +2430,7 @@ private void processOverlappingTask( for (final Pair request : pending) { failedViews.add( - new TUpdateTree2TableViewFailure( + new TAutoSyncViewColumnsFailure( request.getLeft().database, request.getLeft().tableName, request.getRight())); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java index 4975b1b8f3be8..1fe316ebb6e3a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.exception.SemanticException; -import org.apache.iotdb.commons.schema.table.TreeToTableViewOperationType; +import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType; import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; @@ -50,7 +50,7 @@ import java.util.Map; import java.util.Optional; -import static org.apache.iotdb.commons.schema.table.TreeToTableViewOperationType.ALTER_TAG; +import static org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType.ALTER_TAG; import static org.apache.iotdb.commons.schema.table.TreeViewSchema.AUTO_SYNC_SCHEMA; public class AddViewColumnProcedure extends AddTableColumnProcedure { @@ -105,7 +105,7 @@ protected void columnCheck(final ConfigNodeProcedureEnv env) { @Override protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldName) { - final Optional type = tree2TableViewOperationType(); + final Optional type = tree2TableViewOperationType(); if (type.isEmpty()) { return super.buildCommitReleaseTableReq(oldName); } @@ -115,7 +115,7 @@ protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldN return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName, viewInfo); } - private Optional tree2TableViewOperationType() { + private Optional tree2TableViewOperationType() { final Optional propValue = table.getPropValue(AUTO_SYNC_SCHEMA); if (propValue.isPresent() && Boolean.parseBoolean(propValue.get())) { final boolean hasTagColumn = diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java index 082a18fd4497a..4adb7fc00c579 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java @@ -23,7 +23,7 @@ import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.schema.table.TableNodeStatus; -import org.apache.iotdb.commons.schema.table.TreeToTableViewOperationType; +import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType; import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; @@ -137,7 +137,7 @@ protected TUpdateTableReq buildCommitReleaseTableReq() { if (newAutoSync) { // Whether the view is a new one, or replace the old table, just add view final TUpdateTree2TableView viewInfo = - new TUpdateTree2TableView(TreeToTableViewOperationType.ADD.getOperationType()); + new TUpdateTree2TableView(TreeToTableViewTrieOperationType.ADD.getOperationType()); table.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).ifPresent(viewInfo::setTreePath); viewInfo.setIsRestrict(TreeViewSchema.isRestrict(table)); viewInfo.setTagNum(table.getTagNum()); @@ -156,7 +156,7 @@ protected TUpdateTableReq buildCommitReleaseTableReq() { database, table.getTableName(), null, - new TUpdateTree2TableView(TreeToTableViewOperationType.REMOVE.getOperationType())); + new TUpdateTree2TableView(TreeToTableViewTrieOperationType.REMOVE.getOperationType())); } return super.buildCommitReleaseTableReq(); } @@ -179,7 +179,7 @@ protected TUpdateTableReq buildRollbackTableReq() { return super.buildRollbackTableReq(); } final TUpdateTree2TableView viewInfo = - new TUpdateTree2TableView(TreeToTableViewOperationType.ADD.getOperationType()); + new TUpdateTree2TableView(TreeToTableViewTrieOperationType.ADD.getOperationType()); oldView.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).ifPresent(viewInfo::setTreePath); viewInfo.setIsRestrict(TreeViewSchema.isRestrict(oldView)); viewInfo.setTagNum(oldView.getTagNum()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java index 1e141e6f69850..f2117d6c5b927 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java @@ -19,7 +19,7 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table.view; -import org.apache.iotdb.commons.schema.table.TreeToTableViewOperationType; +import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType; import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure; import org.apache.iotdb.confignode.procedure.store.ProcedureType; @@ -50,7 +50,7 @@ protected TUpdateTableReq buildCommitReleaseTableReq(final String oldName) { database, table.getTableName(), null, - new TUpdateTree2TableView(TreeToTableViewOperationType.REMOVE.getOperationType())); + new TUpdateTree2TableView(TreeToTableViewTrieOperationType.REMOVE.getOperationType())); } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java index 407463c5ff6e9..5b163934b1965 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java @@ -19,7 +19,7 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table.view; -import org.apache.iotdb.commons.schema.table.TreeToTableViewOperationType; +import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType; import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableProcedure; import org.apache.iotdb.confignode.procedure.store.ProcedureType; @@ -54,7 +54,7 @@ protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldN } // A rename does not move the view, so the old name is all the DataNode needs. final TUpdateTree2TableView viewInfo = - new TUpdateTree2TableView(TreeToTableViewOperationType.ALTER_RENAME.getOperationType()); + new TUpdateTree2TableView(TreeToTableViewTrieOperationType.ALTER_RENAME.getOperationType()); return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName, viewInfo); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java index 5f1b5c2619551..8e6e85925b992 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java @@ -19,7 +19,7 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table.view; -import org.apache.iotdb.commons.schema.table.TreeToTableViewOperationType; +import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType; import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.confignode.persistence.schema.TreeDeviceViewFieldDetector; @@ -56,12 +56,12 @@ public SetViewPropertiesProcedure( @Override protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldName) { - final Optional type = tree2TableViewOperationType(); + final Optional type = tree2TableViewOperationType(); if (type.isEmpty()) { return super.buildCommitReleaseTableReq(oldName); } final TUpdateTree2TableView viewInfo = new TUpdateTree2TableView(type.get().getOperationType()); - if (TreeToTableViewOperationType.ALTER_AUTO_SYNC == type.get()) { + if (TreeToTableViewTrieOperationType.ALTER_AUTO_SYNC == type.get()) { // Turning the auto detection on mounts the view: the DataNode needs its whole definition. table.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).ifPresent(viewInfo::setTreePath); viewInfo.setIsRestrict(TreeViewSchema.isRestrict(table)); @@ -76,7 +76,7 @@ protected void commitReleaseWithHA(final ConfigNodeProcedureEnv env) { super.commitReleaseWithHA(env); // Turning the auto detection on mounts the view, so the series which already exist under its // path pattern need the same second detection as the ones of a newly created view. - if (TreeToTableViewOperationType.ALTER_AUTO_SYNC + if (TreeToTableViewTrieOperationType.ALTER_AUTO_SYNC == tree2TableViewOperationType().orElse(null)) { reconcileTree2TableViewColumns(env); } @@ -104,11 +104,11 @@ private void reconcileTree2TableViewColumns(final ConfigNodeProcedureEnv env) { * change it. Turning auto_sync_schema on registers the view, turning it off unregisters it, and * any other property leaves the index alone. */ - private Optional tree2TableViewOperationType() { + private Optional tree2TableViewOperationType() { if (!updatedProperties.containsKey(AUTO_SYNC_SCHEMA)) { return Optional.empty(); } - final TreeToTableViewOperationType type = + final TreeToTableViewTrieOperationType type = indexEffectOf(updatedProperties.get(AUTO_SYNC_SCHEMA)); // An absent auto_sync_schema means false, so setting it to false is not a change. return type == indexEffectOf(originalProperties.get(AUTO_SYNC_SCHEMA)) @@ -116,10 +116,11 @@ private Optional tree2TableViewOperationType() { : Optional.of(type); } - private static TreeToTableViewOperationType indexEffectOf(final @Nullable String autoSyncSchema) { + private static TreeToTableViewTrieOperationType indexEffectOf( + final @Nullable String autoSyncSchema) { return Boolean.parseBoolean(autoSyncSchema) - ? TreeToTableViewOperationType.ALTER_AUTO_SYNC - : TreeToTableViewOperationType.REMOVE; + ? TreeToTableViewTrieOperationType.ALTER_AUTO_SYNC + : TreeToTableViewTrieOperationType.REMOVE; } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index f62837bd428ef..489402cb11e6e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -108,6 +108,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerRelationalReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp; import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeHeartbeatReq; @@ -235,8 +237,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp; import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsReq; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsResp; import org.apache.iotdb.confignode.service.ConfigNode; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.consensus.exception.ConsensusGroupNotExistException; @@ -1534,8 +1534,8 @@ public TSStatus createTableView(final TCreateTableViewReq req) { } @Override - public TUpdateTree2TableViewsResp updateTree2TableViews(TUpdateTree2TableViewsReq req) { - return configManager.updateTree2TableViews(req); + public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) { + return configManager.autoSyncViewColumns(req); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index 742539a56874a..f84b907c2563c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -80,7 +80,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateAlignedTimeSeriesPlanImpl; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateTimeSeriesPlanImpl; import org.apache.iotdb.db.schemaengine.table.TreeToTableViewCache; -import org.apache.iotdb.db.schemaengine.table.TriggerAddViewColumnManager; +import org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSync; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -492,7 +492,7 @@ private void syncViewColumns( final String operationId = node.getPlanNodeId().getId(); try { if (TreeToTableViewCache.getInstance().hasView()) { - TriggerAddViewColumnManager.addViewColumnsIfNeeded(createdMeasurements, operationId); + ViewColumnAutoSync.sync(createdMeasurements, operationId); } } catch (final Throwable e) { logger.error(DataNodeSchemaMessages.LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED, operationId, e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 72aeed07483de..02d6c51ef2dcf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -63,6 +63,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerRelationalReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp; import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeHeartbeatReq; @@ -188,8 +190,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp; import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsReq; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsResp; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; @@ -1318,10 +1318,10 @@ public TSStatus createTableView(TCreateTableViewReq req) throws TException { } @Override - public TUpdateTree2TableViewsResp updateTree2TableViews(TUpdateTree2TableViewsReq req) + public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) throws TException { return executeRemoteCallWithRetry( - () -> client.updateTree2TableViews(req), status -> !updateConfigNodeLeader(status.status)); + () -> client.autoSyncViewColumns(req), status -> !updateConfigNodeLeader(status.status)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 53a2722e16a0a..a977f16f2a74d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -199,7 +199,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.table.TreeToTableViewCache; -import org.apache.iotdb.db.schemaengine.table.TriggerAddViewColumnManager; +import org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSync; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUpdateType; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUtil; @@ -411,7 +411,7 @@ import static org.apache.iotdb.commons.client.request.TestConnectionUtils.testConnectionsImpl; import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; import static org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement.AlterType.SET_DATA_TYPE; -import static org.apache.iotdb.db.schemaengine.table.TriggerAddViewColumnManager.ALTER_DEVICE_TEMPLATE_ID_PREFIX; +import static org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSync.EXTEND_DEVICE_TEMPLATE_ID_PREFIX; import static org.apache.iotdb.db.service.RegionMigrateService.REGION_MIGRATE_PROCESS; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onIoTDBException; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onQueryException; @@ -2915,13 +2915,13 @@ public TSStatus updateTemplate(final TUpdateTemplateReq req) { .updateSubtreeMeasurementCountForTemplate(newTemplate.getId(), delta); } if (TreeToTableViewCache.getInstance().hasView()) { - // addViewColumnsIfNeeded, which collects the views from the index and asks CN to add the + // ViewColumnAutoSync.sync, which collects the views from the index and asks CN to add the // columns. try { final Map>> device2Measurements = collectDevicesUsingExtendedTemplate(oldTemplate, newTemplate); - TriggerAddViewColumnManager.addViewColumnsIfNeeded( - device2Measurements, ALTER_DEVICE_TEMPLATE_ID_PREFIX + newTemplate.getId()); + ViewColumnAutoSync.sync( + device2Measurements, EXTEND_DEVICE_TEMPLATE_ID_PREFIX + newTemplate.getId()); } catch (final Exception e) { // todo LOGGER.warn( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index d8c9b28721372..4a1cbef37fcd0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -32,7 +32,7 @@ import org.apache.iotdb.commons.schema.table.NonCommittableTsTable; import org.apache.iotdb.commons.schema.table.PreDeleteTsTable; import org.apache.iotdb.commons.schema.table.TableNodeStatus; -import org.apache.iotdb.commons.schema.table.TreeToTableViewOperationType; +import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType; import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; @@ -432,8 +432,9 @@ private void operateOnTree2TableViewCache( if (Objects.isNull(updateTree2TableViewInfo)) { return; } - final TreeToTableViewOperationType type = - TreeToTableViewOperationType.getType(updateTree2TableViewInfo.getTree2TableOperationType()); + final TreeToTableViewTrieOperationType type = + TreeToTableViewTrieOperationType.getType( + updateTree2TableViewInfo.getTree2TableOperationType()); try { switch (type) { case REMOVE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TriggerAddViewColumnManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java similarity index 89% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TriggerAddViewColumnManager.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java index 2413fe54fc46d..5f6537b891d9c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TriggerAddViewColumnManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java @@ -29,9 +29,9 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewFailure; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsReq; -import org.apache.iotdb.confignode.rpc.thrift.TUpdateTree2TableViewsResp; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsFailure; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp; import org.apache.iotdb.db.i18n.DataNodeSchemaMessages; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; @@ -62,16 +62,18 @@ * Adds to the tree-to-table views the columns which the measurements a schema write has created * need. * - *

The addition is a best-effort side effect of the schema write, see addViewColumnsIfNeeded. + *

It is the automatic counterpart of the {@code ALTER VIEW ... ADD COLUMN} statement: the + * columns come from the measurements the schema write has just created, instead of from the + * statement itself. */ -public class TriggerAddViewColumnManager { +public class ViewColumnAutoSync { - private static final Logger LOGGER = LoggerFactory.getLogger(TriggerAddViewColumnManager.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ViewColumnAutoSync.class); private static final IClientManager CONFIG_NODE_CLIENT_MANAGER = ConfigNodeClientManager.getInstance(); public static final String QUERY_ID_PREFIX = "tree_to_table_add_column"; - public static final String ALTER_DEVICE_TEMPLATE_ID_PREFIX = "template_"; + public static final String EXTEND_DEVICE_TEMPLATE_ID_PREFIX = "template_"; /** * Adds the columns the views covering the given devices need. @@ -87,7 +89,7 @@ public class TriggerAddViewColumnManager { * @param operationId the planNodeId of the plan which triggers this operation, for the log * messages only: it is a per-query index, so it does not identify the operation */ - public static void addViewColumnsIfNeeded( + public static void sync( final Map>> device2Measurements, final String operationId) { if (device2Measurements == null || device2Measurements.isEmpty()) { @@ -104,12 +106,10 @@ public static void addViewColumnsIfNeeded( try (ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(AbstractConfigNodeInfo.CONFIG_REGION_ID)) { List updateViewReqs = - view2Measurements.entrySet().stream() - .map(TriggerAddViewColumnManager::toAlterTableReq) - .toList(); + view2Measurements.entrySet().stream().map(ViewColumnAutoSync::toAlterTableReq).toList(); - TUpdateTree2TableViewsResp resp = - configNodeClient.updateTree2TableViews(new TUpdateTree2TableViewsReq(updateViewReqs)); + TAutoSyncViewColumnsResp resp = + configNodeClient.autoSyncViewColumns(new TAutoSyncViewColumnsReq(updateViewReqs)); if (resp == null || resp.getStatus() == null) { LOGGER.error(DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE, operationId); return; @@ -120,7 +120,7 @@ public static void addViewColumnsIfNeeded( } // the views are updated one by one in ConfigNode, so the failed ones are reported one by one if (resp.getFailedViews() != null) { - for (TUpdateTree2TableViewFailure failedView : resp.getFailedViews()) { + for (TAutoSyncViewColumnsFailure failedView : resp.getFailedViews()) { LOGGER.error( DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW, failedView.getDatabase(), @@ -167,9 +167,7 @@ private static TAlterOrDropTableReq toAlterTableReq( Map.Entry>> view2Measurements) { final TreeToTableViewReference view = view2Measurements.getKey(); final List columnSchemas = - view2Measurements.getValue().stream() - .map(TriggerAddViewColumnManager::toColumnSchema) - .toList(); + view2Measurements.getValue().stream().map(ViewColumnAutoSync::toColumnSchema).toList(); TAlterOrDropTableReq tAlterOrDropTableReq = new TAlterOrDropTableReq( view.getDatabaseName(), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewOperationType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewTrieOperationType.java similarity index 90% rename from iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewOperationType.java rename to iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewTrieOperationType.java index ac7deccfd918e..c8e8855bd8550 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewOperationType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewTrieOperationType.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.i18n.SchemaMessages; -public enum TreeToTableViewOperationType { +public enum TreeToTableViewTrieOperationType { /** Register the view, or replace the entry stored for the same view. */ ADD((byte) 0), /** Unregister the view. */ @@ -36,7 +36,7 @@ public enum TreeToTableViewOperationType { private final byte operationType; - TreeToTableViewOperationType(final byte operationType) { + TreeToTableViewTrieOperationType(final byte operationType) { this.operationType = operationType; } @@ -44,7 +44,7 @@ public byte getOperationType() { return operationType; } - public static TreeToTableViewOperationType getType(final byte type) { + public static TreeToTableViewTrieOperationType getType(final byte type) { switch (type) { case -1: return null; diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 88ecee141dada..a635bf0db9bb7 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1368,19 +1368,19 @@ struct TCreateTableViewReq { 2: required bool replace } - struct TUpdateTree2TableViewFailure { + struct TAutoSyncViewColumnsFailure { 1: required string database 2: required string tableName 3: required common.TSStatus status } -struct TUpdateTree2TableViewsResp{ +struct TAutoSyncViewColumnsResp{ 1: required common.TSStatus status - 2: optional list failedViews + 2: optional list failedViews } -struct TUpdateTree2TableViewsReq{ +struct TAutoSyncViewColumnsReq{ 1:required list viewInfos } @@ -2189,5 +2189,5 @@ service IConfigNodeRPCService { common.TSStatus createTableView(TCreateTableViewReq req) - TUpdateTree2TableViewsResp updateTree2TableViews(TUpdateTree2TableViewsReq req) + TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) } From 616880f022a768c3e0357d5963fd4e7ed3b01842 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Fri, 16 Oct 2026 11:22:34 +0800 Subject: [PATCH 05/24] remove the synchronized lock and add the deep copy when get the TsTable --- .../confignode/i18n/ProcedureMessages.java | 3 + .../confignode/i18n/ProcedureMessages.java | 3 + .../confignode/manager/ConfigManager.java | 4 +- .../confignode/manager/ProcedureManager.java | 69 ++++++++++------ .../manager/schema/ClusterSchemaManager.java | 78 +++++++++---------- .../persistence/schema/ClusterSchemaInfo.java | 15 +++- 6 files changed, 104 insertions(+), 68 deletions(-) diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java index 4d47759231755..8881dc94616ba 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java @@ -1621,6 +1621,9 @@ private ProcedureMessages() {} "rolling back after failure: %s."; public static final String MESSAGE_ROLLING_BACK_AFTER_AN_EARLIER_FAILURE_850D0AF5 = "rolling back after an earlier failure."; + public static final String + MESSAGE_SOME_OTHER_TASK_IS_SETTING_OR_UNSETTING_A_TEMPLATE_ON_TARGET_PATH_ARG_D4FEE38F = + "Some other task is setting or unsetting a template on target path %s."; public static final String EXCEPTION_DATANODE_SETSYSTEMSTATUS_RPC_FAILED_ARG_ARG_ARG_7E8E5F9F = "DataNode setSystemStatus RPC failed: %s: %s: %s; "; public static final String EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4 = diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java index 24ce6192841fa..51d75e1375f01 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java @@ -1511,6 +1511,9 @@ private ProcedureMessages() {} public static final String MESSAGE_ROLLING_BACK_AFTER_FAILURE_ARG_474DF456 = "正在回滚，失败原因：%s。"; public static final String MESSAGE_ROLLING_BACK_AFTER_AN_EARLIER_FAILURE_850D0AF5 = "正在回滚此前发生的失败。"; + public static final String + MESSAGE_SOME_OTHER_TASK_IS_SETTING_OR_UNSETTING_A_TEMPLATE_ON_TARGET_PATH_ARG_D4FEE38F = + "有其他任务正在目标路径 %s 上设置或卸载模板。"; public static final String EXCEPTION_DATANODE_SETSYSTEMSTATUS_RPC_FAILED_ARG_ARG_ARG_7E8E5F9F = "DataNode setSystemStatus RPC 失败：%s：%s：%s； "; public static final String EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4 = diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 97fc460320888..a08b4e87ba187 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -2288,7 +2288,7 @@ public TGetTemplateResp getTemplate(String req) { } @Override - public synchronized TSStatus setSchemaTemplate(TSetSchemaTemplateReq req) { + public TSStatus setSchemaTemplate(TSetSchemaTemplateReq req) { TSStatus status = confirmLeader(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return procedureManager.setSchemaTemplate( @@ -2370,7 +2370,7 @@ public TSStatus deactivateSchemaTemplate(final TDeactivateSchemaTemplateReq req) } @Override - public synchronized TSStatus unsetSchemaTemplate(TUnsetSchemaTemplateReq req) { + public TSStatus unsetSchemaTemplate(TUnsetSchemaTemplateReq req) { TSStatus status = confirmLeader(); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return status; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 1c51132317c73..cecc6dc037f68 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -555,17 +555,27 @@ public TSStatus setSchemaTemplate( SetTemplateProcedure setTemplateProcedure; for (Procedure runningProcedure : executor.getProcedures().values()) { type = ProcedureFactory.getProcedureType(runningProcedure); - if (type == null || !type.equals(ProcedureType.SET_TEMPLATE_PROCEDURE)) { + if (Objects.isNull(type)) { continue; } - setTemplateProcedure = (SetTemplateProcedure) runningProcedure; - if (queryId.equals(setTemplateProcedure.getQueryId())) { - procedure = setTemplateProcedure; - break; - } - if (templateSetPath.equals(setTemplateProcedure.getTemplateSetPath())) { - hasOverlappedTask = true; - break; + if (type.equals(ProcedureType.SET_TEMPLATE_PROCEDURE)) { + setTemplateProcedure = (SetTemplateProcedure) runningProcedure; + if (queryId.equals(setTemplateProcedure.getQueryId())) { + procedure = setTemplateProcedure; + break; + } + if (templateSetPath.equals(setTemplateProcedure.getTemplateSetPath())) { + hasOverlappedTask = true; + break; + } + } else if (type.equals(ProcedureType.UNSET_TEMPLATE_PROCEDURE)) { + // A path is set with one template at a time, so setting and unsetting the same path + // overlap as well. + if (templateSetPath.equals( + ((UnsetTemplateProcedure) runningProcedure).getPath().getFullPath())) { + hasOverlappedTask = true; + break; + } } } @@ -573,7 +583,10 @@ public TSStatus setSchemaTemplate( if (hasOverlappedTask) { return RpcUtils.getStatus( TSStatusCode.OVERLAP_WITH_EXISTING_TASK, - "Some other task is setting template on target path."); + String.format( + ProcedureMessages + .MESSAGE_SOME_OTHER_TASK_IS_SETTING_OR_UNSETTING_A_TEMPLATE_ON_TARGET_PATH_ARG_D4FEE38F, + templateSetPath)); } procedure = new SetTemplateProcedure(queryId, templateName, templateSetPath, isGeneratedByPipe); @@ -639,18 +652,28 @@ public TSStatus unsetSchemaTemplate( UnsetTemplateProcedure unsetTemplateProcedure; for (Procedure runningProcedure : executor.getProcedures().values()) { type = ProcedureFactory.getProcedureType(runningProcedure); - if (type == null || !type.equals(ProcedureType.UNSET_TEMPLATE_PROCEDURE)) { + if (Objects.isNull(type)) { continue; } - unsetTemplateProcedure = (UnsetTemplateProcedure) runningProcedure; - if (queryId.equals(unsetTemplateProcedure.getQueryId())) { - procedure = unsetTemplateProcedure; - break; - } - if (template.getId() == unsetTemplateProcedure.getTemplateId() - && path.equals(unsetTemplateProcedure.getPath())) { - hasOverlappedTask = true; - break; + if (type.equals(ProcedureType.UNSET_TEMPLATE_PROCEDURE)) { + unsetTemplateProcedure = (UnsetTemplateProcedure) runningProcedure; + if (queryId.equals(unsetTemplateProcedure.getQueryId())) { + procedure = unsetTemplateProcedure; + break; + } + if (template.getId() == unsetTemplateProcedure.getTemplateId() + && path.equals(unsetTemplateProcedure.getPath())) { + hasOverlappedTask = true; + break; + } + } else if (type.equals(ProcedureType.SET_TEMPLATE_PROCEDURE)) { + // A path is set with one template at a time, so setting and unsetting the same path + // overlap as well. + if (path.getFullPath() + .equals(((SetTemplateProcedure) runningProcedure).getTemplateSetPath())) { + hasOverlappedTask = true; + break; + } } } @@ -658,8 +681,10 @@ public TSStatus unsetSchemaTemplate( if (hasOverlappedTask) { return RpcUtils.getStatus( TSStatusCode.OVERLAP_WITH_EXISTING_TASK, - "Some other task is unsetting target template from target path " - + path.getFullPath()); + String.format( + ProcedureMessages + .MESSAGE_SOME_OTHER_TASK_IS_SETTING_OR_UNSETTING_A_TEMPLATE_ON_TARGET_PATH_ARG_D4FEE38F, + path.getFullPath())); } procedure = new UnsetTemplateProcedure(queryId, template, path, isGeneratedByPipe); this.executor.submitProcedure(procedure); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 47f405c9fd2d3..3ad40e08f6e7f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -1555,7 +1555,7 @@ public Optional getTableWithUsingStatusIfExists( return Optional.of(tableAndStatus.get().getLeft()); } - public synchronized Pair tableColumnCheckForColumnExtension( + public Pair tableColumnCheckForColumnExtension( final String database, final String tableName, final List columnSchemaList, @@ -1577,8 +1577,7 @@ public synchronized Pair tableColumnCheckForColumnExtension( return result.get(); } - final TsTable expandedTable = new TsTable(originalTable); - + // the table read above is a copy, which this check expands and returns final String errorMsg = String.format( "Column '%s' already exist", @@ -1588,10 +1587,10 @@ public synchronized Pair tableColumnCheckForColumnExtension( final AtomicBoolean hasObject = new AtomicBoolean(false); if (originalTable.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).isEmpty()) { - checkColumnSchema(originalTable, expandedTable, columnSchemaList, hasObject); + checkColumnSchema(originalTable, columnSchemaList, hasObject); } else { // only for tree_to_table view - checkColumnSchemaForTree2TableView(originalTable, expandedTable, columnSchemaList, hasObject); + checkColumnSchemaForTree2TableView(originalTable, columnSchemaList, hasObject); } if (columnSchemaList.isEmpty()) { @@ -1599,23 +1598,20 @@ public synchronized Pair tableColumnCheckForColumnExtension( } if (hasObject.get()) { - expandedTable.checkTableNameAndObjectNames4Object(); + originalTable.checkTableNameAndObjectNames4Object(); } - return new Pair<>(StatusUtils.OK, expandedTable); + return new Pair<>(StatusUtils.OK, originalTable); } private static void checkColumnSchema( - TsTable originalTable, - TsTable expandedTable, - List columnSchemaList, - AtomicBoolean hasObject) { + TsTable table, List columnSchemaList, AtomicBoolean hasObject) { columnSchemaList.removeIf( columnSchema -> { - if (Objects.isNull(originalTable.getColumnSchema(columnSchema.getColumnName()))) { + if (Objects.isNull(table.getColumnSchema(columnSchema.getColumnName()))) { if (columnSchema.getDataType().equals(TSDataType.OBJECT)) { hasObject.set(true); } - expandedTable.addColumnSchema(columnSchema); + table.addColumnSchema(columnSchema); return false; } return true; @@ -1623,14 +1619,13 @@ private static void checkColumnSchema( } private static void checkColumnSchemaForTree2TableView( - final TsTable originalView, - final TsTable expandedView, + final TsTable view, final List columnSchemaList, final AtomicBoolean hasObject) { final Set existingColumnNames = new HashSet<>(); final Set originalNames = new HashSet<>(); - for (final TsTableColumnSchema columnSchema : originalView.getColumnList()) { + for (final TsTableColumnSchema columnSchema : view.getColumnList()) { existingColumnNames.add(columnSchema.getColumnName()); Map props = columnSchema.getProps(); if (props == null) { @@ -1665,12 +1660,12 @@ private static void checkColumnSchemaForTree2TableView( existingColumnNames.add(columnSchema.getColumnName()); originalNames.add(originalName); - expandedView.addColumnSchema(columnSchema); + view.addColumnSchema(columnSchema); return false; }); } - public synchronized Pair tableColumnCheckForColumnAltering( + public Pair tableColumnCheckForColumnAltering( final String database, final String tableName, final String columnName, @@ -1695,8 +1690,8 @@ public synchronized Pair tableColumnCheckForColumnAltering( return new Pair<>(tsStatus, null); } - final TsTable alteredTable = new TsTable(originalTable); - TsTableColumnSchema tsTableColumnSchema = alteredTable.getColumnSchema(columnName); + // the table read above is a copy, which this check alters and returns + TsTableColumnSchema tsTableColumnSchema = originalTable.getColumnSchema(columnName); tsTableColumnSchema.setDataType(dataType); if (tsTableColumnSchema instanceof FieldColumnSchema) { FieldColumnSchema fieldColumnSchema = ((FieldColumnSchema) tsTableColumnSchema); @@ -1704,10 +1699,10 @@ public synchronized Pair tableColumnCheckForColumnAltering( SchemaUtils.getDataTypeCompatibleEncoding(dataType, fieldColumnSchema.getEncoding())); } - return new Pair<>(RpcUtils.SUCCESS_STATUS, alteredTable); + return new Pair<>(RpcUtils.SUCCESS_STATUS, originalTable); } - public synchronized Pair tableColumnCheckForColumnRenaming( + public Pair tableColumnCheckForColumnRenaming( final String database, final String tableName, final String oldName, @@ -1754,14 +1749,13 @@ public synchronized Pair tableColumnCheckForColumnRenaming( null); } - final TsTable expandedTable = new TsTable(originalTable); - - expandedTable.renameColumnSchema(oldName, newName); + // the table read above is a copy, which this check renames and returns + originalTable.renameColumnSchema(oldName, newName); - return new Pair<>(RpcUtils.SUCCESS_STATUS, expandedTable); + return new Pair<>(RpcUtils.SUCCESS_STATUS, originalTable); } - public synchronized Pair tableCheckForRenaming( + public Pair tableCheckForRenaming( final String database, final String tableName, final String newName, @@ -1791,9 +1785,9 @@ public synchronized Pair tableCheckForRenaming( null); } - final TsTable expandedTable = new TsTable(originalTable); - expandedTable.renameTable(newName); - return new Pair<>(RpcUtils.SUCCESS_STATUS, expandedTable); + // the table read above is a copy, which this check renames and returns + originalTable.renameTable(newName); + return new Pair<>(RpcUtils.SUCCESS_STATUS, originalTable); } public TSStatus executePlan(final ConfigPhysicalPlan plan, final boolean isGeneratedByPipe) { @@ -1805,7 +1799,7 @@ public TSStatus executePlan(final ConfigPhysicalPlan plan, final boolean isGener } } - public synchronized TSStatus setTableComment( + public TSStatus setTableComment( final String database, final String tableName, final String comment, @@ -1825,7 +1819,7 @@ public synchronized TSStatus setTableComment( } } - public synchronized TSStatus setTableColumnComment( + public TSStatus setTableColumnComment( final String database, final String tableName, final String columnName, @@ -1845,7 +1839,7 @@ public synchronized TSStatus setTableColumnComment( } } - public synchronized Pair updateTableProperties( + public Pair updateTableProperties( final String database, final String tableName, final Map originalProperties, @@ -1896,24 +1890,24 @@ public synchronized Pair updateTableProperties( throw new MetadataException(e); } - final TsTable updatedTable = new TsTable(originalTable); + // the table read above is a copy, which this check updates and returns updatedProperties.forEach( (k, v) -> { originalProperties.put(k, originalTable.getPropValue(k).orElse(null)); if (Objects.nonNull(v)) { - updatedTable.addProp(k, v); + originalTable.addProp(k, v); } else if (TsTable.NEED_LAST_CACHE_PROPERTY.equals(k) && Objects.nonNull(databaseSchema) && databaseSchema.isSetNeedLastCache()) { - updatedTable.addProp(k, String.valueOf(databaseSchema.isNeedLastCache())); + originalTable.addProp(k, String.valueOf(databaseSchema.isNeedLastCache())); } else { - updatedTable.removeProp(k); + originalTable.removeProp(k); } }); - return new Pair<>(RpcUtils.SUCCESS_STATUS, updatedTable); + return new Pair<>(RpcUtils.SUCCESS_STATUS, originalTable); } - public synchronized Pair updateTableColumnProperties( + public Pair updateTableColumnProperties( final String database, final String tableName, final String columnName, @@ -1944,8 +1938,8 @@ public synchronized Pair updateTableColumnProperties( null); } - final TsTable updatedTable = new TsTable(originalTable); - final TsTableColumnSchema updatedColumn = updatedTable.getColumnSchema(columnName); + // the table read above is a copy, which this check updates and returns + final TsTableColumnSchema updatedColumn = originalTable.getColumnSchema(columnName); updatedProperties .keySet() .removeIf( @@ -1965,7 +1959,7 @@ public synchronized Pair updateTableColumnProperties( } } - return new Pair<>(RpcUtils.SUCCESS_STATUS, updatedTable); + return new Pair<>(RpcUtils.SUCCESS_STATUS, originalTable); } private void invalidateLastCache(final String database) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 6bae27bc7f28a..966b45281025c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -1549,12 +1549,23 @@ public Map> getAllPreDeleteTables() { } } + /** + * The table and its status, or empty when the table does not exist. The table is a copy, which + * the caller owns: the stored table is only written by the plans, under the write lock, and never + * through a reference handed out here. + */ public Optional> getTsTableIfExists( final String database, final String tableName) throws MetadataException { databaseReadWriteLock.readLock().lock(); try { - return tableModelMTree.getTableAndStatusIfExists( - getQualifiedDatabasePartialPath(database), tableName); + final Optional> storedTable = + tableModelMTree.getTableAndStatusIfExists( + getQualifiedDatabasePartialPath(database), tableName); + if (storedTable.isEmpty()) { + return Optional.empty(); + } + return Optional.of( + new Pair<>(new TsTable(storedTable.get().getLeft()), storedTable.get().getRight())); } finally { databaseReadWriteLock.readLock().unlock(); } From d703af080383d59b133333122308f7fd06eb8b1a Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Fri, 16 Oct 2026 15:09:07 +0800 Subject: [PATCH 06/24] fix the i18n --- .../iotdb/db/i18n/DataNodeSchemaMessages.java | 16 ++++++++-------- .../iotdb/db/i18n/DataNodeSchemaMessages.java | 16 ++++++++-------- .../schemaengine/table/DataNodeTableCache.java | 2 +- .../schemaengine/table/TreeToTableViewCache.java | 8 ++++---- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java index 8de068cbbf7e6..839329c225874 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java @@ -564,7 +564,7 @@ public final class DataNodeSchemaMessages { // ======================== DataNodeTableCache ======================== public static final String INIT_TABLE_CACHE_SUCCESS = "Init DataNodeTableCache successfully"; - public static final String LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH_E9D65F63 = "Failed to register the tree to table view %s.%s during init: illegal path"; + public static final String LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH = "Failed to register the tree to table view {}.{} during init: illegal path"; public static final String PRE_UPDATE_TABLE_SUCCESS = "Pre-update table {}.{} successfully"; public static final String PRE_RENAME_OLD_TABLE_SUCCESS = "Pre-rename old table {}.{} successfully"; @@ -741,12 +741,12 @@ private DataNodeSchemaMessages() {} public static final String EXCEPTION_COMMA_50AD1C01 = ", "; public static final String MESSAGE_SKIP_ROLLBACK_RENAMING_OLD_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_664F2456 = "Skip rollback renaming old table {}.{} because it has been handled."; public static final String MESSAGE_SKIP_COMMIT_UPDATE_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_31362A1C = "Skip commit-update table {}.{} because it has been handled."; - public static final String LOG_ADD_VIEW_COLUMN_CN_FAILED = "Failed to add columns to the tree to table views for operation %s, ConfigNode returned %s."; - public static final String LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW = "Failed to add a column to the view %s.%s for operation %s, ConfigNode returned %s."; - public static final String LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE = "Failed to add columns to the tree to table views for operation %s, ConfigNode returned an empty response."; - public static final String LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE = "Failed to add columns to the tree to table views for operation %s, because ConfigNode is not reachable."; - public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED = "Failed to trigger the view column addition of the tree to table views for plan node %s, the schema write is not affected."; - public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_B3449FF3 = "Tree to table view %s.%s is not registered"; - public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE = "Failed to update the tree-to-table view index of view %s.%s"; + public static final String LOG_ADD_VIEW_COLUMN_CN_FAILED = "Failed to add columns to the tree to table views for operation {}, ConfigNode returned {}."; + public static final String LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW = "Failed to add a column to the view {}.{} for operation {}, ConfigNode returned {}."; + public static final String LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE = "Failed to add columns to the tree to table views for operation {}, ConfigNode returned an empty response."; + public static final String LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE = "Failed to add columns to the tree to table views for operation {}, because ConfigNode is not reachable."; + public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED = "Failed to trigger the view column addition of the tree to table views for plan node {}, the schema write is not affected."; + public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED = "Tree to table view {}.{} is not registered"; + public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG = "Failed to update the tree-to-table view index of view {}.{}"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java index 12dfd0e43b57e..0f6e37c83ac67 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java @@ -565,7 +565,7 @@ public final class DataNodeSchemaMessages { // ======================== DataNodeTableCache 相关消息 ======================== public static final String INIT_TABLE_CACHE_SUCCESS = "DataNodeTableCache 初始化成功"; - public static final String LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH_E9D65F63 = "初始化时注册树转表视图 %s.%s 失败：路径非法"; + public static final String LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH = "初始化时注册树转表视图 {}.{} 失败：路径非法"; public static final String PRE_UPDATE_TABLE_SUCCESS = "预更新表 {}.{} 成功"; public static final String PRE_RENAME_OLD_TABLE_SUCCESS = "预重命名旧表 {}.{} 成功"; public static final String ROLLBACK_UPDATE_TABLE_SUCCESS = "回滚更新表 {}.{} 成功"; @@ -729,12 +729,12 @@ private DataNodeSchemaMessages() {} public static final String EXCEPTION_COMMA_50AD1C01 = ", "; public static final String MESSAGE_SKIP_ROLLBACK_RENAMING_OLD_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_664F2456 = "跳过回滚重命名旧表 {}.{}，因为该操作已被处理。"; public static final String MESSAGE_SKIP_COMMIT_UPDATE_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_31362A1C = "跳过提交更新表 {}.{}，因为该操作已被处理。"; - public static final String LOG_ADD_VIEW_COLUMN_CN_FAILED = "为操作 %s 的树转表视图加列失败，ConfigNode 返回 %s。"; - public static final String LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW = "为操作 %s 的视图 %s.%s 加列失败，ConfigNode 返回 %s。"; - public static final String LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE = "为操作 %s 的树转表视图加列失败，ConfigNode 返回了空响应。"; - public static final String LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE = "为操作 %s 的树转表视图加列失败，因为无法访问 ConfigNode。"; - public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED = "计划节点 %s 触发树转表视图加列失败，本次 schema 写入不受影响。"; - public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_B3449FF3 = "树转表视图 %s.%s 未注册"; - public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE = "更新树转表视图 %s.%s 的索引失败"; + public static final String LOG_ADD_VIEW_COLUMN_CN_FAILED = "为操作 {} 的树转表视图加列失败，ConfigNode 返回 {}。"; + public static final String LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW = "为视图 {}.{} 加列失败（操作 {}），ConfigNode 返回 {}。"; + public static final String LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE = "为操作 {} 的树转表视图加列失败，ConfigNode 返回了空响应。"; + public static final String LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE = "为操作 {} 的树转表视图加列失败，因为无法访问 ConfigNode。"; + public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED = "计划节点 {} 触发树转表视图加列失败，本次 schema 写入不受影响。"; + public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED = "树转表视图 {}.{} 未注册"; + public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG = "更新树转表视图 {}.{} 的索引失败"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index 4a1cbef37fcd0..341e1cb89554a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -467,7 +467,7 @@ private void operateOnTree2TableViewCache( } catch (final IllegalPathException e) { LOGGER.warn( DataNodeSchemaMessages - .LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE, + .LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG, database, tableName, e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java index 0b1de30db18db..7b77ba97480d6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java @@ -128,7 +128,7 @@ private void registerView(final String database, final TsTable table) { } catch (final IllegalPathException e) { LOGGER.warn( DataNodeSchemaMessages - .LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH_E9D65F63, + .LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH, database, table.getTableName(), e); @@ -195,7 +195,7 @@ public boolean removeView(String database, String tableName) { final RegisteredView registered = getRegisteredView(database, tableName); if (Objects.isNull(registered)) { LOGGER.warn( - DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_B3449FF3, + DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED, database, tableName); return false; @@ -221,7 +221,7 @@ public boolean alterViewName(String database, String newName, String oldName) { final RegisteredView registered = getRegisteredView(database, oldName); if (Objects.isNull(registered)) { LOGGER.warn( - DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_B3449FF3, + DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED, database, oldName); return false; @@ -249,7 +249,7 @@ public boolean alterViewProperties( final RegisteredView registered = getRegisteredView(database, tableName); if (Objects.isNull(registered)) { LOGGER.warn( - DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_B3449FF3, + DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED, database, tableName); return false; From 148d4f14ef65387b8cde148a213211a69801b734 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Fri, 16 Oct 2026 16:15:13 +0800 Subject: [PATCH 07/24] fix the measurement of detection after creating view --- .../confignode/i18n/ProcedureMessages.java | 2 +- .../confignode/i18n/ProcedureMessages.java | 2 +- .../manager/schema/ClusterSchemaManager.java | 23 ++++--------------- .../schema/TreeDeviceViewFieldDetector.java | 6 ++++- .../table/DataNodeTableCache.java | 3 +-- .../commons/schema/table/TreeViewSchema.java | 19 +++++++++++++++ 6 files changed, 32 insertions(+), 23 deletions(-) diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java index 8881dc94616ba..a49007c876409 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java @@ -1630,5 +1630,5 @@ private ProcedureMessages() {} "ConfigNode trySetNodeStatus failed: %s: %s: %s; "; public static final String LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_9544E1D9 = - "Failed to detect the tree series under the new view %s.%s: %s"; + "Failed to detect the tree series under the new view {}.{}: {}"; } diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java index 51d75e1375f01..9f64aec76bd6f 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java @@ -1520,5 +1520,5 @@ private ProcedureMessages() {} "ConfigNode trySetNodeStatus 失败：%s：%s：%s； "; public static final String LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_9544E1D9 = - "扫描新建视图 %s.%s 下的树序列失败：%s"; + "扫描新建视图 {}.{} 下的树序列失败：{}"; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 3ad40e08f6e7f..b7c6fdc3385c2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -129,7 +129,6 @@ import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -1622,23 +1621,11 @@ private static void checkColumnSchemaForTree2TableView( final TsTable view, final List columnSchemaList, final AtomicBoolean hasObject) { - final Set existingColumnNames = new HashSet<>(); - final Set originalNames = new HashSet<>(); - - for (final TsTableColumnSchema columnSchema : view.getColumnList()) { - existingColumnNames.add(columnSchema.getColumnName()); - Map props = columnSchema.getProps(); - if (props == null) { - originalNames.add(columnSchema.getColumnName()); - continue; - } - String originalName = props.get(TreeViewSchema.ORIGINAL_NAME); - if (originalName == null) { - originalNames.add(columnSchema.getColumnName()); - continue; - } - originalNames.add(originalName); - } + Set existingColumnNames = + view.getColumnList().stream() + .map(TsTableColumnSchema::getColumnName) + .collect(Collectors.toSet()); + final Set originalNames = TreeViewSchema.getSourceNames(view); columnSchemaList.removeIf( columnSchema -> { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java index 8b208f25c20c1..900c7cb88c6ad 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java @@ -131,12 +131,16 @@ public Pair> detectNewFieldColumnsAfterCreat if (result.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return new Pair<>(result.getStatus(), Collections.emptyList()); } + // A source measurement can only be referenced once, so a field whose source the view already + // references under another column name gets no column either. + final Set sourceNames = TreeViewSchema.getSourceNames(table); final List columns = new ArrayList<>(); result .getDeviewViewFieldTypeMap() .forEach( (field, type) -> { - if (Objects.isNull(table.getColumnSchema(field))) { + if (Objects.isNull(table.getColumnSchema(field)) + && !sourceNames.contains(lowerCase2OriginalMap.get(field))) { columns.add(toFieldColumnSchemaWithOriginName(field, type)); } }); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index 341e1cb89554a..1c23bfc6a446e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -466,8 +466,7 @@ private void operateOnTree2TableViewCache( } } catch (final IllegalPathException e) { LOGGER.warn( - DataNodeSchemaMessages - .LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG, + DataNodeSchemaMessages.LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG, database, tableName, e); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java index d0d2d6e9f96c1..0e8d65ca90a9b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java @@ -27,7 +27,9 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.rpc.TSStatusCode; +import java.util.HashSet; import java.util.Objects; +import java.util.Set; public class TreeViewSchema { public static final String ORIGINAL_NAME = "__original_name"; @@ -84,6 +86,23 @@ public static void setOriginalName(final TsTableColumnSchema schema, final Strin schema.getProps().put(ORIGINAL_NAME, name); } + /** + * A source measurement can only be referenced once, so this set decides whether a column may + * still be added for one. + */ + public static Set getSourceNames(final TsTable view) { + final Set sourceNames = new HashSet<>(); + for (final TsTableColumnSchema schema : view.getColumnList()) { + if (Objects.isNull(schema.getProps())) { + // the columns which carry no props reference their own name + sourceNames.add(schema.getColumnName()); + } else { + sourceNames.add(getSourceName(schema)); + } + } + return sourceNames; + } + public static boolean isRestrict(final TsTable table) { return table.getPropValue(RESTRICT).isPresent(); } From cc211aa2b14c6b329f516bbcdad1c4db11247734 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Fri, 16 Oct 2026 17:51:57 +0800 Subject: [PATCH 08/24] defer deserialization when submitAddViewColumn --- .../confignode/manager/ProcedureManager.java | 27 ++++++++++++------- .../schema/table/AddTableColumnProcedure.java | 5 ++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index cecc6dc037f68..6b3bf069def96 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -196,6 +196,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -205,6 +206,7 @@ import java.util.Set; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -2390,7 +2392,7 @@ private TSStatus submitAddViewColumn( addColumnReq.database, addColumnReq.tableName, addColumnReq.queryId, - TsTableColumnSchemaUtil.deserializeColumnSchemaList(addColumnReq.updateInfo), + Collections.emptyList(), false); final Pair submissionResult = @@ -2401,7 +2403,12 @@ private TSStatus submitAddViewColumn( null, addColumnReq.queryId, ProcedureType.ADD_VIEW_COLUMN_PROCEDURE, - procedure); + procedure, + proc -> { + if(proc instanceof AddViewColumnProcedure){ + ((AddViewColumnProcedure)proc).setAddedColumnList(TsTableColumnSchemaUtil.deserializeColumnSchemaList(addColumnReq.updateInfo)); + } + }); final TSStatus status = submissionResult.getLeft(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { procedures.add(new Pair<>(addColumnReq, procedure)); @@ -2809,13 +2816,14 @@ public TSStatus executeWithoutDuplicate( } public Pair executeWithoutDuplicateWithoutWaiting( - final String database, - final TsTable table, - final String tableName, - final @Nullable String newName, - final String queryId, - final ProcedureType thisType, - final Procedure procedure) { + final String database, + final TsTable table, + final String tableName, + final @Nullable String newName, + final String queryId, + final ProcedureType thisType, + final Procedure procedure, + final Consumer> consumer) { final long procedureId; synchronized (this) { final Pair procedureIdDuplicatePair = @@ -2829,6 +2837,7 @@ public Pair executeWithoutDuplicateWithoutWaiting( "Some other task is operating table with same name."); return new Pair<>(status, -1L); } + consumer.accept(procedure); this.executor.submitProcedure(procedure); return new Pair<>(StatusUtils.OK, procedure.getProcId()); } else { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java index b6fb00cd2076d..bd4bfa06fb057 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -242,6 +242,11 @@ public boolean equals(final Object o) { && Objects.equals(this.addedColumnList, ((AddTableColumnProcedure) o).addedColumnList); } + public void setAddedColumnList(List addedColumnList){ + this.addedColumnList = addedColumnList; + } + + @Override public int hashCode() { return Objects.hash(super.hashCode(), addedColumnList); From 28a3a5ae06a039cfc7d6e0b59eed2acb2b084905 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Mon, 19 Oct 2026 10:46:42 +0800 Subject: [PATCH 09/24] fix --- .../confignode/manager/ProcedureManager.java | 25 +++++++++++-------- .../schema/table/AddTableColumnProcedure.java | 3 +-- .../config/TableConfigTaskVisitor.java | 5 ++++ .../iotdb/commons/schema/table/TsTable.java | 4 ++- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 6b3bf069def96..62b90a9845f32 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -2405,10 +2405,13 @@ private TSStatus submitAddViewColumn( ProcedureType.ADD_VIEW_COLUMN_PROCEDURE, procedure, proc -> { - if(proc instanceof AddViewColumnProcedure){ - ((AddViewColumnProcedure)proc).setAddedColumnList(TsTableColumnSchemaUtil.deserializeColumnSchemaList(addColumnReq.updateInfo)); + if (proc instanceof AddViewColumnProcedure) { + ((AddViewColumnProcedure) proc) + .setAddedColumnList( + TsTableColumnSchemaUtil.deserializeColumnSchemaList( + addColumnReq.updateInfo)); } - }); + }); final TSStatus status = submissionResult.getLeft(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { procedures.add(new Pair<>(addColumnReq, procedure)); @@ -2816,14 +2819,14 @@ public TSStatus executeWithoutDuplicate( } public Pair executeWithoutDuplicateWithoutWaiting( - final String database, - final TsTable table, - final String tableName, - final @Nullable String newName, - final String queryId, - final ProcedureType thisType, - final Procedure procedure, - final Consumer> consumer) { + final String database, + final TsTable table, + final String tableName, + final @Nullable String newName, + final String queryId, + final ProcedureType thisType, + final Procedure procedure, + final Consumer> consumer) { final long procedureId; synchronized (this) { final Pair procedureIdDuplicatePair = diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java index bd4bfa06fb057..f459a82335788 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -242,11 +242,10 @@ public boolean equals(final Object o) { && Objects.equals(this.addedColumnList, ((AddTableColumnProcedure) o).addedColumnList); } - public void setAddedColumnList(List addedColumnList){ + public void setAddedColumnList(List addedColumnList) { this.addedColumnList = addedColumnList; } - @Override public int hashCode() { return Objects.hash(super.hashCode(), addedColumnList); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 2f11a37e74930..f3f49a1fe8191 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -1030,6 +1030,11 @@ private Map convertPropertiesToMap( map.put( key, String.valueOf(parseBooleanFromLiteral(value, NEED_LAST_CACHE_PROPERTY))); break; + case TreeViewSchema.AUTO_SYNC_SCHEMA: + map.put( + key, + String.valueOf(parseBooleanFromLiteral(value, TreeViewSchema.AUTO_SYNC_SCHEMA))); + break; default: break; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 681c638bc563e..f4538ed7e54b2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -70,7 +70,9 @@ public class TsTable { public static final String NEED_LAST_CACHE_PROPERTY = "need_last_cache"; public static final Set TABLE_ALLOWED_PROPERTIES = Collections.unmodifiableSet( - new HashSet<>(Arrays.asList(TTL_PROPERTY, NEED_LAST_CACHE_PROPERTY))); + new HashSet<>( + Arrays.asList( + TTL_PROPERTY, NEED_LAST_CACHE_PROPERTY, TreeViewSchema.AUTO_SYNC_SCHEMA))); private static final String OBJECT_STRING_ERROR = "When there are object fields, the %s %s shall not be '.', '..' or contain './', '.\\'."; protected String tableName; From de9cffea887b079110bca33569f95bd3c1fc61c6 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Mon, 19 Oct 2026 15:20:26 +0800 Subject: [PATCH 10/24] fix that create or replace a view would leave the old index in Trie tree --- .../db/schemaengine/table/TreeToTableViewCache.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java index 7b77ba97480d6..8f5fa32c91f56 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java @@ -171,10 +171,12 @@ public boolean addView( current = child; } - final boolean added = - viewReferences - .computeIfAbsent(current, node -> new HashSet<>()) - .add(new TreeToTableViewReference(database, tableName, isRestrict, numOfTags)); + final TreeToTableViewReference reference = + new TreeToTableViewReference(database, tableName, isRestrict, numOfTags); + final Set references = + viewReferences.computeIfAbsent(current, node -> new HashSet<>()); + final boolean added = !references.remove(reference); + references.add(reference); // whether the reference is new or not, this path is covered by a view now hasView = true; return added; From a68f104fa329f35ee4a6f0979082fc53220d5ef0 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Mon, 19 Oct 2026 15:29:14 +0800 Subject: [PATCH 11/24] fix that drop database procedure should include drop the index in TreeToTableCache --- .../table/DataNodeTableCache.java | 1 + .../table/ITreeToTableViewCache.java | 6 ++ .../table/TreeToTableViewCache.java | 59 ++++++++++++++----- 3 files changed, 51 insertions(+), 15 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index 1c23bfc6a446e..27c9b5a5dce57 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -513,6 +513,7 @@ public void invalid(String database) { try { databaseTableMap.remove(database); specialStatusMap.remove(database); + treeToTableViewCache.removeViews(database); instanceVersion.incrementAndGet(); } finally { readWriteLock.writeLock().unlock(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITreeToTableViewCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITreeToTableViewCache.java index a3156da6ca852..b1b675dfc71c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITreeToTableViewCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITreeToTableViewCache.java @@ -47,6 +47,12 @@ boolean addView( boolean removeView(String database, String tableName); + /** + * Removes every view of the database, eg. when the database is dropped. The views are taken from + * the index itself, so the ones the table cache does not hold any more are removed as well. + */ + void removeViews(String database); + boolean alterViewName(String database, String newName, String oldName); /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java index 8f5fa32c91f56..9c06fef301856 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java @@ -35,6 +35,7 @@ import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -194,28 +195,56 @@ public boolean addView( public boolean removeView(String database, String tableName) { readWriteLock.writeLock().lock(); try { - final RegisteredView registered = getRegisteredView(database, tableName); - if (Objects.isNull(registered)) { - LOGGER.warn( - DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED, - database, - tableName); - return false; - } + return removeViewLocked(database, tableName); + } finally { + readWriteLock.writeLock().unlock(); + } + } - registered.references.remove(registered.reference); - if (registered.references.isEmpty()) { - viewReferences.remove(registered.node); - pruneNodesWithoutView(registered.node); - hasView = !viewReferences.isEmpty(); + /** + * Removes every view of the database. A database of the same name may be created again with views + * which have nothing to do with the dropped ones, so the entries of the old database have to go + * together with it. + */ + @Override + public void removeViews(final String database) { + readWriteLock.writeLock().lock(); + try { + // viewPaths is the record of what is registered, so the views which the table cache does not + // hold any more are removed here as well. + final List tableNames = new ArrayList<>(); + for (final Pair view : viewPaths.keySet()) { + if (Objects.equals(database, view.getLeft())) { + tableNames.add(view.getRight()); + } } - viewPaths.remove(new Pair<>(database, tableName)); - return true; + tableNames.forEach(tableName -> removeViewLocked(database, tableName)); } finally { readWriteLock.writeLock().unlock(); } } + /** The body of {@link #removeView}, which the caller runs while holding the write lock. */ + private boolean removeViewLocked(String database, String tableName) { + final RegisteredView registered = getRegisteredView(database, tableName); + if (Objects.isNull(registered)) { + LOGGER.warn( + DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED, + database, + tableName); + return false; + } + + registered.references.remove(registered.reference); + if (registered.references.isEmpty()) { + viewReferences.remove(registered.node); + pruneNodesWithoutView(registered.node); + hasView = !viewReferences.isEmpty(); + } + viewPaths.remove(new Pair<>(database, tableName)); + return true; + } + @Override public boolean alterViewName(String database, String newName, String oldName) { readWriteLock.writeLock().lock(); From 07b35708552dc7a0c953bfea36a875967c42cc90 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Mon, 19 Oct 2026 15:43:21 +0800 Subject: [PATCH 12/24] fix the extension template correctly the trigger the add column --- .../impl/DataNodeInternalRPCServiceImpl.java | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index a977f16f2a74d..304340b1e8f9b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -58,7 +58,6 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.ConfigurationFileUtils; -import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.conf.IoTDBConstant.ClientVersion; import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.consensus.DataRegionId; @@ -2970,25 +2969,31 @@ private Map>> collectDevicesUsingExte final Map>> device2Measurements = new HashMap<>(); Collection allSchemaRegions = SchemaEngine.getInstance().getAllSchemaRegions(); for (final PartialPath setPath : setPaths) { - final String setPathString = setPath.getFullPath(); + // A device which is exactly the mount path has the template too, but "setPath.**" only + // matches its descendants, so the mount path itself is read with a pattern of its own. + final List devicePatterns = + Arrays.asList(setPath, setPath.concatNode(MULTI_LEVEL_PATH_WILDCARD)); for (final ISchemaRegion schemaRegion : allSchemaRegions) { - // A device under this path can only live in the region of the database the path belongs to. - if (!setPathString.startsWith(schemaRegion.getDatabaseFullPath())) { + if (!setPath.startsWithOrPrefixOf( + PathUtils.splitPathToDetachedNodes(schemaRegion.getDatabaseFullPath()))) { continue; } - try (final ISchemaReader deviceReader = - schemaRegion.getDeviceReader( - SchemaRegionReadPlanFactory.getShowDevicesPlan( - setPath.concatNode(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD), - 0, - 0, - false, - newTemplate.getId(), - SchemaConstant.ALL_MATCH_SCOPE))) { - while (deviceReader.hasNext()) { - device2Measurements - .computeIfAbsent(deviceReader.next().getPartialPath(), device -> new ArrayList<>()) - .addAll(extendedMeasurements); + for (final PartialPath devicePattern : devicePatterns) { + try (final ISchemaReader deviceReader = + schemaRegion.getDeviceReader( + SchemaRegionReadPlanFactory.getShowDevicesPlan( + devicePattern, + 0, + 0, + false, + newTemplate.getId(), + SchemaConstant.ALL_MATCH_SCOPE))) { + while (deviceReader.hasNext()) { + device2Measurements + .computeIfAbsent( + deviceReader.next().getPartialPath(), device -> new ArrayList<>()) + .addAll(extendedMeasurements); + } } } } From 53439637bebe7117631f0123a7703efc158c3392 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Mon, 19 Oct 2026 15:59:45 +0800 Subject: [PATCH 13/24] fix the i18n --- .../iotdb/confignode/i18n/ProcedureMessages.java | 2 +- .../iotdb/confignode/i18n/ProcedureMessages.java | 2 +- .../schema/TreeDeviceViewFieldDetector.java | 3 +-- .../iotdb/db/i18n/DataNodeMiscMessages.java | 4 ++-- .../iotdb/db/i18n/DataNodeSchemaMessages.java | 16 ++++++++-------- .../iotdb/db/i18n/DataNodeMiscMessages.java | 4 ++-- .../iotdb/db/i18n/DataNodeSchemaMessages.java | 16 ++++++++-------- .../schemaregion/SchemaExecutionVisitor.java | 3 ++- .../impl/DataNodeInternalRPCServiceImpl.java | 2 +- .../schemaengine/table/DataNodeTableCache.java | 2 +- .../schemaengine/table/TreeToTableViewCache.java | 8 ++++---- .../schemaengine/table/ViewColumnAutoSync.java | 12 ++++++++---- 12 files changed, 39 insertions(+), 35 deletions(-) diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java index a49007c876409..a67211e4a54a7 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java @@ -1629,6 +1629,6 @@ private ProcedureMessages() {} public static final String EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4 = "ConfigNode trySetNodeStatus failed: %s: %s: %s; "; public static final String - LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_9544E1D9 = + LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_8F20CAE7 = "Failed to detect the tree series under the new view {}.{}: {}"; } diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java index 9f64aec76bd6f..91165cba660ff 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java @@ -1519,6 +1519,6 @@ private ProcedureMessages() {} public static final String EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4 = "ConfigNode trySetNodeStatus 失败：%s：%s：%s； "; public static final String - LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_9544E1D9 = + LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_8F20CAE7 = "扫描新建视图 {}.{} 下的树序列失败：{}"; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java index 900c7cb88c6ad..10d052f8776c6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java @@ -171,10 +171,9 @@ public static List detectMissingColumnsAfterCreation( final Pair> detected = new TreeDeviceViewFieldDetector(configManager, table).detectNewFieldColumnsAfterCreation(); if (detected.getLeft().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - // todo, i18N LOGGER.warn( ProcedureMessages - .LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_9544E1D9, + .LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_8F20CAE7, database, table.getTableName(), detected.getLeft()); diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index 5481e61647bfb..567f5ee071892 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1514,7 +1514,7 @@ private DataNodeMiscMessages() {} public static final String MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB = "Multiple types encountered when auto detecting type of measurement '%s', please check"; - public static final String LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_95856581 = - "Failed to collect the devices using the extended template %s"; + public static final String LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_AB32D554 = + "Failed to collect the devices using the extended template {}"; } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java index 839329c225874..d3a92eee16826 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java @@ -564,7 +564,7 @@ public final class DataNodeSchemaMessages { // ======================== DataNodeTableCache ======================== public static final String INIT_TABLE_CACHE_SUCCESS = "Init DataNodeTableCache successfully"; - public static final String LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH = "Failed to register the tree to table view {}.{} during init: illegal path"; + public static final String LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH_359A451E = "Failed to register the tree to table view {}.{} during init: illegal path"; public static final String PRE_UPDATE_TABLE_SUCCESS = "Pre-update table {}.{} successfully"; public static final String PRE_RENAME_OLD_TABLE_SUCCESS = "Pre-rename old table {}.{} successfully"; @@ -741,12 +741,12 @@ private DataNodeSchemaMessages() {} public static final String EXCEPTION_COMMA_50AD1C01 = ", "; public static final String MESSAGE_SKIP_ROLLBACK_RENAMING_OLD_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_664F2456 = "Skip rollback renaming old table {}.{} because it has been handled."; public static final String MESSAGE_SKIP_COMMIT_UPDATE_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_31362A1C = "Skip commit-update table {}.{} because it has been handled."; - public static final String LOG_ADD_VIEW_COLUMN_CN_FAILED = "Failed to add columns to the tree to table views for operation {}, ConfigNode returned {}."; - public static final String LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW = "Failed to add a column to the view {}.{} for operation {}, ConfigNode returned {}."; - public static final String LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE = "Failed to add columns to the tree to table views for operation {}, ConfigNode returned an empty response."; - public static final String LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE = "Failed to add columns to the tree to table views for operation {}, because ConfigNode is not reachable."; - public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED = "Failed to trigger the view column addition of the tree to table views for plan node {}, the schema write is not affected."; - public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED = "Tree to table view {}.{} is not registered"; - public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG = "Failed to update the tree-to-table view index of view {}.{}"; + public static final String LOG_ADD_VIEW_COLUMN_CN_FAILED_FE4F10AC = "Failed to add columns to the tree to table views for operation {}, ConfigNode returned {}."; + public static final String LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW_B3EB8144 = "Failed to add a column to the view {}.{} for operation {}, ConfigNode returned {}."; + public static final String LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE_B9502E26 = "Failed to add columns to the tree to table views for operation {}, ConfigNode returned an empty response."; + public static final String LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE_E9BC7652 = "Failed to add columns to the tree to table views for operation {}, because ConfigNode is not reachable."; + public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED_B909817F = "Failed to trigger the view column addition of the tree to table views for plan node {}, the schema write is not affected."; + public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902 = "Tree to table view {}.{} is not registered"; + public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE = "Failed to update the tree-to-table view index of view {}.{}"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index 40bc72108a093..8ba949c3395bf 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1494,7 +1494,7 @@ private DataNodeMiscMessages() {} public static final String MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB = "自动检测测点 %s 的类型时遇到多种类型，请检查"; - public static final String LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_95856581 = - "收集使用扩展模板 %s 的设备失败"; + public static final String LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_AB32D554 = + "收集使用扩展模板 {} 的设备失败"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java index 0f6e37c83ac67..9966a203d7d2b 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java @@ -565,7 +565,7 @@ public final class DataNodeSchemaMessages { // ======================== DataNodeTableCache 相关消息 ======================== public static final String INIT_TABLE_CACHE_SUCCESS = "DataNodeTableCache 初始化成功"; - public static final String LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH = "初始化时注册树转表视图 {}.{} 失败：路径非法"; + public static final String LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH_359A451E = "初始化时注册树转表视图 {}.{} 失败：路径非法"; public static final String PRE_UPDATE_TABLE_SUCCESS = "预更新表 {}.{} 成功"; public static final String PRE_RENAME_OLD_TABLE_SUCCESS = "预重命名旧表 {}.{} 成功"; public static final String ROLLBACK_UPDATE_TABLE_SUCCESS = "回滚更新表 {}.{} 成功"; @@ -729,12 +729,12 @@ private DataNodeSchemaMessages() {} public static final String EXCEPTION_COMMA_50AD1C01 = ", "; public static final String MESSAGE_SKIP_ROLLBACK_RENAMING_OLD_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_664F2456 = "跳过回滚重命名旧表 {}.{}，因为该操作已被处理。"; public static final String MESSAGE_SKIP_COMMIT_UPDATE_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_31362A1C = "跳过提交更新表 {}.{}，因为该操作已被处理。"; - public static final String LOG_ADD_VIEW_COLUMN_CN_FAILED = "为操作 {} 的树转表视图加列失败，ConfigNode 返回 {}。"; - public static final String LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW = "为视图 {}.{} 加列失败（操作 {}），ConfigNode 返回 {}。"; - public static final String LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE = "为操作 {} 的树转表视图加列失败，ConfigNode 返回了空响应。"; - public static final String LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE = "为操作 {} 的树转表视图加列失败，因为无法访问 ConfigNode。"; - public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED = "计划节点 {} 触发树转表视图加列失败，本次 schema 写入不受影响。"; - public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED = "树转表视图 {}.{} 未注册"; - public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG = "更新树转表视图 {}.{} 的索引失败"; + public static final String LOG_ADD_VIEW_COLUMN_CN_FAILED_FE4F10AC = "为操作 {} 的树转表视图加列失败，ConfigNode 返回 {}。"; + public static final String LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW_B3EB8144 = "为视图 {}.{} 加列失败（操作 {}），ConfigNode 返回 {}。"; + public static final String LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE_B9502E26 = "为操作 {} 的树转表视图加列失败，ConfigNode 返回了空响应。"; + public static final String LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE_E9BC7652 = "为操作 {} 的树转表视图加列失败，因为无法访问 ConfigNode。"; + public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED_B909817F = "计划节点 {} 触发树转表视图加列失败，本次 schema 写入不受影响。"; + public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902 = "树转表视图 {}.{} 未注册"; + public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE = "更新树转表视图 {}.{} 的索引失败"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index f84b907c2563c..deedf119baa29 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -495,7 +495,8 @@ private void syncViewColumns( ViewColumnAutoSync.sync(createdMeasurements, operationId); } } catch (final Throwable e) { - logger.error(DataNodeSchemaMessages.LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED, operationId, e); + logger.error( + DataNodeSchemaMessages.LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED_B909817F, operationId, e); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 304340b1e8f9b..09884aca4f18b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -2925,7 +2925,7 @@ public TSStatus updateTemplate(final TUpdateTemplateReq req) { // todo LOGGER.warn( DataNodeMiscMessages - .LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_95856581, + .LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_AB32D554, newTemplate.getName(), e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index 27c9b5a5dce57..1fd3480762e16 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -466,7 +466,7 @@ private void operateOnTree2TableViewCache( } } catch (final IllegalPathException e) { LOGGER.warn( - DataNodeSchemaMessages.LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG, + DataNodeSchemaMessages.LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE, database, tableName, e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java index 9c06fef301856..ad33d4ffea6ba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java @@ -129,7 +129,7 @@ private void registerView(final String database, final TsTable table) { } catch (final IllegalPathException e) { LOGGER.warn( DataNodeSchemaMessages - .LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH, + .LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH_359A451E, database, table.getTableName(), e); @@ -229,7 +229,7 @@ private boolean removeViewLocked(String database, String tableName) { final RegisteredView registered = getRegisteredView(database, tableName); if (Objects.isNull(registered)) { LOGGER.warn( - DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED, + DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902, database, tableName); return false; @@ -252,7 +252,7 @@ public boolean alterViewName(String database, String newName, String oldName) { final RegisteredView registered = getRegisteredView(database, oldName); if (Objects.isNull(registered)) { LOGGER.warn( - DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED, + DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902, database, oldName); return false; @@ -280,7 +280,7 @@ public boolean alterViewProperties( final RegisteredView registered = getRegisteredView(database, tableName); if (Objects.isNull(registered)) { LOGGER.warn( - DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED, + DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902, database, tableName); return false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java index 5f6537b891d9c..bb2edd2d73f18 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java @@ -111,18 +111,21 @@ public static void sync( TAutoSyncViewColumnsResp resp = configNodeClient.autoSyncViewColumns(new TAutoSyncViewColumnsReq(updateViewReqs)); if (resp == null || resp.getStatus() == null) { - LOGGER.error(DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE, operationId); + LOGGER.error( + DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE_B9502E26, operationId); return; } if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.error( - DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_FAILED, operationId, resp.getStatus()); + DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_FAILED_FE4F10AC, + operationId, + resp.getStatus()); } // the views are updated one by one in ConfigNode, so the failed ones are reported one by one if (resp.getFailedViews() != null) { for (TAutoSyncViewColumnsFailure failedView : resp.getFailedViews()) { LOGGER.error( - DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW, + DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW_B3EB8144, failedView.getDatabase(), failedView.getTableName(), operationId, @@ -130,7 +133,8 @@ public static void sync( } } } catch (ClientManagerException | TException e) { - LOGGER.warn(DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE, operationId, e); + LOGGER.warn( + DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE_E9BC7652, operationId, e); } } From f28ff35cbc509f471159351148032186b0cd5c25 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Mon, 19 Oct 2026 16:03:13 +0800 Subject: [PATCH 14/24] fix the recent IT, now set the template do not contribute the column in the view --- .../relational/it/schema/IoTDBTableIT.java | 25 ++++++------------- .../table/TreeToTableViewCache.java | 4 --- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index f7a5bc4581cd4..037ac00cd1f2b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -1038,24 +1038,13 @@ public void testTreeViewTable() throws Exception { final Statement statement = connection.createStatement()) { statement.execute("use tree_view_db"); - try { - statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"重庆\".\"1\".**"); - fail(); - } catch (final SQLException e) { - assertEquals( - "614: Multiple types encountered when auto detecting type of measurement 'S1', please check", - e.getMessage()); - } - - try { - statement.execute( - "create view tree_table (tag1 tag, tag2 tag, S1 field) as root.\"重庆\".\"1\".**"); - fail(); - } catch (final SQLException e) { - assertEquals( - "614: Multiple types encountered when auto detecting type of measurement 'S1', please check", - e.getMessage()); - } + // Template t1 is set but not activated, so its S1 BOOLEAN is not a created series and does + // not take part in the detection + statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"重庆\".\"1\".**"); + statement.execute("drop view tree_table"); + statement.execute( + "create view tree_table (tag1 tag, tag2 tag, S1 field) as root.\"重庆\".\"1\".**"); + statement.execute("drop view tree_table"); } try (final Connection connection = EnvFactory.getEnv().getConnection(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java index ad33d4ffea6ba..86ca045679bd7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java @@ -228,10 +228,6 @@ public void removeViews(final String database) { private boolean removeViewLocked(String database, String tableName) { final RegisteredView registered = getRegisteredView(database, tableName); if (Objects.isNull(registered)) { - LOGGER.warn( - DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902, - database, - tableName); return false; } From 897276c5afd0a24955bb13ecc6727866942408fa Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Mon, 19 Oct 2026 16:21:45 +0800 Subject: [PATCH 15/24] let the show create view would describe the attribute of the auto_sync_schema --- .../persistence/schema/TreeDeviceViewFieldDetector.java | 2 +- .../impl/schema/table/view/AddViewColumnProcedure.java | 2 +- .../impl/schema/table/view/CreateTableViewProcedure.java | 6 +++--- .../impl/schema/table/view/RenameViewProcedure.java | 2 +- .../schema/table/view/SetViewPropertiesProcedure.java | 2 +- .../plan/execution/config/TableConfigTaskVisitor.java | 6 +++--- .../config/metadata/relational/ShowCreateViewTask.java | 8 +++++++- .../iotdb/db/schemaengine/table/DataNodeTableCache.java | 6 +++--- .../apache/iotdb/commons/schema/table/TreeViewSchema.java | 1 - .../org/apache/iotdb/commons/schema/table/TsTable.java | 4 ++-- 10 files changed, 22 insertions(+), 17 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java index 10d052f8776c6..c6f4f1c527b3e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java @@ -161,7 +161,7 @@ public static List detectMissingColumnsAfterCreation( final ConfigManager configManager, final String database, final TsTable table) { final boolean autoSync = table - .getPropValue(TreeViewSchema.AUTO_SYNC_SCHEMA) + .getPropValue(TsTable.AUTO_SYNC_SCHEMA) .map(Boolean::parseBoolean) .orElse(false); if (!autoSync) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java index 1fe316ebb6e3a..c426558be78b3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java @@ -51,7 +51,7 @@ import java.util.Optional; import static org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType.ALTER_TAG; -import static org.apache.iotdb.commons.schema.table.TreeViewSchema.AUTO_SYNC_SCHEMA; +import static org.apache.iotdb.commons.schema.table.TsTable.AUTO_SYNC_SCHEMA; public class AddViewColumnProcedure extends AddTableColumnProcedure { public AddViewColumnProcedure(final boolean isGeneratedByPipe) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java index 4adb7fc00c579..67152bee7567e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java @@ -131,7 +131,7 @@ protected void checkTableExistence(final ConfigNodeProcedureEnv env) { protected TUpdateTableReq buildCommitReleaseTableReq() { final boolean newAutoSync = table - .getPropValue(TreeViewSchema.AUTO_SYNC_SCHEMA) + .getPropValue(TsTable.AUTO_SYNC_SCHEMA) .map(Boolean::parseBoolean) .orElse(false); if (newAutoSync) { @@ -148,7 +148,7 @@ protected TUpdateTableReq buildCommitReleaseTableReq() { final boolean oldAutoSync = Objects.nonNull(oldView) && oldView - .getPropValue(TreeViewSchema.AUTO_SYNC_SCHEMA) + .getPropValue(TsTable.AUTO_SYNC_SCHEMA) .map(Boolean::parseBoolean) .orElse(false); if (oldAutoSync) { @@ -172,7 +172,7 @@ protected TUpdateTableReq buildRollbackTableReq() { final boolean oldAutoSync = Objects.nonNull(oldView) && oldView - .getPropValue(TreeViewSchema.AUTO_SYNC_SCHEMA) + .getPropValue(TsTable.AUTO_SYNC_SCHEMA) .map(Boolean::parseBoolean) .orElse(false); if (!oldAutoSync) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java index 5b163934b1965..d6dec9689a309 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java @@ -31,7 +31,7 @@ import java.io.DataOutputStream; import java.io.IOException; -import static org.apache.iotdb.commons.schema.table.TreeViewSchema.AUTO_SYNC_SCHEMA; +import static org.apache.iotdb.commons.schema.table.TsTable.AUTO_SYNC_SCHEMA; public class RenameViewProcedure extends RenameTableProcedure { public RenameViewProcedure(final boolean isGeneratedByPipe) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java index 8e6e85925b992..7919f98bc70d3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java @@ -38,7 +38,7 @@ import java.util.Map; import java.util.Optional; -import static org.apache.iotdb.commons.schema.table.TreeViewSchema.AUTO_SYNC_SCHEMA; +import static org.apache.iotdb.commons.schema.table.TsTable.AUTO_SYNC_SCHEMA; public class SetViewPropertiesProcedure extends SetTablePropertiesProcedure { public SetViewPropertiesProcedure(final boolean isGeneratedByPipe) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index f3f49a1fe8191..3526fe9876676 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -1005,7 +1005,7 @@ private Map convertPropertiesToMap( throw new SemanticException(DataNodeQueryMessages.DUPLICATED_PROPERTY + key); } if (TABLE_ALLOWED_PROPERTIES.contains(key)) { - if (TreeViewSchema.AUTO_SYNC_SCHEMA.equals(key) && !allowAutoSyncSchema) { + if (TsTable.AUTO_SYNC_SCHEMA.equals(key) && !allowAutoSyncSchema) { // A property assigned DEFAULT is dropped or reset as well, so the check covers it too throw new SemanticException( EXCEPTION_AUTO_SYNC_SCHEMA_IS_ONLY_SUPPORTED_BY_TREE_TO_TABLE_VIEWS_C6586215); @@ -1030,10 +1030,10 @@ private Map convertPropertiesToMap( map.put( key, String.valueOf(parseBooleanFromLiteral(value, NEED_LAST_CACHE_PROPERTY))); break; - case TreeViewSchema.AUTO_SYNC_SCHEMA: + case TsTable.AUTO_SYNC_SCHEMA: map.put( key, - String.valueOf(parseBooleanFromLiteral(value, TreeViewSchema.AUTO_SYNC_SCHEMA))); + String.valueOf(parseBooleanFromLiteral(value, TsTable.AUTO_SYNC_SCHEMA))); break; default: break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java index a616282e7c4a9..64102a7e46f6c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java @@ -147,7 +147,13 @@ public static String getShowCreateViewSQL(final TsTable table) { if (ttlString.equals(TTL_INFINITE)) { ttlString = "'" + ttlString + "'"; } - builder.append(" WITH (ttl=").append(ttlString).append(")"); + builder.append(" WITH (ttl=").append(ttlString); + // An absent auto_sync_schema means false, so it only has to be given back when it is stored + table + .getPropValue(TsTable.AUTO_SYNC_SCHEMA) + .ifPresent( + autoSyncSchema -> builder.append(", auto_sync_schema=").append(autoSyncSchema)); + builder.append(")"); builder.append(" AS "); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index 1fd3480762e16..6b819362ac6e6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -33,7 +33,6 @@ import org.apache.iotdb.commons.schema.table.PreDeleteTsTable; import org.apache.iotdb.commons.schema.table.TableNodeStatus; import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType; -import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; @@ -180,7 +179,7 @@ private Map> collectInitialViews( */ private static boolean isAutoSyncView(final TsTable table) { final Optional autoSyncSchemaValue = - table.getPropValue(TreeViewSchema.AUTO_SYNC_SCHEMA); + table.getPropValue(TsTable.AUTO_SYNC_SCHEMA); return autoSyncSchemaValue.isPresent() && Boolean.parseBoolean(autoSyncSchemaValue.get()) && table.getPropValue(TREE_PATH_PATTERN).isPresent(); @@ -466,7 +465,8 @@ private void operateOnTree2TableViewCache( } } catch (final IllegalPathException e) { LOGGER.warn( - DataNodeSchemaMessages.LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE, + DataNodeSchemaMessages + .LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE, database, tableName, e); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java index 0e8d65ca90a9b..e1068a21d1558 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java @@ -37,7 +37,6 @@ public class TreeViewSchema { public static final String RESTRICT = "__restrict"; public static final String UNSUPPORTED_NEED_LAST_CACHE_PROPERTY = "The tree view does not support need_last_cache property."; - public static final String AUTO_SYNC_SCHEMA = "auto_sync_schema"; public static boolean isTreeViewTable(final TsTable table) { return table.getPropValue(TREE_PATH_PATTERN).isPresent(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index f4538ed7e54b2..0bcee954fa5c1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -68,11 +68,11 @@ public class TsTable { // No column property key is supported yet. public static final Set COLUMN_ALLOWED_PROPERTIES = Collections.emptySet(); public static final String NEED_LAST_CACHE_PROPERTY = "need_last_cache"; + public static final String AUTO_SYNC_SCHEMA = "auto_sync_schema"; public static final Set TABLE_ALLOWED_PROPERTIES = Collections.unmodifiableSet( new HashSet<>( - Arrays.asList( - TTL_PROPERTY, NEED_LAST_CACHE_PROPERTY, TreeViewSchema.AUTO_SYNC_SCHEMA))); + Arrays.asList(TTL_PROPERTY, NEED_LAST_CACHE_PROPERTY, AUTO_SYNC_SCHEMA))); private static final String OBJECT_STRING_ERROR = "When there are object fields, the %s %s shall not be '.', '..' or contain './', '.\\'."; protected String tableName; From 1a53d7be3cdb232716e44f7d6c2a2fd1758ff85f Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Mon, 19 Oct 2026 16:46:26 +0800 Subject: [PATCH 16/24] use an unambiguous encoding for build queryId --- .../schema/TreeDeviceViewFieldDetector.java | 5 +---- .../table/view/CreateTableViewProcedure.java | 5 +---- .../config/TableConfigTaskVisitor.java | 3 +-- .../relational/ShowCreateViewTask.java | 3 +-- .../schemaengine/table/DataNodeTableCache.java | 3 +-- .../schemaengine/table/ViewColumnAutoSync.java | 18 +++++++++++++----- .../iotdb/commons/schema/table/TsTable.java | 3 +-- 7 files changed, 19 insertions(+), 21 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java index c6f4f1c527b3e..2beca63c2e6f6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java @@ -160,10 +160,7 @@ public Pair> detectNewFieldColumnsAfterCreat public static List detectMissingColumnsAfterCreation( final ConfigManager configManager, final String database, final TsTable table) { final boolean autoSync = - table - .getPropValue(TsTable.AUTO_SYNC_SCHEMA) - .map(Boolean::parseBoolean) - .orElse(false); + table.getPropValue(TsTable.AUTO_SYNC_SCHEMA).map(Boolean::parseBoolean).orElse(false); if (!autoSync) { return Collections.emptyList(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java index 67152bee7567e..bf5551f457b4a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java @@ -130,10 +130,7 @@ protected void checkTableExistence(final ConfigNodeProcedureEnv env) { @Override protected TUpdateTableReq buildCommitReleaseTableReq() { final boolean newAutoSync = - table - .getPropValue(TsTable.AUTO_SYNC_SCHEMA) - .map(Boolean::parseBoolean) - .orElse(false); + table.getPropValue(TsTable.AUTO_SYNC_SCHEMA).map(Boolean::parseBoolean).orElse(false); if (newAutoSync) { // Whether the view is a new one, or replace the old table, just add view final TUpdateTree2TableView viewInfo = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 3526fe9876676..990583f1bbb9b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -1032,8 +1032,7 @@ private Map convertPropertiesToMap( break; case TsTable.AUTO_SYNC_SCHEMA: map.put( - key, - String.valueOf(parseBooleanFromLiteral(value, TsTable.AUTO_SYNC_SCHEMA))); + key, String.valueOf(parseBooleanFromLiteral(value, TsTable.AUTO_SYNC_SCHEMA))); break; default: break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java index 64102a7e46f6c..7d2905e901726 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java @@ -151,8 +151,7 @@ public static String getShowCreateViewSQL(final TsTable table) { // An absent auto_sync_schema means false, so it only has to be given back when it is stored table .getPropValue(TsTable.AUTO_SYNC_SCHEMA) - .ifPresent( - autoSyncSchema -> builder.append(", auto_sync_schema=").append(autoSyncSchema)); + .ifPresent(autoSyncSchema -> builder.append(", auto_sync_schema=").append(autoSyncSchema)); builder.append(")"); builder.append(" AS "); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index 6b819362ac6e6..d58ae384161b4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -178,8 +178,7 @@ private Map> collectInitialViews( * carry no definition at all, so they are never one. */ private static boolean isAutoSyncView(final TsTable table) { - final Optional autoSyncSchemaValue = - table.getPropValue(TsTable.AUTO_SYNC_SCHEMA); + final Optional autoSyncSchemaValue = table.getPropValue(TsTable.AUTO_SYNC_SCHEMA); return autoSyncSchemaValue.isPresent() && Boolean.parseBoolean(autoSyncSchemaValue.get()) && table.getPropValue(TREE_PATH_PATTERN).isPresent(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java index bb2edd2d73f18..4462d91a6915f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java @@ -79,9 +79,7 @@ public class ViewColumnAutoSync { * Adds the columns the views covering the given devices need. * *

It is a best-effort side effect of the write which triggers it: every failure is logged here - * and none of them is propagated, so that the write operation is never affected. The columns - * which are not added this time are only added again when a later schema write covers the same - * measurements. + * and none of them is propagated, so that the write operation is never affected. * * @param device2Measurements the measurements the schema write really created, eg. device01 -> * {s1, s2, s3}; device02 -> {s3, s4}. Only they get a column, so that a creation which failed @@ -198,9 +196,19 @@ private static TsTableColumnSchema toColumnSchema(final Pair } // leader and follower would build the same queryId when apply the same log; so CN could recognize - // the same add view column request + // the same add view column request. The database and table names may contain any character, so + // their lengths are written out to keep the parts apart: without them, the views a_b.c and + // a.b_c would share an id and the request for the second one would be taken for a repeat of the + // first one's. private static String buildQueryId(String columnFingerprint, String database, String table) { - return QUERY_ID_PREFIX + "_" + columnFingerprint + "_" + database + "_" + table; + return String.join( + "_", + QUERY_ID_PREFIX, + columnFingerprint, + String.valueOf(database.length()), + database, + String.valueOf(table.length()), + table); } /** diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 0bcee954fa5c1..3adfdf5f36dc3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -71,8 +71,7 @@ public class TsTable { public static final String AUTO_SYNC_SCHEMA = "auto_sync_schema"; public static final Set TABLE_ALLOWED_PROPERTIES = Collections.unmodifiableSet( - new HashSet<>( - Arrays.asList(TTL_PROPERTY, NEED_LAST_CACHE_PROPERTY, AUTO_SYNC_SCHEMA))); + new HashSet<>(Arrays.asList(TTL_PROPERTY, NEED_LAST_CACHE_PROPERTY, AUTO_SYNC_SCHEMA))); private static final String OBJECT_STRING_ERROR = "When there are object fields, the %s %s shall not be '.', '..' or contain './', '.\\'."; protected String tableName; From ef5d441e18b0e72aaab9f053080897e56b8644b9 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Mon, 19 Oct 2026 17:36:31 +0800 Subject: [PATCH 17/24] dedupe the add view request by check the column if exists in table cache --- .../table/ViewColumnAutoSync.java | 48 +++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java index 4462d91a6915f..7e5982b19a92e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java @@ -25,6 +25,8 @@ import org.apache.iotdb.commons.consensus.ConfigRegionId; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.table.AlterOrDropTableOperationType; +import org.apache.iotdb.commons.schema.table.TreeViewSchema; +import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; @@ -52,10 +54,12 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HexFormat; +import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; /** @@ -137,8 +141,8 @@ public static void sync( } /** - * @return the measurements each view covering the given devices needs; eg: view01 -> {s1, s2, - * s3}, view02 -> {s1, s2, s3, s4} + * @return the measurements each view covering the given devices still needs; eg: view01 -> {s1, + * s2, s3}, view02 -> {s1, s2, s3, s4} */ private static Map>> collectViewMeasurements( @@ -146,7 +150,7 @@ public static void sync( // deduplicate the measurement by set, the measurement with same name and dataType // from different device would count once, eg: dev01.s1(int32) and dev02.s1(int32) - // other deduplication would leave to ConfigMTree in CN + // the measurements which already have a column in the view are dropped below final Map>> view2Measurements = new HashMap<>(); @@ -162,9 +166,47 @@ public static void sync( .computeIfAbsent(view, key -> new LinkedHashSet<>()) .addAll(entry.getValue())); } + + // A measurement the view already has a column for is not needed again: ConfigNode would drop + // it in its column check and reject the request when nothing is left, which costs an RPC and a + // persisted procedure for nothing. A view the table cache does not hold is left as it is, as + // the DataNode cannot tell which columns are missing then. + final Iterator>>> iterator = + view2Measurements.entrySet().iterator(); + while (iterator.hasNext()) { + final Map.Entry>> viewEntry = + iterator.next(); + final TsTable view = getCachedView(viewEntry.getKey()); + if (Objects.isNull(view)) { + continue; + } + removeExistingMeasurements(view, viewEntry.getValue()); + if (viewEntry.getValue().isEmpty()) { + iterator.remove(); + } + } return view2Measurements; } + /** The view as the DataNode caches it, or null when the cache does not hold it. */ + private static TsTable getCachedView(final TreeToTableViewReference view) { + return DataNodeTableCache.getInstance() + .getTable(view.getDatabaseName(), view.getTableName(), false); + } + + /** Drops the measurements which the view already has a column for. */ + private static void removeExistingMeasurements( + final TsTable view, final Set> measurements) { + // The same test ConfigNode applies before it adds the columns: the column name, or the source + // measurement a FROM column references. A measurement the cache does not know yet is kept, as + // the cache may lag behind ConfigNode. + final Set sourceNames = TreeViewSchema.getSourceNames(view); + measurements.removeIf( + measurement -> + Objects.nonNull(view.getColumnSchema(measurement.getLeft().toLowerCase(Locale.ROOT))) + || sourceNames.contains(measurement.getLeft())); + } + private static TAlterOrDropTableReq toAlterTableReq( Map.Entry>> view2Measurements) { final TreeToTableViewReference view = view2Measurements.getKey(); From 7a1b1c5eede72bfd3aacf09a610449103b8cb83d Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Mon, 19 Oct 2026 18:01:34 +0800 Subject: [PATCH 18/24] add some log info --- .../confignode/i18n/ManagerMessages.java | 12 +++++++++++ .../confignode/i18n/ManagerMessages.java | 12 +++++++++++ .../confignode/manager/ProcedureManager.java | 21 +++++++++++++++++++ .../schemaregion/SchemaExecutionVisitor.java | 4 ++-- .../impl/DataNodeInternalRPCServiceImpl.java | 6 +++--- ...nc.java => ViewColumnAutoSyncTrigger.java} | 14 ++++++++----- 6 files changed, 59 insertions(+), 10 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/{ViewColumnAutoSync.java => ViewColumnAutoSyncTrigger.java} (97%) diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 482da8daad126..aad686c5adb57 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -720,4 +720,16 @@ private ManagerMessages() {} public static final String MESSAGE_CONFIGNODE_LEADER_IS_WAITING_FOR_NODE_STATUS_PERSISTENCE_8CA96809 = "ConfigNode leader is waiting for node status persistence."; + + public static final String LOG_RECEIVED_ARG_ADD_VIEW_COLUMN_REQUESTS_6ED6D52F = + "Received {} add view column requests"; + public static final String + LOG_ADD_VIEW_COLUMN_OVERLAP_RETRY_ROUND_ARG_HAS_ARG_PENDING_REQUESTS_43EC58EA = + "Add view column overlap retry round {} has {} pending requests"; + public static final String + LOG_ADD_VIEW_COLUMN_REQUEST_FOR_ARG_ARG_STILL_FAILS_AFTER_ARG_RETRY_ROUNDS_ARG_0181431F = + "Add view column request for {}.{} still fails after {} retry rounds: {}"; + public static final String + LOG_ADD_VIEW_COLUMN_REQUESTS_FOR_ARG_VIEWS_FINISHED_IN_ARG_MS_ARG_VIEWS_FAILED_4A4E0506 = + "Add view column requests for {} views finished in {} ms, {} views failed"; } diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 739ee0029c890..c4087ceca34dd 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -699,4 +699,16 @@ private ManagerMessages() {} public static final String MESSAGE_CONFIGNODE_LEADER_IS_WAITING_FOR_NODE_STATUS_PERSISTENCE_8CA96809 = "ConfigNode leader 正在等待节点状态持久化。"; + + public static final String LOG_RECEIVED_ARG_ADD_VIEW_COLUMN_REQUESTS_6ED6D52F = + "收到 {} 个树转表视图加列请求"; + public static final String + LOG_ADD_VIEW_COLUMN_OVERLAP_RETRY_ROUND_ARG_HAS_ARG_PENDING_REQUESTS_43EC58EA = + "树转表视图加列重叠重试第 {} 轮，仍有 {} 个请求待处理"; + public static final String + LOG_ADD_VIEW_COLUMN_REQUEST_FOR_ARG_ARG_STILL_FAILS_AFTER_ARG_RETRY_ROUNDS_ARG_0181431F = + "视图 {}.{} 的树转表视图加列请求重试 {} 轮后仍然失败：{}"; + public static final String + LOG_ADD_VIEW_COLUMN_REQUESTS_FOR_ARG_VIEWS_FINISHED_IN_ARG_MS_ARG_VIEWS_FAILED_4A4E0506 = + "{} 个视图的树转表视图加列请求在 {} ms 内完成，其中 {} 个失败"; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 62b90a9845f32..d20920250b8bf 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -2313,6 +2313,9 @@ public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) if (req == null || !req.isSetViewInfos()) { return new TAutoSyncViewColumnsResp(RpcUtils.getStatus(TSStatusCode.ILLEGAL_PARAMETER)); } + final long startTime = System.currentTimeMillis(); + LOGGER.info( + ManagerMessages.LOG_RECEIVED_ARG_ADD_VIEW_COLUMN_REQUESTS_6ED6D52F, req.getViewInfosSize()); final List failedViews = new ArrayList<>(); final List>> procedures = @@ -2353,6 +2356,12 @@ public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) if (!failedViews.isEmpty()) { response.setFailedViews(failedViews); } + LOGGER.info( + ManagerMessages + .LOG_ADD_VIEW_COLUMN_REQUESTS_FOR_ARG_VIEWS_FINISHED_IN_ARG_MS_ARG_VIEWS_FAILED_4A4E0506, + req.getViewInfosSize(), + System.currentTimeMillis() - startTime, + failedViews.size()); return response; } @@ -2446,6 +2455,11 @@ private void processOverlappingTask( final List failedViews) { List> pending = overlappingReqs; for (int round = 0; round < OVERLAP_RETRY_ROUNDS && !pending.isEmpty(); round++) { + LOGGER.info( + ManagerMessages + .LOG_ADD_VIEW_COLUMN_OVERLAP_RETRY_ROUND_ARG_HAS_ARG_PENDING_REQUESTS_43EC58EA, + round + 1, + pending.size()); try { Thread.sleep(OVERLAP_RETRY_WAIT_MS); } catch (final InterruptedException e) { @@ -2464,6 +2478,13 @@ private void processOverlappingTask( } for (final Pair request : pending) { + LOGGER.info( + ManagerMessages + .LOG_ADD_VIEW_COLUMN_REQUEST_FOR_ARG_ARG_STILL_FAILS_AFTER_ARG_RETRY_ROUNDS_ARG_0181431F, + request.getLeft().database, + request.getLeft().tableName, + OVERLAP_RETRY_ROUNDS, + request.getRight()); failedViews.add( new TAutoSyncViewColumnsFailure( request.getLeft().database, request.getLeft().tableName, request.getRight())); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index deedf119baa29..954fc2031e472 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -80,7 +80,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateAlignedTimeSeriesPlanImpl; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateTimeSeriesPlanImpl; import org.apache.iotdb.db.schemaengine.table.TreeToTableViewCache; -import org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSync; +import org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSyncTrigger; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -492,7 +492,7 @@ private void syncViewColumns( final String operationId = node.getPlanNodeId().getId(); try { if (TreeToTableViewCache.getInstance().hasView()) { - ViewColumnAutoSync.sync(createdMeasurements, operationId); + ViewColumnAutoSyncTrigger.sync(createdMeasurements, operationId); } } catch (final Throwable e) { logger.error( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 09884aca4f18b..41484b9dfa83c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -198,7 +198,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.table.TreeToTableViewCache; -import org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSync; +import org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSyncTrigger; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUpdateType; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUtil; @@ -410,7 +410,7 @@ import static org.apache.iotdb.commons.client.request.TestConnectionUtils.testConnectionsImpl; import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; import static org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement.AlterType.SET_DATA_TYPE; -import static org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSync.EXTEND_DEVICE_TEMPLATE_ID_PREFIX; +import static org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSyncTrigger.EXTEND_DEVICE_TEMPLATE_ID_PREFIX; import static org.apache.iotdb.db.service.RegionMigrateService.REGION_MIGRATE_PROCESS; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onIoTDBException; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onQueryException; @@ -2919,7 +2919,7 @@ public TSStatus updateTemplate(final TUpdateTemplateReq req) { try { final Map>> device2Measurements = collectDevicesUsingExtendedTemplate(oldTemplate, newTemplate); - ViewColumnAutoSync.sync( + ViewColumnAutoSyncTrigger.sync( device2Measurements, EXTEND_DEVICE_TEMPLATE_ID_PREFIX + newTemplate.getId()); } catch (final Exception e) { // todo diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSyncTrigger.java similarity index 97% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSyncTrigger.java index 7e5982b19a92e..999996c554134 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSync.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSyncTrigger.java @@ -70,9 +70,9 @@ * columns come from the measurements the schema write has just created, instead of from the * statement itself. */ -public class ViewColumnAutoSync { +public class ViewColumnAutoSyncTrigger { - private static final Logger LOGGER = LoggerFactory.getLogger(ViewColumnAutoSync.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ViewColumnAutoSyncTrigger.class); private static final IClientManager CONFIG_NODE_CLIENT_MANAGER = ConfigNodeClientManager.getInstance(); @@ -108,7 +108,9 @@ public static void sync( try (ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(AbstractConfigNodeInfo.CONFIG_REGION_ID)) { List updateViewReqs = - view2Measurements.entrySet().stream().map(ViewColumnAutoSync::toAlterTableReq).toList(); + view2Measurements.entrySet().stream() + .map(ViewColumnAutoSyncTrigger::toAlterTableReq) + .toList(); TAutoSyncViewColumnsResp resp = configNodeClient.autoSyncViewColumns(new TAutoSyncViewColumnsReq(updateViewReqs)); @@ -118,7 +120,7 @@ public static void sync( return; } if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - LOGGER.error( + LOGGER.warn( DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_FAILED_FE4F10AC, operationId, resp.getStatus()); @@ -211,7 +213,9 @@ private static TAlterOrDropTableReq toAlterTableReq( Map.Entry>> view2Measurements) { final TreeToTableViewReference view = view2Measurements.getKey(); final List columnSchemas = - view2Measurements.getValue().stream().map(ViewColumnAutoSync::toColumnSchema).toList(); + view2Measurements.getValue().stream() + .map(ViewColumnAutoSyncTrigger::toColumnSchema) + .toList(); TAlterOrDropTableReq tAlterOrDropTableReq = new TAlterOrDropTableReq( view.getDatabaseName(), From 45ea4fe2d2e4a4e346557823769cfec60ae3a465 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Tue, 20 Oct 2026 09:45:13 +0800 Subject: [PATCH 19/24] fix recent ut --- .../statemachine/dataregion/IoTConsensusShutdownTest.java | 2 +- .../iotdb/db/storageengine/load/LoadTsFileTableSchemaTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusShutdownTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusShutdownTest.java index bf2f5c21e6ade..8b3217bad2f98 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusShutdownTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusShutdownTest.java @@ -595,7 +595,7 @@ private void checkShutdownDrainsTableDeletion(final boolean failDeletion) throws DataNodeTableCache.getInstance() .preUpdateTable(DATABASE, StatementTestUtils.genTsTable(), null); DataNodeTableCache.getInstance() - .commitUpdateTable(DATABASE, StatementTestUtils.tableName(), null); + .commitUpdateTable(DATABASE, StatementTestUtils.tableName(), null, null); assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), stateMachine diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileTableSchemaTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileTableSchemaTest.java index 306f3050c4882..602ab575f6ec1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileTableSchemaTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileTableSchemaTest.java @@ -87,7 +87,7 @@ public void setUp() throws Exception { table.addColumnSchema( new FieldColumnSchema("value", TSDataType.INT32, TSEncoding.PLAIN, CompressionType.LZ4)); DataNodeTableCache.getInstance().preUpdateTable(DATABASE, table, null); - DataNodeTableCache.getInstance().commitUpdateTable(DATABASE, TABLE, null); + DataNodeTableCache.getInstance().commitUpdateTable(DATABASE, TABLE, null, null); } @After From 953741d5f95219d78a02e591d81cb4e696adb7d0 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Tue, 20 Oct 2026 10:56:25 +0800 Subject: [PATCH 20/24] Integrate MetadataLeaseManager framework management --- .../db/schemaengine/lease/MetadataLeaseManager.java | 4 +++- .../db/schemaengine/table/DataNodeTableCache.java | 12 ------------ .../db/schemaengine/table/TreeToTableViewCache.java | 13 +++++++++++-- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManager.java index 7f7365a1550e3..5f3c7357c885d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManager.java @@ -30,6 +30,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.db.schemaengine.table.TreeToTableViewCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,7 +109,8 @@ private static List defaultClearCacheList() { () -> ClusterPartitionFetcher.getInstance().invalidAllCache(), () -> DataNodeTableCache.getInstance().invalidateAll(), () -> TreeDeviceSchemaCacheManager.getInstance().cleanUp(), - AuthorityChecker::invalidateAllCache); + AuthorityChecker::invalidateAllCache, + () -> TreeToTableViewCache.getInstance().clear()); } private static List defaultPullMetaList() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index d58ae384161b4..71bd1c1417873 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -202,18 +202,6 @@ public void reloadTableCacheAfterLeaseRecovery() { } } - /** - * The case that pre update Table and pre delete Table procedures targeting the same table are - * executed serially by CN. - * - *

Consider the scenario: - * - *

    - *
  1. Drop the table first: DN executed pre update but missed the commit phase - *
  2. Create table second: DN executed pre update, which overwrites the result of the drop - * table procedure in {@link #specialStatusMap} - *
- */ @Override public void preUpdateTable(String database, final TsTable table, final String oldName) { database = PathUtils.unQualifyDatabaseName(database); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java index 86ca045679bd7..a41e00b6878ff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java @@ -19,12 +19,14 @@ package org.apache.iotdb.db.schemaengine.table; import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.exception.MetadataLeaseFencedException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternUtil; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.db.i18n.DataNodeSchemaMessages; +import org.apache.iotdb.db.schemaengine.lease.MetadataLeaseManager; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader; import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference; @@ -142,7 +144,6 @@ private void registerView(final String database, final TsTable table) { * @parm Partial path, eg: root.db.dev01.**; * @return true if a new reference was registered; false if it was already present */ - // todo, log; @Override public boolean addView( final String database, @@ -364,11 +365,13 @@ private static TreeToTableViewReference findReference( */ @Override public Set getTreeToTableViews(PartialPath path) { + // A fenced DataNode must not answer from its possibly stale index, not even with "no view + // covers this path": the caller has to know that the cache is unavailable rather than empty. + failIfMetadataLeaseFenced(); if (!hasView) { // the answer for every path when no view is registered at all return Collections.emptySet(); } - final String[] nodes = path.getNodes(); readWriteLock.readLock().lock(); try { @@ -446,6 +449,12 @@ private void pruneNodesWithoutView(final IMemMNode node) { } } + void failIfMetadataLeaseFenced() { + MetadataLeaseManager.getInstance() + .failIfMetadataLeaseFenced( + MetadataLeaseFencedException.LeaseFencedRetryPolicy.RETRY_UNTIL_SUCCESS); + } + private static final class TreeToTableViewCacheHolder { private static final TreeToTableViewCache INSTANCE = new TreeToTableViewCache(); From ef9f73cc98444515f58a4db0c7163133fcb7857a Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Tue, 20 Oct 2026 18:19:21 +0800 Subject: [PATCH 21/24] add operation log for TreeToTableViewCache --- .../iotdb/db/i18n/DataNodeSchemaMessages.java | 10 +++ .../iotdb/db/i18n/DataNodeSchemaMessages.java | 10 +++ .../table/DataNodeTableCache.java | 83 +++++++++++++++---- .../table/TreeToTableViewCache.java | 2 +- .../table/DataNodeTableCacheTest.java | 4 +- 5 files changed, 92 insertions(+), 17 deletions(-) diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java index d3a92eee16826..858ba4f4be2de 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java @@ -748,5 +748,15 @@ private DataNodeSchemaMessages() {} public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED_B909817F = "Failed to trigger the view column addition of the tree to table views for plan node {}, the schema write is not affected."; public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902 = "Tree to table view {}.{} is not registered"; public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE = "Failed to update the tree-to-table view index of view {}.{}"; + public static final String + LOG_REGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_AT_ARG_WITH_RESTRICT_ARG_AND_NUM_OF_TAGS_ARG_7538BCB2 = + "Registered the tree to table view {}.{} at {} with restrict {} and num of tags {}"; + public static final String LOG_UNREGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_C90B661A = + "Unregistered the tree to table view {}.{}"; + public static final String LOG_RENAMED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_FROM_ARG_B5D52D41 = + "Renamed the tree to table view {}.{} from {}"; + public static final String + LOG_UPDATED_THE_PROPERTIES_OF_THE_TREE_TO_TABLE_VIEW_ARG_ARG_TO_ARG_1C3DE984 = + "Updated the properties of the tree to table view {}.{} to {}"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java index 9966a203d7d2b..cbcb0331b0d33 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java @@ -736,5 +736,15 @@ private DataNodeSchemaMessages() {} public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED_B909817F = "计划节点 {} 触发树转表视图加列失败，本次 schema 写入不受影响。"; public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902 = "树转表视图 {}.{} 未注册"; public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE = "更新树转表视图 {}.{} 的索引失败"; + public static final String + LOG_REGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_AT_ARG_WITH_RESTRICT_ARG_AND_NUM_OF_TAGS_ARG_7538BCB2 = + "注册树转表视图 {}.{}，路径 {}，restrict {}，tag 数 {}"; + public static final String LOG_UNREGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_C90B661A = + "注销树转表视图 {}.{}"; + public static final String LOG_RENAMED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_FROM_ARG_B5D52D41 = + "树转表视图 {}.{} 由 {} 重命名而来"; + public static final String + LOG_UPDATED_THE_PROPERTIES_OF_THE_TREE_TO_TABLE_VIEW_ARG_ARG_TO_ARG_1C3DE984 = + "树转表视图 {}.{} 的属性更新为 {}"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index e3ba07fa2c44d..1f10cb7f4b54f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -421,34 +421,38 @@ private void operateOnTree2TableViewCache( final TreeToTableViewTrieOperationType type = TreeToTableViewTrieOperationType.getType( updateTree2TableViewInfo.getTree2TableOperationType()); + final boolean changed; try { switch (type) { case REMOVE: // Unregistering a view needs nothing but its name. - treeToTableViewCache.removeView(database, tableName); + changed = treeToTableViewCache.removeView(database, tableName); break; case ALTER_RENAME: // A rename keeps the place of the view. - treeToTableViewCache.alterViewName(database, tableName, oldTableName); + changed = treeToTableViewCache.alterViewName(database, tableName, oldTableName); break; case ALTER_TAG: - final Map properties = new HashMap<>(); - properties.put( - TreeToTableViewReference.NUM_OF_TAGS, - String.valueOf(updateTree2TableViewInfo.getTagNum())); - treeToTableViewCache.alterViewProperties(database, tableName, properties); + changed = + treeToTableViewCache.alterViewProperties( + database, + tableName, + Collections.singletonMap( + TreeToTableViewReference.NUM_OF_TAGS, + String.valueOf(updateTree2TableViewInfo.getTagNum()))); break; case ADD: case ALTER_AUTO_SYNC: - treeToTableViewCache.addView( - database, - tableName, - updateTree2TableViewInfo.isRestrict, - updateTree2TableViewInfo.getTagNum(), - new PartialPath(updateTree2TableViewInfo.getTreePath())); + changed = + treeToTableViewCache.addView( + database, + tableName, + updateTree2TableViewInfo.isRestrict, + updateTree2TableViewInfo.getTagNum(), + new PartialPath(updateTree2TableViewInfo.getTreePath())); break; default: - break; + return; } } catch (final IllegalPathException e) { LOGGER.warn( @@ -457,6 +461,57 @@ private void operateOnTree2TableViewCache( database, tableName, e); + return; + } + if (changed) { + logTreeToTableViewChange(type, database, tableName, oldTableName, updateTree2TableViewInfo); + } + } + + /** Logs the change an operation really applied to the tree-to-table view index. */ + private static void logTreeToTableViewChange( + final TreeToTableViewTrieOperationType type, + final String database, + final String tableName, + final @Nullable String oldTableName, + final TUpdateTree2TableView updateTree2TableViewInfo) { + switch (type) { + case REMOVE: + TreeToTableViewCache.LOGGER.info( + DataNodeSchemaMessages.LOG_UNREGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_C90B661A, + database, + tableName); + break; + case ALTER_RENAME: + TreeToTableViewCache.LOGGER.info( + DataNodeSchemaMessages.LOG_RENAMED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_FROM_ARG_B5D52D41, + database, + tableName, + oldTableName); + break; + case ALTER_TAG: + TreeToTableViewCache.LOGGER.info( + DataNodeSchemaMessages + .LOG_UPDATED_THE_PROPERTIES_OF_THE_TREE_TO_TABLE_VIEW_ARG_ARG_TO_ARG_1C3DE984, + database, + tableName, + Collections.singletonMap( + TreeToTableViewReference.NUM_OF_TAGS, + String.valueOf(updateTree2TableViewInfo.getTagNum()))); + break; + case ADD: + case ALTER_AUTO_SYNC: + TreeToTableViewCache.LOGGER.info( + DataNodeSchemaMessages + .LOG_REGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_AT_ARG_WITH_RESTRICT_ARG_AND_NUM_OF_TAGS_ARG_7538BCB2, + database, + tableName, + updateTree2TableViewInfo.getTreePath(), + updateTree2TableViewInfo.isRestrict, + updateTree2TableViewInfo.getTagNum()); + break; + default: + break; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java index a41e00b6878ff..473b2266f54a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java @@ -63,7 +63,7 @@ */ public class TreeToTableViewCache implements ITreeToTableViewCache { - private static final Logger LOGGER = LoggerFactory.getLogger(TreeToTableViewCache.class); + public static final Logger LOGGER = LoggerFactory.getLogger(TreeToTableViewCache.class); private final IMNodeFactory nodeFactory = MNodeFactoryLoader.getInstance().getMemMNodeIMNodeFactory(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java index 3345c46dcd38c..18ee416d2e843 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java @@ -60,12 +60,12 @@ public void localReadUsesCommittedSchemaWithoutResolvingPendingUpdates() { final TsTable initial = createTable(TABLE_NAME); cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, initial, null); Assert.assertNull(cache.getTableInCache(TABLE_CACHE_TEST_DATABASE, TABLE_NAME)); - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null); Assert.assertSame(initial, cache.getTableInCache(TABLE_CACHE_TEST_DATABASE, TABLE_NAME)); final TsTable updated = new TsTable(initial); cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, updated, null); Assert.assertSame(initial, cache.getTableInCache(TABLE_CACHE_TEST_DATABASE, TABLE_NAME)); - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null); Assert.assertSame(updated, cache.getTableInCache(TABLE_CACHE_TEST_DATABASE, TABLE_NAME)); } finally { cache.invalid(TABLE_CACHE_TEST_DATABASE); From 326d21dc8fb24fd192e7153b72db5168be652279 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Wed, 21 Oct 2026 10:16:58 +0800 Subject: [PATCH 22/24] integrate the logs of ViewColumnAutoSyncTrigger into a solo log files --- .../confignode/i18n/ManagerMessages.java | 12 ---- .../confignode/i18n/ManagerMessages.java | 12 ---- .../confignode/manager/ProcedureManager.java | 21 ------ .../resources/conf/logback-datanode.xml | 21 ++++++ .../iotdb/db/i18n/DataNodeSchemaMessages.java | 3 + .../iotdb/db/i18n/DataNodeSchemaMessages.java | 3 + .../schemaregion/SchemaExecutionVisitor.java | 4 +- .../impl/DataNodeInternalRPCServiceImpl.java | 3 +- .../table/ViewColumnAutoSyncTrigger.java | 66 +++++++++++++++++-- 9 files changed, 93 insertions(+), 52 deletions(-) diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java index aad686c5adb57..482da8daad126 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -720,16 +720,4 @@ private ManagerMessages() {} public static final String MESSAGE_CONFIGNODE_LEADER_IS_WAITING_FOR_NODE_STATUS_PERSISTENCE_8CA96809 = "ConfigNode leader is waiting for node status persistence."; - - public static final String LOG_RECEIVED_ARG_ADD_VIEW_COLUMN_REQUESTS_6ED6D52F = - "Received {} add view column requests"; - public static final String - LOG_ADD_VIEW_COLUMN_OVERLAP_RETRY_ROUND_ARG_HAS_ARG_PENDING_REQUESTS_43EC58EA = - "Add view column overlap retry round {} has {} pending requests"; - public static final String - LOG_ADD_VIEW_COLUMN_REQUEST_FOR_ARG_ARG_STILL_FAILS_AFTER_ARG_RETRY_ROUNDS_ARG_0181431F = - "Add view column request for {}.{} still fails after {} retry rounds: {}"; - public static final String - LOG_ADD_VIEW_COLUMN_REQUESTS_FOR_ARG_VIEWS_FINISHED_IN_ARG_MS_ARG_VIEWS_FAILED_4A4E0506 = - "Add view column requests for {} views finished in {} ms, {} views failed"; } diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java index c4087ceca34dd..739ee0029c890 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -699,16 +699,4 @@ private ManagerMessages() {} public static final String MESSAGE_CONFIGNODE_LEADER_IS_WAITING_FOR_NODE_STATUS_PERSISTENCE_8CA96809 = "ConfigNode leader 正在等待节点状态持久化。"; - - public static final String LOG_RECEIVED_ARG_ADD_VIEW_COLUMN_REQUESTS_6ED6D52F = - "收到 {} 个树转表视图加列请求"; - public static final String - LOG_ADD_VIEW_COLUMN_OVERLAP_RETRY_ROUND_ARG_HAS_ARG_PENDING_REQUESTS_43EC58EA = - "树转表视图加列重叠重试第 {} 轮，仍有 {} 个请求待处理"; - public static final String - LOG_ADD_VIEW_COLUMN_REQUEST_FOR_ARG_ARG_STILL_FAILS_AFTER_ARG_RETRY_ROUNDS_ARG_0181431F = - "视图 {}.{} 的树转表视图加列请求重试 {} 轮后仍然失败：{}"; - public static final String - LOG_ADD_VIEW_COLUMN_REQUESTS_FOR_ARG_VIEWS_FINISHED_IN_ARG_MS_ARG_VIEWS_FAILED_4A4E0506 = - "{} 个视图的树转表视图加列请求在 {} ms 内完成，其中 {} 个失败"; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index d20920250b8bf..62b90a9845f32 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -2313,9 +2313,6 @@ public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) if (req == null || !req.isSetViewInfos()) { return new TAutoSyncViewColumnsResp(RpcUtils.getStatus(TSStatusCode.ILLEGAL_PARAMETER)); } - final long startTime = System.currentTimeMillis(); - LOGGER.info( - ManagerMessages.LOG_RECEIVED_ARG_ADD_VIEW_COLUMN_REQUESTS_6ED6D52F, req.getViewInfosSize()); final List failedViews = new ArrayList<>(); final List>> procedures = @@ -2356,12 +2353,6 @@ public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) if (!failedViews.isEmpty()) { response.setFailedViews(failedViews); } - LOGGER.info( - ManagerMessages - .LOG_ADD_VIEW_COLUMN_REQUESTS_FOR_ARG_VIEWS_FINISHED_IN_ARG_MS_ARG_VIEWS_FAILED_4A4E0506, - req.getViewInfosSize(), - System.currentTimeMillis() - startTime, - failedViews.size()); return response; } @@ -2455,11 +2446,6 @@ private void processOverlappingTask( final List failedViews) { List> pending = overlappingReqs; for (int round = 0; round < OVERLAP_RETRY_ROUNDS && !pending.isEmpty(); round++) { - LOGGER.info( - ManagerMessages - .LOG_ADD_VIEW_COLUMN_OVERLAP_RETRY_ROUND_ARG_HAS_ARG_PENDING_REQUESTS_43EC58EA, - round + 1, - pending.size()); try { Thread.sleep(OVERLAP_RETRY_WAIT_MS); } catch (final InterruptedException e) { @@ -2478,13 +2464,6 @@ private void processOverlappingTask( } for (final Pair request : pending) { - LOGGER.info( - ManagerMessages - .LOG_ADD_VIEW_COLUMN_REQUEST_FOR_ARG_ARG_STILL_FAILS_AFTER_ARG_RETRY_ROUNDS_ARG_0181431F, - request.getLeft().database, - request.getLeft().tableName, - OVERLAP_RETRY_ROUNDS, - request.getRight()); failedViews.add( new TAutoSyncViewColumnsFailure( request.getLeft().database, request.getLeft().tableName, request.getRight())); diff --git a/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml b/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml index 500d1d9c46020..e5bbbdba23041 100644 --- a/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml +++ b/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml @@ -91,6 +91,24 @@ DENY + + + ${IOTDB_HOME}/logs/log_datanode_view_column_auto_sync_error.log + + ${IOTDB_HOME}/logs/log-datanode-view-column-auto-sync-error-%d{yyyyMMdd}.log.gz + 30 + + true + + %d [%t] %-5p %C{25}:%L - %m %n + utf-8 + + + ERROR + ACCEPT + DENY + + System.out @@ -218,6 +236,9 @@ + + + diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java index 858ba4f4be2de..2502aef79dc8a 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java @@ -758,5 +758,8 @@ private DataNodeSchemaMessages() {} public static final String LOG_UPDATED_THE_PROPERTIES_OF_THE_TREE_TO_TABLE_VIEW_ARG_ARG_TO_ARG_1C3DE984 = "Updated the properties of the tree to table view {}.{} to {}"; + public static final String + LOG_THE_METADATA_LEASE_IS_FENCED_WHILE_ADDING_VIEW_COLUMNS_GIVING_UP_THIS_ROUND_THE_MEASUREMENTS_ARE_ARG_5D1B2D44 = + "The metadata lease is fenced while adding view columns, giving up this round; the measurements are {}"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java index cbcb0331b0d33..18d6443388f5e 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java @@ -746,5 +746,8 @@ private DataNodeSchemaMessages() {} public static final String LOG_UPDATED_THE_PROPERTIES_OF_THE_TREE_TO_TABLE_VIEW_ARG_ARG_TO_ARG_1C3DE984 = "树转表视图 {}.{} 的属性更新为 {}"; + public static final String + LOG_THE_METADATA_LEASE_IS_FENCED_WHILE_ADDING_VIEW_COLUMNS_GIVING_UP_THIS_ROUND_THE_MEASUREMENTS_ARE_ARG_5D1B2D44 = + "自动扩列时元数据租约已隔离，本次放弃；涉及的测点为 {}"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index 954fc2031e472..47b353aefa8c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -495,7 +495,9 @@ private void syncViewColumns( ViewColumnAutoSyncTrigger.sync(createdMeasurements, operationId); } } catch (final Throwable e) { - logger.error( + // Logged with the logger of the feature, so that every failure of the auto column addition, + // including the ones the trigger itself does not see, reaches its dedicated log file. + ViewColumnAutoSyncTrigger.LOGGER.error( DataNodeSchemaMessages.LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED_B909817F, operationId, e); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 41484b9dfa83c..5d3bce4e4c32b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -2922,8 +2922,7 @@ public TSStatus updateTemplate(final TUpdateTemplateReq req) { ViewColumnAutoSyncTrigger.sync( device2Measurements, EXTEND_DEVICE_TEMPLATE_ID_PREFIX + newTemplate.getId()); } catch (final Exception e) { - // todo - LOGGER.warn( + ViewColumnAutoSyncTrigger.LOGGER.warn( DataNodeMiscMessages .LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_AB32D554, newTemplate.getName(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSyncTrigger.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSyncTrigger.java index 999996c554134..9a937fcd315f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSyncTrigger.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSyncTrigger.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.consensus.ConfigRegionId; +import org.apache.iotdb.commons.exception.MetadataLeaseFencedException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.table.AlterOrDropTableOperationType; import org.apache.iotdb.commons.schema.table.TreeViewSchema; @@ -72,13 +73,20 @@ */ public class ViewColumnAutoSyncTrigger { - private static final Logger LOGGER = LoggerFactory.getLogger(ViewColumnAutoSyncTrigger.class); + // Public so that the schema write path, which invokes sync and catches its failures, logs them + // with the logger of the feature; logback routes this logger to its own error file. + public static final Logger LOGGER = LoggerFactory.getLogger(ViewColumnAutoSyncTrigger.class); private static final IClientManager CONFIG_NODE_CLIENT_MANAGER = ConfigNodeClientManager.getInstance(); public static final String QUERY_ID_PREFIX = "tree_to_table_add_column"; public static final String EXTEND_DEVICE_TEMPLATE_ID_PREFIX = "template_"; + /** How many devices, and how many measurements per device, a fenced lease log line shows. */ + private static final int MAX_LOGGED_DEVICES = 10; + + private static final int MAX_LOGGED_MEASUREMENTS_PER_DEVICE = 10; + /** * Adds the columns the views covering the given devices need. * @@ -99,8 +107,18 @@ public static void sync( } // the views to be added columns - final Map>> view2Measurements = - collectViewMeasurements(device2Measurements); + final Map>> view2Measurements; + try { + view2Measurements = collectViewMeasurements(device2Measurements); + } catch (final MetadataLeaseFencedException e) { + // The DataNode is isolated: neither the index nor the table cache it consults may be used. + // Give up this round and record the measurements which cannot be added while it is fenced. + LOGGER.error( + DataNodeSchemaMessages + .LOG_THE_METADATA_LEASE_IS_FENCED_WHILE_ADDING_VIEW_COLUMNS_GIVING_UP_THIS_ROUND_THE_MEASUREMENTS_ARE_ARG_5D1B2D44, + formatDeviceMeasurements(device2Measurements)); + return; + } if (view2Measurements.isEmpty()) { return; } @@ -120,7 +138,7 @@ public static void sync( return; } if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - LOGGER.warn( + LOGGER.error( DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_FAILED_FE4F10AC, operationId, resp.getStatus()); @@ -142,6 +160,46 @@ public static void sync( } } + /** + * A bounded, human readable form of the measurements, so that the log line of a batch write stays + * short: at most {@link #MAX_LOGGED_DEVICES} devices and {@link + * #MAX_LOGGED_MEASUREMENTS_PER_DEVICE} measurements per device are written, the rest is + * summarized. + */ + private static String formatDeviceMeasurements( + final Map>> device2Measurements) { + final StringBuilder builder = new StringBuilder("{"); + int loggedDevices = 0; + for (final Map.Entry>> entry : + device2Measurements.entrySet()) { + if (loggedDevices == MAX_LOGGED_DEVICES) { + builder.append("... (").append(device2Measurements.size()).append(" devices in total)}"); + return builder.toString(); + } + if (loggedDevices > 0) { + builder.append(", "); + } + builder.append(entry.getKey().getFullPath()).append("=["); + final List> measurements = entry.getValue(); + for (int i = 0; i < measurements.size(); i++) { + if (i == MAX_LOGGED_MEASUREMENTS_PER_DEVICE) { + builder.append("... (").append(measurements.size()).append(" measurements in total)"); + break; + } + if (i > 0) { + builder.append(", "); + } + builder + .append(measurements.get(i).getLeft()) + .append(':') + .append(measurements.get(i).getRight()); + } + builder.append(']'); + loggedDevices++; + } + return builder.append('}').toString(); + } + /** * @return the measurements each view covering the given devices still needs; eg: view01 -> {s1, * s2, s3}, view02 -> {s1, s2, s3, s4} From dedf4abb4e922a9da573da798b2d86258d5977d7 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Wed, 21 Oct 2026 16:12:48 +0800 Subject: [PATCH 23/24] add IT --- .../IoTDBTreeToTableViewAutoSyncIT.java | 717 ++++++++++++++++++ .../config/TableConfigTaskVisitor.java | 4 - .../iotdb/commons/schema/table/TsTable.java | 2 +- 3 files changed, 718 insertions(+), 5 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTreeToTableViewAutoSyncIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTreeToTableViewAutoSyncIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTreeToTableViewAutoSyncIT.java new file mode 100644 index 0000000000000..17dfee8483602 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTreeToTableViewAutoSyncIT.java @@ -0,0 +1,717 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.schema; + +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBTreeToTableViewAutoSyncIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + // ==================== the property and its index (Trie) ==================== + + /** + * The property defaults to false: when the view is created, only the series which exist become + * columns. An explicit false behaves the same, and SHOW CREATE VIEW only shows the property when + * it is stored. + */ + @Test + @Ignore + public void testAutoSyncSchemaDisabledByDefault() throws Exception { + final String treeDb = "root.as01"; + final String viewDb = "as01"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s1 INT32"); + + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute(table, "CREATE VIEW v1 (dev STRING TAG) AS " + treeDb + ".**"); + // the creation always scans the series which already exist + awaitFields(table, viewDb, "v1", fields("s1", "INT32")); + Assert.assertFalse(showCreateView(table, viewDb, "v1").contains("auto_sync_schema")); + + // the default is false, so a new measurement is not added + execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s2 INT32"); + awaitAbsent(table, viewDb, "v1", "s2"); + + execute( + table, + "CREATE VIEW v2 (dev STRING TAG) WITH (auto_sync_schema=false) AS " + treeDb + ".**"); + awaitFields(table, viewDb, "v2", fields("s1", "INT32", "s2", "INT32")); + Assert.assertTrue(showCreateView(table, viewDb, "v2").contains("auto_sync_schema=false")); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s3 INT32"); + awaitAbsent(table, viewDb, "v2", "s3"); + + // an explicit true follows the new measurements, both the existing and the later ones + execute( + table, + "CREATE VIEW v3 (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".**"); + awaitFields(table, viewDb, "v3", fields("s1", "INT32", "s2", "INT32", "s3", "INT32")); + Assert.assertTrue(showCreateView(table, viewDb, "v3").contains("auto_sync_schema=true")); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s4 INT32"); + awaitFields( + table, viewDb, "v3", fields("s1", "INT32", "s2", "INT32", "s3", "INT32", "s4", "INT32")); + } + } + + /** The property is only accepted by the views created on a tree path, not by the tables. */ + @Test + @Ignore + public void testAutoSyncSchemaOnlyForTreeToTableViews() throws Exception { + final String viewDb = "as02"; + try (Connection table = tableConnection()) { + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute(table, "CREATE TABLE t1 (tag1 STRING TAG, s1 INT32 FIELD)"); + + assertSqlError( + table, + "CREATE TABLE t2 (tag1 STRING TAG, s1 INT32 FIELD) WITH (auto_sync_schema=true)", + "auto_sync_schema"); + assertSqlError( + table, "ALTER TABLE t1 SET PROPERTIES auto_sync_schema=true", "auto_sync_schema"); + } + } + + /** + * A measurement whose name only differs in case from an existing column, or whose name is the + * same but whose type is different, does not change the view; the other measurements of the same + * write are still added. + */ + @Test + @Ignore + public void testConflictingMeasurementsDoNotChangeTheView() throws Exception { + final String treeDb = "root.as03"; + final String viewDb = "as03"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s1 DOUBLE"); + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute( + table, + "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".**"); + awaitFields(table, viewDb, "v", fields("s1", "DOUBLE")); + + // the same name with another type: the series is created, the column is not touched + execute(tree, "CREATE TIMESERIES " + treeDb + ".d2.s1 INT32"); + // a new measurement of the same write is still added + execute(tree, "CREATE TIMESERIES " + treeDb + ".d2.s2 INT32"); + awaitFields(table, viewDb, "v", fields("s1", "DOUBLE", "s2", "INT32")); + + // the same name in another case is a conflict as well: no column is added + execute(tree, "CREATE TIMESERIES " + treeDb + ".d3.S2 INT32"); + awaitFields(table, viewDb, "v", fields("s1", "DOUBLE", "s2", "INT32")); + } + } + + /** + * A restrict view matches a device with exactly numOfTags levels under its prefix, a non-restrict + * view matches every device with at most numOfTags levels. A device may be covered by several + * views at once. + */ + @Test + @Ignore + public void testTrieMatchingOfRestrictAndNonRestrictViews() throws Exception { + final String treeDb = "root.as04"; + final String viewDb = "as04"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + treeDb); + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute( + table, + "CREATE VIEW v_restrict (a STRING TAG, b STRING TAG) RESTRICT " + + "WITH (auto_sync_schema=true) AS " + + treeDb + + ".**"); + execute( + table, + "CREATE VIEW v_non_restrict (a STRING TAG) " + + "WITH (auto_sync_schema=true) AS " + + treeDb + + ".**"); + + // one level under the prefix: only the non-restrict view matches + execute(tree, "CREATE TIMESERIES " + treeDb + ".p.s1 INT32"); + awaitFields(table, viewDb, "v_non_restrict", fields("s1", "INT32")); + awaitAbsent(table, viewDb, "v_restrict", "s1"); + + // two levels: only the restrict view matches + execute(tree, "CREATE TIMESERIES " + treeDb + ".p.q.s2 INT32"); + awaitFields(table, viewDb, "v_restrict", fields("s2", "INT32")); + awaitFields(table, viewDb, "v_non_restrict", fields("s1", "INT32")); + + // three levels: neither view matches + execute(tree, "CREATE TIMESERIES " + treeDb + ".p.q.r.s3 INT32"); + awaitAbsent(table, viewDb, "v_restrict", "s3"); + awaitAbsent(table, viewDb, "v_non_restrict", "s3"); + } + } + + /** Renaming a view keeps its place in the index; the property can be switched on and off. */ + @Test + @Ignore + public void testTrieRenameAndToggleAutoSyncSchema() throws Exception { + final String treeDb = "root.as05"; + final String viewDb = "as05"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + treeDb); + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute( + table, + "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".**"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s1 INT32"); + awaitFields(table, viewDb, "v", fields("s1", "INT32")); + + execute(table, "ALTER VIEW v RENAME TO v2"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d2.s2 INT32"); + awaitFields(table, viewDb, "v2", fields("s1", "INT32", "s2", "INT32")); + assertSqlError(table, "DESC v", "does not exist"); + + // turned off: the new measurements are ignored + execute(table, "ALTER VIEW v2 SET PROPERTIES auto_sync_schema=false"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d3.s3 INT32"); + awaitAbsent(table, viewDb, "v2", "s3"); + + // turned on again: the missing columns are backfilled, and the later ones are followed + execute(table, "ALTER VIEW v2 SET PROPERTIES auto_sync_schema=true"); + awaitFields(table, viewDb, "v2", fields("s1", "INT32", "s2", "INT32", "s3", "INT32")); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d4.s4 INT32"); + awaitFields( + table, viewDb, "v2", fields("s1", "INT32", "s2", "INT32", "s3", "INT32", "s4", "INT32")); + } + } + + /** + * CREATE OR REPLACE registers the new definition of the view: the old path stops being followed, + * and the new matching attributes take effect. + */ + @Test + @Ignore + public void testTrieReplaceView() throws Exception { + final String treeDb = "root.as06"; + final String viewDb = "as06"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + treeDb); + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute( + table, + "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".old.**"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".old.a.s1 INT32"); + awaitFields(table, viewDb, "v", fields("s1", "INT32")); + + // another source path: the new definition replaces the old one + execute( + table, + "CREATE OR REPLACE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + + treeDb + + ".new.**"); + awaitFields(table, viewDb, "v", fields()); + execute(tree, "CREATE TIMESERIES " + treeDb + ".old.b.s2 INT32"); + awaitAbsent(table, viewDb, "v", "s2"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".new.c.s3 INT32"); + awaitFields(table, viewDb, "v", fields("s3", "INT32")); + + // the same path with another tag count: the new matching attributes replace the old ones + execute( + table, + "CREATE VIEW w (a STRING TAG) RESTRICT WITH (auto_sync_schema=true) AS " + + treeDb + + ".replace.**"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".replace.p.s1 INT32"); + awaitFields(table, viewDb, "w", fields("s1", "INT32")); + execute( + table, + "CREATE OR REPLACE VIEW w (a STRING TAG, b STRING TAG) RESTRICT " + + "WITH (auto_sync_schema=true) AS " + + treeDb + + ".replace.**"); + awaitFields(table, viewDb, "w", fields()); + execute(tree, "CREATE TIMESERIES " + treeDb + ".replace.p.q.s2 INT32"); + awaitFields(table, viewDb, "w", fields("s2", "INT32")); + execute(tree, "CREATE TIMESERIES " + treeDb + ".replace.m.s3 INT32"); + awaitAbsent(table, viewDb, "w", "s3"); + } + } + + /** Dropping a view or its database removes the entry from the index of the DataNodes. */ + @Test + @Ignore + public void testTrieDropViewAndDropDatabase() throws Exception { + final String treeDb = "root.as07"; + final String viewDb = "as07"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + treeDb); + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute( + table, + "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".old.**"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".old.a.s1 INT32"); + awaitFields(table, viewDb, "v", fields("s1", "INT32")); + + execute(table, "DROP VIEW v"); + assertSqlError(table, "DESC v", "does not exist"); + // a view of another path follows only its own path + execute( + table, + "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".new.**"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".old.b.s2 INT32"); + awaitAbsent(table, viewDb, "v", "s2"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".new.c.s3 INT32"); + awaitFields(table, viewDb, "v", fields("s3", "INT32")); + } + + // dropping the database removes its views from the index: a view of the same name created + // again with another source path is not polluted by the old one + final String replacedTreeDb = "root.as07b"; + final String replacedViewDb = "as07b"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + replacedTreeDb); + execute(tree, "CREATE TIMESERIES " + replacedTreeDb + ".old.d1.s1 INT32"); + execute(table, "CREATE DATABASE " + replacedViewDb); + execute(table, "USE " + replacedViewDb); + execute( + table, + "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + + replacedTreeDb + + ".old.**"); + awaitFields(table, replacedViewDb, "v", fields("s1", "INT32")); + + execute(table, "DROP DATABASE " + replacedViewDb); + execute(table, "CREATE DATABASE " + replacedViewDb); + execute(table, "USE " + replacedViewDb); + execute( + table, + "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + + replacedTreeDb + + ".new.**"); + + execute(tree, "CREATE TIMESERIES " + replacedTreeDb + ".old.d2.s2 INT32"); + awaitAbsent(table, replacedViewDb, "v", "s2"); + execute(tree, "CREATE TIMESERIES " + replacedTreeDb + ".new.d3.s3 INT32"); + awaitFields(table, replacedViewDb, "v", fields("s3", "INT32")); + } + } + + /** Adding a tag column changes the tag count the view matches devices with. */ + @Test + @Ignore + public void testTrieAddingTagColumnUpdatesTheTagCount() throws Exception { + final String treeDb = "root.as08"; + final String viewDb = "as08"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + treeDb); + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute( + table, + "CREATE VIEW v (a STRING TAG) RESTRICT WITH (auto_sync_schema=true) AS " + + treeDb + + ".**"); + + // the tag count of the view, hence the devices it matches, changes with the new tag column + execute(table, "ALTER VIEW v ADD COLUMN b STRING TAG"); + + // two levels under the prefix now match the restrict view + execute(tree, "CREATE TIMESERIES " + treeDb + ".p.q.s2 INT32"); + awaitFields(table, viewDb, "v", fields("s2", "INT32")); + // one level does not match any more + execute(tree, "CREATE TIMESERIES " + treeDb + ".p.s1 INT32"); + awaitAbsent(table, viewDb, "v", "s1"); + // three levels do not match either + execute(tree, "CREATE TIMESERIES " + treeDb + ".p.q.r.s3 INT32"); + awaitAbsent(table, viewDb, "v", "s3"); + } + } + + // ==================== the interfaces which create measurements ==================== + + @Test + @Ignore + public void testCreateTimeSeriesAddsColumn() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + final String treeDb = "root.as10"; + final String viewDb = "as10"; + prepareView(tree, table, treeDb, viewDb); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s1 INT32"); + awaitFields(table, viewDb, "v", fields("s1", "INT32")); + } + } + + @Test + @Ignore + public void testCreateAlignedTimeSeriesAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + final String treeDb = "root.as11"; + final String viewDb = "as11"; + prepareView(tree, table, treeDb, viewDb); + execute(tree, "CREATE ALIGNED TIMESERIES " + treeDb + ".d1 (s1 INT32, s2 DOUBLE)"); + awaitFields(table, viewDb, "v", fields("s1", "INT32", "s2", "DOUBLE")); + } + } + + @Test + @Ignore + public void testCreateMultiTimeseriesAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection(); + ISession session = EnvFactory.getEnv().getSessionConnection()) { + final String treeDb = "root.as12"; + final String viewDb = "as12"; + prepareView(tree, table, treeDb, viewDb); + session.createMultiTimeseries( + Arrays.asList(treeDb + ".d1.s1", treeDb + ".d1.s2"), + Arrays.asList(TSDataType.INT32, TSDataType.DOUBLE), + Arrays.asList(TSEncoding.PLAIN, TSEncoding.PLAIN), + Arrays.asList(CompressionType.LZ4, CompressionType.LZ4), + null, + null, + null, + null); + awaitFields(table, viewDb, "v", fields("s1", "INT32", "s2", "DOUBLE")); + } + } + + @Test + @Ignore + public void testInsertAutoCreatesColumn() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + final String treeDb = "root.as13"; + final String viewDb = "as13"; + prepareView(tree, table, treeDb, viewDb); + execute(tree, "INSERT INTO " + treeDb + ".d1(time, s1) VALUES (1, 1.0)"); + awaitFields(table, viewDb, "v", fields("s1", "DOUBLE")); + } + } + + @Test + @Ignore + public void testInsertMultipleDevicesAutoCreatesColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection(); + ISession session = EnvFactory.getEnv().getSessionConnection()) { + final String treeDb = "root.as14"; + final String viewDb = "as14"; + prepareView(tree, table, treeDb, viewDb); + final List valuesOfD1 = Collections.singletonList(1); + final List valuesOfD2 = Collections.singletonList(2.0); + session.insertRecords( + Arrays.asList(treeDb + ".d1", treeDb + ".d2"), + Arrays.asList(1L, 1L), + Arrays.asList(Collections.singletonList("s1"), Collections.singletonList("s2")), + Arrays.asList( + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(TSDataType.DOUBLE)), + Arrays.asList(valuesOfD1, valuesOfD2)); + awaitFields(table, viewDb, "v", fields("s1", "INT32", "s2", "DOUBLE")); + } + } + + @Test + @Ignore + public void testActivateTemplateAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + final String treeDb = "root.as15"; + final String viewDb = "as15"; + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE DEVICE TEMPLATE t_as15 (s1 FLOAT, s2 DOUBLE)"); + execute(tree, "SET DEVICE TEMPLATE t_as15 TO " + treeDb); + createView(table, viewDb, treeDb); + + execute(tree, "CREATE TIMESERIES USING DEVICE TEMPLATE ON " + treeDb + ".d1"); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE")); + } + } + + @Test + @Ignore + public void testBatchActivateTemplateAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection(); + ISession session = EnvFactory.getEnv().getSessionConnection()) { + final String treeDb = "root.as16"; + final String viewDb = "as16"; + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE DEVICE TEMPLATE t_as16 (s1 FLOAT, s2 DOUBLE)"); + execute(tree, "SET DEVICE TEMPLATE t_as16 TO " + treeDb); + createView(table, viewDb, treeDb); + + session.createTimeseriesUsingSchemaTemplate(Arrays.asList(treeDb + ".d1", treeDb + ".d2")); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE")); + } + } + + @Test + @Ignore + public void testInternalBatchActivateTemplateAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection(); + ISession session = EnvFactory.getEnv().getSessionConnection()) { + final String treeDb = "root.as17"; + final String viewDb = "as17"; + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE DEVICE TEMPLATE t_as17 (s1 FLOAT, s2 DOUBLE)"); + execute(tree, "SET DEVICE TEMPLATE t_as17 TO " + treeDb); + createView(table, viewDb, treeDb); + + // the insert activates the template on both devices and creates the template measurements + final List valuesOfD1 = Arrays.asList(1.0f, 1.0); + final List valuesOfD2 = Arrays.asList(2.0f, 2.0); + session.insertRecords( + Arrays.asList(treeDb + ".d1", treeDb + ".d2"), + Arrays.asList(1L, 1L), + Arrays.asList(Arrays.asList("s1", "s2"), Arrays.asList("s1", "s2")), + Arrays.asList( + Arrays.asList(TSDataType.FLOAT, TSDataType.DOUBLE), + Arrays.asList(TSDataType.FLOAT, TSDataType.DOUBLE)), + Arrays.asList(valuesOfD1, valuesOfD2)); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE")); + } + } + + @Test + @Ignore + public void testAlterTemplateAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + final String treeDb = "root.as18"; + final String viewDb = "as18"; + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE DEVICE TEMPLATE t_as18 (s1 FLOAT)"); + execute(tree, "SET DEVICE TEMPLATE t_as18 TO " + treeDb); + createView(table, viewDb, treeDb); + execute(tree, "CREATE TIMESERIES USING DEVICE TEMPLATE ON " + treeDb + ".d1"); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT")); + + execute(tree, "ALTER DEVICE TEMPLATE t_as18 ADD (s2 DOUBLE)"); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE")); + } + } + + /** A device which is exactly the path the template is set on uses the template as well. */ + @Test + @Ignore + public void testAlterTemplateOnDevicePathAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + final String treeDb = "root.as19"; + final String viewDb = "as19"; + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE DEVICE TEMPLATE t_as19 (s1 FLOAT)"); + execute(tree, "SET DEVICE TEMPLATE t_as19 TO " + treeDb + ".exact.d"); + createView(table, viewDb, treeDb, "a STRING TAG, b STRING TAG, c STRING TAG"); + + execute(tree, "CREATE TIMESERIES USING DEVICE TEMPLATE ON " + treeDb + ".exact.d"); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT")); + + execute(tree, "ALTER DEVICE TEMPLATE t_as19 ADD (s2 DOUBLE)"); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE")); + } + } + + /** The template may be set above the database which holds the activated devices. */ + @Test + @Ignore + public void testAlterTemplateOnAncestorPathAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + final String treeDb = "root.as20.db"; + final String viewDb = "as20"; + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE DEVICE TEMPLATE t_as20 (s1 FLOAT)"); + execute(tree, "SET DEVICE TEMPLATE t_as20 TO root.as20"); + createView(table, viewDb, "root.as20", "a STRING TAG, b STRING TAG"); + + execute(tree, "CREATE TIMESERIES USING DEVICE TEMPLATE ON " + treeDb + ".d1"); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT")); + + execute(tree, "ALTER DEVICE TEMPLATE t_as20 ADD (s2 DOUBLE)"); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE")); + } + } + + // ==================== helpers ==================== + + private static Connection treeConnection() throws SQLException { + return EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); + } + + private static Connection tableConnection() throws SQLException { + return EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + } + + private static void execute(final Connection connection, final String sql) throws SQLException { + try (Statement statement = connection.createStatement()) { + statement.execute(sql); + } + } + + private static void prepareView( + final Connection tree, final Connection table, final String treeDb, final String viewDb) + throws SQLException { + execute(tree, "CREATE DATABASE " + treeDb); + createView(table, viewDb, treeDb); + } + + private static void createView(final Connection table, final String viewDb, final String treeDb) + throws SQLException { + createView(table, viewDb, treeDb, "dev STRING TAG"); + } + + private static void createView( + final Connection table, + final String viewDb, + final String treeDb, + final String columnDefinitions) + throws SQLException { + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute( + table, + "CREATE VIEW v (" + + columnDefinitions + + ") WITH (auto_sync_schema=true) AS " + + treeDb + + ".**"); + } + + /** The FIELD columns of the view, column name to data type. */ + private static Map fieldColumns( + final Connection table, final String viewDb, final String view) throws SQLException { + execute(table, "USE " + viewDb); + final Map fields = new HashMap<>(); + try (Statement statement = table.createStatement(); + ResultSet resultSet = statement.executeQuery("DESC " + view)) { + while (resultSet.next()) { + if ("FIELD".equalsIgnoreCase(resultSet.getString("Category"))) { + fields.put(resultSet.getString("ColumnName"), resultSet.getString("DataType")); + } + } + } + return fields; + } + + private static String showCreateView( + final Connection table, final String viewDb, final String view) throws SQLException { + execute(table, "USE " + viewDb); + try (Statement statement = table.createStatement(); + ResultSet resultSet = statement.executeQuery("SHOW CREATE VIEW " + view)) { + Assert.assertTrue(resultSet.next()); + return resultSet.getString(2); + } + } + + private static void assertSqlError( + final Connection connection, final String sql, final String expectedMessage) { + try (Statement statement = connection.createStatement()) { + statement.execute(sql); + Assert.fail("Expected an error for: " + sql); + } catch (final SQLException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains(expectedMessage)); + } + } + + private static Map fields(final String... nameTypePairs) { + final Map fields = new HashMap<>(); + for (int i = 0; i < nameTypePairs.length; i += 2) { + fields.put(nameTypePairs[i], nameTypePairs[i + 1]); + } + return fields; + } + + /** The column addition runs on the DataNode and the procedure may finish a bit later. */ + private static void awaitFields( + final Connection table, + final String viewDb, + final String view, + final Map expected) + throws Exception { + final long deadline = System.currentTimeMillis() + 10_000L; + Map actual = fieldColumns(table, viewDb, view); + while (!expected.equals(actual) && System.currentTimeMillis() < deadline) { + Thread.sleep(100L); + actual = fieldColumns(table, viewDb, view); + } + Assert.assertEquals(expected, actual); + } + + /** A column which must not be added; waits a little so that a late addition is caught too. */ + private static void awaitAbsent( + final Connection table, final String viewDb, final String view, final String columnName) + throws Exception { + Thread.sleep(500L); + Assert.assertFalse( + "Column " + columnName + " should not be added to " + view, + fieldColumns(table, viewDb, view).containsKey(columnName)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 820cc2a2f8ee2..553779d21191b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -1030,10 +1030,6 @@ private Map convertPropertiesToMap( map.put( key, String.valueOf(parseBooleanFromLiteral(value, NEED_LAST_CACHE_PROPERTY))); break; - case TsTable.AUTO_SYNC_SCHEMA: - map.put( - key, String.valueOf(parseBooleanFromLiteral(value, TsTable.AUTO_SYNC_SCHEMA))); - break; default: break; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 3adfdf5f36dc3..402045012e4fd 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -71,7 +71,7 @@ public class TsTable { public static final String AUTO_SYNC_SCHEMA = "auto_sync_schema"; public static final Set TABLE_ALLOWED_PROPERTIES = Collections.unmodifiableSet( - new HashSet<>(Arrays.asList(TTL_PROPERTY, NEED_LAST_CACHE_PROPERTY, AUTO_SYNC_SCHEMA))); + new HashSet<>(Arrays.asList(TTL_PROPERTY, NEED_LAST_CACHE_PROPERTY))); private static final String OBJECT_STRING_ERROR = "When there are object fields, the %s %s shall not be '.', '..' or contain './', '.\\'."; protected String tableName; From 6e0836aa00b15455176947f4f5c3329a4e9df8b5 Mon Sep 17 00:00:00 2001 From: Yaobin Chen Date: Thu, 22 Oct 2026 00:23:38 +0800 Subject: [PATCH 24/24] fix ut --- .../iotdb/db/schemaengine/metric/SchemaMemMetricTableTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/metric/SchemaMemMetricTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/metric/SchemaMemMetricTableTest.java index f9e92207f34fb..a14d7b21244f2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/metric/SchemaMemMetricTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/metric/SchemaMemMetricTableTest.java @@ -115,7 +115,7 @@ public void testSchemaRegionSeriesMetricsIncludeTableMeasurements() { table.addColumnSchema( new FieldColumnSchema("s2", TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); tableCache.preUpdateTable(DATABASE, table, null); - tableCache.commitUpdateTable(DATABASE, TABLE, null); + tableCache.commitUpdateTable(DATABASE, TABLE, null, null); final MemSchemaRegionStatistics statistics = new MemSchemaRegionStatistics(SCHEMA_REGION_ID, new MemSchemaEngineStatistics());