diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index f7a5bc4581cd4..037ac00cd1f2b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -1038,24 +1038,13 @@ public void testTreeViewTable() throws Exception { final Statement statement = connection.createStatement()) { statement.execute("use tree_view_db"); - try { - statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"重庆\".\"1\".**"); - fail(); - } catch (final SQLException e) { - assertEquals( - "614: Multiple types encountered when auto detecting type of measurement 'S1', please check", - e.getMessage()); - } - - try { - statement.execute( - "create view tree_table (tag1 tag, tag2 tag, S1 field) as root.\"重庆\".\"1\".**"); - fail(); - } catch (final SQLException e) { - assertEquals( - "614: Multiple types encountered when auto detecting type of measurement 'S1', please check", - e.getMessage()); - } + // Template t1 is set but not activated, so its S1 BOOLEAN is not a created series and does + // not take part in the detection + statement.execute("create view tree_table (tag1 tag, tag2 tag) as root.\"重庆\".\"1\".**"); + statement.execute("drop view tree_table"); + statement.execute( + "create view tree_table (tag1 tag, tag2 tag, S1 field) as root.\"重庆\".\"1\".**"); + statement.execute("drop view tree_table"); } try (final Connection connection = EnvFactory.getEnv().getConnection(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTreeToTableViewAutoSyncIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTreeToTableViewAutoSyncIT.java new file mode 100644 index 0000000000000..17dfee8483602 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTreeToTableViewAutoSyncIT.java @@ -0,0 +1,717 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.schema; + +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBTreeToTableViewAutoSyncIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + // ==================== the property and its index (Trie) ==================== + + /** + * The property defaults to false: when the view is created, only the series which exist become + * columns. An explicit false behaves the same, and SHOW CREATE VIEW only shows the property when + * it is stored. + */ + @Test + @Ignore + public void testAutoSyncSchemaDisabledByDefault() throws Exception { + final String treeDb = "root.as01"; + final String viewDb = "as01"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s1 INT32"); + + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute(table, "CREATE VIEW v1 (dev STRING TAG) AS " + treeDb + ".**"); + // the creation always scans the series which already exist + awaitFields(table, viewDb, "v1", fields("s1", "INT32")); + Assert.assertFalse(showCreateView(table, viewDb, "v1").contains("auto_sync_schema")); + + // the default is false, so a new measurement is not added + execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s2 INT32"); + awaitAbsent(table, viewDb, "v1", "s2"); + + execute( + table, + "CREATE VIEW v2 (dev STRING TAG) WITH (auto_sync_schema=false) AS " + treeDb + ".**"); + awaitFields(table, viewDb, "v2", fields("s1", "INT32", "s2", "INT32")); + Assert.assertTrue(showCreateView(table, viewDb, "v2").contains("auto_sync_schema=false")); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s3 INT32"); + awaitAbsent(table, viewDb, "v2", "s3"); + + // an explicit true follows the new measurements, both the existing and the later ones + execute( + table, + "CREATE VIEW v3 (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".**"); + awaitFields(table, viewDb, "v3", fields("s1", "INT32", "s2", "INT32", "s3", "INT32")); + Assert.assertTrue(showCreateView(table, viewDb, "v3").contains("auto_sync_schema=true")); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s4 INT32"); + awaitFields( + table, viewDb, "v3", fields("s1", "INT32", "s2", "INT32", "s3", "INT32", "s4", "INT32")); + } + } + + /** The property is only accepted by the views created on a tree path, not by the tables. */ + @Test + @Ignore + public void testAutoSyncSchemaOnlyForTreeToTableViews() throws Exception { + final String viewDb = "as02"; + try (Connection table = tableConnection()) { + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute(table, "CREATE TABLE t1 (tag1 STRING TAG, s1 INT32 FIELD)"); + + assertSqlError( + table, + "CREATE TABLE t2 (tag1 STRING TAG, s1 INT32 FIELD) WITH (auto_sync_schema=true)", + "auto_sync_schema"); + assertSqlError( + table, "ALTER TABLE t1 SET PROPERTIES auto_sync_schema=true", "auto_sync_schema"); + } + } + + /** + * A measurement whose name only differs in case from an existing column, or whose name is the + * same but whose type is different, does not change the view; the other measurements of the same + * write are still added. + */ + @Test + @Ignore + public void testConflictingMeasurementsDoNotChangeTheView() throws Exception { + final String treeDb = "root.as03"; + final String viewDb = "as03"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s1 DOUBLE"); + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute( + table, + "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".**"); + awaitFields(table, viewDb, "v", fields("s1", "DOUBLE")); + + // the same name with another type: the series is created, the column is not touched + execute(tree, "CREATE TIMESERIES " + treeDb + ".d2.s1 INT32"); + // a new measurement of the same write is still added + execute(tree, "CREATE TIMESERIES " + treeDb + ".d2.s2 INT32"); + awaitFields(table, viewDb, "v", fields("s1", "DOUBLE", "s2", "INT32")); + + // the same name in another case is a conflict as well: no column is added + execute(tree, "CREATE TIMESERIES " + treeDb + ".d3.S2 INT32"); + awaitFields(table, viewDb, "v", fields("s1", "DOUBLE", "s2", "INT32")); + } + } + + /** + * A restrict view matches a device with exactly numOfTags levels under its prefix, a non-restrict + * view matches every device with at most numOfTags levels. A device may be covered by several + * views at once. + */ + @Test + @Ignore + public void testTrieMatchingOfRestrictAndNonRestrictViews() throws Exception { + final String treeDb = "root.as04"; + final String viewDb = "as04"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + treeDb); + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute( + table, + "CREATE VIEW v_restrict (a STRING TAG, b STRING TAG) RESTRICT " + + "WITH (auto_sync_schema=true) AS " + + treeDb + + ".**"); + execute( + table, + "CREATE VIEW v_non_restrict (a STRING TAG) " + + "WITH (auto_sync_schema=true) AS " + + treeDb + + ".**"); + + // one level under the prefix: only the non-restrict view matches + execute(tree, "CREATE TIMESERIES " + treeDb + ".p.s1 INT32"); + awaitFields(table, viewDb, "v_non_restrict", fields("s1", "INT32")); + awaitAbsent(table, viewDb, "v_restrict", "s1"); + + // two levels: only the restrict view matches + execute(tree, "CREATE TIMESERIES " + treeDb + ".p.q.s2 INT32"); + awaitFields(table, viewDb, "v_restrict", fields("s2", "INT32")); + awaitFields(table, viewDb, "v_non_restrict", fields("s1", "INT32")); + + // three levels: neither view matches + execute(tree, "CREATE TIMESERIES " + treeDb + ".p.q.r.s3 INT32"); + awaitAbsent(table, viewDb, "v_restrict", "s3"); + awaitAbsent(table, viewDb, "v_non_restrict", "s3"); + } + } + + /** Renaming a view keeps its place in the index; the property can be switched on and off. */ + @Test + @Ignore + public void testTrieRenameAndToggleAutoSyncSchema() throws Exception { + final String treeDb = "root.as05"; + final String viewDb = "as05"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + treeDb); + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute( + table, + "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".**"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s1 INT32"); + awaitFields(table, viewDb, "v", fields("s1", "INT32")); + + execute(table, "ALTER VIEW v RENAME TO v2"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d2.s2 INT32"); + awaitFields(table, viewDb, "v2", fields("s1", "INT32", "s2", "INT32")); + assertSqlError(table, "DESC v", "does not exist"); + + // turned off: the new measurements are ignored + execute(table, "ALTER VIEW v2 SET PROPERTIES auto_sync_schema=false"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d3.s3 INT32"); + awaitAbsent(table, viewDb, "v2", "s3"); + + // turned on again: the missing columns are backfilled, and the later ones are followed + execute(table, "ALTER VIEW v2 SET PROPERTIES auto_sync_schema=true"); + awaitFields(table, viewDb, "v2", fields("s1", "INT32", "s2", "INT32", "s3", "INT32")); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d4.s4 INT32"); + awaitFields( + table, viewDb, "v2", fields("s1", "INT32", "s2", "INT32", "s3", "INT32", "s4", "INT32")); + } + } + + /** + * CREATE OR REPLACE registers the new definition of the view: the old path stops being followed, + * and the new matching attributes take effect. + */ + @Test + @Ignore + public void testTrieReplaceView() throws Exception { + final String treeDb = "root.as06"; + final String viewDb = "as06"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + treeDb); + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute( + table, + "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".old.**"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".old.a.s1 INT32"); + awaitFields(table, viewDb, "v", fields("s1", "INT32")); + + // another source path: the new definition replaces the old one + execute( + table, + "CREATE OR REPLACE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + + treeDb + + ".new.**"); + awaitFields(table, viewDb, "v", fields()); + execute(tree, "CREATE TIMESERIES " + treeDb + ".old.b.s2 INT32"); + awaitAbsent(table, viewDb, "v", "s2"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".new.c.s3 INT32"); + awaitFields(table, viewDb, "v", fields("s3", "INT32")); + + // the same path with another tag count: the new matching attributes replace the old ones + execute( + table, + "CREATE VIEW w (a STRING TAG) RESTRICT WITH (auto_sync_schema=true) AS " + + treeDb + + ".replace.**"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".replace.p.s1 INT32"); + awaitFields(table, viewDb, "w", fields("s1", "INT32")); + execute( + table, + "CREATE OR REPLACE VIEW w (a STRING TAG, b STRING TAG) RESTRICT " + + "WITH (auto_sync_schema=true) AS " + + treeDb + + ".replace.**"); + awaitFields(table, viewDb, "w", fields()); + execute(tree, "CREATE TIMESERIES " + treeDb + ".replace.p.q.s2 INT32"); + awaitFields(table, viewDb, "w", fields("s2", "INT32")); + execute(tree, "CREATE TIMESERIES " + treeDb + ".replace.m.s3 INT32"); + awaitAbsent(table, viewDb, "w", "s3"); + } + } + + /** Dropping a view or its database removes the entry from the index of the DataNodes. */ + @Test + @Ignore + public void testTrieDropViewAndDropDatabase() throws Exception { + final String treeDb = "root.as07"; + final String viewDb = "as07"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + treeDb); + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute( + table, + "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".old.**"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".old.a.s1 INT32"); + awaitFields(table, viewDb, "v", fields("s1", "INT32")); + + execute(table, "DROP VIEW v"); + assertSqlError(table, "DESC v", "does not exist"); + // a view of another path follows only its own path + execute( + table, + "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + treeDb + ".new.**"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".old.b.s2 INT32"); + awaitAbsent(table, viewDb, "v", "s2"); + execute(tree, "CREATE TIMESERIES " + treeDb + ".new.c.s3 INT32"); + awaitFields(table, viewDb, "v", fields("s3", "INT32")); + } + + // dropping the database removes its views from the index: a view of the same name created + // again with another source path is not polluted by the old one + final String replacedTreeDb = "root.as07b"; + final String replacedViewDb = "as07b"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + replacedTreeDb); + execute(tree, "CREATE TIMESERIES " + replacedTreeDb + ".old.d1.s1 INT32"); + execute(table, "CREATE DATABASE " + replacedViewDb); + execute(table, "USE " + replacedViewDb); + execute( + table, + "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + + replacedTreeDb + + ".old.**"); + awaitFields(table, replacedViewDb, "v", fields("s1", "INT32")); + + execute(table, "DROP DATABASE " + replacedViewDb); + execute(table, "CREATE DATABASE " + replacedViewDb); + execute(table, "USE " + replacedViewDb); + execute( + table, + "CREATE VIEW v (dev STRING TAG) WITH (auto_sync_schema=true) AS " + + replacedTreeDb + + ".new.**"); + + execute(tree, "CREATE TIMESERIES " + replacedTreeDb + ".old.d2.s2 INT32"); + awaitAbsent(table, replacedViewDb, "v", "s2"); + execute(tree, "CREATE TIMESERIES " + replacedTreeDb + ".new.d3.s3 INT32"); + awaitFields(table, replacedViewDb, "v", fields("s3", "INT32")); + } + } + + /** Adding a tag column changes the tag count the view matches devices with. */ + @Test + @Ignore + public void testTrieAddingTagColumnUpdatesTheTagCount() throws Exception { + final String treeDb = "root.as08"; + final String viewDb = "as08"; + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + execute(tree, "CREATE DATABASE " + treeDb); + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute( + table, + "CREATE VIEW v (a STRING TAG) RESTRICT WITH (auto_sync_schema=true) AS " + + treeDb + + ".**"); + + // the tag count of the view, hence the devices it matches, changes with the new tag column + execute(table, "ALTER VIEW v ADD COLUMN b STRING TAG"); + + // two levels under the prefix now match the restrict view + execute(tree, "CREATE TIMESERIES " + treeDb + ".p.q.s2 INT32"); + awaitFields(table, viewDb, "v", fields("s2", "INT32")); + // one level does not match any more + execute(tree, "CREATE TIMESERIES " + treeDb + ".p.s1 INT32"); + awaitAbsent(table, viewDb, "v", "s1"); + // three levels do not match either + execute(tree, "CREATE TIMESERIES " + treeDb + ".p.q.r.s3 INT32"); + awaitAbsent(table, viewDb, "v", "s3"); + } + } + + // ==================== the interfaces which create measurements ==================== + + @Test + @Ignore + public void testCreateTimeSeriesAddsColumn() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + final String treeDb = "root.as10"; + final String viewDb = "as10"; + prepareView(tree, table, treeDb, viewDb); + execute(tree, "CREATE TIMESERIES " + treeDb + ".d1.s1 INT32"); + awaitFields(table, viewDb, "v", fields("s1", "INT32")); + } + } + + @Test + @Ignore + public void testCreateAlignedTimeSeriesAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + final String treeDb = "root.as11"; + final String viewDb = "as11"; + prepareView(tree, table, treeDb, viewDb); + execute(tree, "CREATE ALIGNED TIMESERIES " + treeDb + ".d1 (s1 INT32, s2 DOUBLE)"); + awaitFields(table, viewDb, "v", fields("s1", "INT32", "s2", "DOUBLE")); + } + } + + @Test + @Ignore + public void testCreateMultiTimeseriesAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection(); + ISession session = EnvFactory.getEnv().getSessionConnection()) { + final String treeDb = "root.as12"; + final String viewDb = "as12"; + prepareView(tree, table, treeDb, viewDb); + session.createMultiTimeseries( + Arrays.asList(treeDb + ".d1.s1", treeDb + ".d1.s2"), + Arrays.asList(TSDataType.INT32, TSDataType.DOUBLE), + Arrays.asList(TSEncoding.PLAIN, TSEncoding.PLAIN), + Arrays.asList(CompressionType.LZ4, CompressionType.LZ4), + null, + null, + null, + null); + awaitFields(table, viewDb, "v", fields("s1", "INT32", "s2", "DOUBLE")); + } + } + + @Test + @Ignore + public void testInsertAutoCreatesColumn() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + final String treeDb = "root.as13"; + final String viewDb = "as13"; + prepareView(tree, table, treeDb, viewDb); + execute(tree, "INSERT INTO " + treeDb + ".d1(time, s1) VALUES (1, 1.0)"); + awaitFields(table, viewDb, "v", fields("s1", "DOUBLE")); + } + } + + @Test + @Ignore + public void testInsertMultipleDevicesAutoCreatesColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection(); + ISession session = EnvFactory.getEnv().getSessionConnection()) { + final String treeDb = "root.as14"; + final String viewDb = "as14"; + prepareView(tree, table, treeDb, viewDb); + final List valuesOfD1 = Collections.singletonList(1); + final List valuesOfD2 = Collections.singletonList(2.0); + session.insertRecords( + Arrays.asList(treeDb + ".d1", treeDb + ".d2"), + Arrays.asList(1L, 1L), + Arrays.asList(Collections.singletonList("s1"), Collections.singletonList("s2")), + Arrays.asList( + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(TSDataType.DOUBLE)), + Arrays.asList(valuesOfD1, valuesOfD2)); + awaitFields(table, viewDb, "v", fields("s1", "INT32", "s2", "DOUBLE")); + } + } + + @Test + @Ignore + public void testActivateTemplateAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + final String treeDb = "root.as15"; + final String viewDb = "as15"; + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE DEVICE TEMPLATE t_as15 (s1 FLOAT, s2 DOUBLE)"); + execute(tree, "SET DEVICE TEMPLATE t_as15 TO " + treeDb); + createView(table, viewDb, treeDb); + + execute(tree, "CREATE TIMESERIES USING DEVICE TEMPLATE ON " + treeDb + ".d1"); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE")); + } + } + + @Test + @Ignore + public void testBatchActivateTemplateAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection(); + ISession session = EnvFactory.getEnv().getSessionConnection()) { + final String treeDb = "root.as16"; + final String viewDb = "as16"; + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE DEVICE TEMPLATE t_as16 (s1 FLOAT, s2 DOUBLE)"); + execute(tree, "SET DEVICE TEMPLATE t_as16 TO " + treeDb); + createView(table, viewDb, treeDb); + + session.createTimeseriesUsingSchemaTemplate(Arrays.asList(treeDb + ".d1", treeDb + ".d2")); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE")); + } + } + + @Test + @Ignore + public void testInternalBatchActivateTemplateAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection(); + ISession session = EnvFactory.getEnv().getSessionConnection()) { + final String treeDb = "root.as17"; + final String viewDb = "as17"; + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE DEVICE TEMPLATE t_as17 (s1 FLOAT, s2 DOUBLE)"); + execute(tree, "SET DEVICE TEMPLATE t_as17 TO " + treeDb); + createView(table, viewDb, treeDb); + + // the insert activates the template on both devices and creates the template measurements + final List valuesOfD1 = Arrays.asList(1.0f, 1.0); + final List valuesOfD2 = Arrays.asList(2.0f, 2.0); + session.insertRecords( + Arrays.asList(treeDb + ".d1", treeDb + ".d2"), + Arrays.asList(1L, 1L), + Arrays.asList(Arrays.asList("s1", "s2"), Arrays.asList("s1", "s2")), + Arrays.asList( + Arrays.asList(TSDataType.FLOAT, TSDataType.DOUBLE), + Arrays.asList(TSDataType.FLOAT, TSDataType.DOUBLE)), + Arrays.asList(valuesOfD1, valuesOfD2)); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE")); + } + } + + @Test + @Ignore + public void testAlterTemplateAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + final String treeDb = "root.as18"; + final String viewDb = "as18"; + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE DEVICE TEMPLATE t_as18 (s1 FLOAT)"); + execute(tree, "SET DEVICE TEMPLATE t_as18 TO " + treeDb); + createView(table, viewDb, treeDb); + execute(tree, "CREATE TIMESERIES USING DEVICE TEMPLATE ON " + treeDb + ".d1"); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT")); + + execute(tree, "ALTER DEVICE TEMPLATE t_as18 ADD (s2 DOUBLE)"); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE")); + } + } + + /** A device which is exactly the path the template is set on uses the template as well. */ + @Test + @Ignore + public void testAlterTemplateOnDevicePathAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + final String treeDb = "root.as19"; + final String viewDb = "as19"; + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE DEVICE TEMPLATE t_as19 (s1 FLOAT)"); + execute(tree, "SET DEVICE TEMPLATE t_as19 TO " + treeDb + ".exact.d"); + createView(table, viewDb, treeDb, "a STRING TAG, b STRING TAG, c STRING TAG"); + + execute(tree, "CREATE TIMESERIES USING DEVICE TEMPLATE ON " + treeDb + ".exact.d"); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT")); + + execute(tree, "ALTER DEVICE TEMPLATE t_as19 ADD (s2 DOUBLE)"); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE")); + } + } + + /** The template may be set above the database which holds the activated devices. */ + @Test + @Ignore + public void testAlterTemplateOnAncestorPathAddsColumns() throws Exception { + try (Connection tree = treeConnection(); + Connection table = tableConnection()) { + final String treeDb = "root.as20.db"; + final String viewDb = "as20"; + execute(tree, "CREATE DATABASE " + treeDb); + execute(tree, "CREATE DEVICE TEMPLATE t_as20 (s1 FLOAT)"); + execute(tree, "SET DEVICE TEMPLATE t_as20 TO root.as20"); + createView(table, viewDb, "root.as20", "a STRING TAG, b STRING TAG"); + + execute(tree, "CREATE TIMESERIES USING DEVICE TEMPLATE ON " + treeDb + ".d1"); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT")); + + execute(tree, "ALTER DEVICE TEMPLATE t_as20 ADD (s2 DOUBLE)"); + awaitFields(table, viewDb, "v", fields("s1", "FLOAT", "s2", "DOUBLE")); + } + } + + // ==================== helpers ==================== + + private static Connection treeConnection() throws SQLException { + return EnvFactory.getEnv().getConnection(BaseEnv.TREE_SQL_DIALECT); + } + + private static Connection tableConnection() throws SQLException { + return EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + } + + private static void execute(final Connection connection, final String sql) throws SQLException { + try (Statement statement = connection.createStatement()) { + statement.execute(sql); + } + } + + private static void prepareView( + final Connection tree, final Connection table, final String treeDb, final String viewDb) + throws SQLException { + execute(tree, "CREATE DATABASE " + treeDb); + createView(table, viewDb, treeDb); + } + + private static void createView(final Connection table, final String viewDb, final String treeDb) + throws SQLException { + createView(table, viewDb, treeDb, "dev STRING TAG"); + } + + private static void createView( + final Connection table, + final String viewDb, + final String treeDb, + final String columnDefinitions) + throws SQLException { + execute(table, "CREATE DATABASE " + viewDb); + execute(table, "USE " + viewDb); + execute( + table, + "CREATE VIEW v (" + + columnDefinitions + + ") WITH (auto_sync_schema=true) AS " + + treeDb + + ".**"); + } + + /** The FIELD columns of the view, column name to data type. */ + private static Map fieldColumns( + final Connection table, final String viewDb, final String view) throws SQLException { + execute(table, "USE " + viewDb); + final Map fields = new HashMap<>(); + try (Statement statement = table.createStatement(); + ResultSet resultSet = statement.executeQuery("DESC " + view)) { + while (resultSet.next()) { + if ("FIELD".equalsIgnoreCase(resultSet.getString("Category"))) { + fields.put(resultSet.getString("ColumnName"), resultSet.getString("DataType")); + } + } + } + return fields; + } + + private static String showCreateView( + final Connection table, final String viewDb, final String view) throws SQLException { + execute(table, "USE " + viewDb); + try (Statement statement = table.createStatement(); + ResultSet resultSet = statement.executeQuery("SHOW CREATE VIEW " + view)) { + Assert.assertTrue(resultSet.next()); + return resultSet.getString(2); + } + } + + private static void assertSqlError( + final Connection connection, final String sql, final String expectedMessage) { + try (Statement statement = connection.createStatement()) { + statement.execute(sql); + Assert.fail("Expected an error for: " + sql); + } catch (final SQLException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains(expectedMessage)); + } + } + + private static Map fields(final String... nameTypePairs) { + final Map fields = new HashMap<>(); + for (int i = 0; i < nameTypePairs.length; i += 2) { + fields.put(nameTypePairs[i], nameTypePairs[i + 1]); + } + return fields; + } + + /** The column addition runs on the DataNode and the procedure may finish a bit later. */ + private static void awaitFields( + final Connection table, + final String viewDb, + final String view, + final Map expected) + throws Exception { + final long deadline = System.currentTimeMillis() + 10_000L; + Map actual = fieldColumns(table, viewDb, view); + while (!expected.equals(actual) && System.currentTimeMillis() < deadline) { + Thread.sleep(100L); + actual = fieldColumns(table, viewDb, view); + } + Assert.assertEquals(expected, actual); + } + + /** A column which must not be added; waits a little so that a late addition is caught too. */ + private static void awaitAbsent( + final Connection table, final String viewDb, final String view, final String columnName) + throws Exception { + Thread.sleep(500L); + Assert.assertFalse( + "Column " + columnName + " should not be added to " + view, + fieldColumns(table, viewDb, view).containsKey(columnName)); + } +} diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index da9b4388691e8..0da39cf56047c 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -52,6 +52,7 @@ public enum TSStatusCode { DISPATCH_ERROR(306), LICENSE_ERROR(307), THRIFT_FRAME_OVERSIZE(308), + REPEATED_WITH_EXISTING_TASK(309), // Client, REDIRECTION_RECOMMEND(400), diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 183ab36c47816..482da8daad126 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -614,6 +614,9 @@ private ManagerMessages() {} public static final String LOG_GOT_IOEXCEPTION_CONSTRUCT_DATABASE_TREE_49436621 = "Got IOException when construct database Tree"; public static final String LOG_GOT_IOEXCEPTION_DESERIALIZE_TEMPLATE_INFO_49EE617E = "Got IOException when deserialize template info"; public static final String MESSAGE_MEASUREMENTS_NOT_FOUND_ARG_CANNOT_AUTO_DETECT_980D7D44 = "Measurements not found for %s, cannot auto detect"; + public static final String EXCEPTION_THE_DUPLICATED_SOURCE_MEASUREMENT_ARG_IS_UNSUPPORTED_YET_873AF566 = "The duplicated source measurement %s is unsupported yet."; + public static final String MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB = "Multiple types encountered when auto detecting type of measurement '%s', please check"; + public static final String MESSAGE_THE_MEASUREMENTS_ARG_AND_ARG_SHARE_THE_SAME_LOWER_CASE_WHEN_AUTO_DETECTING_TYPE_PLEASE_CHECK_7225D062 = "The measurements %s and %s share the same lower case when auto detecting type, please check"; public static final String LOG_FAILED_TAKE_SNAPSHOT_BECAUSE_SNAPSHOT_FILE_ARG_ALREADY_EXIST_EB2A6093 = "Failed to take snapshot, because snapshot file [{}] is already exist."; public static final String LOG_FAILED_LOAD_SNAPSHOT_SNAPSHOT_FILE_ARG_NOT_EXIST_8828CFBA = "Failed to load snapshot,snapshot file [{}] is not exist."; public static final String LOG_YOU_MAYBE_NEED_RENAME_SIMPLE_DIR_0_0_MANUALLY_2A12C5C9 = "you maybe need to rename the simple dir to 0_0 manually."; diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java index 7154b0b4c82d2..488e7cc759841 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java @@ -490,7 +490,8 @@ public final class ProcedureMessages { "Failed to rollback pipe plugin [%s] on data nodes"; public static final String FAILED_TO_ROLLBACK_PRE_RELEASE_FOR_TABLE_INFO_TO_DATANODE = "Failed to rollback pre-release {} for table {}.{} info to DataNode, failure results: {}"; - public static final String FAILED_TO_PROVE_AN_UNREACHABLE_DN_IS_FENCED = "Failed to prove an unreachable DataNode is fenced"; + public static final String FAILED_TO_PROVE_AN_UNREACHABLE_DN_IS_FENCED = + "Failed to prove an unreachable DataNode is fenced"; public static final String FAILED_TO_ROLLBACK_PRE_RELEASE_TEMPLATE_INFO_OF_TEMPLATE_SET = "Failed to rollback pre release template info of template {} set on path {} on DataNode {}"; public static final String FAILED_TO_ROLLBACK_PRE_SET_TEMPLATE_ON_PATH_DUE_TO = @@ -1435,10 +1436,16 @@ private ProcedureMessages() {} "rolling back after failure: %s."; public static final String MESSAGE_ROLLING_BACK_AFTER_AN_EARLIER_FAILURE_850D0AF5 = "rolling back after an earlier failure."; + public static final String + MESSAGE_SOME_OTHER_TASK_IS_SETTING_OR_UNSETTING_A_TEMPLATE_ON_TARGET_PATH_ARG_D4FEE38F = + "Some other task is setting or unsetting a template on target path %s."; public static final String EXCEPTION_DATANODE_SETSYSTEMSTATUS_RPC_FAILED_ARG_ARG_ARG_7E8E5F9F = "DataNode setSystemStatus RPC failed: %s: %s: %s; "; public static final String EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4 = "ConfigNode trySetNodeStatus failed: %s: %s: %s; "; + public static final String + LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_8F20CAE7 = + "Failed to detect the tree series under the new view {}.{}: {}"; public static final String LOG_PROCEDURE_ARG_IS_PAST_ITS_COMMIT_POINT_KEEPING_EVERY_STATE_IT_EXECUTED_CA21D7E9 = "Procedure {} is past its commit point; keeping every state it executed."; diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java index fa3d63dddb3d2..739ee0029c890 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -606,6 +606,9 @@ private ManagerMessages() {} public static final String LOG_GOT_IOEXCEPTION_CONSTRUCT_DATABASE_TREE_49436621 = "构建数据库树时发生 IOException"; public static final String LOG_GOT_IOEXCEPTION_DESERIALIZE_TEMPLATE_INFO_49EE617E = "反序列化模板信息时发生 IOException"; public static final String MESSAGE_MEASUREMENTS_NOT_FOUND_ARG_CANNOT_AUTO_DETECT_980D7D44 = "未找到 %s 的测点，无法自动检测"; + public static final String EXCEPTION_THE_DUPLICATED_SOURCE_MEASUREMENT_ARG_IS_UNSUPPORTED_YET_873AF566 = "暂不支持重复的源测点 %s。"; + public static final String MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB = "自动检测测点 %s 的类型时遇到多种类型，请检查"; + public static final String MESSAGE_THE_MEASUREMENTS_ARG_AND_ARG_SHARE_THE_SAME_LOWER_CASE_WHEN_AUTO_DETECTING_TYPE_PLEASE_CHECK_7225D062 = "自动检测类型时，测点 %s 和 %s 的小写形式相同，请检查"; public static final String LOG_FAILED_TAKE_SNAPSHOT_BECAUSE_SNAPSHOT_FILE_ARG_ALREADY_EXIST_EB2A6093 = "获取快照失败，原因：快照文件 [{}] 已存在。"; public static final String LOG_FAILED_LOAD_SNAPSHOT_SNAPSHOT_FILE_ARG_NOT_EXIST_8828CFBA = "无法加载快照，快照文件 [{}] 不存在。"; public static final String LOG_YOU_MAYBE_NEED_RENAME_SIMPLE_DIR_0_0_MANUALLY_2A12C5C9 = "可能需要手动将 simple 目录重命名为 0_0。"; diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java index 973241cd431d2..d65ee58ec53a7 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java @@ -1520,10 +1520,16 @@ private ProcedureMessages() {} public static final String MESSAGE_ROLLING_BACK_AFTER_FAILURE_ARG_474DF456 = "正在回滚，失败原因：%s。"; public static final String MESSAGE_ROLLING_BACK_AFTER_AN_EARLIER_FAILURE_850D0AF5 = "正在回滚此前发生的失败。"; + public static final String + MESSAGE_SOME_OTHER_TASK_IS_SETTING_OR_UNSETTING_A_TEMPLATE_ON_TARGET_PATH_ARG_D4FEE38F = + "有其他任务正在目标路径 %s 上设置或卸载模板。"; public static final String EXCEPTION_DATANODE_SETSYSTEMSTATUS_RPC_FAILED_ARG_ARG_ARG_7E8E5F9F = "DataNode setSystemStatus RPC 失败：%s：%s：%s； "; public static final String EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4 = "ConfigNode trySetNodeStatus 失败：%s：%s：%s； "; + public static final String + LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_8F20CAE7 = + "扫描新建视图 {}.{} 下的树序列失败：{}"; public static final String LOG_PROCEDURE_ARG_IS_PAST_ITS_COMMIT_POINT_KEEPING_EVERY_STATE_IT_EXECUTED_CA21D7E9 = "流程 {} 已越过提交点，保留它执行过的所有状态。"; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index f82d8187fab33..a08b4e87ba187 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -158,6 +158,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterTimeSeriesReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterTopicReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TClusterParameters; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; @@ -2286,7 +2288,7 @@ public TGetTemplateResp getTemplate(String req) { } @Override - public synchronized TSStatus setSchemaTemplate(TSetSchemaTemplateReq req) { + public TSStatus setSchemaTemplate(TSetSchemaTemplateReq req) { TSStatus status = confirmLeader(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return procedureManager.setSchemaTemplate( @@ -2368,7 +2370,7 @@ public TSStatus deactivateSchemaTemplate(final TDeactivateSchemaTemplateReq req) } @Override - public synchronized TSStatus unsetSchemaTemplate(TUnsetSchemaTemplateReq req) { + public TSStatus unsetSchemaTemplate(TUnsetSchemaTemplateReq req) { TSStatus status = confirmLeader(); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return status; @@ -3378,6 +3380,14 @@ public TSStatus alterOrDropTable(final TAlterOrDropTableReq req) { } } + @Override + public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) { + final TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? procedureManager.autoSyncViewColumns(req) + : new TAutoSyncViewColumnsResp(status); + } + @Override public TDeleteTableDeviceResp deleteDevice(final TDeleteTableDeviceReq req) { final TSStatus status = confirmLeader(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 37cc7c72247cc..9d2ea4c725d03 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -72,6 +72,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterTimeSeriesReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterTopicReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp; @@ -919,6 +921,8 @@ TPermissionInfoResp login( TSStatus alterOrDropTable(final TAlterOrDropTableReq req); + TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req); + TDeleteTableDeviceResp deleteDevice(final TDeleteTableDeviceReq req); TShowTableResp showTables(final String database, final boolean isDetails); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 18f65a98481da..62b90a9845f32 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -39,6 +39,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta; import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.commons.schema.table.AlterOrDropTableOperationType; import org.apache.iotdb.commons.schema.table.TableNodeStatus; import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.TsTable; @@ -151,6 +152,9 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterTopicReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsFailure; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; @@ -192,6 +196,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -201,6 +206,7 @@ import java.util.Set; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -216,6 +222,15 @@ public class ProcedureManager { private static final String PROCEDURE_TIMEOUT_MESSAGE = "Timed out to wait for procedure return. The procedure is still running."; + /** One addColumnForTree2TableViews call waits at most this long for its procedures to finish. */ + private static final long ADD_VIEW_COLUMN_WAIT_TIME_OUT_MS = 30_000L; + + /** How many times a request which overlapped with another task is submitted again. */ + private static final int OVERLAP_RETRY_ROUNDS = 4; + + /** How long a retry round waits for the overlapping task to finish before submitting again. */ + private static final long OVERLAP_RETRY_WAIT_MS = 2_000L; + private final ConfigManager configManager; private ProcedureExecutor executor; private ProcedureScheduler scheduler; @@ -542,17 +557,27 @@ public TSStatus setSchemaTemplate( SetTemplateProcedure setTemplateProcedure; for (Procedure runningProcedure : executor.getProcedures().values()) { type = ProcedureFactory.getProcedureType(runningProcedure); - if (type == null || !type.equals(ProcedureType.SET_TEMPLATE_PROCEDURE)) { + if (Objects.isNull(type)) { continue; } - setTemplateProcedure = (SetTemplateProcedure) runningProcedure; - if (queryId.equals(setTemplateProcedure.getQueryId())) { - procedure = setTemplateProcedure; - break; - } - if (templateSetPath.equals(setTemplateProcedure.getTemplateSetPath())) { - hasOverlappedTask = true; - break; + if (type.equals(ProcedureType.SET_TEMPLATE_PROCEDURE)) { + setTemplateProcedure = (SetTemplateProcedure) runningProcedure; + if (queryId.equals(setTemplateProcedure.getQueryId())) { + procedure = setTemplateProcedure; + break; + } + if (templateSetPath.equals(setTemplateProcedure.getTemplateSetPath())) { + hasOverlappedTask = true; + break; + } + } else if (type.equals(ProcedureType.UNSET_TEMPLATE_PROCEDURE)) { + // A path is set with one template at a time, so setting and unsetting the same path + // overlap as well. + if (templateSetPath.equals( + ((UnsetTemplateProcedure) runningProcedure).getPath().getFullPath())) { + hasOverlappedTask = true; + break; + } } } @@ -560,7 +585,10 @@ public TSStatus setSchemaTemplate( if (hasOverlappedTask) { return RpcUtils.getStatus( TSStatusCode.OVERLAP_WITH_EXISTING_TASK, - "Some other task is setting template on target path."); + String.format( + ProcedureMessages + .MESSAGE_SOME_OTHER_TASK_IS_SETTING_OR_UNSETTING_A_TEMPLATE_ON_TARGET_PATH_ARG_D4FEE38F, + templateSetPath)); } procedure = new SetTemplateProcedure(queryId, templateName, templateSetPath, isGeneratedByPipe); @@ -626,18 +654,28 @@ public TSStatus unsetSchemaTemplate( UnsetTemplateProcedure unsetTemplateProcedure; for (Procedure runningProcedure : executor.getProcedures().values()) { type = ProcedureFactory.getProcedureType(runningProcedure); - if (type == null || !type.equals(ProcedureType.UNSET_TEMPLATE_PROCEDURE)) { + if (Objects.isNull(type)) { continue; } - unsetTemplateProcedure = (UnsetTemplateProcedure) runningProcedure; - if (queryId.equals(unsetTemplateProcedure.getQueryId())) { - procedure = unsetTemplateProcedure; - break; - } - if (template.getId() == unsetTemplateProcedure.getTemplateId() - && path.equals(unsetTemplateProcedure.getPath())) { - hasOverlappedTask = true; - break; + if (type.equals(ProcedureType.UNSET_TEMPLATE_PROCEDURE)) { + unsetTemplateProcedure = (UnsetTemplateProcedure) runningProcedure; + if (queryId.equals(unsetTemplateProcedure.getQueryId())) { + procedure = unsetTemplateProcedure; + break; + } + if (template.getId() == unsetTemplateProcedure.getTemplateId() + && path.equals(unsetTemplateProcedure.getPath())) { + hasOverlappedTask = true; + break; + } + } else if (type.equals(ProcedureType.SET_TEMPLATE_PROCEDURE)) { + // A path is set with one template at a time, so setting and unsetting the same path + // overlap as well. + if (path.getFullPath() + .equals(((SetTemplateProcedure) runningProcedure).getTemplateSetPath())) { + hasOverlappedTask = true; + break; + } } } @@ -645,8 +683,10 @@ public TSStatus unsetSchemaTemplate( if (hasOverlappedTask) { return RpcUtils.getStatus( TSStatusCode.OVERLAP_WITH_EXISTING_TASK, - "Some other task is unsetting target template from target path " - + path.getFullPath()); + String.format( + ProcedureMessages + .MESSAGE_SOME_OTHER_TASK_IS_SETTING_OR_UNSETTING_A_TEMPLATE_ON_TARGET_PATH_ARG_D4FEE38F, + path.getFullPath())); } procedure = new UnsetTemplateProcedure(queryId, template, path, isGeneratedByPipe); this.executor.submitProcedure(procedure); @@ -2269,6 +2309,167 @@ public TSStatus alterTableAddColumn(final TAlterOrDropTableReq req) { false)); } + public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) { + if (req == null || !req.isSetViewInfos()) { + return new TAutoSyncViewColumnsResp(RpcUtils.getStatus(TSStatusCode.ILLEGAL_PARAMETER)); + } + + final List failedViews = new ArrayList<>(); + final List>> procedures = + new ArrayList<>(); + // The requests which overlapped with another task on the same table, together with the status + // of the last submission attempt, see processOverlappingTask. + final List> overlappingReqs = new ArrayList<>(); + + // Submit every view before waiting, so procedures for different views can run concurrently. + for (final TAlterOrDropTableReq addColumnReq : req.getViewInfos()) { + if (!isAddViewColumnReq(addColumnReq)) { + failedViews.add(illegalAddViewColumnFailure(addColumnReq)); + continue; + } + + final TSStatus submissionStatus = submitAddViewColumn(addColumnReq, procedures, failedViews); + if (submissionStatus.getCode() == TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode()) { + overlappingReqs.add(new Pair<>(addColumnReq, submissionStatus)); + } + } + + processOverlappingTask(overlappingReqs, procedures, failedViews); + + final long waitDeadline = System.currentTimeMillis() + ADD_VIEW_COLUMN_WAIT_TIME_OUT_MS; + for (final Pair> procedure : + procedures) { + final long remainingWaitTime = Math.max(0, waitDeadline - System.currentTimeMillis()); + final TSStatus status = waitingProcedureFinished(procedure.getRight(), remainingWaitTime); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + failedViews.add( + new TAutoSyncViewColumnsFailure( + procedure.getLeft().database, procedure.getLeft().tableName, status)); + } + } + + final TSStatus status = failedViews.isEmpty() ? StatusUtils.OK : StatusUtils.INTERNAL_ERROR; + final TAutoSyncViewColumnsResp response = new TAutoSyncViewColumnsResp(status); + if (!failedViews.isEmpty()) { + response.setFailedViews(failedViews); + } + return response; + } + + /** Whether the request carries everything an add-column operation on a view needs. */ + private static boolean isAddViewColumnReq(final @Nullable TAlterOrDropTableReq addColumnReq) { + return Objects.nonNull(addColumnReq) + && addColumnReq.isSetDatabase() + && addColumnReq.isSetTableName() + && addColumnReq.isSetQueryId() + && addColumnReq.isSetOperationType() + && addColumnReq.getOperationType() + == AlterOrDropTableOperationType.ADD_COLUMN.getTypeValue() + && addColumnReq.isSetUpdateInfo() + && addColumnReq.isSetIsView() + && addColumnReq.isIsView(); + } + + private static TAutoSyncViewColumnsFailure illegalAddViewColumnFailure( + final TAlterOrDropTableReq addColumnReq) { + return new TAutoSyncViewColumnsFailure( + Objects.isNull(addColumnReq) ? "" : Objects.toString(addColumnReq.getDatabase(), ""), + Objects.isNull(addColumnReq) ? "" : Objects.toString(addColumnReq.getTableName(), ""), + RpcUtils.getStatus(TSStatusCode.ILLEGAL_PARAMETER)); + } + + /** + * Submits the add-column procedure of one view and settles everything but an overlap with another + * task on the same table. + */ + private TSStatus submitAddViewColumn( + final TAlterOrDropTableReq addColumnReq, + final List>> procedures, + final List failedViews) { + final AddViewColumnProcedure procedure; + procedure = + new AddViewColumnProcedure( + addColumnReq.database, + addColumnReq.tableName, + addColumnReq.queryId, + Collections.emptyList(), + false); + + final Pair submissionResult = + executeWithoutDuplicateWithoutWaiting( + addColumnReq.database, + null, + addColumnReq.tableName, + null, + addColumnReq.queryId, + ProcedureType.ADD_VIEW_COLUMN_PROCEDURE, + procedure, + proc -> { + if (proc instanceof AddViewColumnProcedure) { + ((AddViewColumnProcedure) proc) + .setAddedColumnList( + TsTableColumnSchemaUtil.deserializeColumnSchemaList( + addColumnReq.updateInfo)); + } + }); + final TSStatus status = submissionResult.getLeft(); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + procedures.add(new Pair<>(addColumnReq, procedure)); + } else if (status.getCode() == TSStatusCode.REPEATED_WITH_EXISTING_TASK.getStatusCode()) { + final Procedure repeatedProcedure = + executor.getProcedures().get(submissionResult.getRight()); + // null means the procedure could have finished at this moment + if (Objects.isNull(repeatedProcedure)) { + failedViews.add( + new TAutoSyncViewColumnsFailure( + addColumnReq.database, + addColumnReq.tableName, + RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR))); + } else { + procedures.add(new Pair<>(addColumnReq, repeatedProcedure)); + } + } else if (status.getCode() != TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode()) { + failedViews.add( + new TAutoSyncViewColumnsFailure(addColumnReq.database, addColumnReq.tableName, status)); + } + return status; + } + + /** + * Retries the requests which overlapped with another task on the same table, A request which + * still overlaps after the last round is reported as a failure, with the status of its last + * attempt. + */ + private void processOverlappingTask( + final List> overlappingReqs, + final List>> procedures, + final List failedViews) { + List> pending = overlappingReqs; + for (int round = 0; round < OVERLAP_RETRY_ROUNDS && !pending.isEmpty(); round++) { + try { + Thread.sleep(OVERLAP_RETRY_WAIT_MS); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + final List> stillOverlapping = + new ArrayList<>(pending.size()); + for (final Pair request : pending) { + final TSStatus status = submitAddViewColumn(request.getLeft(), procedures, failedViews); + if (status.getCode() == TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode()) { + stillOverlapping.add(new Pair<>(request.getLeft(), status)); + } + } + pending = stillOverlapping; + } + + for (final Pair request : pending) { + failedViews.add( + new TAutoSyncViewColumnsFailure( + request.getLeft().database, request.getLeft().tableName, request.getRight())); + } + } + public TSStatus alterTableSetProperties(final TAlterOrDropTableReq req) { final boolean isView = req.isSetIsView() && req.isIsView(); return executeWithoutDuplicate( @@ -2617,6 +2818,38 @@ public TSStatus executeWithoutDuplicate( return waitingProcedureFinished(procedureToWait); } + public Pair executeWithoutDuplicateWithoutWaiting( + final String database, + final TsTable table, + final String tableName, + final @Nullable String newName, + final String queryId, + final ProcedureType thisType, + final Procedure procedure, + final Consumer> consumer) { + final long procedureId; + synchronized (this) { + final Pair procedureIdDuplicatePair = + checkDuplicateTableTask(database, table, tableName, newName, queryId, thisType); + procedureId = procedureIdDuplicatePair.getLeft(); + if (procedureId == -1) { + if (Boolean.TRUE.equals(procedureIdDuplicatePair.getRight())) { + TSStatus status = + RpcUtils.getStatus( + TSStatusCode.OVERLAP_WITH_EXISTING_TASK, + "Some other task is operating table with same name."); + return new Pair<>(status, -1L); + } + consumer.accept(procedure); + this.executor.submitProcedure(procedure); + return new Pair<>(StatusUtils.OK, procedure.getProcId()); + } else { + TSStatus status = RpcUtils.getStatus(TSStatusCode.REPEATED_WITH_EXISTING_TASK, ""); + return new Pair<>(status, procedureId); + } + } + } + public Pair checkDuplicateTableTask( final @Nonnull String database, final TsTable table, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index e7f96322a5346..b7c6fdc3385c2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -1554,7 +1554,7 @@ public Optional getTableWithUsingStatusIfExists( return Optional.of(tableAndStatus.get().getLeft()); } - public synchronized Pair tableColumnCheckForColumnExtension( + public Pair tableColumnCheckForColumnExtension( final String database, final String tableName, final List columnSchemaList, @@ -1576,8 +1576,7 @@ public synchronized Pair tableColumnCheckForColumnExtension( return result.get(); } - final TsTable expandedTable = new TsTable(originalTable); - + // the table read above is a copy, which this check expands and returns final String errorMsg = String.format( "Column '%s' already exist", @@ -1586,29 +1585,74 @@ public synchronized Pair tableColumnCheckForColumnExtension( .collect(Collectors.joining(", "))); final AtomicBoolean hasObject = new AtomicBoolean(false); + if (originalTable.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).isEmpty()) { + checkColumnSchema(originalTable, columnSchemaList, hasObject); + } else { + // only for tree_to_table view + checkColumnSchemaForTree2TableView(originalTable, columnSchemaList, hasObject); + } + + if (columnSchemaList.isEmpty()) { + return new Pair<>(RpcUtils.getStatus(TSStatusCode.COLUMN_ALREADY_EXISTS, errorMsg), null); + } + + if (hasObject.get()) { + originalTable.checkTableNameAndObjectNames4Object(); + } + return new Pair<>(StatusUtils.OK, originalTable); + } + + private static void checkColumnSchema( + TsTable table, List columnSchemaList, AtomicBoolean hasObject) { columnSchemaList.removeIf( columnSchema -> { - if (Objects.isNull(originalTable.getColumnSchema(columnSchema.getColumnName()))) { + if (Objects.isNull(table.getColumnSchema(columnSchema.getColumnName()))) { if (columnSchema.getDataType().equals(TSDataType.OBJECT)) { hasObject.set(true); } - expandedTable.addColumnSchema(columnSchema); + table.addColumnSchema(columnSchema); return false; } return true; }); + } - if (columnSchemaList.isEmpty()) { - return new Pair<>(RpcUtils.getStatus(TSStatusCode.COLUMN_ALREADY_EXISTS, errorMsg), null); - } + private static void checkColumnSchemaForTree2TableView( + final TsTable view, + final List columnSchemaList, + final AtomicBoolean hasObject) { + Set existingColumnNames = + view.getColumnList().stream() + .map(TsTableColumnSchema::getColumnName) + .collect(Collectors.toSet()); + final Set originalNames = TreeViewSchema.getSourceNames(view); - if (hasObject.get()) { - expandedTable.checkTableNameAndObjectNames4Object(); - } - return new Pair<>(StatusUtils.OK, expandedTable); + columnSchemaList.removeIf( + columnSchema -> { + if (existingColumnNames.contains(columnSchema.getColumnName())) { + // todo log + return true; + } + + // The original time series can only be referenced once. + final String originalName = TreeViewSchema.getSourceName(columnSchema); + if (originalNames.contains(originalName)) { + // todo log + return true; + } + + if (columnSchema.getDataType().equals(TSDataType.OBJECT)) { + hasObject.set(true); + } + + existingColumnNames.add(columnSchema.getColumnName()); + originalNames.add(originalName); + view.addColumnSchema(columnSchema); + return false; + }); } - public synchronized Pair tableColumnCheckForColumnAltering( + public Pair tableColumnCheckForColumnAltering( final String database, final String tableName, final String columnName, @@ -1633,8 +1677,8 @@ public synchronized Pair tableColumnCheckForColumnAltering( return new Pair<>(tsStatus, null); } - final TsTable alteredTable = new TsTable(originalTable); - TsTableColumnSchema tsTableColumnSchema = alteredTable.getColumnSchema(columnName); + // the table read above is a copy, which this check alters and returns + TsTableColumnSchema tsTableColumnSchema = originalTable.getColumnSchema(columnName); tsTableColumnSchema.setDataType(dataType); if (tsTableColumnSchema instanceof FieldColumnSchema) { FieldColumnSchema fieldColumnSchema = ((FieldColumnSchema) tsTableColumnSchema); @@ -1642,10 +1686,10 @@ public synchronized Pair tableColumnCheckForColumnAltering( SchemaUtils.getDataTypeCompatibleEncoding(dataType, fieldColumnSchema.getEncoding())); } - return new Pair<>(RpcUtils.SUCCESS_STATUS, alteredTable); + return new Pair<>(RpcUtils.SUCCESS_STATUS, originalTable); } - public synchronized Pair tableColumnCheckForColumnRenaming( + public Pair tableColumnCheckForColumnRenaming( final String database, final String tableName, final String oldName, @@ -1692,14 +1736,13 @@ public synchronized Pair tableColumnCheckForColumnRenaming( null); } - final TsTable expandedTable = new TsTable(originalTable); - - expandedTable.renameColumnSchema(oldName, newName); + // the table read above is a copy, which this check renames and returns + originalTable.renameColumnSchema(oldName, newName); - return new Pair<>(RpcUtils.SUCCESS_STATUS, expandedTable); + return new Pair<>(RpcUtils.SUCCESS_STATUS, originalTable); } - public synchronized Pair tableCheckForRenaming( + public Pair tableCheckForRenaming( final String database, final String tableName, final String newName, @@ -1729,9 +1772,9 @@ public synchronized Pair tableCheckForRenaming( null); } - final TsTable expandedTable = new TsTable(originalTable); - expandedTable.renameTable(newName); - return new Pair<>(RpcUtils.SUCCESS_STATUS, expandedTable); + // the table read above is a copy, which this check renames and returns + originalTable.renameTable(newName); + return new Pair<>(RpcUtils.SUCCESS_STATUS, originalTable); } public TSStatus executePlan(final ConfigPhysicalPlan plan, final boolean isGeneratedByPipe) { @@ -1743,7 +1786,7 @@ public TSStatus executePlan(final ConfigPhysicalPlan plan, final boolean isGener } } - public synchronized TSStatus setTableComment( + public TSStatus setTableComment( final String database, final String tableName, final String comment, @@ -1763,7 +1806,7 @@ public synchronized TSStatus setTableComment( } } - public synchronized TSStatus setTableColumnComment( + public TSStatus setTableColumnComment( final String database, final String tableName, final String columnName, @@ -1783,7 +1826,7 @@ public synchronized TSStatus setTableColumnComment( } } - public synchronized Pair updateTableProperties( + public Pair updateTableProperties( final String database, final String tableName, final Map originalProperties, @@ -1834,24 +1877,24 @@ public synchronized Pair updateTableProperties( throw new MetadataException(e); } - final TsTable updatedTable = new TsTable(originalTable); + // the table read above is a copy, which this check updates and returns updatedProperties.forEach( (k, v) -> { originalProperties.put(k, originalTable.getPropValue(k).orElse(null)); if (Objects.nonNull(v)) { - updatedTable.addProp(k, v); + originalTable.addProp(k, v); } else if (TsTable.NEED_LAST_CACHE_PROPERTY.equals(k) && Objects.nonNull(databaseSchema) && databaseSchema.isSetNeedLastCache()) { - updatedTable.addProp(k, String.valueOf(databaseSchema.isNeedLastCache())); + originalTable.addProp(k, String.valueOf(databaseSchema.isNeedLastCache())); } else { - updatedTable.removeProp(k); + originalTable.removeProp(k); } }); - return new Pair<>(RpcUtils.SUCCESS_STATUS, updatedTable); + return new Pair<>(RpcUtils.SUCCESS_STATUS, originalTable); } - public synchronized Pair updateTableColumnProperties( + public Pair updateTableColumnProperties( final String database, final String tableName, final String columnName, @@ -1882,8 +1925,8 @@ public synchronized Pair updateTableColumnProperties( null); } - final TsTable updatedTable = new TsTable(originalTable); - final TsTableColumnSchema updatedColumn = updatedTable.getColumnSchema(columnName); + // the table read above is a copy, which this check updates and returns + final TsTableColumnSchema updatedColumn = originalTable.getColumnSchema(columnName); updatedProperties .keySet() .removeIf( @@ -1903,7 +1946,7 @@ public synchronized Pair updateTableColumnProperties( } } - return new Pair<>(RpcUtils.SUCCESS_STATUS, updatedTable); + return new Pair<>(RpcUtils.SUCCESS_STATUS, originalTable); } private void invalidateLastCache(final String database) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 6bae27bc7f28a..966b45281025c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -1549,12 +1549,23 @@ public Map> getAllPreDeleteTables() { } } + /** + * The table and its status, or empty when the table does not exist. The table is a copy, which + * the caller owns: the stored table is only written by the plans, under the write lock, and never + * through a reference handed out here. + */ public Optional> getTsTableIfExists( final String database, final String tableName) throws MetadataException { databaseReadWriteLock.readLock().lock(); try { - return tableModelMTree.getTableAndStatusIfExists( - getQualifiedDatabasePartialPath(database), tableName); + final Optional> storedTable = + tableModelMTree.getTableAndStatusIfExists( + getQualifiedDatabasePartialPath(database), tableName); + if (storedTable.isEmpty()) { + return Optional.empty(); + } + return Optional.of( + new Pair<>(new TsTable(storedTable.get().getLeft()), storedTable.get().getRight())); } finally { databaseReadWriteLock.readLock().unlock(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java index f407d4d8ecdce..2beca63c2e6f6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TreeDeviceViewFieldDetector.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.table.TreeViewSchema; @@ -32,6 +33,7 @@ import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.confignode.i18n.ManagerMessages; +import org.apache.iotdb.confignode.i18n.ProcedureMessages; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.procedure.impl.schema.DataNodeRegionTaskExecutor; import org.apache.iotdb.mpp.rpc.thrift.TDeviceViewReq; @@ -40,17 +42,16 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -64,28 +65,32 @@ public class TreeDeviceViewFieldDetector { private final ConfigManager configManager; private final PartialPath path; private final TsTable table; - private final Map> fields; private TDeviceViewResp result = new TDeviceViewResp(StatusUtils.OK, new ConcurrentHashMap<>()); private final Map lowerCase2OriginalMap = new HashMap<>(); - public TreeDeviceViewFieldDetector( - final ConfigManager configManager, - final TsTable table, - final Map> fields) { + public TreeDeviceViewFieldDetector(final ConfigManager configManager, final TsTable table) { this.configManager = configManager; this.path = TreeViewSchema.getPrefixPattern(table); this.table = table; - this.fields = fields; } - public TSStatus detectMissingFieldTypes() { - if (table.getFieldNum() == 0 && Objects.isNull(fields)) { + /** + * Fills the fields of the view which is being created: a view may declare no field at all, in + * which case every field under its tree path pattern becomes a column, or declare fields whose + * type is left to the detection. + */ + public TSStatus detectFieldsForCreation() { + if (table.getFieldNum() == 0) { + // for the first time creation of the table, user do not assign any field; + // just detect all the fields of prefix path new TreeDeviceViewFieldDetectionTaskExecutor( configManager, getLatestSchemaRegionMap(), table.getTagNum(), - TreeViewSchema.isRestrict(table)) + TreeViewSchema.isRestrict(table), + null, + false) .execute(); if (result.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return result.getStatus(); @@ -93,69 +98,158 @@ public TSStatus detectMissingFieldTypes() { result .getDeviewViewFieldTypeMap() .forEach( - (field, type) -> { - final FieldColumnSchema columnSchema = - new FieldColumnSchema(field, TSDataType.getTsDataType(type)); - if (!field.equals(lowerCase2OriginalMap.get(field))) { - TreeViewSchema.setOriginalName(columnSchema, lowerCase2OriginalMap.get(field)); - } - table.addColumnSchema(columnSchema); - }); - } else { - final Map> unknownFields; - if (Objects.isNull(fields)) { - unknownFields = new HashMap<>(); - for (final TsTableColumnSchema schema : table.getColumnList()) { - if (!(schema instanceof FieldColumnSchema) - || schema.getDataType() != TSDataType.UNKNOWN) { - continue; - } - final String key = TreeViewSchema.getSourceName(schema); - if (!unknownFields.containsKey(key)) { - unknownFields.put(key, new HashSet<>()); - } - unknownFields.get(key).add((FieldColumnSchema) schema); - } - } else { - unknownFields = fields; - } + (field, type) -> + table.addColumnSchema(toFieldColumnSchemaWithOriginName(field, type))); + return StatusUtils.OK; + } + + return detectFieldTypes(collectUnknownFieldColumns()); + } + + /** + * Resolves the type of the field columns which are being added to an existing view. The columns + * to resolve are given by the caller, as only it knows which ones it is adding. + */ + public TSStatus detectFieldTypesForColumnAddition(final Map fields) { + return detectFieldTypes(fields); + } + + /** + * The fields which exist under the view's tree path pattern but are not columns of the view yet, + * as the columns to add. Used after a view is created, to pick up the series which were created + * before the view was mounted on the DataNodes, and which the DataNodes therefore never reported. + */ + public Pair> detectNewFieldColumnsAfterCreation() { + new TreeDeviceViewFieldDetectionTaskExecutor( + configManager, + getLatestSchemaRegionMap(), + table.getTagNum(), + TreeViewSchema.isRestrict(table), + null, + true) + .execute(); + if (result.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return new Pair<>(result.getStatus(), Collections.emptyList()); + } + // A source measurement can only be referenced once, so a field whose source the view already + // references under another column name gets no column either. + final Set sourceNames = TreeViewSchema.getSourceNames(table); + final List columns = new ArrayList<>(); + result + .getDeviewViewFieldTypeMap() + .forEach( + (field, type) -> { + if (Objects.isNull(table.getColumnSchema(field)) + && !sourceNames.contains(lowerCase2OriginalMap.get(field))) { + columns.add(toFieldColumnSchemaWithOriginName(field, type)); + } + }); + return new Pair<>(StatusUtils.OK, columns); + } + + /** + * The columns which are missing from a view which auto-syncs its schema: the series which already + * existed under its tree path pattern were created before the DataNodes mounted the view on their + * tree-to-table view index, so they never triggered the addition of their column. Used right + * after the view is mounted, whether it has just been created or its auto detection has just been + * turned on. + * + * @return the columns to add; empty when the view does not auto-sync its schema, or when the + * detection itself failed + */ + public static List detectMissingColumnsAfterCreation( + final ConfigManager configManager, final String database, final TsTable table) { + final boolean autoSync = + table.getPropValue(TsTable.AUTO_SYNC_SCHEMA).map(Boolean::parseBoolean).orElse(false); + if (!autoSync) { + return Collections.emptyList(); + } - if (unknownFields.isEmpty()) { - return StatusUtils.OK; + final Pair> detected = + new TreeDeviceViewFieldDetector(configManager, table).detectNewFieldColumnsAfterCreation(); + if (detected.getLeft().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.warn( + ProcedureMessages + .LOG_FAILED_TO_DETECT_THE_TREE_SERIES_UNDER_THE_NEW_VIEW_ARG_ARG_ARG_8F20CAE7, + database, + table.getTableName(), + detected.getLeft()); + return Collections.emptyList(); + } + return detected.getRight(); + } + + /** The field columns of the view whose type is not known yet, keyed by their source name. */ + private Map collectUnknownFieldColumns() { + final Map unknownFields = new HashMap<>(); + for (final TsTableColumnSchema schema : table.getColumnList()) { + if (!(schema instanceof FieldColumnSchema) || schema.getDataType() != TSDataType.UNKNOWN) { + continue; } - new TreeDeviceViewFieldDetectionTaskExecutor( - configManager, - getLatestSchemaRegionMap(), - table.getTagNum(), - TreeViewSchema.isRestrict(table), - unknownFields.keySet()) - .execute(); - if (result.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return result.getStatus(); + final String key = TreeViewSchema.getSourceName(schema); + if (unknownFields.containsKey(key)) { + // A table which references the same source measurement twice is rejected when it is + // parsed, so a table in the schema must not reach this point. + throw new SemanticException( + String.format( + ManagerMessages + .EXCEPTION_THE_DUPLICATED_SOURCE_MEASUREMENT_ARG_IS_UNSUPPORTED_YET_873AF566, + key)); } - for (final Map.Entry> unknownField : - unknownFields.entrySet()) { - if (result.getDeviewViewFieldTypeMap().containsKey(unknownField.getKey())) { - unknownField - .getValue() - .forEach( - field -> - field.setDataType( - TSDataType.getTsDataType( - result.getDeviewViewFieldTypeMap().get(unknownField.getKey())))); - } else { - return new TSStatus(TSStatusCode.TYPE_NOT_FOUND.getStatusCode()) - .setMessage( - String.format( - ManagerMessages - .MESSAGE_MEASUREMENTS_NOT_FOUND_ARG_CANNOT_AUTO_DETECT_980D7D44, - unknownField.getKey())); - } + unknownFields.put(key, (FieldColumnSchema) schema); + } + return unknownFields; + } + + /** + * Resolves the type of every field column in {@code unknownFields} from the schema of the regions + * the view covers. A measurement which the detection cannot find fails the whole detection. + */ + private TSStatus detectFieldTypes(final Map unknownFields) { + if (unknownFields.isEmpty()) { + return StatusUtils.OK; + } + new TreeDeviceViewFieldDetectionTaskExecutor( + configManager, + getLatestSchemaRegionMap(), + table.getTagNum(), + TreeViewSchema.isRestrict(table), + unknownFields.keySet(), + false) + .execute(); + if (result.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return result.getStatus(); + } + for (final Map.Entry unknownField : unknownFields.entrySet()) { + // The detection keys the field types by the lower case of the measurement name, while the + // column keeps the source name as the statement spelled it. + final String sourceName = unknownField.getKey().toLowerCase(Locale.ENGLISH); + if (result.getDeviewViewFieldTypeMap().containsKey(sourceName)) { + unknownField + .getValue() + .setDataType( + TSDataType.getTsDataType(result.getDeviewViewFieldTypeMap().get(sourceName))); + } else { + return new TSStatus(TSStatusCode.TYPE_NOT_FOUND.getStatusCode()) + .setMessage( + String.format( + ManagerMessages.MESSAGE_MEASUREMENTS_NOT_FOUND_ARG_CANNOT_AUTO_DETECT_980D7D44, + unknownField.getKey())); } } return StatusUtils.OK; } + private TsTableColumnSchema toFieldColumnSchemaWithOriginName( + final String field, final byte type) { + final FieldColumnSchema columnSchema = + new FieldColumnSchema(field, TSDataType.getTsDataType(type)); + if (!field.equals(lowerCase2OriginalMap.get(field))) { + TreeViewSchema.setOriginalName(columnSchema, lowerCase2OriginalMap.get(field)); + } + return columnSchema; + } + private Map getLatestSchemaRegionMap() { final PathPatternTree tree = new PathPatternTree(); tree.appendPathPattern(path); @@ -166,59 +260,37 @@ private Map getLatestSchemaRegionMap() { private class TreeDeviceViewFieldDetectionTaskExecutor extends DataNodeRegionTaskExecutor { - protected TreeDeviceViewFieldDetectionTaskExecutor( - final ConfigManager configManager, - final Map targetRegionGroup, - final int tagNumber, - final boolean restrict) { - super( - configManager, - targetRegionGroup, - false, - CnToDnAsyncRequestType.DETECT_TREE_DEVICE_VIEW_FIELD_TYPE, - ((dataNodeLocation, consensusGroupIdList) -> - new TDeviceViewReq( - consensusGroupIdList, Arrays.asList(path.getNodes()), tagNumber, restrict))); - configManager - .getClusterSchemaManager() - .getTemplateSetInfo(Collections.singletonList(path)) - .getPatternTemplateMap() - .values() - .stream() - .flatMap(Collection::stream) - .flatMap(template -> template.getSchemaMap().values().stream()) - .forEach( - schema -> - mergeMeasurementAndType(schema.getMeasurementName(), schema.getTypeInByte())); - } + /** + * Whether several types of the same measurement are tolerated. When they are, the type the + * detection meets first is kept instead of failing the whole detection. + */ + private final boolean tolerateTypeMismatch; protected TreeDeviceViewFieldDetectionTaskExecutor( final ConfigManager configManager, final Map targetRegionGroup, final int tagNumber, final boolean restrict, - final @Nonnull Set measurements) { + final @Nullable Set measurements, + final boolean tolerateTypeMismatch) { super( configManager, targetRegionGroup, false, CnToDnAsyncRequestType.DETECT_TREE_DEVICE_VIEW_FIELD_TYPE, - ((dataNodeLocation, consensusGroupIdList) -> - new TDeviceViewReq( - consensusGroupIdList, Arrays.asList(path.getNodes()), tagNumber, restrict) - .setRequiredMeasurements(measurements))); - configManager - .getClusterSchemaManager() - .getTemplateSetInfo(Collections.singletonList(path)) - .getPatternTemplateMap() - .values() - .stream() - .flatMap(Collection::stream) - .flatMap(template -> template.getSchemaMap().values().stream()) - .filter(schema -> measurements.contains(schema.getMeasurementName())) - .forEach( - schema -> - mergeMeasurementAndType(schema.getMeasurementName(), schema.getTypeInByte())); + ((dataNodeLocation, consensusGroupIdList) -> { + final TDeviceViewReq req = + new TDeviceViewReq( + consensusGroupIdList, Arrays.asList(path.getNodes()), tagNumber, restrict); + if (Objects.nonNull(measurements)) { + req.setRequiredMeasurements(measurements); + } + if (tolerateTypeMismatch) { + req.setTolerateTypeMismatch(true); + } + return req; + })); + this.tolerateTypeMismatch = tolerateTypeMismatch; } @Override @@ -273,25 +345,30 @@ private void mergeMeasurementAndType(final String measurement, final byte type) // Field type collection if (!result.getDeviewViewFieldTypeMap().containsKey(fieldName)) { result.getDeviewViewFieldTypeMap().put(fieldName, type); - } else if (!Objects.equals(result.getDeviewViewFieldTypeMap().get(fieldName), type)) { + } else if (!Objects.equals(result.getDeviewViewFieldTypeMap().get(fieldName), type) + && !tolerateTypeMismatch) { result.setStatus( RpcUtils.getStatus( TSStatusCode.DATA_TYPE_MISMATCH, String.format( - "Multiple types encountered when auto detecting type of measurement '%s', please check", + ManagerMessages + .MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB, measurement))); } // Field name detection if (!lowerCase2OriginalMap.containsKey(fieldName)) { lowerCase2OriginalMap.put(fieldName, measurement); - } else if (!Objects.equals(lowerCase2OriginalMap.get(fieldName), measurement)) { + } else if (!Objects.equals(lowerCase2OriginalMap.get(fieldName), measurement) + && !tolerateTypeMismatch) { result.setStatus( RpcUtils.getStatus( TSStatusCode.MEASUREMENT_NAME_CONFLICT, String.format( - "The measurements %s and %s share the same lower case when auto detecting type, please check", - lowerCase2OriginalMap.get(fieldName), measurement))); + ManagerMessages + .MESSAGE_THE_MEASUREMENTS_ARG_AND_ARG_SHARE_THE_SAME_LOWER_CASE_WHEN_AUTO_DETECTING_TYPE_PLEASE_CHECK_7225D062, + lowerCase2OriginalMap.get(fieldName), + measurement))); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java index 241eea8a70201..ad01ffc399174 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java @@ -48,6 +48,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TCountPathsUsingTemplateResp; import org.apache.iotdb.mpp.rpc.thrift.TInvalidateMatchedSchemaCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -272,6 +273,21 @@ public static Map broadcastTableUpdate( return clientHandler.getResponseMap(); } + /** + * Broadcasts a table change at commit-release through {@link ClusterCachePropagator}: proceed + * once every unreachable DataNode is provably self-fenced, instead of failing on the first one. A + * fenced DataNode fails closed and resyncs on lease recovery, which rebuilds the tree-to-table + * view index it missed; a DataNode which is not provably fenced may still serve clients, so the + * commit is reported as unproven instead. + * + * @return whether it is safe to proceed + */ + public static boolean commitReleaseTableWithHA( + final TUpdateTableReq req, final ConfigManager configManager) { + return new ClusterCachePropagator(filterFencedDataNode(configManager)) + .propagate(targets -> broadcastTableUpdate(req, targets)); + } + private static Map failedOnly(final Map responses) { return responses.entrySet().stream() .filter(entry -> entry.getValue().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) @@ -308,24 +324,22 @@ public static TUpdateTableReq buildCommitUpdateTableReq( return req; } - public static Map commitReleaseTable( + /** + * Builds the COMMIT_UPDATE_TABLE request of a change which also affects the tree-to-table view + * index of the DataNodes. + * + *

The caller declares the view attributes the DataNode needs to apply the change, so that a + * DataNode never reads them from the table it caches: that cache may hold a placeholder, in which + * case the DataNode could not compute the entry at all. + */ + public static TUpdateTableReq buildCommitUpdateTableReq( final String database, final String tableName, - final ConfigManager configManager, - final @Nullable String oldName) { + final @Nullable String oldName, + final TUpdateTree2TableView viewInfo) { final TUpdateTableReq req = buildCommitUpdateTableReq(database, tableName, oldName); - - final DataNodeAsyncRequestContext clientHandler = - new DataNodeAsyncRequestContext<>( - CnToDnAsyncRequestType.UPDATE_TABLE, req, filterFencedDataNode(configManager)); - CnToDnInternalServiceAsyncRequestManager.getInstance() - .sendAsyncRequest( - clientHandler, - ClusterCachePropagator.BROADCAST_RPC_RETRY, - ClusterCachePropagator.BROADCAST_RPC_TIMEOUT_MS); - return clientHandler.getResponseMap().entrySet().stream() - .filter(entry -> entry.getValue().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + req.setUpdateTree2TableViewInfo(viewInfo); + return req; } /** Build the ROLLBACK_UPDATE_TABLE request used to roll back a pre-released table change. */ @@ -345,6 +359,16 @@ public static TUpdateTableReq rollbackUpdateTableReq( return req; } + public static TUpdateTableReq rollbackUpdateTableReq( + final String database, + final String tableName, + final String oldName, + final TUpdateTree2TableView viewInfo) { + final TUpdateTableReq req = rollbackUpdateTableReq(database, tableName, oldName); + req.setUpdateTree2TableViewInfo(viewInfo); + return req; + } + /** * Broadcast an INVALIDATE_MATCHED_SCHEMA_CACHE to all DataNodes through {@link * ClusterCachePropagator}: proceed once every unreachable DataNode is provably self-fenced (it diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java index b4aab9949c28e..510519110f972 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java @@ -22,7 +22,6 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; -import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; @@ -118,31 +117,22 @@ protected void preRelease(final ConfigNodeProcedureEnv env, final @Nullable Stri } } - protected void commitRelease(final ConfigNodeProcedureEnv env) { - commitRelease(env, null); + /** + * The request broadcast at commit-release. A subclass whose change also affects the tree-to-table + * view index of the DataNodes overrides this to declare the effect. + */ + protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldName) { + return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName); } - protected void commitRelease(final ConfigNodeProcedureEnv env, final @Nullable String oldName) { - final Map failedResults = - SchemaUtils.commitReleaseTable( - database, table.getTableName(), env.getConfigManager(), oldName); - if (!failedResults.isEmpty()) { - LOGGER.warn( - ProcedureMessages.FAILED_TO_FOR_TABLE_TO_DATANODE_FAILURE_RESULTS, - getActionMessage(), - database, - table.getTableName(), - failedResults); - } + protected void commitReleaseWithHA(final ConfigNodeProcedureEnv env) { + commitReleaseWithHA(env, null); } protected void commitReleaseWithHA( final ConfigNodeProcedureEnv env, final @Nullable String oldName) { - final TUpdateTableReq req = - SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName); - final boolean proceeded = - new ClusterCachePropagator(SchemaUtils.filterFencedDataNode(env.getConfigManager())) - .propagate(targets -> SchemaUtils.broadcastTableUpdate(req, targets)); + final TUpdateTableReq req = buildCommitReleaseTableReq(oldName); + final boolean proceeded = SchemaUtils.commitReleaseTableWithHA(req, env.getConfigManager()); if (!proceeded) { LOGGER.warn( @@ -165,11 +155,11 @@ protected void rollbackPreRelease(final ConfigNodeProcedureEnv env) { } protected void rollbackPreRelease( - final ConfigNodeProcedureEnv env, final @Nullable String tableName) { + final ConfigNodeProcedureEnv env, final @Nullable String oldName) { // A down DataNode must not block rollback either: proceed past provably-fenced DataNodes (which // resync on recovery) and only fail on an unacked DataNode that is not provably fenced. final TUpdateTableReq req = - SchemaUtils.rollbackUpdateTableReq(database, table.getTableName(), tableName); + SchemaUtils.rollbackUpdateTableReq(database, table.getTableName(), oldName); final boolean proceeded = new ClusterCachePropagator(SchemaUtils.filterFencedDataNode(env.getConfigManager())) .propagate(targets -> SchemaUtils.broadcastTableUpdate(req, targets)); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractSetPropertiesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractSetPropertiesProcedure.java index a354f0ff3a393..79d69a179979b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractSetPropertiesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractSetPropertiesProcedure.java @@ -101,7 +101,7 @@ protected Flow executeFromState( LOGGER.info(ProcedureMessages.SET_PROPERTIES_TO_TABLE, database, tableName); break; case COMMIT_RELEASE: - commitRelease(env); + commitReleaseWithHA(env); LOGGER.info( ProcedureMessages.COMMIT_RELEASE_INFO_OF_TABLE_WHEN_SETTING_PROPERTIES, database, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java index 07bdbf44c55c9..61d35137cf88b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -90,7 +90,7 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final AddTable ProcedureMessages.COMMIT_RELEASE_INFO_OF_TABLE_WHEN_ADDING_COLUMN, database, tableName); - commitRelease(env); + commitReleaseWithHA(env); return Flow.NO_MORE_STATE; default: setFailure( @@ -259,6 +259,10 @@ public boolean equals(final Object o) { && Objects.equals(this.addedColumnList, ((AddTableColumnProcedure) o).addedColumnList); } + public void setAddedColumnList(List addedColumnList) { + this.addedColumnList = addedColumnList; + } + @Override public int hashCode() { return Objects.hash(super.hashCode(), addedColumnList); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AlterTableColumnDataTypeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AlterTableColumnDataTypeProcedure.java index 1cfda63fcf736..3a17a7de5d7c8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AlterTableColumnDataTypeProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AlterTableColumnDataTypeProcedure.java @@ -103,7 +103,7 @@ protected Flow executeFromState( ProcedureMessages.COMMIT_RELEASE_INFO_OF_TABLE_WHEN_ALTERING_COLUMN, database, tableName); - commitRelease(env); + commitReleaseWithHA(env); return Flow.NO_MORE_STATE; default: setFailure( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java index ac573463ebb23..5737b433150e6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java @@ -47,7 +47,6 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Map; import java.util.Objects; import static org.apache.iotdb.rpc.TSStatusCode.TABLE_ALREADY_EXISTS; @@ -98,6 +97,7 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final CreateTa case COMMIT_RELEASE: LOGGER.info(ProcedureMessages.COMMIT_RELEASE_TABLE, database, table.getTableName()); commitReleaseTable(env); + reconcileTree2TableViewColumns(env); return Flow.NO_MORE_STATE; default: setFailure( @@ -156,13 +156,13 @@ protected void preCreateTable(final ConfigNodeProcedureEnv env) { } } - private void preReleaseTable(final ConfigNodeProcedureEnv env) { + protected void preReleaseTable(final ConfigNodeProcedureEnv env) { // Broadcast the pre-update to all DataNodes. Instead of failing whenever any DataNode is // unreachable, proceed once every unacked DataNode is provably self-fenced: such a DataNode // fails closed on its (now-stale) table cache and resyncs on lease recovery, so it cannot serve // dirty schema. Only fail if an unacked DataNode is not provably fenced (it may still be // serving clients). - final TUpdateTableReq req = SchemaUtils.buildPreUpdateTableReq(database, table, null); + final TUpdateTableReq req = buildPreReleaseTableReq(); final boolean proceeded = new ClusterCachePropagator(SchemaUtils.filterFencedDataNode(env.getConfigManager())) .propagate(targets -> SchemaUtils.broadcastTableUpdate(req, targets)); @@ -181,6 +181,10 @@ private void preReleaseTable(final ConfigNodeProcedureEnv env) { setNextState(CreateTableState.COMMIT_CREATE); } + protected TUpdateTableReq buildPreReleaseTableReq() { + return SchemaUtils.buildPreUpdateTableReq(database, table, null); + } + private void commitCreateTable(final ConfigNodeProcedureEnv env) { final TSStatus status = SchemaUtils.executeInConsensusLayer( @@ -196,20 +200,35 @@ private void commitCreateTable(final ConfigNodeProcedureEnv env) { } } + /** + * The request broadcast at commit-release. A subclass whose change also affects the tree-to-table + * view index of the DataNodes overrides this to declare the effect. + */ + protected TUpdateTableReq buildCommitReleaseTableReq() { + return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), null); + } + private void commitReleaseTable(final ConfigNodeProcedureEnv env) { - final Map failedResults = - SchemaUtils.commitReleaseTable( - database, table.getTableName(), env.getConfigManager(), null); + final TUpdateTableReq req = buildCommitReleaseTableReq(); + final boolean proceeded = SchemaUtils.commitReleaseTableWithHA(req, env.getConfigManager()); - if (!failedResults.isEmpty()) { + if (!proceeded) { LOGGER.warn( ProcedureMessages.FAILED_TO_SYNC_TABLE_COMMIT_CREATE_INFO_TO_DATANODE_FAILURE, database, table.getTableName(), - failedResults); + ProcedureMessages.FAILED_TO_PROVE_AN_UNREACHABLE_DN_IS_FENCED); } } + /** + * Adds the columns of the tree series which already exist under a newly created tree-to-table + * view's path. A plain table has no such path, so the base procedure does nothing. + */ + protected void reconcileTree2TableViewColumns(final ConfigNodeProcedureEnv env) { + // do nothing here + } + @Override protected boolean isRollbackSupported(final CreateTableState state) { switch (state) { @@ -259,10 +278,9 @@ protected void rollbackCreate(final ConfigNodeProcedureEnv env) { } } - private void rollbackPreRelease(final ConfigNodeProcedureEnv env) { + protected void rollbackPreRelease(final ConfigNodeProcedureEnv env) { // A down DataNode must not block rollback if it is already provably self-fenced. - final TUpdateTableReq req = - SchemaUtils.rollbackUpdateTableReq(database, table.getTableName(), null); + final TUpdateTableReq req = buildRollbackTableReq(); final boolean proceeded = new ClusterCachePropagator(SchemaUtils.filterFencedDataNode(env.getConfigManager())) .propagate(targets -> SchemaUtils.broadcastTableUpdate(req, targets)); @@ -279,6 +297,10 @@ private void rollbackPreRelease(final ConfigNodeProcedureEnv env) { } } + protected TUpdateTableReq buildRollbackTableReq() { + return SchemaUtils.rollbackUpdateTableReq(database, table.getTableName(), null); + } + @Override protected CreateTableState getState(final int stateId) { return CreateTableState.values()[stateId]; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 54217212a1151..00ec6655f7c4c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -254,7 +254,9 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final DropTabl ProcedureMessages.COMMIT_RELEASE_DELETE_TABLE_WHEN_DROPPING_TABLE, database, tableName); - commitRelease(env); + // A DataNode that misses this commit keeps the table in its pre-delete state, which only + // a later read of that table repairs. Retry until every reachable DataNode acks. + commitReleaseWithHA(env); return Flow.NO_MORE_STATE; default: setFailure(new ProcedureException(ProcedureMessages.UNRECOGNIZED_DROPTABLESTATE + state)); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java index 03088935ce616..6d8329e476acc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java @@ -94,7 +94,7 @@ protected Flow executeFromState( ProcedureMessages.COMMIT_RELEASE_INFO_OF_TABLE_WHEN_RENAMING_COLUMN, database, tableName); - commitRelease(env); + commitReleaseWithHA(env); return Flow.NO_MORE_STATE; default: setFailure( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java index 2bce5180505c2..c426558be78b3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/AddViewColumnProcedure.java @@ -21,26 +21,37 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType; import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.persistence.schema.TreeDeviceViewFieldDetector; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; import org.apache.iotdb.confignode.procedure.state.ProcedureState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; +import javax.annotation.Nullable; + import java.io.DataOutputStream; import java.io.IOException; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.Optional; + +import static org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType.ALTER_TAG; +import static org.apache.iotdb.commons.schema.table.TsTable.AUTO_SYNC_SCHEMA; public class AddViewColumnProcedure extends AddTableColumnProcedure { public AddViewColumnProcedure(final boolean isGeneratedByPipe) { @@ -64,28 +75,56 @@ protected void columnCheck(final ConfigNodeProcedureEnv env) { return; } - final Map> fields2Detect = new HashMap<>(); + final Map fields2Detect = new HashMap<>(); for (final TsTableColumnSchema schema : addedColumnList) { if (!(schema instanceof FieldColumnSchema) || schema.getDataType() != TSDataType.UNKNOWN) { continue; } final String key = TreeViewSchema.getSourceName(schema); - if (!fields2Detect.containsKey(key)) { - fields2Detect.put(key, new HashSet<>()); + if (fields2Detect.containsKey(key)) { + // The column check has already dropped the columns which refer to an already referenced + // source measurement, so the added columns must not share one. + throw new SemanticException( + String.format( + ManagerMessages + .EXCEPTION_THE_DUPLICATED_SOURCE_MEASUREMENT_ARG_IS_UNSUPPORTED_YET_873AF566, + key)); } - fields2Detect.get(key).add((FieldColumnSchema) schema); + fields2Detect.put(key, (FieldColumnSchema) schema); } if (!fields2Detect.isEmpty()) { final TSStatus status = - new TreeDeviceViewFieldDetector(env.getConfigManager(), table, fields2Detect) - .detectMissingFieldTypes(); + new TreeDeviceViewFieldDetector(env.getConfigManager(), table) + .detectFieldTypesForColumnAddition(fields2Detect); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { setFailure(new ProcedureException(new IoTDBException(status))); } } } + @Override + protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldName) { + final Optional type = tree2TableViewOperationType(); + if (type.isEmpty()) { + return super.buildCommitReleaseTableReq(oldName); + } + // Adding a tag column changes the tag count a device path is matched with. + final TUpdateTree2TableView viewInfo = new TUpdateTree2TableView(type.get().getOperationType()); + viewInfo.setTagNum(table.getTagNum()); + return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName, viewInfo); + } + + private Optional tree2TableViewOperationType() { + final Optional propValue = table.getPropValue(AUTO_SYNC_SCHEMA); + if (propValue.isPresent() && Boolean.parseBoolean(propValue.get())) { + final boolean hasTagColumn = + addedColumnList.stream().anyMatch(c -> c instanceof TagColumnSchema); + return hasTagColumn ? Optional.of(ALTER_TAG) : Optional.empty(); + } + return Optional.empty(); + } + @Override protected String getActionMessage() { return "add view column"; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java index f699a88a6edf2..bf5551f457b4a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/CreateTableViewProcedure.java @@ -23,8 +23,10 @@ import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.schema.table.TableNodeStatus; +import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType; import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.confignode.consensus.request.write.table.view.PreCreateTableViewPlan; import org.apache.iotdb.confignode.exception.DatabaseNotExistsException; import org.apache.iotdb.confignode.i18n.ProcedureMessages; @@ -36,6 +38,8 @@ import org.apache.iotdb.confignode.procedure.state.schema.CreateTableState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.Pair; @@ -46,6 +50,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.List; import java.util.Objects; import java.util.Optional; @@ -116,13 +121,88 @@ protected void checkTableExistence(final ConfigNodeProcedureEnv env) { } } final TSStatus status = - new TreeDeviceViewFieldDetector(env.getConfigManager(), table, null) - .detectMissingFieldTypes(); + new TreeDeviceViewFieldDetector(env.getConfigManager(), table).detectFieldsForCreation(); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { setFailure(new ProcedureException(new IoTDBException(status))); } } + @Override + protected TUpdateTableReq buildCommitReleaseTableReq() { + final boolean newAutoSync = + table.getPropValue(TsTable.AUTO_SYNC_SCHEMA).map(Boolean::parseBoolean).orElse(false); + if (newAutoSync) { + // Whether the view is a new one, or replace the old table, just add view + final TUpdateTree2TableView viewInfo = + new TUpdateTree2TableView(TreeToTableViewTrieOperationType.ADD.getOperationType()); + table.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).ifPresent(viewInfo::setTreePath); + viewInfo.setIsRestrict(TreeViewSchema.isRestrict(table)); + viewInfo.setTagNum(table.getTagNum()); + return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), null, viewInfo); + } + // Creating, or replacing with the auto detection turned off: the view the old definition had + // registered has to go. drop a non-existing view is allowed. + final boolean oldAutoSync = + Objects.nonNull(oldView) + && oldView + .getPropValue(TsTable.AUTO_SYNC_SCHEMA) + .map(Boolean::parseBoolean) + .orElse(false); + if (oldAutoSync) { + return SchemaUtils.buildCommitUpdateTableReq( + database, + table.getTableName(), + null, + new TUpdateTree2TableView(TreeToTableViewTrieOperationType.REMOVE.getOperationType())); + } + return super.buildCommitReleaseTableReq(); + } + + /** + * A rolled-back replace restores the view the old definition had. A DataNode which restarted + * while the replace was running does not hold it: the table such a DataNode pulls is only + * pre-created, and a pre-created table does not enter the index. Registering the old view again + * is idempotent, so a DataNode which never lost it is not affected. + */ + @Override + protected TUpdateTableReq buildRollbackTableReq() { + final boolean oldAutoSync = + Objects.nonNull(oldView) + && oldView + .getPropValue(TsTable.AUTO_SYNC_SCHEMA) + .map(Boolean::parseBoolean) + .orElse(false); + if (!oldAutoSync) { + return super.buildRollbackTableReq(); + } + final TUpdateTree2TableView viewInfo = + new TUpdateTree2TableView(TreeToTableViewTrieOperationType.ADD.getOperationType()); + oldView.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).ifPresent(viewInfo::setTreePath); + viewInfo.setIsRestrict(TreeViewSchema.isRestrict(oldView)); + viewInfo.setTagNum(oldView.getTagNum()); + return SchemaUtils.rollbackUpdateTableReq(database, table.getTableName(), null, viewInfo); + } + + @Override + protected void reconcileTree2TableViewColumns(final ConfigNodeProcedureEnv env) { + // The series which already existed under the path pattern were created before the DataNodes + // mounted the view, so they never triggered the addition of their column. + final List missingColumns = + TreeDeviceViewFieldDetector.detectMissingColumnsAfterCreation( + env.getConfigManager(), database, table); + if (missingColumns.isEmpty()) { + return; + } + + addChildProcedure( + new AddViewColumnProcedure( + database, + table.getTableName(), + String.format("create_view_reconcile_%s.%s", database, table.getTableName()), + missingColumns, + false)); + } + @Override protected void preCreateTable(final ConfigNodeProcedureEnv env) { final TSStatus status = diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java index 163150c0a09e4..c365d1a306549 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/DropViewProcedure.java @@ -19,9 +19,13 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table.view; +import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType; +import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure; import org.apache.iotdb.confignode.procedure.state.schema.DropTableState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; import java.io.DataOutputStream; import java.io.IOException; @@ -39,6 +43,17 @@ public DropViewProcedure( super(database, tableName, queryId, isGeneratedByPipe); } + /** A dropped view has to leave the tree-to-table view index of every DataNode. */ + @Override + protected TUpdateTableReq buildCommitReleaseTableReq(final String oldName) { + // Unregistering a view needs nothing but its name, which the request already carries. + return SchemaUtils.buildCommitUpdateTableReq( + database, + table.getTableName(), + null, + new TUpdateTree2TableView(TreeToTableViewTrieOperationType.REMOVE.getOperationType())); + } + /** * A view holds no data, so the procedure goes straight from PRE_DELETE to DROP_TABLE. The view is * removed from the ConfigNode once DROP_TABLE has completed; before that it can still be diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java index 62f77fb3fd0de..d6dec9689a309 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/RenameViewProcedure.java @@ -19,12 +19,20 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table.view; +import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType; +import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableProcedure; import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; + +import javax.annotation.Nullable; import java.io.DataOutputStream; import java.io.IOException; +import static org.apache.iotdb.commons.schema.table.TsTable.AUTO_SYNC_SCHEMA; + public class RenameViewProcedure extends RenameTableProcedure { public RenameViewProcedure(final boolean isGeneratedByPipe) { super(isGeneratedByPipe); @@ -39,6 +47,17 @@ public RenameViewProcedure( super(database, tableName, queryId, newName, isGeneratedByPipe); } + @Override + protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldName) { + if (!Boolean.parseBoolean(table.getPropValue(AUTO_SYNC_SCHEMA).orElse(null))) { + return super.buildCommitReleaseTableReq(oldName); + } + // A rename does not move the view, so the old name is all the DataNode needs. + final TUpdateTree2TableView viewInfo = + new TUpdateTree2TableView(TreeToTableViewTrieOperationType.ALTER_RENAME.getOperationType()); + return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName, viewInfo); + } + @Override protected String getActionMessage() { return "rename view"; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java index ce5b223cbb181..7919f98bc70d3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/view/SetViewPropertiesProcedure.java @@ -19,12 +19,26 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table.view; +import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType; +import org.apache.iotdb.commons.schema.table.TreeViewSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.confignode.persistence.schema.TreeDeviceViewFieldDetector; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.impl.schema.table.SetTablePropertiesProcedure; import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; + +import javax.annotation.Nullable; import java.io.DataOutputStream; import java.io.IOException; +import java.util.List; import java.util.Map; +import java.util.Optional; + +import static org.apache.iotdb.commons.schema.table.TsTable.AUTO_SYNC_SCHEMA; public class SetViewPropertiesProcedure extends SetTablePropertiesProcedure { public SetViewPropertiesProcedure(final boolean isGeneratedByPipe) { @@ -40,6 +54,75 @@ public SetViewPropertiesProcedure( super(database, tableName, queryId, properties, isGeneratedByPipe); } + @Override + protected TUpdateTableReq buildCommitReleaseTableReq(final @Nullable String oldName) { + final Optional type = tree2TableViewOperationType(); + if (type.isEmpty()) { + return super.buildCommitReleaseTableReq(oldName); + } + final TUpdateTree2TableView viewInfo = new TUpdateTree2TableView(type.get().getOperationType()); + if (TreeToTableViewTrieOperationType.ALTER_AUTO_SYNC == type.get()) { + // Turning the auto detection on mounts the view: the DataNode needs its whole definition. + table.getPropValue(TreeViewSchema.TREE_PATH_PATTERN).ifPresent(viewInfo::setTreePath); + viewInfo.setIsRestrict(TreeViewSchema.isRestrict(table)); + viewInfo.setTagNum(table.getTagNum()); + } + // turn down the auto_sync_schema + return SchemaUtils.buildCommitUpdateTableReq(database, table.getTableName(), oldName, viewInfo); + } + + @Override + protected void commitReleaseWithHA(final ConfigNodeProcedureEnv env) { + super.commitReleaseWithHA(env); + // Turning the auto detection on mounts the view, so the series which already exist under its + // path pattern need the same second detection as the ones of a newly created view. + if (TreeToTableViewTrieOperationType.ALTER_AUTO_SYNC + == tree2TableViewOperationType().orElse(null)) { + reconcileTree2TableViewColumns(env); + } + } + + /** Adds the columns which the view, just mounted by the commit, is still missing. */ + private void reconcileTree2TableViewColumns(final ConfigNodeProcedureEnv env) { + final List missingColumns = + TreeDeviceViewFieldDetector.detectMissingColumnsAfterCreation( + env.getConfigManager(), database, table); + if (missingColumns.isEmpty()) { + return; + } + addChildProcedure( + new AddViewColumnProcedure( + database, + table.getTableName(), + String.format("set_view_properties_reconcile_%s.%s", database, table.getTableName()), + missingColumns, + false)); + } + + /** + * The effect of the new properties on the tree-to-table view index, or empty when they do not + * change it. Turning auto_sync_schema on registers the view, turning it off unregisters it, and + * any other property leaves the index alone. + */ + private Optional tree2TableViewOperationType() { + if (!updatedProperties.containsKey(AUTO_SYNC_SCHEMA)) { + return Optional.empty(); + } + final TreeToTableViewTrieOperationType type = + indexEffectOf(updatedProperties.get(AUTO_SYNC_SCHEMA)); + // An absent auto_sync_schema means false, so setting it to false is not a change. + return type == indexEffectOf(originalProperties.get(AUTO_SYNC_SCHEMA)) + ? Optional.empty() + : Optional.of(type); + } + + private static TreeToTableViewTrieOperationType indexEffectOf( + final @Nullable String autoSyncSchema) { + return Boolean.parseBoolean(autoSyncSchema) + ? TreeToTableViewTrieOperationType.ALTER_AUTO_SYNC + : TreeToTableViewTrieOperationType.REMOVE; + } + @Override protected String getActionMessage() { return "set view properties"; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index 1f89e39d44883..489402cb11e6e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -108,6 +108,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerRelationalReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp; import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeHeartbeatReq; @@ -1531,6 +1533,11 @@ public TSStatus createTableView(final TCreateTableViewReq req) { return configManager.createTableView(req); } + @Override + public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) { + return configManager.autoSyncViewColumns(req); + } + @Override public TSStatus pushHeartbeat(final int dataNodeId, final TPipeHeartbeatResp resp) { return configManager.pushHeartbeat(dataNodeId, resp); diff --git a/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml b/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml index 500d1d9c46020..e5bbbdba23041 100644 --- a/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml +++ b/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml @@ -91,6 +91,24 @@ DENY + + + ${IOTDB_HOME}/logs/log_datanode_view_column_auto_sync_error.log + + ${IOTDB_HOME}/logs/log-datanode-view-column-auto-sync-error-%d{yyyyMMdd}.log.gz + 30 + + true + + %d [%t] %-5p %C{25}:%L - %m %n + utf-8 + + + ERROR + ACCEPT + DENY + + System.out @@ -218,6 +236,9 @@ + + + diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index b4271e3162f48..f5ecb62b048f3 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1544,7 +1544,8 @@ private DataNodeMiscMessages() {} public static final String MESSAGE_CHANGEREGIONLEADER_SUCCESSFULLY_CHANGE_LEADER_REGIONGROUP_462797B4 = "[ChangeRegionLeader] Successfully change the leader of RegionGroup: "; public static final String MESSAGE_MESSAGE_3501C7E6 = " to "; public static final String MESSAGE_EXECUTEFASTLASTDATAQUERYFORONEPREFIXPATH_DOS_NOT_SUPPORT_WILDCARDS_8E8F44F5 = "The \"executeFastLastDataQueryForOnePrefixPath\" dos not support wildcards."; - public static final String MESSAGE_TEMPLATE_NULL_TRYING_ACTIVATE_TEMPLATE_MAY_TEMPLATE_BEING_UNSET_1CE92779 = "The template is null when trying to activate template, may be the template is being unset."; + public static final String MESSAGE_THE_TEMPLATE_IS_NULL_WHEN_TRYING_TO_ACTIVATE_TEMPLATE_MAY_BE_THE_TEMPLATE_IS_BEING_UNSET_THE_TEMPLATE_IDS_ARE_ARG_7B5F4F72 = + "The template is null when trying to activate template, may be the template is being unset. The template ids are %s."; public static final String MESSAGE_DOT_9D9B854A = "."; public static final String MESSAGE_ERROR_OCCURRED_WHILE_PARSING_SQL_TO_PHYSICAL_PLAN_COLON_5C9F2C59 = "Error occurred while parsing SQL to physical plan: "; public static final String MESSAGE_ERROR_OCCURRED_IN_READ_PROCESS_COLON_CD184195 = "Error occurred in read process: "; @@ -1598,4 +1599,10 @@ private DataNodeMiscMessages() {} LOG_MPP_DATA_EXCHANGE_MAX_PAYLOAD_SIZE_ARG_IS_BELOW_MINIMUM_ALLOWED_VALUE_ARG_USING_ARG_794ABC76 = "mpp_data_exchange_max_payload_size_in_bytes (%d) is below the minimum allowed value %d, using %d"; + public static final String MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB = + "Multiple types encountered when auto detecting type of measurement '%s', please check"; + + public static final String LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_AB32D554 = + "Failed to collect the devices using the extended template {}"; + } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index d3282b7895321..3f9abbc40ff46 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -4064,6 +4064,8 @@ private DataNodeQueryMessages() {} "Only InMemoryDeviceEntryDataSet supports get inline device entries"; public static final String EXCEPTION_LOAD_CONSENSUS_INVALID_PIECE_REF_F3498507 = "Invalid LOAD consensus piece ref: path %s, offset %d, size %d"; + public static final String EXCEPTION_AUTO_SYNC_SCHEMA_IS_ONLY_SUPPORTED_BY_TREE_TO_TABLE_VIEWS_C6586215 = + "auto_sync_schema is only supported by tree to table views"; public static final String MESSAGE_COPYTO_ARG_38343199 = "CopyTo-%s"; public static final String MESSAGE_TARGETFILE_ARG_EC484B24 = "TargetFile: %s"; diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java index c0e0d67346b63..2502aef79dc8a 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java @@ -564,6 +564,7 @@ public final class DataNodeSchemaMessages { // ======================== DataNodeTableCache ======================== public static final String INIT_TABLE_CACHE_SUCCESS = "Init DataNodeTableCache successfully"; + public static final String LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH_359A451E = "Failed to register the tree to table view {}.{} during init: illegal path"; public static final String PRE_UPDATE_TABLE_SUCCESS = "Pre-update table {}.{} successfully"; public static final String PRE_RENAME_OLD_TABLE_SUCCESS = "Pre-rename old table {}.{} successfully"; @@ -740,5 +741,25 @@ private DataNodeSchemaMessages() {} public static final String EXCEPTION_COMMA_50AD1C01 = ", "; public static final String MESSAGE_SKIP_ROLLBACK_RENAMING_OLD_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_664F2456 = "Skip rollback renaming old table {}.{} because it has been handled."; public static final String MESSAGE_SKIP_COMMIT_UPDATE_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_31362A1C = "Skip commit-update table {}.{} because it has been handled."; + public static final String LOG_ADD_VIEW_COLUMN_CN_FAILED_FE4F10AC = "Failed to add columns to the tree to table views for operation {}, ConfigNode returned {}."; + public static final String LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW_B3EB8144 = "Failed to add a column to the view {}.{} for operation {}, ConfigNode returned {}."; + public static final String LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE_B9502E26 = "Failed to add columns to the tree to table views for operation {}, ConfigNode returned an empty response."; + public static final String LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE_E9BC7652 = "Failed to add columns to the tree to table views for operation {}, because ConfigNode is not reachable."; + public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED_B909817F = "Failed to trigger the view column addition of the tree to table views for plan node {}, the schema write is not affected."; + public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902 = "Tree to table view {}.{} is not registered"; + public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE = "Failed to update the tree-to-table view index of view {}.{}"; + public static final String + LOG_REGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_AT_ARG_WITH_RESTRICT_ARG_AND_NUM_OF_TAGS_ARG_7538BCB2 = + "Registered the tree to table view {}.{} at {} with restrict {} and num of tags {}"; + public static final String LOG_UNREGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_C90B661A = + "Unregistered the tree to table view {}.{}"; + public static final String LOG_RENAMED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_FROM_ARG_B5D52D41 = + "Renamed the tree to table view {}.{} from {}"; + public static final String + LOG_UPDATED_THE_PROPERTIES_OF_THE_TREE_TO_TABLE_VIEW_ARG_ARG_TO_ARG_1C3DE984 = + "Updated the properties of the tree to table view {}.{} to {}"; + public static final String + LOG_THE_METADATA_LEASE_IS_FENCED_WHILE_ADDING_VIEW_COLUMNS_GIVING_UP_THIS_ROUND_THE_MEASUREMENTS_ARE_ARG_5D1B2D44 = + "The metadata lease is fenced while adding view columns, giving up this round; the measurements are {}"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index 0139e97dcf6e1..26e6a16a55584 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1524,8 +1524,8 @@ private DataNodeMiscMessages() {} "[ChangeRegionLeader] 成功切换 RegionGroup 的 leader："; public static final String MESSAGE_MESSAGE_3501C7E6 = " 到 "; public static final String MESSAGE_EXECUTEFASTLASTDATAQUERYFORONEPREFIXPATH_DOS_NOT_SUPPORT_WILDCARDS_8E8F44F5 = "\"executeFastLastDataQueryForOnePrefixPath\" 不支持通配符。"; - public static final String MESSAGE_TEMPLATE_NULL_TRYING_ACTIVATE_TEMPLATE_MAY_TEMPLATE_BEING_UNSET_1CE92779 = - "尝试激活模板时模板为空，可能正在取消设置该模板。"; + public static final String MESSAGE_THE_TEMPLATE_IS_NULL_WHEN_TRYING_TO_ACTIVATE_TEMPLATE_MAY_BE_THE_TEMPLATE_IS_BEING_UNSET_THE_TEMPLATE_IDS_ARE_ARG_7B5F4F72 = + "尝试激活模板时模板为空，可能正在取消设置该模板。模板 ID 为 %s。"; public static final String MESSAGE_DOT_9D9B854A = "."; public static final String MESSAGE_ERROR_OCCURRED_WHILE_PARSING_SQL_TO_PHYSICAL_PLAN_COLON_5C9F2C59 = "将 SQL 解析为物理计划时发生错误："; @@ -1581,4 +1581,10 @@ private DataNodeMiscMessages() {} LOG_MPP_DATA_EXCHANGE_MAX_PAYLOAD_SIZE_ARG_IS_BELOW_MINIMUM_ALLOWED_VALUE_ARG_USING_ARG_794ABC76 = "mpp_data_exchange_max_payload_size_in_bytes（%d）低于允许的最小值 %d，将使用 %d"; + public static final String MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB = + "自动检测测点 %s 的类型时遇到多种类型，请检查"; + + public static final String LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_AB32D554 = + "收集使用扩展模板 {} 的设备失败"; + } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 6a50a255cf60b..12cb3b07a0598 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -4820,6 +4820,8 @@ private DataNodeQueryMessages() {} "只有 InMemoryDeviceEntryDataSet 支持获取内存中的设备条目"; public static final String EXCEPTION_LOAD_CONSENSUS_INVALID_PIECE_REF_F3498507 = "无效的 LOAD 共识分片引用：路径 %s，偏移 %d，大小 %d"; + public static final String EXCEPTION_AUTO_SYNC_SCHEMA_IS_ONLY_SUPPORTED_BY_TREE_TO_TABLE_VIEWS_C6586215 = + "auto_sync_schema 仅支持树转表视图"; public static final String MESSAGE_COPYTO_ARG_38343199 = "CopyTo-%s"; public static final String MESSAGE_TARGETFILE_ARG_EC484B24 = "目标文件：%s"; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java index 417f547573a01..18d6443388f5e 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeSchemaMessages.java @@ -565,6 +565,7 @@ public final class DataNodeSchemaMessages { // ======================== DataNodeTableCache 相关消息 ======================== public static final String INIT_TABLE_CACHE_SUCCESS = "DataNodeTableCache 初始化成功"; + public static final String LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH_359A451E = "初始化时注册树转表视图 {}.{} 失败：路径非法"; public static final String PRE_UPDATE_TABLE_SUCCESS = "预更新表 {}.{} 成功"; public static final String PRE_RENAME_OLD_TABLE_SUCCESS = "预重命名旧表 {}.{} 成功"; public static final String ROLLBACK_UPDATE_TABLE_SUCCESS = "回滚更新表 {}.{} 成功"; @@ -728,5 +729,25 @@ private DataNodeSchemaMessages() {} public static final String EXCEPTION_COMMA_50AD1C01 = ", "; public static final String MESSAGE_SKIP_ROLLBACK_RENAMING_OLD_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_664F2456 = "跳过回滚重命名旧表 {}.{}，因为该操作已被处理。"; public static final String MESSAGE_SKIP_COMMIT_UPDATE_TABLE_ARG_ARG_BECAUSE_IT_HAS_BEEN_HANDLED_31362A1C = "跳过提交更新表 {}.{}，因为该操作已被处理。"; + public static final String LOG_ADD_VIEW_COLUMN_CN_FAILED_FE4F10AC = "为操作 {} 的树转表视图加列失败，ConfigNode 返回 {}。"; + public static final String LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW_B3EB8144 = "为视图 {}.{} 加列失败（操作 {}），ConfigNode 返回 {}。"; + public static final String LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE_B9502E26 = "为操作 {} 的树转表视图加列失败，ConfigNode 返回了空响应。"; + public static final String LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE_E9BC7652 = "为操作 {} 的树转表视图加列失败，因为无法访问 ConfigNode。"; + public static final String LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED_B909817F = "计划节点 {} 触发树转表视图加列失败，本次 schema 写入不受影响。"; + public static final String LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902 = "树转表视图 {}.{} 未注册"; + public static final String LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE = "更新树转表视图 {}.{} 的索引失败"; + public static final String + LOG_REGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_AT_ARG_WITH_RESTRICT_ARG_AND_NUM_OF_TAGS_ARG_7538BCB2 = + "注册树转表视图 {}.{}，路径 {}，restrict {}，tag 数 {}"; + public static final String LOG_UNREGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_C90B661A = + "注销树转表视图 {}.{}"; + public static final String LOG_RENAMED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_FROM_ARG_B5D52D41 = + "树转表视图 {}.{} 由 {} 重命名而来"; + public static final String + LOG_UPDATED_THE_PROPERTIES_OF_THE_TREE_TO_TABLE_VIEW_ARG_ARG_TO_ARG_1C3DE984 = + "树转表视图 {}.{} 的属性更新为 {}"; + public static final String + LOG_THE_METADATA_LEASE_IS_FENCED_WHILE_ADDING_VIEW_COLUMNS_GIVING_UP_THIS_ROUND_THE_MEASUREMENTS_ARE_ARG_5D1B2D44 = + "自动扩列时元数据租约已隔离，本次放弃；涉及的测点为 {}"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index a77b0895e4c0b..47b353aefa8c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -79,6 +79,8 @@ import org.apache.iotdb.db.schemaengine.schemaregion.write.req.SchemaRegionWritePlanFactory; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateAlignedTimeSeriesPlanImpl; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateTimeSeriesPlanImpl; +import org.apache.iotdb.db.schemaengine.table.TreeToTableViewCache; +import org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSyncTrigger; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -94,9 +96,14 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; /** Schema write {@link PlanNode} visitor */ public class SchemaExecutionVisitor implements PlanVisitor { @@ -111,6 +118,10 @@ public TSStatus visitCreateTimeSeries( logMetaDataException(String.format("%s: MetaData error: ", IoTDBConstant.GLOBAL_DB_NAME), e); return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); } + final MeasurementPath path = node.getPath(); + final Map>> createdMeasurement = + toSingleMeasurement(path.getDevicePath(), path.getMeasurement(), node.getDataType()); + syncViewColumns(node, createdMeasurement); return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Execute successfully"); } @@ -138,6 +149,11 @@ public TSStatus visitCreateAlignedTimeSeries( logMetaDataException(String.format("%s: MetaData error: ", IoTDBConstant.GLOBAL_DB_NAME), e); return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); } + + final Map>> createdMeasurements = + Collections.singletonMap( + node.getDevicePath(), toMeasurements(node.getMeasurements(), node.getDataTypes())); + syncViewColumns(node, createdMeasurements); return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Execute successfully"); } @@ -145,6 +161,7 @@ public TSStatus visitCreateAlignedTimeSeries( public TSStatus visitCreateMultiTimeSeries( final CreateMultiTimeSeriesNode node, final ISchemaRegion schemaRegion) { final Map measurementGroupMap = node.getMeasurementGroupMap(); + final Map>> createdMeasurements = new HashMap<>(); final List failingStatus = new ArrayList<>(); PartialPath devicePath; MeasurementGroup measurementGroup; @@ -160,6 +177,11 @@ public TSStatus visitCreateMultiTimeSeries( transformToCreateTimeSeriesPlan(devicePath, measurementGroup, i); ((CreateTimeSeriesPlanImpl) createTimeSeriesPlan).setWithMerge(node.isGeneratedByPipe()); schemaRegion.createTimeSeries(createTimeSeriesPlan, -1); + recordCreatedMeasurement( + createdMeasurements, + devicePath, + measurementGroup.getMeasurements().get(i), + measurementGroup.getDataTypes().get(i)); } catch (final MetadataException e) { logMetaDataException( String.format("%s: MetaData error: ", IoTDBConstant.GLOBAL_DB_NAME), e); @@ -168,6 +190,10 @@ public TSStatus visitCreateMultiTimeSeries( } } + // The measurements which were created need their columns even when another one of the same + // statement failed: no later write of them triggers this addition again. + syncViewColumns(node, createdMeasurements); + if (!failingStatus.isEmpty()) { return RpcUtils.getStatus(failingStatus); } @@ -194,9 +220,11 @@ private ICreateTimeSeriesPlan transformToCreateTimeSeriesPlan( @Override public TSStatus visitInternalCreateTimeSeries( final InternalCreateTimeSeriesNode node, final ISchemaRegion schemaRegion) { + final PartialPath devicePath = node.getDevicePath(); final MeasurementGroup measurementGroup = node.getMeasurementGroup(); + final Map>> createdMeasurements = new HashMap<>(); final List existingTimeSeriesAndAlignmentMismatch = new ArrayList<>(); final List failingStatus = new ArrayList<>(); @@ -207,7 +235,8 @@ public TSStatus visitInternalCreateTimeSeries( schemaRegion, existingTimeSeriesAndAlignmentMismatch, failingStatus, - node.isGeneratedByPipe()); + node.isGeneratedByPipe(), + createdMeasurements); } else { executeInternalCreateTimeSeries( devicePath, @@ -215,9 +244,12 @@ public TSStatus visitInternalCreateTimeSeries( schemaRegion, existingTimeSeriesAndAlignmentMismatch, failingStatus, - node.isGeneratedByPipe()); + node.isGeneratedByPipe(), + createdMeasurements); } + syncViewColumns(node, createdMeasurements); + if (!failingStatus.isEmpty()) { return RpcUtils.getStatus(failingStatus); } @@ -232,6 +264,8 @@ public TSStatus visitInternalCreateTimeSeries( @Override public TSStatus visitInternalCreateMultiTimeSeries( final InternalCreateMultiTimeSeriesNode node, final ISchemaRegion schemaRegion) { + + final Map>> createdMeasurements = new HashMap<>(); PartialPath devicePath; MeasurementGroup measurementGroup; @@ -249,7 +283,8 @@ public TSStatus visitInternalCreateMultiTimeSeries( schemaRegion, existingTimeSeriesAndAlignmentMismatch, failingStatus, - node.isGeneratedByPipe()); + node.isGeneratedByPipe(), + createdMeasurements); } else { executeInternalCreateTimeSeries( devicePath, @@ -257,10 +292,13 @@ public TSStatus visitInternalCreateMultiTimeSeries( schemaRegion, existingTimeSeriesAndAlignmentMismatch, failingStatus, - node.isGeneratedByPipe()); + node.isGeneratedByPipe(), + createdMeasurements); } } + syncViewColumns(node, createdMeasurements); + if (!failingStatus.isEmpty()) { return RpcUtils.getStatus(failingStatus); } @@ -278,7 +316,8 @@ private void executeInternalCreateTimeSeries( final ISchemaRegion schemaRegion, final List existingTimeSeriesAndAlignmentMismatch, final List failingStatus, - final boolean withMerge) { + final boolean withMerge, + final Map>> createdMeasurements) { final int size = measurementGroup.getMeasurements().size(); // todo implement batch creation of one device in SchemaRegion boolean alignedIsSet = false; @@ -292,6 +331,13 @@ private void executeInternalCreateTimeSeries( // Thus the original ones are not altered ((CreateTimeSeriesPlanImpl) createTimeSeriesPlan).setWithMerge(withMerge); schemaRegion.createTimeSeries(createTimeSeriesPlan, -1); + // The alignment mismatch below does not undo the creation, so the measurement is created + // either way. + recordCreatedMeasurement( + createdMeasurements, + devicePath, + measurementGroup.getMeasurements().get(i), + measurementGroup.getDataTypes().get(i)); if (((CreateTimeSeriesPlanImpl) createTimeSeriesPlan).getAligned().get() && !alignedIsSet) { existingTimeSeriesAndAlignmentMismatch.add( new TSStatus(TSStatusCode.ALIGNED_TIMESERIES_ERROR.getStatusCode()) @@ -316,7 +362,8 @@ private void executeInternalCreateAlignedTimeSeries( final ISchemaRegion schemaRegion, final List existingTimeSeriesAndAlignmentMismatch, final List failingStatus, - final boolean withMerge) { + final boolean withMerge, + final Map>> createdMeasurements) { final List measurementList = measurementGroup.getMeasurements(); final List dataTypeList = measurementGroup.getDataTypes(); final List encodingList = measurementGroup.getEncodings(); @@ -344,8 +391,19 @@ private void executeInternalCreateAlignedTimeSeries( boolean shouldRetry = true; while (shouldRetry) { + // The retry trims the plan, so the measurements of this attempt are the ones the plan holds + // now. They are captured before the call, as the call itself trims the measurements it merges + // instead of creating. + final List attemptedMeasurements = + new ArrayList<>(createAlignedTimeSeriesPlan.getMeasurements()); + final List attemptedDataTypes = + new ArrayList<>(createAlignedTimeSeriesPlan.getDataTypes()); try { schemaRegion.createAlignedTimeSeries(createAlignedTimeSeriesPlan); + // An aligned batch is validated as a whole before it is created, so a call which returns + // without an exception has created every measurement it was given. + recordCreatedMeasurements( + createdMeasurements, devicePath, attemptedMeasurements, attemptedDataTypes); shouldRetry = false; } catch (final MeasurementAlreadyExistException e) { // The existence check will be executed before truly creation @@ -411,6 +469,81 @@ private void executeInternalCreateAlignedTimeSeries( } } + /** + * Adds to the tree-to-table views the columns the measurements a schema write has created need. + * + *

It is a best-effort side effect of the write which triggers it: every failure is logged here + * and none of them is propagated, so that the write operation is never affected. + * + *

A plan which a pipe pushed is skipped: the cluster the measurements come from has already + * added their columns, and those columns reach this cluster with the view schema itself, so + * asking ConfigNode for them again can only be rejected as a duplicate. + * + * @param node the plan which triggers the addition + * @param createdMeasurements the measurements this operation really created; only they get a + * column, so that a creation which failed leaves no column behind + */ + private void syncViewColumns( + final WritePlanNode node, + final Map>> createdMeasurements) { + if (node.isGeneratedByPipe()) { + return; + } + final String operationId = node.getPlanNodeId().getId(); + try { + if (TreeToTableViewCache.getInstance().hasView()) { + ViewColumnAutoSyncTrigger.sync(createdMeasurements, operationId); + } + } catch (final Throwable e) { + // Logged with the logger of the feature, so that every failure of the auto column addition, + // including the ones the trigger itself does not see, reaches its dedicated log file. + ViewColumnAutoSyncTrigger.LOGGER.error( + DataNodeSchemaMessages.LOG_TRIGGER_ADD_VIEW_COLUMN_FAILED_B909817F, operationId, e); + } + } + + private static void recordCreatedMeasurement( + final Map>> createdMeasurements, + final PartialPath devicePath, + final String measurement, + final TSDataType dataType) { + createdMeasurements + .computeIfAbsent(devicePath, key -> new ArrayList<>()) + .add(new Pair<>(measurement, dataType)); + } + + private static void recordCreatedMeasurements( + final Map>> createdMeasurements, + final PartialPath devicePath, + final List measurements, + final List dataTypes) { + for (int i = 0; i < measurements.size(); i++) { + recordCreatedMeasurement( + createdMeasurements, devicePath, measurements.get(i), dataTypes.get(i)); + } + } + + private static Map>> toSingleMeasurement( + final PartialPath devicePath, final String measurement, final TSDataType dataType) { + return Collections.singletonMap( + devicePath, Collections.singletonList(new Pair<>(measurement, dataType))); + } + + private static List> toMeasurements(final Template template) { + return template.getSchemaMap().values().stream() + .map(schema -> new Pair<>(schema.getMeasurementName(), schema.getType())) + .toList(); + } + + private static List> toMeasurements( + final List measurements, final List dataTypes) { + final List> pairs = new ArrayList<>(measurements.size()); + for (int i = 0; i < measurements.size(); i++) { + pairs.add(new Pair<>(measurements.get(i), dataTypes.get(i))); + } + return pairs; + } + @Override public TSStatus visitAlterTimeSeries( final AlterTimeSeriesNode node, final ISchemaRegion schemaRegion) { @@ -465,43 +598,49 @@ public TSStatus visitAlterTimeSeries( @Override public TSStatus visitActivateTemplate( final ActivateTemplateNode node, final ISchemaRegion schemaRegion) { + final Template template = + ClusterTemplateManager.getInstance().getTemplate(node.getTemplateId()); + if (Objects.isNull(template)) { + return getUndefinedTemplateStatus(Collections.singleton(node.getTemplateId())); + } try { - final Template template = - ClusterTemplateManager.getInstance().getTemplate(node.getTemplateId()); - if (Objects.isNull(template)) { - return new TSStatus(TSStatusCode.UNDEFINED_TEMPLATE.getStatusCode()) - .setMessage( - DataNodeMiscMessages - .MESSAGE_TEMPLATE_NULL_TRYING_ACTIVATE_TEMPLATE_MAY_TEMPLATE_BEING_UNSET_1CE92779); - } schemaRegion.activateSchemaTemplate(node, template); - return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } catch (final MetadataException e) { logMetaDataException(e); return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); } + final PartialPath devicePath = node.getActivatePath(); + final Map>> activatedMeasurements = + Collections.singletonMap(devicePath, toMeasurements(template)); + syncViewColumns(node, activatedMeasurements); + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } @Override public TSStatus visitBatchActivateTemplate( final BatchActivateTemplateNode node, final ISchemaRegion schemaRegion) { + final Map templateMap = new HashMap<>(); + final Set missingTemplateIds = new HashSet<>(); + collectTemplates(node.getTemplateActivationMap(), templateMap, missingTemplateIds); + + final Map>> activatedMeasurements = new HashMap<>(); final List statusList = new ArrayList<>(); final List alreadyActivatedDeviceList = new ArrayList<>(); for (final Map.Entry> entry : node.getTemplateActivationMap().entrySet()) { - final Template template = - ClusterTemplateManager.getInstance().getTemplate(entry.getValue().left); + final Template template = templateMap.get(entry.getValue().left); if (Objects.isNull(template)) { - return new TSStatus(TSStatusCode.UNDEFINED_TEMPLATE.getStatusCode()) - .setMessage( - DataNodeMiscMessages - .MESSAGE_TEMPLATE_NULL_TRYING_ACTIVATE_TEMPLATE_MAY_TEMPLATE_BEING_UNSET_1CE92779); + // its id is in missingTemplateIds, which is reported at the end + continue; } try { schemaRegion.activateSchemaTemplate( SchemaRegionWritePlanFactory.getActivateTemplateInClusterPlan( entry.getKey(), entry.getValue().right, entry.getValue().left), template); + // A device which already carries the template is reported below and gets no column here: + // its measurements were already covered when the template was activated on it. + activatedMeasurements.put(entry.getKey(), toMeasurements(template)); } catch (final MetadataException e) { if (e.getErrorCode() == TSStatusCode.TEMPLATE_IS_IN_USE.getStatusCode()) { alreadyActivatedDeviceList.add(entry.getKey()); @@ -511,34 +650,44 @@ public TSStatus visitBatchActivateTemplate( } } } + syncViewColumns(node, activatedMeasurements); if (!alreadyActivatedDeviceList.isEmpty()) { final TemplateIsInUseException e = new TemplateIsInUseException(alreadyActivatedDeviceList.toString()); logMetaDataException(e); statusList.add(RpcUtils.getStatus(e.getErrorCode(), e.getMessage())); } + if (!missingTemplateIds.isEmpty()) { + statusList.add(getUndefinedTemplateStatus(missingTemplateIds)); + } return statusList.isEmpty() ? RpcUtils.SUCCESS_STATUS : RpcUtils.getStatus(statusList); } @Override public TSStatus visitInternalBatchActivateTemplate( final InternalBatchActivateTemplateNode node, final ISchemaRegion schemaRegion) { + final Map templateMap = new HashMap<>(); + final Set missingTemplateIds = new HashSet<>(); + collectTemplates(node.getTemplateActivationMap(), templateMap, missingTemplateIds); + + final Map>> activatedMeasurements = new HashMap<>(); + for (final Map.Entry> entry : node.getTemplateActivationMap().entrySet()) { - final Template template = - ClusterTemplateManager.getInstance().getTemplate(entry.getValue().left); + final Template template = templateMap.get(entry.getValue().left); if (Objects.isNull(template)) { - return new TSStatus(TSStatusCode.UNDEFINED_TEMPLATE.getStatusCode()) - .setMessage( - DataNodeMiscMessages - .MESSAGE_TEMPLATE_NULL_TRYING_ACTIVATE_TEMPLATE_MAY_TEMPLATE_BEING_UNSET_1CE92779); + // its id is in missingTemplateIds, which is reported at the end + continue; } try { schemaRegion.activateSchemaTemplate( SchemaRegionWritePlanFactory.getActivateTemplateInClusterPlan( entry.getKey(), entry.getValue().right, entry.getValue().left), template); + activatedMeasurements.put(entry.getKey(), toMeasurements(template)); } catch (final TemplateIsInUseException e) { + // The measurements of a device which already carries the template were already covered when + // the template was activated on it. logger.info( String.format( DataNodeSchemaMessages.DEVICE_TEMPLATE_ALREADY_ACTIVATED_ON_PATH_FMT, @@ -548,9 +697,44 @@ public TSStatus visitInternalBatchActivateTemplate( return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); } } + syncViewColumns(node, activatedMeasurements); + if (!missingTemplateIds.isEmpty()) { + return getUndefinedTemplateStatus(missingTemplateIds); + } return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } + private static void collectTemplates( + final Map> templateActivationMap, + final Map templateMap, + final Set missingTemplateIds) { + final ClusterTemplateManager templateManager = ClusterTemplateManager.getInstance(); + for (final Pair activation : templateActivationMap.values()) { + final int templateId = activation.left; + if (templateMap.containsKey(templateId) || missingTemplateIds.contains(templateId)) { + // already resolved for another path of this plan + continue; + } + final Template template = templateManager.getTemplate(templateId); + if (Objects.isNull(template)) { + missingTemplateIds.add(templateId); + } else { + templateMap.put(templateId, template); + } + } + } + + private static TSStatus getUndefinedTemplateStatus(final Set missingTemplateIds) { + return new TSStatus(TSStatusCode.UNDEFINED_TEMPLATE.getStatusCode()) + .setMessage( + String.format( + DataNodeMiscMessages + .MESSAGE_THE_TEMPLATE_IS_NULL_WHEN_TRYING_TO_ACTIVATE_TEMPLATE_MAY_BE_THE_TEMPLATE_IS_BEING_UNSET_THE_TEMPLATE_IDS_ARE_ARG_7B5F4F72, + missingTemplateIds.stream() + .map(String::valueOf) + .collect(Collectors.joining(", ")))); + } + @Override public TSStatus visitConstructSchemaBlackList( final ConstructSchemaBlackListNode node, final ISchemaRegion schemaRegion) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 9f1e90fda10af..02d6c51ef2dcf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -63,6 +63,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerRelationalReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp; import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeHeartbeatReq; @@ -1315,6 +1317,13 @@ public TSStatus createTableView(TCreateTableViewReq req) throws TException { () -> client.createTableView(req), status -> !updateConfigNodeLeader(status)); } + @Override + public TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) + throws TException { + return executeRemoteCallWithRetry( + () -> client.autoSyncViewColumns(req), status -> !updateConfigNodeLeader(status.status)); + } + @Override public TSStatus pushHeartbeat(final int dataNodeId, final TPipeHeartbeatResp resp) throws TException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index e9b26334f66b2..5d3bce4e4c32b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -192,9 +192,13 @@ import org.apache.iotdb.db.schemaengine.SchemaEngine; import org.apache.iotdb.db.schemaengine.lease.MetadataLeaseManager; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.schemaengine.schemaregion.read.req.SchemaRegionReadPlanFactory; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.db.schemaengine.table.TreeToTableViewCache; +import org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSyncTrigger; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUpdateType; import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUtil; @@ -406,6 +410,7 @@ import static org.apache.iotdb.commons.client.request.TestConnectionUtils.testConnectionsImpl; import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; import static org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement.AlterType.SET_DATA_TYPE; +import static org.apache.iotdb.db.schemaengine.table.ViewColumnAutoSyncTrigger.EXTEND_DEVICE_TEMPLATE_ID_PREFIX; import static org.apache.iotdb.db.service.RegionMigrateService.REGION_MIGRATE_PROCESS; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onIoTDBException; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onQueryException; @@ -1959,14 +1964,16 @@ public TSStatus updateTable(final TUpdateTableReq req) { .rollbackUpdateTable( ReadWriteIOUtils.readString(req.tableInfo), ReadWriteIOUtils.readString(req.tableInfo), - req.oldName); + req.oldName, + req.updateTree2TableViewInfo); break; case COMMIT_UPDATE_TABLE: DataNodeTableCache.getInstance() .commitUpdateTable( ReadWriteIOUtils.readString(req.tableInfo), ReadWriteIOUtils.readString(req.tableInfo), - req.oldName); + req.oldName, + req.updateTree2TableViewInfo); break; default: LOGGER.warn(DataNodeMiscMessages.UNSUPPORTED_TYPE_UPDATING_TABLE, req.type); @@ -2149,13 +2156,15 @@ public TDeviceViewResp detectTreeDeviceViewFieldType(final TDeviceViewReq req) { for (int i = 0; i < req.getPrefixPattern().size() - 1; ++i) { nodes[i] = req.getPrefixPattern().get(i); } + final ExtendedPartialPath pathPattern = + new ExtendedPartialPath(nodes, req.isRestrict()); final ISchemaSource schemaSource = SchemaSourceFactory.getTimeSeriesSchemaCountSource( - new ExtendedPartialPath(nodes, req.isRestrict()), + pathPattern, false, // Does not support logical view currently SchemaFilterFactory.createViewTypeFilter(ViewType.BASE), - null, + ClusterTemplateManager.getInstance().checkAllRelatedTemplate(pathPattern), SchemaConstant.ALL_MATCH_SCOPE, true, true); @@ -2173,11 +2182,13 @@ public TDeviceViewResp detectTreeDeviceViewFieldType(final TDeviceViewReq req) { } if (!updateMap.containsKey(measurementName)) { updateMap.put(measurementName, schema.getTypeInByte()); - } else if (schema.getTypeInByte() != updateMap.get(measurementName)) { + } else if (schema.getTypeInByte() != updateMap.get(measurementName) + && !req.isTolerateTypeMismatch()) { return RpcUtils.getStatus( TSStatusCode.DATA_TYPE_MISMATCH, String.format( - "Multiple types encountered when auto detecting type of measurement '%s', please check", + DataNodeMiscMessages + .MESSAGE_MULTIPLE_TYPES_ENCOUNTERED_WHEN_AUTO_DETECTING_TYPE_OF_MEASUREMENT_ARG_PLEASE_CHECK_6EDF48DB, measurementName)); } } @@ -2902,6 +2913,23 @@ public TSStatus updateTemplate(final TUpdateTemplateReq req) { SchemaEngine.getInstance() .updateSubtreeMeasurementCountForTemplate(newTemplate.getId(), delta); } + if (TreeToTableViewCache.getInstance().hasView()) { + // ViewColumnAutoSync.sync, which collects the views from the index and asks CN to add the + // columns. + try { + final Map>> device2Measurements = + collectDevicesUsingExtendedTemplate(oldTemplate, newTemplate); + ViewColumnAutoSyncTrigger.sync( + device2Measurements, EXTEND_DEVICE_TEMPLATE_ID_PREFIX + newTemplate.getId()); + } catch (final Exception e) { + ViewColumnAutoSyncTrigger.LOGGER.warn( + DataNodeMiscMessages + .LOG_FAILED_TO_COLLECT_THE_DEVICES_USING_THE_EXTENDED_TEMPLATE_ARG_AB32D554, + newTemplate.getName(), + e); + } + } + break; default: LOGGER.warn(DataNodeMiscMessages.UNSUPPORTED_TYPE_UPDATING_TEMPLATE, req.type); @@ -2910,6 +2938,68 @@ public TSStatus updateTemplate(final TUpdateTemplateReq req) { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } + /** + * The devices which activated the template, each with the measurements the extension added to it: + * the tree-to-table views covering these devices need them as columns. + * + *

The measurements come from comparing the new template with the old one. The devices come + * from the schema regions, where every device which uses a template records its id, see {@code + * EntityTraverser}; the template manager only knows where a template is mounted, so it cannot + * tell which devices use it. + */ + private Map>> collectDevicesUsingExtendedTemplate( + final Template oldTemplate, final Template newTemplate) throws Exception { + final List> extendedMeasurements = + newTemplate.getSchemaMap().values().stream() + .filter( + schema -> + Objects.isNull(oldTemplate) + || !oldTemplate.getSchemaMap().containsKey(schema.getMeasurementName())) + .map(schema -> new Pair<>(schema.getMeasurementName(), schema.getType())) + .toList(); + final Pair> templateSetInfo = + ClusterTemplateManager.getInstance().getAllPathsSetTemplate(newTemplate.getName()); + final List setPaths = + Objects.isNull(templateSetInfo) ? null : templateSetInfo.getRight(); + if (extendedMeasurements.isEmpty() || Objects.isNull(setPaths)) { + return Collections.emptyMap(); + } + + final Map>> device2Measurements = new HashMap<>(); + Collection allSchemaRegions = SchemaEngine.getInstance().getAllSchemaRegions(); + for (final PartialPath setPath : setPaths) { + // A device which is exactly the mount path has the template too, but "setPath.**" only + // matches its descendants, so the mount path itself is read with a pattern of its own. + final List devicePatterns = + Arrays.asList(setPath, setPath.concatNode(MULTI_LEVEL_PATH_WILDCARD)); + for (final ISchemaRegion schemaRegion : allSchemaRegions) { + if (!setPath.startsWithOrPrefixOf( + PathUtils.splitPathToDetachedNodes(schemaRegion.getDatabaseFullPath()))) { + continue; + } + for (final PartialPath devicePattern : devicePatterns) { + try (final ISchemaReader deviceReader = + schemaRegion.getDeviceReader( + SchemaRegionReadPlanFactory.getShowDevicesPlan( + devicePattern, + 0, + 0, + false, + newTemplate.getId(), + SchemaConstant.ALL_MATCH_SCOPE))) { + while (deviceReader.hasNext()) { + device2Measurements + .computeIfAbsent( + deviceReader.next().getPartialPath(), device -> new ArrayList<>()) + .addAll(extendedMeasurements); + } + } + } + } + } + return device2Measurements; + } + @Override public TSStatus deleteRegion(TConsensusGroupId tconsensusGroupId) { TSStatus consensusStatus = waitForConsensusStarted(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 033ba02836c63..49ddfda6fac81 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -304,6 +304,7 @@ import static org.apache.iotdb.commons.schema.table.TsTable.TABLE_ALLOWED_PROPERTIES; import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME; import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY; +import static org.apache.iotdb.db.i18n.DataNodeQueryMessages.EXCEPTION_AUTO_SYNC_SCHEMA_IS_ONLY_SUPPORTED_BY_TREE_TO_TABLE_VIEWS_C6586215; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AbstractDatabaseTask.NEED_LAST_CACHE_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.MAX_DATA_REGION_GROUP_NUM_KEY; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.MAX_SCHEMA_REGION_GROUP_NUM_KEY; @@ -633,8 +634,7 @@ private Pair parseTable4CreateTableOrView( context.getSession().getUserName(), new QualifiedObjectName(database, tableName), context); final TsTable table = new TsTable(tableName); - - table.setProps(convertPropertiesToMap(node.getProperties(), false)); + table.setProps(convertPropertiesToMap(node.getProperties(), false, node instanceof CreateView)); if (Objects.nonNull(node.getComment())) { table.addProp(TsTable.COMMENT_KEY, node.getComment()); } @@ -902,7 +902,7 @@ public IConfigTask visitSetProperties(final SetProperties node, final MPPQueryCo return new AlterTableSetPropertiesTask( database, tableName, - convertPropertiesToMap(node.getProperties(), true), + convertPropertiesToMap(node.getProperties(), true, isTreeView), context.getQueryId().getId(), node.ifExists(), isTreeView); @@ -985,8 +985,20 @@ private void validateTreeViewProperties(final List propertyList) { } } + /** + * Converts the property assignments of a CREATE TABLE/VIEW or a SET PROPERTIES statement into the + * props of the TsTable. + * + * @param propertyList the property assignments to convert + * @param serializeDefault whether a property assigned {@code DEFAULT} shall be kept in the result + * as a null value, which resets the property, instead of being dropped + * @param allowAutoSyncSchema whether the statement targets a tree to table view, the only kind of + * table whose schema can be synchronized from a tree path prefix + */ private Map convertPropertiesToMap( - final List propertyList, final boolean serializeDefault) { + final List propertyList, + final boolean serializeDefault, + final boolean allowAutoSyncSchema) { final Map map = new HashMap<>(); final Set deduplicate = new HashSet<>(); for (final Property property : propertyList) { @@ -995,6 +1007,11 @@ private Map convertPropertiesToMap( throw new SemanticException(DataNodeQueryMessages.DUPLICATED_PROPERTY + key); } if (TABLE_ALLOWED_PROPERTIES.contains(key)) { + if (TsTable.AUTO_SYNC_SCHEMA.equals(key) && !allowAutoSyncSchema) { + // A property assigned DEFAULT is dropped or reset as well, so the check covers it too + throw new SemanticException( + EXCEPTION_AUTO_SYNC_SCHEMA_IS_ONLY_SUPPORTED_BY_TREE_TO_TABLE_VIEWS_C6586215); + } if (!property.isSetToDefault()) { final Expression value = property.getNonDefaultValue(); switch (key) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java index a616282e7c4a9..7d2905e901726 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowCreateViewTask.java @@ -147,7 +147,12 @@ public static String getShowCreateViewSQL(final TsTable table) { if (ttlString.equals(TTL_INFINITE)) { ttlString = "'" + ttlString + "'"; } - builder.append(" WITH (ttl=").append(ttlString).append(")"); + builder.append(" WITH (ttl=").append(ttlString); + // An absent auto_sync_schema means false, so it only has to be given back when it is stored + table + .getPropValue(TsTable.AUTO_SYNC_SCHEMA) + .ifPresent(autoSyncSchema -> builder.append(", auto_sync_schema=").append(autoSyncSchema)); + builder.append(")"); builder.append(" AS "); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManager.java index 7f7365a1550e3..5f3c7357c885d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManager.java @@ -30,6 +30,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.db.schemaengine.table.TreeToTableViewCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,7 +109,8 @@ private static List defaultClearCacheList() { () -> ClusterPartitionFetcher.getInstance().invalidAllCache(), () -> DataNodeTableCache.getInstance().invalidateAll(), () -> TreeDeviceSchemaCacheManager.getInstance().cleanUp(), - AuthorityChecker::invalidateAllCache); + AuthorityChecker::invalidateAllCache, + () -> TreeToTableViewCache.getInstance().clear()); } private static List defaultPullMetaList() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/view/TreeToTableViewReference.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/view/TreeToTableViewReference.java new file mode 100644 index 0000000000000..e9eb73ae8022a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/view/TreeToTableViewReference.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.view; + +import java.util.Objects; + +/** A view identity and its matching metadata. Equality depends only on database and table names. */ +public final class TreeToTableViewReference { + + public static final String NUM_OF_TAGS = "num_of_tags"; + + private final String databaseName; + + private final String tableName; + + private final boolean restrict; + + private final int numOfTags; + + public TreeToTableViewReference( + String databaseName, String tableName, boolean restrict, int numOfTags) { + this.databaseName = databaseName; + this.tableName = tableName; + this.restrict = restrict; + this.numOfTags = numOfTags; + } + + public String getDatabaseName() { + return databaseName; + } + + public String getTableName() { + return tableName; + } + + public boolean isRestrict() { + return restrict; + } + + public int getNumOfTags() { + return numOfTags; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof TreeToTableViewReference)) { + return false; + } + TreeToTableViewReference that = (TreeToTableViewReference) obj; + return Objects.equals(databaseName, that.databaseName) + && Objects.equals(tableName, that.tableName); + } + + @Override + public int hashCode() { + return Objects.hash(databaseName, tableName); + } + + @Override + public String toString() { + return "TreeToTableViewReference{" + + "databaseName='" + + databaseName + + '\'' + + ", tableName='" + + tableName + + '\'' + + ", restrict=" + + restrict + + ", numOfTags=" + + numOfTags + + '}'; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index 3db7af6f89a4d..1f10cb7f4b54f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -23,13 +23,16 @@ import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.consensus.ConfigRegionId; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.iotdb.commons.exception.MetadataLeaseFencedException.LeaseFencedRetryPolicy; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.i18n.CommonMessages; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.table.NonCommittableTsTable; import org.apache.iotdb.commons.schema.table.PreDeleteTsTable; import org.apache.iotdb.commons.schema.table.TableNodeStatus; +import org.apache.iotdb.commons.schema.table.TreeToTableViewTrieOperationType; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; @@ -44,6 +47,8 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.executor.ClusterConfigTaskExecutor; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.schemaengine.lease.MetadataLeaseManager; +import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.TException; @@ -55,11 +60,13 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; @@ -69,6 +76,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.apache.iotdb.commons.schema.table.TreeViewSchema.TREE_PATH_PATTERN; import static org.apache.iotdb.db.i18n.DataNodeSchemaMessages.FAILED_TO_REFRESH_CACHE_FROM_CN; /** It contains all tables' latest column schema */ @@ -77,6 +85,7 @@ public class DataNodeTableCache implements ITableCache { private static final Logger LOGGER = LoggerFactory.getLogger(DataNodeTableCache.class); private static final IClientManager CONFIG_NODE_CLIENT_MANAGER = ConfigNodeClientManager.getInstance(); + private final ITreeToTableViewCache treeToTableViewCache = TreeToTableViewCache.getInstance(); /** Instance-specific version counter for optimistic locking mechanisms. */ private final AtomicLong instanceVersion = new AtomicLong(0); @@ -142,12 +151,39 @@ public void init(final byte[] tableInitializationBytes) { table -> new Pair<>(table, 0L), (v1, v2) -> v2, ConcurrentHashMap::new)))); + treeToTableViewCache.init(collectInitialViews(usingMap)); LOGGER.info(DataNodeSchemaMessages.INIT_TABLE_CACHE_SUCCESS); } finally { readWriteLock.writeLock().unlock(); } } + private Map> collectInitialViews( + final Map> usingMap) { + final Map> initialViews = new HashMap<>(); + usingMap.forEach( + (key, value) -> { + final List views = + value.stream().filter(DataNodeTableCache::isAutoSyncView).toList(); + if (!views.isEmpty()) { + initialViews.put(PathUtils.unQualifyDatabaseName(key), new ArrayList<>(views)); + } + }); + return initialViews; + } + + /** + * Whether the table belongs in the tree-to-table view index: it is a tree-to-table view which + * auto-syncs its schema. A NonCommittableTsTable and a PreDeleteTsTable are placeholders which + * carry no definition at all, so they are never one. + */ + private static boolean isAutoSyncView(final TsTable table) { + final Optional autoSyncSchemaValue = table.getPropValue(TsTable.AUTO_SYNC_SCHEMA); + return autoSyncSchemaValue.isPresent() + && Boolean.parseBoolean(autoSyncSchemaValue.get()) + && table.getPropValue(TREE_PATH_PATTERN).isPresent(); + } + // No need to acquire a lock here; reloadTableCacheAfterLeaseRecovery is within a critical section // protected by metadataLeaseManager @Override @@ -166,18 +202,6 @@ public void reloadTableCacheAfterLeaseRecovery() { } } - /** - * The case that pre update Table and pre delete Table procedures targeting the same table are - * executed serially by CN. - * - *

Consider the scenario: - * - *

    - *
  1. Drop the table first: DN executed pre update but missed the commit phase - *
  2. Create table second: DN executed pre update, which overwrites the result of the drop - * table procedure in {@link #specialStatusMap} - *
- */ @Override public void preUpdateTable(String database, final TsTable table, final String oldName) { database = PathUtils.unQualifyDatabaseName(database); @@ -203,12 +227,12 @@ public void preUpdateTable(String database, final TsTable table, final String ol if (!table.getCachedNeedLastCache()) { TableDeviceSchemaCache.getInstance().invalidateLastCache(database, table.getTableName()); } - if (table instanceof PreDeleteTsTable) { if (databaseTableMap.containsKey(database)) { databaseTableMap.get(database).remove(table.getTableName()); } } + // If rename table if (Objects.nonNull(oldName)) { final TsTable oldTable = databaseTableMap.get(database).remove(oldName); @@ -233,11 +257,17 @@ public void preUpdateTable(String database, final TsTable table, final String ol } @Override - public void rollbackUpdateTable(String database, final String tableName, final String oldName) { + public void rollbackUpdateTable( + String database, + final String tableName, + final String oldName, + TUpdateTree2TableView updateTree2TableViewInfo) { database = PathUtils.unQualifyDatabaseName(database); readWriteLock.writeLock().lock(); try { failIfMetadataLeaseFenced(LeaseFencedRetryPolicy.RETRY_UNTIL_SUCCESS); + // for create or replace the view, only triggered when rollback + operateOnTree2TableViewCache(database, tableName, oldName, updateTree2TableViewInfo); // if rollback the drop table procedure, do nothing, // wait for triggering the action of pull table from CN final TsTable table = getTableFromSpecialStatusMap(database, tableName); @@ -316,11 +346,16 @@ private void removeTableFromSpecialStatusMap(final String database, final String @Override public void commitUpdateTable( - String database, final String tableName, final @Nullable String oldName) { + String database, + final String tableName, + final @Nullable String oldName, + TUpdateTree2TableView updateTree2TableViewInfo) { database = PathUtils.unQualifyDatabaseName(database); readWriteLock.writeLock().lock(); try { failIfMetadataLeaseFenced(LeaseFencedRetryPolicy.RETRY_UNTIL_SUCCESS); + operateOnTree2TableViewCache(database, tableName, oldName, updateTree2TableViewInfo); + // for update tree-to-table view final TsTable newTable = getTableFromSpecialStatusMap(database, tableName); if (Objects.isNull(newTable)) { LOGGER.info( @@ -370,6 +405,116 @@ public void commitUpdateTable( } } + /** + * Applies the effect of a committed change on the tree-to-table view index. Every attribute the + * operation needs comes with the request: the table this DataNode caches may be a placeholder, so + * the entry cannot be computed from it. + */ + private void operateOnTree2TableViewCache( + String database, + String tableName, + @Nullable String oldTableName, + TUpdateTree2TableView updateTree2TableViewInfo) { + if (Objects.isNull(updateTree2TableViewInfo)) { + return; + } + final TreeToTableViewTrieOperationType type = + TreeToTableViewTrieOperationType.getType( + updateTree2TableViewInfo.getTree2TableOperationType()); + final boolean changed; + try { + switch (type) { + case REMOVE: + // Unregistering a view needs nothing but its name. + changed = treeToTableViewCache.removeView(database, tableName); + break; + case ALTER_RENAME: + // A rename keeps the place of the view. + changed = treeToTableViewCache.alterViewName(database, tableName, oldTableName); + break; + case ALTER_TAG: + changed = + treeToTableViewCache.alterViewProperties( + database, + tableName, + Collections.singletonMap( + TreeToTableViewReference.NUM_OF_TAGS, + String.valueOf(updateTree2TableViewInfo.getTagNum()))); + break; + case ADD: + case ALTER_AUTO_SYNC: + changed = + treeToTableViewCache.addView( + database, + tableName, + updateTree2TableViewInfo.isRestrict, + updateTree2TableViewInfo.getTagNum(), + new PartialPath(updateTree2TableViewInfo.getTreePath())); + break; + default: + return; + } + } catch (final IllegalPathException e) { + LOGGER.warn( + DataNodeSchemaMessages + .LOG_FAILED_TO_UPDATE_THE_TREE_TO_TABLE_VIEW_INDEX_OF_VIEW_ARG_ARG_E86EAEDE, + database, + tableName, + e); + return; + } + if (changed) { + logTreeToTableViewChange(type, database, tableName, oldTableName, updateTree2TableViewInfo); + } + } + + /** Logs the change an operation really applied to the tree-to-table view index. */ + private static void logTreeToTableViewChange( + final TreeToTableViewTrieOperationType type, + final String database, + final String tableName, + final @Nullable String oldTableName, + final TUpdateTree2TableView updateTree2TableViewInfo) { + switch (type) { + case REMOVE: + TreeToTableViewCache.LOGGER.info( + DataNodeSchemaMessages.LOG_UNREGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_C90B661A, + database, + tableName); + break; + case ALTER_RENAME: + TreeToTableViewCache.LOGGER.info( + DataNodeSchemaMessages.LOG_RENAMED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_FROM_ARG_B5D52D41, + database, + tableName, + oldTableName); + break; + case ALTER_TAG: + TreeToTableViewCache.LOGGER.info( + DataNodeSchemaMessages + .LOG_UPDATED_THE_PROPERTIES_OF_THE_TREE_TO_TABLE_VIEW_ARG_ARG_TO_ARG_1C3DE984, + database, + tableName, + Collections.singletonMap( + TreeToTableViewReference.NUM_OF_TAGS, + String.valueOf(updateTree2TableViewInfo.getTagNum()))); + break; + case ADD: + case ALTER_AUTO_SYNC: + TreeToTableViewCache.LOGGER.info( + DataNodeSchemaMessages + .LOG_REGISTERED_THE_TREE_TO_TABLE_VIEW_ARG_ARG_AT_ARG_WITH_RESTRICT_ARG_AND_NUM_OF_TAGS_ARG_7538BCB2, + database, + tableName, + updateTree2TableViewInfo.getTreePath(), + updateTree2TableViewInfo.isRestrict, + updateTree2TableViewInfo.getTagNum()); + break; + default: + break; + } + } + /** * Repairs the cache after this DataNode missed the pre-update of a rename procedure. The * ConfigNode has already renamed the table, so the new name can only be re-fetched, and the old @@ -410,6 +555,7 @@ public void invalid(String database) { try { databaseTableMap.remove(database); specialStatusMap.remove(database); + treeToTableViewCache.removeViews(database); instanceVersion.incrementAndGet(); } finally { readWriteLock.writeLock().unlock(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java index b87b6db7147b2..307df70b42712 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.exception.MetadataLeaseFencedException.LeaseFencedRetryPolicy; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.mpp.rpc.thrift.TUpdateTree2TableView; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -36,10 +37,16 @@ public interface ITableCache { void preUpdateTable(final String database, final TsTable table, final @Nullable String oldName); void rollbackUpdateTable( - final String database, final String tableName, final @Nullable String oldName); + final String database, + final String tableName, + final @Nullable String oldName, + TUpdateTree2TableView updateTree2TableViewInfo); void commitUpdateTable( - final String database, final String tableName, final @Nullable String oldName); + final String database, + final String tableName, + final @Nullable String oldName, + TUpdateTree2TableView updateTree2TableViewInfo); /** * @param database shouldn't start with `root.` diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITreeToTableViewCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITreeToTableViewCache.java new file mode 100644 index 0000000000000..b1b675dfc71c8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITreeToTableViewCache.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.table; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public interface ITreeToTableViewCache { + + void init(Map> initialTables); + + void clear(); + + /** + * Registers the view at {@code path}, the fixed prefix of its tree path pattern. {@code + * isRestrict} and {@code numOfTags} are the two attributes a device path is matched with, see + * {@link TreeToTableViewReference}. + */ + boolean addView( + String database, String tableName, boolean isRestrict, int numOfTags, PartialPath path) + throws IllegalPathException; + + boolean removeView(String database, String tableName); + + /** + * Removes every view of the database, eg. when the database is dropped. The views are taken from + * the index itself, so the ones the table cache does not hold any more are removed as well. + */ + void removeViews(String database); + + boolean alterViewName(String database, String newName, String oldName); + + /** + * Replaces the fields named in {@code properties} on the registered view. Only the restrict flag + * and the tag count can be replaced today, and a field whose key is absent keeps its value. + */ + boolean alterViewProperties(String database, String tableName, Map properties); + + @Nullable + PartialPath getViewPath(String database, String tableName); + + Set getTreeToTableViews(PartialPath path); + + boolean hasView(); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java new file mode 100644 index 0000000000000..473b2266f54a8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCache.java @@ -0,0 +1,501 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.schemaengine.table; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.exception.MetadataLeaseFencedException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternUtil; +import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; +import org.apache.iotdb.commons.schema.table.TreeViewSchema; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.db.i18n.DataNodeSchemaMessages; +import org.apache.iotdb.db.schemaengine.lease.MetadataLeaseManager; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader; +import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference; + +import org.apache.tsfile.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; + +/** + * The index of the tree to table views on DataNode, keyed by the fixed prefix of each view's tree + * path pattern. + * + *

Only DataNodeTableCache writes this index, and it does so while holding its own write lock, so + * every write here also holds the write lock of this cache and the lock order is always + * DataNodeTableCache's write lock and then this cache's write lock. A reader only takes this + * cache's read lock, so the two locks can never form a cycle. + */ +public class TreeToTableViewCache implements ITreeToTableViewCache { + + public static final Logger LOGGER = LoggerFactory.getLogger(TreeToTableViewCache.class); + + private final IMNodeFactory nodeFactory = + MNodeFactoryLoader.getInstance().getMemMNodeIMNodeFactory(); + // This index describes path prefixes, not the database/device roles in a SchemaRegion. clear() + // replaces it with a fresh one. + private IMemMNode rootNode = nodeFactory.createInternalMNode(null, PATH_ROOT); + + private final Map> viewReferences = + new IdentityHashMap<>(); + private final Map, PartialPath> viewPaths = new HashMap<>(); + private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(); + + /** + * Whether any view is registered. A cluster without any view answers "no view covers this path" + * without taking the lock or walking the trie. It is written only under the write lock, and read + * without any lock. + */ + private volatile boolean hasView = false; + + private TreeToTableViewCache() {} + + @Override + public void clear() { + readWriteLock.writeLock().lock(); + try { + // A node is only reachable from its parent, so a fresh root drops the whole trie at once. + rootNode = nodeFactory.createInternalMNode(null, PATH_ROOT); + viewReferences.clear(); + viewPaths.clear(); + hasView = false; + } finally { + readWriteLock.writeLock().unlock(); + } + } + + @Override + public void init(final Map> initialTables) { + readWriteLock.writeLock().lock(); + try { + // The index is rebuilt from scratch, so the entry of a view which is gone does not survive. + clear(); + initialTables.forEach( + (database, tables) -> tables.forEach(table -> registerView(database, table))); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + /** + * Registers the view at the path of its tree path pattern. The caller has already picked the + * tables which belong in the index, see DataNodeTableCache#collectInitialViews. + */ + private void registerView(final String database, final TsTable table) { + final Optional treePath = table.getPropValue(TreeViewSchema.TREE_PATH_PATTERN); + if (treePath.isEmpty()) { + return; + } + try { + addView( + database, + table.getTableName(), + TreeViewSchema.isRestrict(table), + table.getTagNum(), + new PartialPath(treePath.get())); + } catch (final IllegalPathException e) { + LOGGER.warn( + DataNodeSchemaMessages + .LOG_FAILED_TO_REGISTER_THE_TREE_TO_TABLE_VIEW_ARG_ARG_DURING_INIT_ILLEGAL_PATH_359A451E, + database, + table.getTableName(), + e); + } + } + + /** + * Register a view at a fixed prefix path. + * + * @parm Partial path, eg: root.db.dev01.**; + * @return true if a new reference was registered; false if it was already present + */ + @Override + public boolean addView( + final String database, + final String tableName, + final boolean isRestrict, + final int numOfTags, + final PartialPath path) + throws IllegalPathException { + int prefixLength = checkPartialPath(path); + final String[] nodes = path.getNodes(); + readWriteLock.writeLock().lock(); + try { + final PartialPath registeredPath = viewPaths.get(new Pair<>(database, tableName)); + if (Objects.nonNull(registeredPath) && !Objects.equals(registeredPath, path)) { + // A view is registered at exactly one path, so the entry left at the previous path has to + // go, or removing this view by name later would miss it. + removeView(database, tableName); + } + viewPaths.put(new Pair<>(database, tableName), path); + + IMemMNode current = rootNode; + for (int i = 1; i < prefixLength; i++) { + IMemMNode child = current.getChild(nodes[i]); + if (child == null) { + child = current.addChild(nodes[i], nodeFactory.createInternalMNode(current, nodes[i])); + } + current = child; + } + + final TreeToTableViewReference reference = + new TreeToTableViewReference(database, tableName, isRestrict, numOfTags); + final Set references = + viewReferences.computeIfAbsent(current, node -> new HashSet<>()); + final boolean added = !references.remove(reference); + references.add(reference); + // whether the reference is new or not, this path is covered by a view now + hasView = true; + return added; + } finally { + readWriteLock.writeLock().unlock(); + } + } + + /** + * Removes a view from the trie. + * + * @return true if the view was registered and has been removed; false otherwise + */ + @Override + public boolean removeView(String database, String tableName) { + readWriteLock.writeLock().lock(); + try { + return removeViewLocked(database, tableName); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + /** + * Removes every view of the database. A database of the same name may be created again with views + * which have nothing to do with the dropped ones, so the entries of the old database have to go + * together with it. + */ + @Override + public void removeViews(final String database) { + readWriteLock.writeLock().lock(); + try { + // viewPaths is the record of what is registered, so the views which the table cache does not + // hold any more are removed here as well. + final List tableNames = new ArrayList<>(); + for (final Pair view : viewPaths.keySet()) { + if (Objects.equals(database, view.getLeft())) { + tableNames.add(view.getRight()); + } + } + tableNames.forEach(tableName -> removeViewLocked(database, tableName)); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + /** The body of {@link #removeView}, which the caller runs while holding the write lock. */ + private boolean removeViewLocked(String database, String tableName) { + final RegisteredView registered = getRegisteredView(database, tableName); + if (Objects.isNull(registered)) { + return false; + } + + registered.references.remove(registered.reference); + if (registered.references.isEmpty()) { + viewReferences.remove(registered.node); + pruneNodesWithoutView(registered.node); + hasView = !viewReferences.isEmpty(); + } + viewPaths.remove(new Pair<>(database, tableName)); + return true; + } + + @Override + public boolean alterViewName(String database, String newName, String oldName) { + readWriteLock.writeLock().lock(); + try { + final RegisteredView registered = getRegisteredView(database, oldName); + if (Objects.isNull(registered)) { + LOGGER.warn( + DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902, + database, + oldName); + return false; + } + + final TreeToTableViewReference oldReference = registered.reference; + registered.references.remove(oldReference); + registered.references.add( + new TreeToTableViewReference( + database, newName, oldReference.isRestrict(), oldReference.getNumOfTags())); + // A rename does not move the view, so the path of the old name follows the new name. + final PartialPath path = viewPaths.remove(new Pair<>(database, oldName)); + viewPaths.put(new Pair<>(database, newName), path); + return true; + } finally { + readWriteLock.writeLock().unlock(); + } + } + + @Override + public boolean alterViewProperties( + final String database, final String tableName, final Map properties) { + readWriteLock.writeLock().lock(); + try { + final RegisteredView registered = getRegisteredView(database, tableName); + if (Objects.isNull(registered)) { + LOGGER.warn( + DataNodeSchemaMessages.LOG_TREE_TO_TABLE_VIEW_ARG_ARG_IS_NOT_REGISTERED_BBD78902, + database, + tableName); + return false; + } + + final TreeToTableViewReference oldReference = registered.reference; + final String restrict = properties.get(TreeViewSchema.RESTRICT); + final String numOfTags = properties.get(TreeToTableViewReference.NUM_OF_TAGS); + registered.references.remove(oldReference); + registered.references.add( + new TreeToTableViewReference( + database, + tableName, + Objects.isNull(restrict) ? oldReference.isRestrict() : Boolean.parseBoolean(restrict), + Objects.isNull(numOfTags) + ? oldReference.getNumOfTags() + : Integer.parseInt(numOfTags))); + return true; + } finally { + readWriteLock.writeLock().unlock(); + } + } + + /** + * The place a registered view lives: the trie node it sits at, the references of that node, and + * the reference of the view itself. + */ + private static class RegisteredView { + IMemMNode node; + Set references; + TreeToTableViewReference reference; + + private RegisteredView( + final IMemMNode node, + final Set references, + final TreeToTableViewReference reference) { + this.node = node; + this.references = references; + this.reference = reference; + } + } + + /** + * The registered view of these names, or null when no view of them is registered. The node may + * also exist as an intermediate node of another view's path, without any view of its own. + */ + private RegisteredView getRegisteredView(final String database, final String tableName) { + final IMemMNode node = getViewNode(database, tableName); + final Set references = + Objects.isNull(node) ? null : viewReferences.get(node); + if (Objects.isNull(references)) { + return null; + } + final TreeToTableViewReference reference = findReference(references, database, tableName); + return Objects.isNull(reference) ? null : new RegisteredView(node, references, reference); + } + + /** The trie node the view of these names is registered at, or null if it is not registered. */ + private IMemMNode getViewNode(final String database, final String tableName) { + final PartialPath path = viewPaths.get(new Pair<>(database, tableName)); + if (Objects.isNull(path)) { + return null; + } + final String[] nodes = path.getNodes(); + return findNode(nodes, nodes.length - 1); + } + + /** The reference of the view of these names in {@code references}, or null if it holds none. */ + private static TreeToTableViewReference findReference( + final Set references, + final String database, + final String tableName) { + for (final TreeToTableViewReference reference : references) { + if (Objects.equals(database, reference.getDatabaseName()) + && Objects.equals(tableName, reference.getTableName())) { + return reference; + } + } + return null; + } + + /** + * @param path the device path, eg, root.db01.plant01.dev01 + * @return the views covering the device path, empty if there is none + */ + @Override + public Set getTreeToTableViews(PartialPath path) { + // A fenced DataNode must not answer from its possibly stale index, not even with "no view + // covers this path": the caller has to know that the cache is unavailable rather than empty. + failIfMetadataLeaseFenced(); + if (!hasView) { + // the answer for every path when no view is registered at all + return Collections.emptySet(); + } + final String[] nodes = path.getNodes(); + readWriteLock.readLock().lock(); + try { + Set result = new HashSet<>(); + IMemMNode current = rootNode; + int currentLevel = 1; + while (current != null && currentLevel < (nodes.length + 1)) { + // maybe has the qualified nodes, check and collect + if (viewReferences.containsKey(current)) { + checkAndCollectView(viewReferences.get(current), nodes.length - currentLevel, result); + } + if (currentLevel == nodes.length) { + break; + } + current = current.getChild(nodes[currentLevel]); + currentLevel++; + } + return result; + } finally { + readWriteLock.readLock().unlock(); + } + } + + @Override + public boolean hasView() { + return hasView; + } + + /** + * The path a view is registered at, so that a caller which knows only the names of the view can + * still locate and remove it. + * + * @return the path the view is registered at, or null if no view of these names is registered + */ + @Override + public @Nullable PartialPath getViewPath(final String database, final String tableName) { + readWriteLock.readLock().lock(); + try { + return viewPaths.get(new Pair<>(database, tableName)); + } finally { + readWriteLock.readLock().unlock(); + } + } + + /** + * Walks the fixed prefix of a checked path, which is the path without the leading "root" and the + * trailing multi level wildcard, eg, nodes of root.db.dev01.** give the node of root.db.dev01. + * + * @return the node the prefix ends at, or null if the trie does not hold that prefix + */ + private IMemMNode findNode(final String[] nodes, final int prefixLength) { + IMemMNode current = rootNode; + for (int i = 1; i < prefixLength && current != null; i++) { + current = current.getChild(nodes[i]); + } + return current; + } + + /** + * Drops the nodes which have neither a child nor a view of their own any more, starting at {@code + * node} and walking up to the first node that is still needed. Removing a view is rare, so the + * walk is affordable. + */ + private void pruneNodesWithoutView(final IMemMNode node) { + IMemMNode current = node; + while (current != rootNode) { + final IMemMNode parent = current.getParent(); + if (parent == null + || !current.getChildren().isEmpty() + || viewReferences.containsKey(current)) { + return; + } + parent.deleteChild(current.getName()); + current = parent; + } + } + + void failIfMetadataLeaseFenced() { + MetadataLeaseManager.getInstance() + .failIfMetadataLeaseFenced( + MetadataLeaseFencedException.LeaseFencedRetryPolicy.RETRY_UNTIL_SUCCESS); + } + + private static final class TreeToTableViewCacheHolder { + private static final TreeToTableViewCache INSTANCE = new TreeToTableViewCache(); + + private TreeToTableViewCacheHolder() {} + } + + public static ITreeToTableViewCache getInstance() { + return TreeToTableViewCacheHolder.INSTANCE; + } + + // the path has to be like root.db.dev01.**; + private int checkPartialPath(PartialPath path) throws IllegalPathException { + final String[] nodes = path.getNodes(); + if (nodes.length == 0 || !PATH_ROOT.equals(nodes[0])) { + throw new IllegalPathException(path.getFullPath()); + } + if (!MULTI_LEVEL_PATH_WILDCARD.equals(nodes[nodes.length - 1])) { + throw new IllegalPathException(path.getFullPath()); + } + final int prefixLength = nodes.length - 1; + // Validate before modifying the trie. Wildcards inside the fixed prefix are not supported. + for (int i = 1; i < prefixLength; i++) { + if (nodes[i] == null || nodes[i].isEmpty() || PathPatternUtil.hasWildcard(nodes[i])) { + throw new IllegalPathException(path.getFullPath()); + } + } + return prefixLength; + } + + private void checkAndCollectView( + Set treeToTableViewReferences, + int remainingLevel, + Set result) { + treeToTableViewReferences.forEach( + view -> { + int numOfTags = view.getNumOfTags(); + boolean match = + view.isRestrict() ? numOfTags == remainingLevel : numOfTags >= remainingLevel; + if (match) { + result.add(view); + } + }); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSyncTrigger.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSyncTrigger.java new file mode 100644 index 0000000000000..9a937fcd315f8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ViewColumnAutoSyncTrigger.java @@ -0,0 +1,337 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.table; + +import org.apache.iotdb.commons.client.AbstractConfigNodeInfo; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.consensus.ConfigRegionId; +import org.apache.iotdb.commons.exception.MetadataLeaseFencedException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.table.AlterOrDropTableOperationType; +import org.apache.iotdb.commons.schema.table.TreeViewSchema; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; +import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsFailure; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsReq; +import org.apache.iotdb.confignode.rpc.thrift.TAutoSyncViewColumnsResp; +import org.apache.iotdb.db.i18n.DataNodeSchemaMessages; +import org.apache.iotdb.db.protocol.client.ConfigNodeClient; +import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator; +import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HexFormat; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Adds to the tree-to-table views the columns which the measurements a schema write has created + * need. + * + *

It is the automatic counterpart of the {@code ALTER VIEW ... ADD COLUMN} statement: the + * columns come from the measurements the schema write has just created, instead of from the + * statement itself. + */ +public class ViewColumnAutoSyncTrigger { + + // Public so that the schema write path, which invokes sync and catches its failures, logs them + // with the logger of the feature; logback routes this logger to its own error file. + public static final Logger LOGGER = LoggerFactory.getLogger(ViewColumnAutoSyncTrigger.class); + private static final IClientManager CONFIG_NODE_CLIENT_MANAGER = + ConfigNodeClientManager.getInstance(); + + public static final String QUERY_ID_PREFIX = "tree_to_table_add_column"; + public static final String EXTEND_DEVICE_TEMPLATE_ID_PREFIX = "template_"; + + /** How many devices, and how many measurements per device, a fenced lease log line shows. */ + private static final int MAX_LOGGED_DEVICES = 10; + + private static final int MAX_LOGGED_MEASUREMENTS_PER_DEVICE = 10; + + /** + * Adds the columns the views covering the given devices need. + * + *

It is a best-effort side effect of the write which triggers it: every failure is logged here + * and none of them is propagated, so that the write operation is never affected. + * + * @param device2Measurements the measurements the schema write really created, eg. device01 -> + * {s1, s2, s3}; device02 -> {s3, s4}. Only they get a column, so that a creation which failed + * cannot leave a column behind. + * @param operationId the planNodeId of the plan which triggers this operation, for the log + * messages only: it is a per-query index, so it does not identify the operation + */ + public static void sync( + final Map>> device2Measurements, + final String operationId) { + if (device2Measurements == null || device2Measurements.isEmpty()) { + return; + } + + // the views to be added columns + final Map>> view2Measurements; + try { + view2Measurements = collectViewMeasurements(device2Measurements); + } catch (final MetadataLeaseFencedException e) { + // The DataNode is isolated: neither the index nor the table cache it consults may be used. + // Give up this round and record the measurements which cannot be added while it is fenced. + LOGGER.error( + DataNodeSchemaMessages + .LOG_THE_METADATA_LEASE_IS_FENCED_WHILE_ADDING_VIEW_COLUMNS_GIVING_UP_THIS_ROUND_THE_MEASUREMENTS_ARE_ARG_5D1B2D44, + formatDeviceMeasurements(device2Measurements)); + return; + } + if (view2Measurements.isEmpty()) { + return; + } + + try (ConfigNodeClient configNodeClient = + CONFIG_NODE_CLIENT_MANAGER.borrowClient(AbstractConfigNodeInfo.CONFIG_REGION_ID)) { + List updateViewReqs = + view2Measurements.entrySet().stream() + .map(ViewColumnAutoSyncTrigger::toAlterTableReq) + .toList(); + + TAutoSyncViewColumnsResp resp = + configNodeClient.autoSyncViewColumns(new TAutoSyncViewColumnsReq(updateViewReqs)); + if (resp == null || resp.getStatus() == null) { + LOGGER.error( + DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_EMPTY_RESPONSE_B9502E26, operationId); + return; + } + if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.error( + DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_FAILED_FE4F10AC, + operationId, + resp.getStatus()); + } + // the views are updated one by one in ConfigNode, so the failed ones are reported one by one + if (resp.getFailedViews() != null) { + for (TAutoSyncViewColumnsFailure failedView : resp.getFailedViews()) { + LOGGER.error( + DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_FAILED_FOR_VIEW_B3EB8144, + failedView.getDatabase(), + failedView.getTableName(), + operationId, + failedView.getStatus()); + } + } + } catch (ClientManagerException | TException e) { + LOGGER.warn( + DataNodeSchemaMessages.LOG_ADD_VIEW_COLUMN_CN_UNREACHABLE_E9BC7652, operationId, e); + } + } + + /** + * A bounded, human readable form of the measurements, so that the log line of a batch write stays + * short: at most {@link #MAX_LOGGED_DEVICES} devices and {@link + * #MAX_LOGGED_MEASUREMENTS_PER_DEVICE} measurements per device are written, the rest is + * summarized. + */ + private static String formatDeviceMeasurements( + final Map>> device2Measurements) { + final StringBuilder builder = new StringBuilder("{"); + int loggedDevices = 0; + for (final Map.Entry>> entry : + device2Measurements.entrySet()) { + if (loggedDevices == MAX_LOGGED_DEVICES) { + builder.append("... (").append(device2Measurements.size()).append(" devices in total)}"); + return builder.toString(); + } + if (loggedDevices > 0) { + builder.append(", "); + } + builder.append(entry.getKey().getFullPath()).append("=["); + final List> measurements = entry.getValue(); + for (int i = 0; i < measurements.size(); i++) { + if (i == MAX_LOGGED_MEASUREMENTS_PER_DEVICE) { + builder.append("... (").append(measurements.size()).append(" measurements in total)"); + break; + } + if (i > 0) { + builder.append(", "); + } + builder + .append(measurements.get(i).getLeft()) + .append(':') + .append(measurements.get(i).getRight()); + } + builder.append(']'); + loggedDevices++; + } + return builder.append('}').toString(); + } + + /** + * @return the measurements each view covering the given devices still needs; eg: view01 -> {s1, + * s2, s3}, view02 -> {s1, s2, s3, s4} + */ + private static Map>> + collectViewMeasurements( + Map>> device2Measurements) { + + // deduplicate the measurement by set, the measurement with same name and dataType + // from different device would count once, eg: dev01.s1(int32) and dev02.s1(int32) + // the measurements which already have a column in the view are dropped below + final Map>> view2Measurements = + new HashMap<>(); + + for (Map.Entry>> entry : + device2Measurements.entrySet()) { + PartialPath path = entry.getKey(); + Set treeToTableViews = + TreeToTableViewCache.getInstance().getTreeToTableViews(path); + + treeToTableViews.forEach( + view -> + view2Measurements + .computeIfAbsent(view, key -> new LinkedHashSet<>()) + .addAll(entry.getValue())); + } + + // A measurement the view already has a column for is not needed again: ConfigNode would drop + // it in its column check and reject the request when nothing is left, which costs an RPC and a + // persisted procedure for nothing. A view the table cache does not hold is left as it is, as + // the DataNode cannot tell which columns are missing then. + final Iterator>>> iterator = + view2Measurements.entrySet().iterator(); + while (iterator.hasNext()) { + final Map.Entry>> viewEntry = + iterator.next(); + final TsTable view = getCachedView(viewEntry.getKey()); + if (Objects.isNull(view)) { + continue; + } + removeExistingMeasurements(view, viewEntry.getValue()); + if (viewEntry.getValue().isEmpty()) { + iterator.remove(); + } + } + return view2Measurements; + } + + /** The view as the DataNode caches it, or null when the cache does not hold it. */ + private static TsTable getCachedView(final TreeToTableViewReference view) { + return DataNodeTableCache.getInstance() + .getTable(view.getDatabaseName(), view.getTableName(), false); + } + + /** Drops the measurements which the view already has a column for. */ + private static void removeExistingMeasurements( + final TsTable view, final Set> measurements) { + // The same test ConfigNode applies before it adds the columns: the column name, or the source + // measurement a FROM column references. A measurement the cache does not know yet is kept, as + // the cache may lag behind ConfigNode. + final Set sourceNames = TreeViewSchema.getSourceNames(view); + measurements.removeIf( + measurement -> + Objects.nonNull(view.getColumnSchema(measurement.getLeft().toLowerCase(Locale.ROOT))) + || sourceNames.contains(measurement.getLeft())); + } + + private static TAlterOrDropTableReq toAlterTableReq( + Map.Entry>> view2Measurements) { + final TreeToTableViewReference view = view2Measurements.getKey(); + final List columnSchemas = + view2Measurements.getValue().stream() + .map(ViewColumnAutoSyncTrigger::toColumnSchema) + .toList(); + TAlterOrDropTableReq tAlterOrDropTableReq = + new TAlterOrDropTableReq( + view.getDatabaseName(), + view.getTableName(), + buildQueryId( + columnFingerprint(columnSchemas), view.getDatabaseName(), view.getTableName()), + AlterOrDropTableOperationType.ADD_COLUMN.getTypeValue(), + ByteBuffer.wrap(TsTableColumnSchemaUtil.serialize(columnSchemas))); + tAlterOrDropTableReq.setIsView(true); + return tAlterOrDropTableReq; + } + + // transform the upperCase to the from-attribute + private static TsTableColumnSchema toColumnSchema(final Pair measurement) { + final String measurementName = measurement.getLeft(); + final String from = + measurementName.chars().anyMatch(Character::isUpperCase) ? measurementName : null; + return TableHeaderSchemaValidator.generateColumnSchema( + TsTableColumnCategory.FIELD, + measurementName.toLowerCase(Locale.ROOT), + measurement.getRight(), + null, + from); + } + + // leader and follower would build the same queryId when apply the same log; so CN could recognize + // the same add view column request. The database and table names may contain any character, so + // their lengths are written out to keep the parts apart: without them, the views a_b.c and + // a.b_c would share an id and the request for the second one would be taken for a repeat of the + // first one's. + private static String buildQueryId(String columnFingerprint, String database, String table) { + return String.join( + "_", + QUERY_ID_PREFIX, + columnFingerprint, + String.valueOf(database.length()), + database, + String.valueOf(table.length()), + table); + } + + /** + * The columns are sorted first so that the same column set always yields the same fingerprint, no + * matter in which order the measurements were collected. + */ + private static String columnFingerprint(final List columnSchemas) { + final List sortedColumnSchemas = new ArrayList<>(columnSchemas); + sortedColumnSchemas.sort(Comparator.comparing(TsTableColumnSchema::getColumnName)); + return digestOf(TsTableColumnSchemaUtil.serialize(sortedColumnSchemas)); + } + + private static String digestOf(final byte[] content) { + try { + // 4 bytes are enough here, the number of distinct column sets of one view is rather small + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content), 0, 4); + } catch (final NoSuchAlgorithmException e) { + // SHA-256 is supported by every Java platform + throw new IllegalStateException(e); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusShutdownTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusShutdownTest.java index bf2f5c21e6ade..8b3217bad2f98 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusShutdownTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/consensus/statemachine/dataregion/IoTConsensusShutdownTest.java @@ -595,7 +595,7 @@ private void checkShutdownDrainsTableDeletion(final boolean failDeletion) throws DataNodeTableCache.getInstance() .preUpdateTable(DATABASE, StatementTestUtils.genTsTable(), null); DataNodeTableCache.getInstance() - .commitUpdateTable(DATABASE, StatementTestUtils.tableName(), null); + .commitUpdateTable(DATABASE, StatementTestUtils.tableName(), null, null); assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), stateMachine diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java index d60a8f2cc3cca..98ef2dc61c012 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java @@ -290,7 +290,7 @@ public void testRecoverAfterCreateAndUpdateDevice() throws Exception { new TagColumnSchema(columnHeader.getColumnName(), columnHeader.getColumnType()))); testTable.addColumnSchema(new AttributeColumnSchema(attributeName, TSDataType.STRING)); DataNodeTableCache.getInstance().preUpdateTable(database, testTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null, null); schemaRegion.updateTableDeviceAttribute( new TableDeviceAttributeUpdateNode( @@ -337,7 +337,7 @@ public void testUpdateDeviceRejectsFieldAssignment() throws Exception { testTable.addColumnSchema(new AttributeColumnSchema("attr", TSDataType.STRING)); testTable.addColumnSchema(new FieldColumnSchema("field", TSDataType.STRING)); DataNodeTableCache.getInstance().preUpdateTable(database, testTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null, null); try { final ISchemaRegion schemaRegion = getSchemaRegion(database, 0); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java index 744bda9ce5b84..cc7f97984715f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java @@ -159,7 +159,7 @@ public void setUp() throws MetadataException, IOException, WriteProcessException TreeViewSchema.TREE_PATH_PATTERN, NON_ALIGNED_TREE_DEVICE_VIEW_SCAN_OPERATOR_TREE_TEST + ".**"); DataNodeTableCache.getInstance().preUpdateTable(tableDbName, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(tableDbName, tableViewName, null); + DataNodeTableCache.getInstance().commitUpdateTable(tableDbName, tableViewName, null, null); } @After diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java index 9c9918ddbc8c4..bc4fd02b192d7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java @@ -1149,7 +1149,7 @@ public void testPipeTransferredUpdateRejectsInvalidTarget() { tsTable.addColumnSchema(new AttributeColumnSchema("attr1", TSDataType.STRING)); tsTable.addColumnSchema(new FieldColumnSchema("s1", TSDataType.INT64)); DataNodeTableCache.getInstance().preUpdateTable(database, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, table, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, table, null, null); try { final Update statement = @@ -1211,7 +1211,7 @@ public void testSqlUpdateRejectsInvalidColumnsWithExplicitErrors() { tsTable.addColumnSchema(new AttributeColumnSchema("attr1", TSDataType.STRING)); tsTable.addColumnSchema(new FieldColumnSchema("s1", TSDataType.INT64)); DataNodeTableCache.getInstance().preUpdateTable(database, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, table, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, table, null, null); try { assertUpdateSemanticException( @@ -1276,7 +1276,7 @@ public void testInternalShowDeviceKeepsPreparedTraversalState() { tsTable.addColumnSchema(new TagColumnSchema("tag1", TSDataType.STRING)); tsTable.addColumnSchema(new AttributeColumnSchema("attr1", TSDataType.STRING)); DataNodeTableCache.getInstance().preUpdateTable(database, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, table, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, table, null, null); try { final ShowDevice statement = new ShowDevice(database, table); @@ -1316,7 +1316,7 @@ private void assertDeviceQueryCanBeAnalyzedAgain(final String sql) { tsTable.addColumnSchema(new AttributeColumnSchema("attr1", TSDataType.STRING)); tsTable.addColumnSchema(new AttributeColumnSchema("attr2", TSDataType.STRING)); DataNodeTableCache.getInstance().preUpdateTable(database, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, table, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, table, null, null); try { final SqlParser sqlParser = new SqlParser(); @@ -1370,7 +1370,7 @@ private void assertParsedDeviceStatementUnchanged( private Metadata mockMetadataForInsertion() { final TsTable tsTable = StatementTestUtils.genTsTable(); DataNodeTableCache.getInstance().preUpdateTable(database, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, table, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, table, null, null); return new TestMetadata() { @Override public Optional validateTableHeaderSchema4TsFile( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TreeViewTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TreeViewTest.java index 0c154850ca5a5..74667017f25e5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TreeViewTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TreeViewTest.java @@ -73,7 +73,8 @@ public void setup() { tsTable.addProp(TsTable.TTL_PROPERTY, Long.MAX_VALUE + ""); tsTable.addProp(TreeViewSchema.TREE_PATH_PATTERN, "root.test" + ".**"); DataNodeTableCache.getInstance().preUpdateTable(TREE_VIEW_DB, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(TREE_VIEW_DB, DEVICE_VIEW_TEST_TABLE, null); + DataNodeTableCache.getInstance() + .commitUpdateTable(TREE_VIEW_DB, DEVICE_VIEW_TEST_TABLE, null, null); } @After diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java index 2b8909f38613e..a59d36839029d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java @@ -107,10 +107,10 @@ public static void prepareEnvironment() { new FieldColumnSchema( measurement6, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); DataNodeTableCache.getInstance().preUpdateTable(database1, testTable1, null); - DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null); + DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null, null); DataNodeTableCache.getInstance().preUpdateTable(database2, testTable1, null); - DataNodeTableCache.getInstance().commitUpdateTable(database2, table1, null); + DataNodeTableCache.getInstance().commitUpdateTable(database2, table1, null, null); final TsTable testTable2 = new TsTable(table2); columnHeaderList.forEach( @@ -139,7 +139,7 @@ public static void prepareEnvironment() { new FieldColumnSchema( measurement6, TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); DataNodeTableCache.getInstance().preUpdateTable(database1, testTable2, null); - DataNodeTableCache.getInstance().commitUpdateTable(database1, table2, null); + DataNodeTableCache.getInstance().commitUpdateTable(database1, table2, null, null); final TsTable testTableNeedLastCacheDisabled = new TsTable(tableNeedLastCacheDisabled); columnHeaderList.forEach( @@ -173,7 +173,8 @@ public static void prepareEnvironment() { TsTable.NEED_LAST_CACHE_PROPERTY, Boolean.FALSE.toString()); DataNodeTableCache.getInstance() .preUpdateTable(database1, testTableNeedLastCacheDisabled, null); - DataNodeTableCache.getInstance().commitUpdateTable(database1, tableNeedLastCacheDisabled, null); + DataNodeTableCache.getInstance() + .commitUpdateTable(database1, tableNeedLastCacheDisabled, null, null); } @AfterClass @@ -628,7 +629,7 @@ public void testLastCacheIsInvalidatedWhenTableNeedLastCacheTurnsFalse() { new TsTable(DataNodeTableCache.getInstance().getTable(database1, table1)); disabledTable.addProp(TsTable.NEED_LAST_CACHE_PROPERTY, Boolean.FALSE.toString()); DataNodeTableCache.getInstance().preUpdateTable(database1, disabledTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null); + DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null, null); try { Assert.assertNull(cache.getLastEntry(database1, deviceID, measurement1)); @@ -636,7 +637,7 @@ public void testLastCacheIsInvalidatedWhenTableNeedLastCacheTurnsFalse() { final TsTable enabledTable = new TsTable(disabledTable); enabledTable.removeProp(TsTable.NEED_LAST_CACHE_PROPERTY); DataNodeTableCache.getInstance().preUpdateTable(database1, enabledTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null); + DataNodeTableCache.getInstance().commitUpdateTable(database1, table1, null, null); } } @@ -654,7 +655,7 @@ public void testLastCacheIsInvalidatedWhenTableNeedLastCacheSetFalseWithoutCache new TsTable(DataNodeTableCache.getInstance().getTable(database1, table1)); enabledTable.renameTable(tableName); DataNodeTableCache.getInstance().preUpdateTable(database, enabledTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null); + DataNodeTableCache.getInstance().commitUpdateTable(database, tableName, null, null); final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); updateLastCache4Query(cache, database, deviceID, measurements, values); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TopKRuntimeFilterTreeViewPlanTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TopKRuntimeFilterTreeViewPlanTest.java index 91b16f17d5ee3..305f6dd706c4f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TopKRuntimeFilterTreeViewPlanTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TopKRuntimeFilterTreeViewPlanTest.java @@ -48,7 +48,8 @@ public void setup() { tsTable.addProp(TsTable.TTL_PROPERTY, Long.MAX_VALUE + ""); tsTable.addProp(TreeViewSchema.TREE_PATH_PATTERN, "root.test.**"); DataNodeTableCache.getInstance().preUpdateTable(TREE_VIEW_DB, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(TREE_VIEW_DB, DEVICE_VIEW_TEST_TABLE, null); + DataNodeTableCache.getInstance() + .commitUpdateTable(TREE_VIEW_DB, DEVICE_VIEW_TEST_TABLE, null, null); } @After diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java index bf42e64460982..e0ddc58039fa2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/InsertStatementTest.java @@ -99,7 +99,7 @@ public void setUp() throws Exception { TsTable tsTable = convertTableSchemaToTsTable(tableSchema); DataNodeTableCache.getInstance().preUpdateTable("test", tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null); + DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null, null); when(metadata.validateTableHeaderSchema4TsFile( any(String.class), @@ -252,7 +252,7 @@ public void testConflictCategory() { TsTable tsTable = convertTableSchemaToTsTable(tableSchema); DataNodeTableCache.getInstance().preUpdateTable("test", tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null); + DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null, null); when(metadata.validateTableHeaderSchema4TsFile( any(String.class), @@ -288,7 +288,7 @@ public void testMissingIdColumn() { TsTable tsTable = convertTableSchemaToTsTable(tableSchema); DataNodeTableCache.getInstance().preUpdateTable("test", tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null); + DataNodeTableCache.getInstance().commitUpdateTable("test", "table1", null, null); when(metadata.validateTableHeaderSchema4TsFile( any(String.class), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/metric/SchemaMemMetricTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/metric/SchemaMemMetricTableTest.java index f9e92207f34fb..a14d7b21244f2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/metric/SchemaMemMetricTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/metric/SchemaMemMetricTableTest.java @@ -115,7 +115,7 @@ public void testSchemaRegionSeriesMetricsIncludeTableMeasurements() { table.addColumnSchema( new FieldColumnSchema("s2", TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); tableCache.preUpdateTable(DATABASE, table, null); - tableCache.commitUpdateTable(DATABASE, TABLE, null); + tableCache.commitUpdateTable(DATABASE, TABLE, null, null); final MemSchemaRegionStatistics statistics = new MemSchemaRegionStatistics(SCHEMA_REGION_ID, new MemSchemaEngineStatistics()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheLeaseTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheLeaseTest.java index 865ef633248b9..91cfff1d6359f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheLeaseTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheLeaseTest.java @@ -81,8 +81,8 @@ public void fencedLeaseFailsClosedForUpdateApis() { nowNanos.addAndGet((T_FENCE_MS + 1) * 1_000_000L); assertLeaseFenced(() -> tableCache.preUpdateTable("root.db", table, null)); - assertLeaseFenced(() -> tableCache.rollbackUpdateTable("root.db", "t", null)); - assertLeaseFenced(() -> tableCache.commitUpdateTable("root.db", "t", null)); + assertLeaseFenced(() -> tableCache.rollbackUpdateTable("root.db", "t", null, null)); + assertLeaseFenced(() -> tableCache.commitUpdateTable("root.db", "t", null, null)); } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java index db9b36c7555af..18ee416d2e843 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCacheTest.java @@ -60,12 +60,12 @@ public void localReadUsesCommittedSchemaWithoutResolvingPendingUpdates() { final TsTable initial = createTable(TABLE_NAME); cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, initial, null); Assert.assertNull(cache.getTableInCache(TABLE_CACHE_TEST_DATABASE, TABLE_NAME)); - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null); Assert.assertSame(initial, cache.getTableInCache(TABLE_CACHE_TEST_DATABASE, TABLE_NAME)); final TsTable updated = new TsTable(initial); cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, updated, null); Assert.assertSame(initial, cache.getTableInCache(TABLE_CACHE_TEST_DATABASE, TABLE_NAME)); - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null); Assert.assertSame(updated, cache.getTableInCache(TABLE_CACHE_TEST_DATABASE, TABLE_NAME)); } finally { cache.invalid(TABLE_CACHE_TEST_DATABASE); @@ -100,8 +100,8 @@ public void commitUpdateTableIsIdempotent() { try { cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, createTable(TABLE_NAME), null); - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null); - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null); Assert.assertEquals( TABLE_NAME, cache.getTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME).getTableName()); @@ -117,8 +117,8 @@ public void commitAfterRollbackUpdateTableIsIgnored() { try { cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, createTable(TABLE_NAME), null); - cache.rollbackUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null); - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null); + cache.rollbackUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null); Assert.assertNull(cache.getTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, false)); } finally { @@ -133,10 +133,10 @@ public void rollbackRenameTableRestoresOldName() { cache.invalid(TABLE_CACHE_TEST_DATABASE); try { cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, createTable(TABLE_NAME), null); - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME, null, null); cache.preUpdateTable(TABLE_CACHE_TEST_DATABASE, createTable(newTableName), TABLE_NAME); - cache.rollbackUpdateTable(TABLE_CACHE_TEST_DATABASE, newTableName, TABLE_NAME); + cache.rollbackUpdateTable(TABLE_CACHE_TEST_DATABASE, newTableName, TABLE_NAME, null); Assert.assertEquals( TABLE_NAME, cache.getTable(TABLE_CACHE_TEST_DATABASE, TABLE_NAME).getTableName()); @@ -164,9 +164,9 @@ public void commitRenameRepairsCacheWhenPreReleaseWasMissed() throws Exception { TsTableInternalRPCUtil.serializeTableInitializationInfo(usingTables, specialTables)); // The commit is the only sign of the rename, and the HA broadcast may send it more than once. - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, newName, oldName); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, newName, oldName, null); assertRenameRepaired(cache, oldName, newName); - cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, newName, oldName); + cache.commitUpdateTable(TABLE_CACHE_TEST_DATABASE, newName, oldName, null); assertRenameRepaired(cache, oldName, newName); } finally { cache.invalid(TABLE_CACHE_TEST_DATABASE); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCacheTest.java new file mode 100644 index 0000000000000..58ec3c7b4aa43 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/table/TreeToTableViewCacheTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.table; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.view.TreeToTableViewReference; + +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.Set; + +public class TreeToTableViewCacheTest { + + /** + * The cache is a singleton, so every case registers its own views and removes them again, and + * every case uses its own database. + */ + private final ITreeToTableViewCache viewCache = TreeToTableViewCache.getInstance(); + + @Test + public void testAddAlterAndRemoveView() throws Exception { + final String database = "add_test"; + final PartialPath viewPath = new PartialPath("root.add_test.dev01.**"); + final PartialPath devicePath = new PartialPath("root.add_test.dev01"); + try { + Assert.assertTrue(viewCache.getTreeToTableViews(devicePath).isEmpty()); + + Assert.assertTrue(viewCache.addView(database, "view1", false, 0, viewPath)); + // the same view at the same path twice does not add a second reference + Assert.assertFalse(viewCache.addView(database, "view1", false, 0, viewPath)); + Assert.assertTrue(isCovered(devicePath, "view1")); + + Assert.assertTrue(viewCache.alterViewName(database, "view2", "view1")); + Assert.assertFalse(isCovered(devicePath, "view1")); + Assert.assertTrue(isCovered(devicePath, "view2")); + + Assert.assertTrue(viewCache.removeView(database, "view2")); + Assert.assertTrue(viewCache.getTreeToTableViews(devicePath).isEmpty()); + // the view is gone, so removing it again fails + Assert.assertFalse(viewCache.removeView(database, "view2")); + } finally { + viewCache.removeView(database, "view1"); + viewCache.removeView(database, "view2"); + } + } + + @Test + public void testRemovingOneViewKeepsTheOthers() throws Exception { + final String database = "remove_test"; + final PartialPath viewPath = new PartialPath("root.remove_test.dev01.**"); + final PartialPath devicePath = new PartialPath("root.remove_test.dev01"); + try { + Assert.assertTrue(viewCache.addView(database, "view3", false, 0, viewPath)); + Assert.assertTrue(viewCache.addView(database, "view4", false, 0, viewPath)); + + Assert.assertTrue(viewCache.removeView(database, "view3")); + + final Set views = viewCache.getTreeToTableViews(devicePath); + Assert.assertEquals(1, views.size()); + Assert.assertEquals("view4", views.iterator().next().getTableName()); + } finally { + viewCache.removeView(database, "view3"); + viewCache.removeView(database, "view4"); + } + } + + /** The nodes created for the path of the removed view have to be dropped together with it. */ + @Test + public void testRemoveViewPrunesTheTrie() throws Exception { + final String database = "prune_test"; + final PartialPath viewPath = new PartialPath("root.prune_test.dev01.**"); + final PartialPath devicePath = new PartialPath("root.prune_test.dev01"); + try { + Assert.assertTrue(viewCache.addView(database, "view5", false, 0, viewPath)); + + Assert.assertTrue(viewCache.removeView(database, "view5")); + + // the path of the removed view is not held by the trie any more + Assert.assertNull(getRootNode().getChild(database)); + Assert.assertTrue(viewCache.getTreeToTableViews(devicePath).isEmpty()); + } finally { + viewCache.removeView(database, "view5"); + } + } + + @Test + public void testRemoveViewOfAnotherDatabase() throws Exception { + final String database = "database_test"; + final PartialPath viewPath = new PartialPath("root.database_test.dev01.**"); + final PartialPath devicePath = new PartialPath("root.database_test.dev01"); + try { + Assert.assertTrue(viewCache.addView(database, "view6", false, 0, viewPath)); + + // the view of another database is another view, even if the name and the path are the same + Assert.assertFalse(viewCache.removeView("another_database", "view6")); + Assert.assertTrue(isCovered(devicePath, "view6")); + } finally { + viewCache.removeView(database, "view6"); + } + } + + private boolean isCovered(final PartialPath devicePath, final String tableName) { + return viewCache.getTreeToTableViews(devicePath).stream() + .anyMatch(view -> tableName.equals(view.getTableName())); + } + + private static IMemMNode getRootNode() throws Exception { + final Field field = TreeToTableViewCache.class.getDeclaredField("rootNode"); + field.setAccessible(true); + return (IMemMNode) field.get(TreeToTableViewCache.getInstance()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index f7a594659e771..a70315cca3491 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -177,7 +177,8 @@ public void setUp() throws Exception { DataNodeTableCache.getInstance() .preUpdateTable(dataRegion.getDatabaseName(), StatementTestUtils.genTsTable(), null); DataNodeTableCache.getInstance() - .commitUpdateTable(dataRegion.getDatabaseName(), StatementTestUtils.tableName(), null); + .commitUpdateTable( + dataRegion.getDatabaseName(), StatementTestUtils.tableName(), null, null); } @After diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java index e9ea0025250ed..9aa2143ab8ea1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java @@ -188,8 +188,11 @@ private void createTable(String tableName, TSDataType dataType) { tsTable.addColumnSchema(new TagColumnSchema("id_column", TSDataType.STRING)); tsTable.addColumnSchema( new FieldColumnSchema("s1", dataType, TSEncoding.PLAIN, CompressionType.LZ4)); + DataNodeTableCache.getInstance().preUpdateTable(this.COMPACTION_TEST_SG, tsTable, null); + DataNodeTableCache.getInstance() + .commitUpdateTable(this.COMPACTION_TEST_SG, tableName, null, null); DataNodeTableCache.getInstance().preUpdateTable(TABLE_MODEL_TEST_SG, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null); + DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null, null); } private TsFileResource generateInt32TableFile(TimeRange timeRange, boolean seq) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java index 877756b73d5b7..e372b0b43ace9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java @@ -170,6 +170,6 @@ public void createTable(String tableName, long ttl) { new FieldColumnSchema("s1", TSDataType.STRING, TSEncoding.PLAIN, CompressionType.LZ4)); tsTable.addProp(TsTable.TTL_PROPERTY, ttl + ""); DataNodeTableCache.getInstance().preUpdateTable(TABLE_MODEL_TEST_SG, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null); + DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null, null); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java index 49f22af864070..8d43c77970cd4 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformerTest.java @@ -190,7 +190,7 @@ public void testRedoRelationalInsertPlan() throws Exception { testTable1.addColumnSchema( new FieldColumnSchema("m1", TSDataType.INT32, TSEncoding.RLE, CompressionType.GZIP)); DataNodeTableCache.getInstance().preUpdateTable(SG_NAME, testTable1, null); - DataNodeTableCache.getInstance().commitUpdateTable(SG_NAME, "table1", null); + DataNodeTableCache.getInstance().commitUpdateTable(SG_NAME, "table1", null, null); // generate crashed .tsfile File file = new File(FILE_NAME); Files.createDirectories(file.getParentFile().toPath()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileTableSchemaTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileTableSchemaTest.java index 306f3050c4882..602ab575f6ec1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileTableSchemaTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileTableSchemaTest.java @@ -87,7 +87,7 @@ public void setUp() throws Exception { table.addColumnSchema( new FieldColumnSchema("value", TSDataType.INT32, TSEncoding.PLAIN, CompressionType.LZ4)); DataNodeTableCache.getInstance().preUpdateTable(DATABASE, table, null); - DataNodeTableCache.getInstance().commitUpdateTable(DATABASE, TABLE, null); + DataNodeTableCache.getInstance().commitUpdateTable(DATABASE, TABLE, null, null); } @After diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/SchemaMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/SchemaMessages.java index 58e7f6fe22012..966512e4bfb2d 100644 --- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/SchemaMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/SchemaMessages.java @@ -38,6 +38,8 @@ public final class SchemaMessages { public static final String CANNOT_REMOVE_TAG_COLUMN = "Cannot remove an tag column: %s"; public static final String UNKNOWN_TABLE_UPDATE_OPERATION_TYPE = "Unknown table update operation type%s"; + public static final String EXCEPTION_UNKNOWN_TREE_TO_TABLE_VIEW_OPERATION_TYPE_ARG_C13CAC3F = + "Unknown tree to table view operation type: %s"; // Column messages public static final String UNSUPPORTED_COLUMN_TYPE_IN_TSFILE = diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/SchemaMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/SchemaMessages.java index 776690108a0df..dcbdec3a1b6c7 100644 --- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/SchemaMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/SchemaMessages.java @@ -38,6 +38,8 @@ public final class SchemaMessages { public static final String CANNOT_REMOVE_TAG_COLUMN = "无法移除 tag 列：%s"; public static final String UNKNOWN_TABLE_UPDATE_OPERATION_TYPE = "未知的 table 更新操作类型%s"; + public static final String EXCEPTION_UNKNOWN_TREE_TO_TABLE_VIEW_OPERATION_TYPE_ARG_C13CAC3F = + "未知的树转表视图操作类型：%s"; // Column 相关消息 public static final String UNSUPPORTED_COLUMN_TYPE_IN_TSFILE = diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewTrieOperationType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewTrieOperationType.java new file mode 100644 index 0000000000000..c8e8855bd8550 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeToTableViewTrieOperationType.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.schema.table; + +import org.apache.iotdb.commons.i18n.SchemaMessages; + +public enum TreeToTableViewTrieOperationType { + /** Register the view, or replace the entry stored for the same view. */ + ADD((byte) 0), + /** Unregister the view. */ + REMOVE((byte) 1), + + /** Replace the registered entry of the view with the one computed from the new table. */ + ALTER_TAG((byte) 2), + + ALTER_RENAME((byte) 3), + + ALTER_AUTO_SYNC((byte) 4); + + private final byte operationType; + + TreeToTableViewTrieOperationType(final byte operationType) { + this.operationType = operationType; + } + + public byte getOperationType() { + return operationType; + } + + public static TreeToTableViewTrieOperationType getType(final byte type) { + switch (type) { + case -1: + return null; + case 0: + return ADD; + case 1: + return REMOVE; + case 2: + return ALTER_TAG; + case 3: + return ALTER_RENAME; + case 4: + return ALTER_AUTO_SYNC; + default: + throw new IllegalArgumentException( + String.format( + SchemaMessages.EXCEPTION_UNKNOWN_TREE_TO_TABLE_VIEW_OPERATION_TYPE_ARG_C13CAC3F, + type)); + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java index 2acbb993254d7..e1068a21d1558 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TreeViewSchema.java @@ -27,7 +27,9 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.rpc.TSStatusCode; +import java.util.HashSet; import java.util.Objects; +import java.util.Set; public class TreeViewSchema { public static final String ORIGINAL_NAME = "__original_name"; @@ -83,6 +85,23 @@ public static void setOriginalName(final TsTableColumnSchema schema, final Strin schema.getProps().put(ORIGINAL_NAME, name); } + /** + * A source measurement can only be referenced once, so this set decides whether a column may + * still be added for one. + */ + public static Set getSourceNames(final TsTable view) { + final Set sourceNames = new HashSet<>(); + for (final TsTableColumnSchema schema : view.getColumnList()) { + if (Objects.isNull(schema.getProps())) { + // the columns which carry no props reference their own name + sourceNames.add(schema.getColumnName()); + } else { + sourceNames.add(getSourceName(schema)); + } + } + return sourceNames; + } + public static boolean isRestrict(final TsTable table) { return table.getPropValue(RESTRICT).isPresent(); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 681c638bc563e..402045012e4fd 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -68,6 +68,7 @@ public class TsTable { // No column property key is supported yet. public static final Set COLUMN_ALLOWED_PROPERTIES = Collections.emptySet(); public static final String NEED_LAST_CACHE_PROPERTY = "need_last_cache"; + public static final String AUTO_SYNC_SCHEMA = "auto_sync_schema"; public static final Set TABLE_ALLOWED_PROPERTIES = Collections.unmodifiableSet( new HashSet<>(Arrays.asList(TTL_PROPERTY, NEED_LAST_CACHE_PROPERTY))); diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 0b733829d87b1..a635bf0db9bb7 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1368,6 +1368,23 @@ struct TCreateTableViewReq { 2: required bool replace } + struct TAutoSyncViewColumnsFailure { + 1: required string database + 2: required string tableName + 3: required common.TSStatus status + } + + +struct TAutoSyncViewColumnsResp{ + 1: required common.TSStatus status + 2: optional list failedViews +} + +struct TAutoSyncViewColumnsReq{ + 1:required list viewInfos +} + + service IConfigNodeRPCService { // ====================================================== @@ -2171,4 +2188,6 @@ service IConfigNodeRPCService { // Table view common.TSStatus createTableView(TCreateTableViewReq req) + + TAutoSyncViewColumnsResp autoSyncViewColumns(TAutoSyncViewColumnsReq req) } diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 76c0860a24fa8..57afa70e72846 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -356,6 +356,14 @@ struct TUpdateTableReq { 1: required byte type 2: required binary tableInfo 3: optional string oldName + 4: optional TUpdateTree2TableView updateTree2TableViewInfo +} + +struct TUpdateTree2TableView{ + 1: required byte tree2TableOperationType + 2: optional string treePath + 3: optional bool isRestrict + 4: optional i32 tagNum } struct TInvalidateTableCacheReq { @@ -450,6 +458,7 @@ struct TDeviceViewReq { 3: required i32 tagNumber 4: required bool restrict 5: optional set requiredMeasurements + 6: optional bool tolerateTypeMismatch } struct TDeviceViewResp {