diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java index 90174941e54e4..e5db58936828d 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java @@ -56,6 +56,12 @@ public DataNodeConfig setMetricReporterType(List metricReporterTypes) { return this; } + @Override + public DataNodeConfig setMetricPrometheusReporterAsyncUpdate(boolean asyncUpdate) { + properties.setProperty("prometheus_reporter_async_update", String.valueOf(asyncUpdate)); + return this; + } + @Override public DataNodeConfig setMetricPrometheusReporterUsername(String username) { properties.setProperty("metric_prometheus_reporter_username", username); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java index 95e4b23b3e73f..0eb829a265e97 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java @@ -28,6 +28,11 @@ public DataNodeConfig setMetricReporterType(List metricReporterTypes) { return this; } + @Override + public DataNodeConfig setMetricPrometheusReporterAsyncUpdate(boolean asyncUpdate) { + return this; + } + @Override public DataNodeConfig setMetricPrometheusReporterUsername(String username) { return this; diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java index 6f65e18cdeac8..c7d148bc2cbfe 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java @@ -25,6 +25,8 @@ public interface DataNodeConfig { DataNodeConfig setMetricReporterType(List metricReporterTypes); + DataNodeConfig setMetricPrometheusReporterAsyncUpdate(boolean asyncUpdate); + DataNodeConfig setMetricPrometheusReporterUsername(String username); DataNodeConfig setMetricPrometheusReporterPassword(String password); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeNumericRowIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeNumericRowIT.java new file mode 100644 index 0000000000000..cc2e614b43a86 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeNumericRowIT.java @@ -0,0 +1,443 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.cluster.RegionRoleType; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.NumericRowPreparedStatement; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.session.TableSessionBuilder; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.LocalDate; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +@RunWith(IoTDBTestRunner.class) +@Category(MultiClusterIT2DualTableManualBasic.class) +public class IoTDBPipeNumericRowIT extends AbstractPipeTableModelDualManualIT { + private static final String DATABASE = "numeric_sync"; + private static final List NAMES = + Arrays.asList("d", "device", "f", "l", "region", "i", "b", "event_date", "event_time"); + private static final List TYPES = + Arrays.asList( + TSDataType.DOUBLE, + TSDataType.STRING, + TSDataType.FLOAT, + TSDataType.INT64, + TSDataType.STRING, + TSDataType.INT32, + TSDataType.BOOLEAN, + TSDataType.DATE, + TSDataType.TIMESTAMP); + private static final List CATEGORIES = + Arrays.asList( + ColumnCategory.FIELD, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + + @Override + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + setupConfig(); + senderEnv + .getConfig() + .getCommonConfig() + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setDataRegionGroupExtensionPolicy("CUSTOM") + .setDefaultDataRegionGroupNumPerDatabase(1) + .setDataReplicationFactor(3) + .setSchemaReplicationFactor(3); + senderEnv + .getConfig() + .getDataNodeConfig() + .setMetricReporterType(Collections.singletonList("PROMETHEUS")); + receiverEnv + .getConfig() + .getCommonConfig() + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + senderEnv.initClusterEnvironment(1, 3); + receiverEnv.initClusterEnvironment(1, 1); + } + + @Test + public void testRealtimeWithoutBatch() throws Exception { + // Unflushed prepared rows must arrive through individual InsertNode requests. + testRealtime(false); + } + + @Test + public void testRealtimeWithBatch() throws Exception { + // Batch transfer must preserve all numeric types, FIELD null bits, and null/empty STRING TAGs. + testRealtime(true); + } + + private void testRealtime(boolean batch) throws Exception { + createTable(); + createPipe(batch, false); + writeRows(0, 32); + assertRowsEventually(receiverEnv, 0, 32); + // The destination must remain readable after its own MemTable has been flushed. + try (ITableSession session = receiverEnv.getTableSessionConnection()) { + session.executeNonQueryStatement("FLUSH"); + } + assertRowsEventually(receiverEnv, 0, 32); + } + + @Test + public void testHistoricalTsFile() throws Exception { + // Start the pipe after flushing so the historical TsFile path also carries numeric-row data. + createTable(); + writeRows(0, 32); + try (ITableSession session = senderEnv.getTableSessionConnection()) { + session.executeNonQueryStatement("FLUSH"); + } + createPipe(true, true); + assertRowsEventually(receiverEnv, 0, 32); + } + + @Test + public void testIoTConsensusReplicaAndRestart() throws Exception { + // Pin writes to the actual DataRegion leader, then stop that leader before querying followers. + // Merely stopping a client entry node can still leave all reads on the original leader. + createTable(); + writeRows(0, 16); + int leader = findDataRegionLeader(); + awaitReplication(leader, 16); + int offlineFollower = (leader + 1) % 3; + senderEnv.shutdownDataNode(offlineFollower); + try { + awaitNodeStatus(offlineFollower, NodeStatus.Unknown); + writeRows(16, 32, leader); + } finally { + senderEnv.startDataNode(offlineFollower); + } + awaitNodeStatus(offlineFollower, NodeStatus.Running); + // Running only confirms node health; IoTConsensus catch-up must finish before losing its + // sender. + awaitReplication(leader, 32); + // Resolve the leader again after restart instead of assuming its assignment is unchanged. + int stoppedLeader = findDataRegionLeader(); + assertEquals( + "The primary that accepted the second write batch must be stopped", leader, stoppedLeader); + senderEnv.shutdownDataNode(stoppedLeader); + try { + assertFalse(senderEnv.getDataNodeWrapper(stoppedLeader).isAlive()); + awaitNodeStatus(stoppedLeader, NodeStatus.Unknown); + for (int follower = 0; follower < 3; follower++) { + if (follower == stoppedLeader) { + continue; + } + assertFalse(senderEnv.getDataNodeWrapper(stoppedLeader).isAlive()); + assertRowsEventually(senderEnv, follower, 32); + assertFalse(senderEnv.getDataNodeWrapper(stoppedLeader).isAlive()); + System.out.printf( + "NumericRow IoTConsensus: stopped primary port=%d, query follower port=%d, verified rows=32%n", + senderEnv.getDataNodeWrapper(stoppedLeader).getPort(), + senderEnv.getDataNodeWrapper(follower).getPort()); + } + } finally { + senderEnv.startDataNode(stoppedLeader); + } + } + + private void awaitReplication(int leader, int expectedWrites) { + String metricsUrl = + "http://" + + senderEnv.getDataNodeWrapper(leader).getIp() + + ":" + + senderEnv.getDataNodeWrapper(leader).getMetricPort() + + "/metrics"; + await() + .atMost(90, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .untilAsserted( + () -> { + String metrics = senderEnv.getUrlContent(metricsUrl, null); + assertNotNull(metrics); + // Require the latest write index too: an old zero-lag sample is not catch-up proof. + // This database has exactly one DataRegion and each numeric row adds one write. + double searchIndex = -1; + double safeIndex = -1; + for (String line : metrics.split("\\R")) { + if (!line.startsWith("iot_consensus{")) { + continue; + } + double value = Double.parseDouble(line.substring(line.lastIndexOf(' ') + 1)); + if (line.contains("type=\"searchIndex\"")) { + searchIndex = value; + } else if (line.contains("type=\"safeIndex\"")) { + safeIndex = value; + } + } + assertTrue( + "Missing or stale searchIndex at " + metricsUrl, searchIndex >= expectedWrites); + assertEquals("Replication is incomplete at " + metricsUrl, searchIndex, safeIndex, 0); + }); + } + + private void awaitNodeStatus(int node, NodeStatus expected) { + // BaseEnv.ensureNodeStatus uses EnvFactory, which cannot be used with MultiCluster + // environments. + await() + .atMost(90, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + TShowClusterResp response = client.showCluster(); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode()); + List matches = + response.getDataNodeList().stream() + .filter( + location -> + location.getClientRpcEndPoint().getPort() + == senderEnv.getDataNodeWrapper(node).getPort()) + .toList(); + assertEquals(1, matches.size()); + assertEquals( + expected.getStatus(), + response.getNodeStatus().get(matches.get(0).getDataNodeId())); + } + }); + } + + private int findDataRegionLeader() throws Exception { + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + TShowRegionResp response = + client.showRegion( + new TShowRegionReq() + .setConsensusGroupType(TConsensusGroupType.DataRegion) + .setDatabases(Collections.singletonList(DATABASE)) + .setIsTableModel(true)); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode()); + // Exactly one three-replica region ensures stopping this leader covers all inserted rows. + List replicas = response.getRegionInfoList(); + assertEquals(3, replicas.size()); + for (TRegionInfo replica : replicas) { + assertEquals(replicas.get(0).getConsensusGroupId(), replica.getConsensusGroupId()); + } + List leaders = + replicas.stream() + .filter(replica -> RegionRoleType.Leader.getRoleType().equals(replica.getRoleType())) + .toList(); + assertEquals(1, leaders.size()); + TRegionInfo leader = leaders.get(0); + for (int node = 0; node < 3; node++) { + if (senderEnv.getDataNodeWrapper(node).getPort() == leader.getClientRpcPort()) { + System.out.printf( + "NumericRow IoTConsensus: region=%d, primary DataNodeId=%d, port=%d%n", + leader.getConsensusGroupId().getId(), + leader.getDataNodeId(), + leader.getClientRpcPort()); + return node; + } + } + throw new AssertionError("No wrapper for DataRegion leader " + leader); + } + } + + private void createTable() throws Exception { + try (ITableSession session = senderEnv.getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE " + DATABASE); + session.executeNonQueryStatement("USE " + DATABASE); + session.executeNonQueryStatement( + "CREATE TABLE readings(region STRING TAG, device STRING TAG, missing STRING TAG, " + + "d DOUBLE FIELD, f FLOAT FIELD, l INT64 FIELD, i INT32 FIELD, b BOOLEAN FIELD, " + + "event_date DATE FIELD, event_time TIMESTAMP FIELD)"); + } + } + + private void createPipe(boolean batch, boolean history) throws Exception { + Map source = new HashMap<>(); + source.put("capture.table", "true"); + source.put("capture.tree", "false"); + source.put("__system.sql-dialect", "table"); + source.put("user", "root"); + source.put("database-name", DATABASE); + source.put("table-name", "readings"); + source.put("source.realtime.mode", "log"); + source.put("source.history.enable", Boolean.toString(history)); + Map sink = new HashMap<>(); + sink.put("sink", "iotdb-thrift-sink"); + sink.put("sink.ip", receiverEnv.getDataNodeWrapper(0).getIp()); + sink.put("sink.port", Integer.toString(receiverEnv.getDataNodeWrapper(0).getPort())); + sink.put("sink.batch.enable", Boolean.toString(batch)); + sink.put("sink.batch.max-delay-seconds", "1"); + sink.put("sink.format", history ? "tsfile" : "tablet"); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("numeric_pipe", sink) + .setExtractorAttributes(source) + .setProcessorAttributes(Collections.emptyMap())) + .getCode()); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("numeric_pipe").getCode()); + } + } + + private void writeRows(int start, int end) throws Exception { + writeRows(start, end, 0); + } + + private void writeRows(int start, int end, int node) throws Exception { + // Pin the client to the selected endpoint; its index is not necessarily the DataRegion leader. + try (ITableSession session = + new TableSessionBuilder() + .nodeUrls( + Collections.singletonList( + senderEnv.getDataNodeWrapper(node).getIpAndPortString())) + .username(SessionConfig.DEFAULT_USER) + .password(SessionConfig.DEFAULT_PASSWORD) + .database(DATABASE) + .enableRedirection(false) + .build(); + NumericRowPreparedStatement prepared = + session.prepareInsertNumericRow("readings", NAMES, TYPES, CATEGORIES)) { + for (int row = start; row < end; row++) { + long[] values = { + Double.doubleToRawLongBits(row + 0.25), + Float.floatToRawIntBits(-row - 0.5f), + Long.MIN_VALUE + row, + Integer.MIN_VALUE + row, + row % 2, + 20260924, + 1000L + row + }; + String[] tags = {device(row), "region"}; + BitMap nulls = new BitMap(7); + if (row % 3 == 0) { + nulls.mark(row % 7); + } + session.executePreparedInsertNumericRow(prepared, row, values, tags, nulls); + } + } + } + + private static String device(int row) { + return row % 3 == 0 ? null : row % 3 == 1 ? "" : "device-\u8bbe\u5907"; + } + + private void assertRowsEventually(BaseEnv env, int node, int count) { + await() + // Restart and leader failover can temporarily reject JDBC connections or query RPCs. + .ignoreExceptionsInstanceOf(SQLException.class) + .atMost(90, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try (Connection connection = + env.getConnection( + env.getDataNodeWrapper(node), + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + "table"); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE); + try (ResultSet result = + statement.executeQuery( + "SELECT time,region,device,missing,d,f,l,i,b,event_date,event_time FROM readings ORDER BY time")) { + for (int row = 0; row < count; row++) { + assertTrue("Missing row " + row, result.next()); + assertEquals(row, result.getLong(1)); + assertEquals("region", result.getString(2)); + assertEquals(device(row), result.getString(3)); + assertNull(result.getObject(4)); + for (int field = 0; field < 7; field++) { + int column = field + 5; + if (row % 3 == 0 && field == row % 7) { + assertNull(result.getObject(column)); + continue; + } + assertNotNull(result.getObject(column)); + switch (field) { + case 0 -> assertEquals(row + 0.25, result.getDouble(column), 0); + case 1 -> assertEquals(-row - 0.5f, result.getFloat(column), 0); + case 2 -> assertEquals(Long.MIN_VALUE + row, result.getLong(column)); + case 3 -> assertEquals(Integer.MIN_VALUE + row, result.getInt(column)); + case 4 -> assertEquals(row % 2 != 0, result.getBoolean(column)); + case 5 -> + assertEquals( + LocalDate.of(2026, 9, 24), result.getDate(column).toLocalDate()); + case 6 -> assertEquals(1000L + row, result.getLong(column)); + default -> throw new AssertionError(field); + } + } + } + assertFalse(result.next()); + } + } + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java index b3014ff56ee62..6f6ee44f129a0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java @@ -20,23 +20,55 @@ package org.apache.iotdb.relational.it.session; import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.NumericRowPreparedStatement; import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.ITableSessionPool; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.TableClusterIT; import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.jdbc.Config; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.UrlUtils; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.write.record.Tablet; import org.junit.After; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.lang.reflect.Field; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.SplittableRandom; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.IntConsumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showTablesColumnHeaders; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -45,6 +77,27 @@ public class IoTDBTableModelSessionIT { @Before public void setUp() throws Exception { + if (Boolean.getBoolean("iotdb.numeric.row.insert.perf.enabled")) { + // A saturated single DataNode can delay ConfigNode heartbeats while the benchmark is running. + // Keep the test focused on write-path throughput instead of fencing the node for that delay. + EnvFactory.getEnv().getConfig().getConfigNodeConfig().setMetadataLeaseFenceMs(120_000); + if (Boolean.getBoolean("iotdb.numeric.row.insert.perf.breakdown.enabled")) { + EnvFactory.getEnv() + .getConfig() + .getDataNodeConfig() + .setMetricReporterType(Collections.singletonList("PROMETHEUS")) + .setMetricPrometheusReporterAsyncUpdate(false); + } + if (Boolean.getBoolean("iotdb.numeric.row.insert.perf.io.enabled")) { + // Isolate per-round storage accounting from compaction of earlier rounds. + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false); + } + } EnvFactory.getEnv().initClusterEnvironment(); } @@ -53,6 +106,1045 @@ public void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + @Test + public void testPreparedNumericRowInsert() throws Exception { + // Exercise direct and pooled table sessions, schema reordering and missing STRING columns. + // The same values must remain readable from MemTable and after flushing to TsFile. + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE numeric_rows"); + session.executeNonQueryStatement("USE numeric_rows"); + session.executeNonQueryStatement( + "CREATE TABLE readings(tag STRING TAG, attr STRING ATTRIBUTE, " + + "event_time TIMESTAMP FIELD, event_date DATE FIELD, d DOUBLE FIELD, " + + "f FLOAT FIELD, l INT64 FIELD, i INT32 FIELD, b BOOLEAN FIELD, missing STRING FIELD)"); + List names = Arrays.asList("b", "i", "l", "f", "d", "event_date", "event_time"); + List types = + Arrays.asList( + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.DATE, + TSDataType.TIMESTAMP); + long[] values = { + -1, + Integer.MIN_VALUE, + Long.MIN_VALUE, + Float.floatToRawIntBits(-1.25f), + Double.doubleToRawLongBits(2.5), + 20260910, + 1234 + }; + try (NumericRowPreparedStatement rowStatement = + session.prepareInsertNumericRow( + "readings", names, types, Collections.nCopies(types.size(), ColumnCategory.FIELD))) { + session.executePreparedInsertNumericRow(rowStatement, 1, values, new String[0]); + } + try (ITableSessionPool pool = EnvFactory.getEnv().getTableSessionPool(1, "numeric_rows"); + ITableSession pooledSession = pool.getSession(); + NumericRowPreparedStatement pooledRow = + pooledSession.prepareInsertNumericRow( + "readings", + names, + types, + Collections.nCopies(types.size(), ColumnCategory.FIELD))) { + pooledSession.executePreparedInsertNumericRow(pooledRow, 2, values, new String[0]); + } + assertNumericRows(session); + session.executeNonQueryStatement("FLUSH"); + assertNumericRows(session); + } + } + + @Test + public void testPreparedNumericRowInsertWithTags() throws Exception { + // Different TAGs at the same timestamp must stay separate, with identical behavior after flush. + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE numeric_tag_rows"); + session.executeNonQueryStatement("USE numeric_tag_rows"); + session.executeNonQueryStatement( + "CREATE TABLE readings(region STRING TAG, device STRING TAG, " + + "missing STRING TAG, v INT64 FIELD, enabled BOOLEAN FIELD)"); + List names = Arrays.asList("enabled", "device", "v", "region"); + List types = + Arrays.asList(TSDataType.BOOLEAN, TSDataType.STRING, TSDataType.INT64, TSDataType.STRING); + List categories = + Arrays.asList( + ColumnCategory.FIELD, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.TAG); + try (NumericRowPreparedStatement rowStatement = + session.prepareInsertNumericRow("readings", names, types, categories)) { + session.executePreparedInsertNumericRow( + rowStatement, 1, new long[] {1, 10}, new String[] {"设备甲😀", "华北"}); + } + try (ITableSessionPool pool = EnvFactory.getEnv().getTableSessionPool(1, "numeric_tag_rows"); + ITableSession pooled = pool.getSession()) { + try (NumericRowPreparedStatement rowStatement = + pooled.prepareInsertNumericRow("readings", names, types, categories)) { + pooled.executePreparedInsertNumericRow( + rowStatement, 1, new long[] {0, 20}, new String[] {"设备乙", "华北"}); + } + } + try (NumericRowPreparedStatement rowStatement = + session.prepareInsertNumericRow("readings", names, types, categories)) { + session.executePreparedInsertNumericRow( + rowStatement, 2, new long[] {1, 30}, new String[] {"", null}); + } + try (NumericRowPreparedStatement rowStatement = + session.prepareInsertNumericRow("readings", names, types, categories)) { + session.executePreparedInsertNumericRow( + rowStatement, 2, new long[] {0, 40}, new String[] {null, null}); + } + assertNumericTagRows(session); + session.executeNonQueryStatement("FLUSH"); + assertNumericTagRows(session); + } + } + + @Test + public void testPreparedNumericRowSchemaVersionChange() throws Exception { + // Refresh compatible FIELD additions, validate TAG padding, then verify a flush/read round + // trip. + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE numeric_version"); + session.executeNonQueryStatement("USE numeric_version"); + session.executeNonQueryStatement("CREATE TABLE readings (tag STRING TAG, v DOUBLE FIELD)"); + NumericRowPreparedStatement prepared = + session.prepareInsertNumericRow( + "readings", + Arrays.asList("tag", "v"), + Arrays.asList(TSDataType.STRING, TSDataType.DOUBLE), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); + for (int time = 1; time <= 5; time++) { + if (time == 2) { + session.executeNonQueryStatement("ALTER TABLE readings ADD COLUMN extra INT64 FIELD"); + } + if (time == 4) { + session.executeNonQueryStatement("ALTER TABLE readings ADD COLUMN tag2 STRING TAG"); + } + session.executePreparedInsertNumericRow( + prepared, + time, + new long[] {Double.doubleToRawLongBits(time + 0.5)}, + new String[] {"device"}); + } + session.executeNonQueryStatement("FLUSH"); + try (SessionDataSet dataSet = + session.executeQueryStatement("SELECT time,tag,tag2,v FROM readings ORDER BY time")) { + for (int time = 1; time <= 5; time++) { + assertTrue(dataSet.hasNext()); + RowRecord row = dataSet.next(); + assertEquals(time, row.getFields().get(0).getLongV()); + assertEquals("device", row.getFields().get(1).getStringValue()); + assertNull(row.getFields().get(2).getDataType()); + assertEquals(time + 0.5, row.getFields().get(3).getDoubleV(), 0); + } + assertFalse(dataSet.hasNext()); + } + } + } + + @Test + public void testNumericRowNullBitmap() throws Exception { + // Exercise bits 0/7/8, interleaved TAG metadata, prepared fast/fallback paths and pooled + // sessions. + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE numeric_nulls"); + session.executeNonQueryStatement("USE numeric_nulls"); + StringBuilder ddl = new StringBuilder("CREATE TABLE readings(tag STRING TAG"); + List names = new ArrayList<>(); + List types = new ArrayList<>(); + List categories = new ArrayList<>(); + long[] values = new long[9]; + for (int i = 0; i < 9; i++) { + ddl.append(", f").append(i).append(" DOUBLE FIELD"); + names.add("f" + i); + types.add(TSDataType.DOUBLE); + categories.add(ColumnCategory.FIELD); + values[i] = Double.doubleToRawLongBits(i + 0.25); + } + session.executeNonQueryStatement(ddl.append(")").toString()); + names.add(1, "tag"); + types.add(1, TSDataType.STRING); + categories.add(1, ColumnCategory.TAG); + BitMap nulls = new BitMap(9); + nulls.mark(0); + nulls.mark(7); + nulls.mark(8); + String[] tags = {"device"}; + try (NumericRowPreparedStatement prepared = + session.prepareInsertNumericRow("readings", names, types, categories)) { + session.executePreparedInsertNumericRow(prepared, 1, values, tags, nulls); + session.executePreparedInsertNumericRow(prepared, 2, values, tags, nulls); + session.executePreparedInsertNumericRow(prepared, 3, values, tags); + // All-null table rows retain their timestamp/TAGs; null slots must never become zeros. + BitMap allNulls = new BitMap(9); + allNulls.markAll(); + session.executePreparedInsertNumericRow(prepared, 3, values, tags, allNulls); + session.executePreparedInsertNumericRow(prepared, 10, values, tags, allNulls); + session.executePreparedInsertNumericRow(prepared, 11, values, tags, allNulls); + session.executeNonQueryStatement("ALTER TABLE readings ADD COLUMN tag2 STRING TAG"); + session.executePreparedInsertNumericRow(prepared, 4, values, tags, nulls); + } + try (ITableSessionPool pool = EnvFactory.getEnv().getTableSessionPool(1, "numeric_nulls"); + ITableSession pooled = pool.getSession(); + NumericRowPreparedStatement prepared = + pooled.prepareInsertNumericRow("readings", names, types, categories)) { + pooled.executePreparedInsertNumericRow(prepared, 5, values, tags, nulls); + pooled.executePreparedInsertNumericRow(prepared, 6, values, tags, nulls); + } + assertNumericNullRows(session); + session.executeNonQueryStatement("FLUSH"); + assertNumericNullRows(session); + } + } + + // Failed schema validation closes only its handle; fixing the table still requires a new prepare. + @Test + public void testPreparedNumericRowClosesOnSchemaFailure() throws Exception { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE numeric_invalid"); + session.executeNonQueryStatement("USE numeric_invalid"); + session.executeNonQueryStatement("CREATE TABLE readings(tag STRING TAG, v DOUBLE FIELD)"); + List names = Arrays.asList("tag", "v"); + List types = Arrays.asList(TSDataType.STRING, TSDataType.DOUBLE); + List categories = Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD); + long[] values = {Double.doubleToRawLongBits(2.5)}; + String[] tags = {"device"}; + try (NumericRowPreparedStatement prepared = + session.prepareInsertNumericRow("readings", names, types, categories); + NumericRowPreparedStatement other = + session.prepareInsertNumericRow("readings", names, types, categories)) { + session.executeNonQueryStatement("DROP TABLE readings"); + session.executeNonQueryStatement("CREATE TABLE readings(tag STRING TAG, v BOOLEAN FIELD)"); + StatementExecutionException schemaError = + assertThrows( + StatementExecutionException.class, + () -> session.executePreparedInsertNumericRow(prepared, 1, values, tags)); + assertTrue(schemaError.getMessage().contains("v")); + session.executeNonQueryStatement("DROP TABLE readings"); + session.executeNonQueryStatement("CREATE TABLE readings(tag STRING TAG, v DOUBLE FIELD)"); + IoTDBConnectionException closedError = + assertThrows( + IoTDBConnectionException.class, + () -> session.executePreparedInsertNumericRow(prepared, 2, values, tags)); + assertTrue(prepared.isClosed()); + assertTrue(closedError.getMessage().contains("closed")); + session.executePreparedInsertNumericRow(other, 3, values, tags); + try (NumericRowPreparedStatement replacement = + session.prepareInsertNumericRow("readings", names, types, categories)) { + session.executePreparedInsertNumericRow(replacement, 4, values, tags); + } + try (SessionDataSet dataSet = + session.executeQueryStatement("SELECT v FROM readings ORDER BY time")) { + for (int i = 0; i < 2; i++) { + assertTrue(dataSet.hasNext()); + assertEquals(2.5, dataSet.next().getFields().get(0).getDoubleV(), 0); + } + assertFalse(dataSet.hasNext()); + } + } + } + } + + private void assertNumericNullRows(ITableSession session) throws Exception { + try (SessionDataSet dataSet = + session.executeQueryStatement( + "SELECT time,tag,tag2,f0,f1,f2,f3,f4,f5,f6,f7,f8 FROM readings ORDER BY time")) { + for (int time : new int[] {1, 2, 3, 4, 5, 6, 10, 11}) { + assertTrue(dataSet.hasNext()); + RowRecord row = dataSet.next(); + assertEquals(time, row.getFields().get(0).getLongV()); + assertEquals("device", row.getFields().get(1).getStringValue()); + assertNull(row.getFields().get(2).getDataType()); + for (int field = 0; field < 9; field++) { + if (time >= 10 || (time != 3 && (field == 0 || field == 7 || field == 8))) { + assertNull(row.getFields().get(field + 3).getDataType()); + } else { + assertEquals(field + 0.25, row.getFields().get(field + 3).getDoubleV(), 0); + } + } + } + assertFalse(dataSet.hasNext()); + } + } + + // Pin clients to every DataNode so at least one must learn a remote leader in a cluster. + @Test + public void testPreparedNumericRowRedirection() throws Exception { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE numeric_redirect"); + session.executeNonQueryStatement("USE numeric_redirect"); + session.executeNonQueryStatement( + "CREATE TABLE readings(tag STRING TAG, v INT64 FIELD, n INT64 FIELD)"); + int clientIndex = 0; + boolean redirected = false; + for (var node : EnvFactory.getEnv().getDataNodeWrapperList()) { + try (ITableSession pinned = + EnvFactory.getEnv().getTableSessionConnection(List.of(node.getIpAndPortString()))) { + pinned.executeNonQueryStatement("USE numeric_redirect"); + redirected |= writeRedirectedNumericRows(pinned, clientIndex++ * 1000L); + } + } + try (ITableSessionPool pool = EnvFactory.getEnv().getTableSessionPool(1, "numeric_redirect"); + ITableSession pooled = pool.getSession()) { + redirected |= writeRedirectedNumericRows(pooled, clientIndex++ * 1000L); + } + if (EnvFactory.getEnv().getDataNodeWrapperList().size() > 1) { + assertTrue("Expected a prepared handle on a remote leader", redirected); + } + session.executeNonQueryStatement("FLUSH"); + try (SessionDataSet result = + session.executeQueryStatement("SELECT count(v),count(n) FROM readings")) { + assertTrue(result.hasNext()); + RowRecord counts = result.next(); + assertEquals(clientIndex * 32L, counts.getFields().get(0).getLongV()); + assertEquals(clientIndex * 16L, counts.getFields().get(1).getLongV()); + assertFalse(result.hasNext()); + } + } + } + + private boolean writeRedirectedNumericRows(ITableSession session, long start) throws Exception { + try (NumericRowPreparedStatement prepared = + session.prepareInsertNumericRow( + "readings", + List.of("tag", "v", "n"), + List.of(TSDataType.STRING, TSDataType.INT64, TSDataType.INT64), + List.of(ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD))) { + BitMap nulls = new BitMap(2); + nulls.mark(1); + for (int round = 0; round < 4; round++) { + for (int device = 0; device < 8; device++) { + long time = start + round * 8 + device; + session.executePreparedInsertNumericRow( + prepared, + time, + new long[] {time, time}, + new String[] {"d" + device}, + round % 2 == 0 ? nulls : null); + } + } + // Count actual physical bindings: row counts alone cannot prove the client followed a hint. + Field bindings = NumericRowPreparedStatement.class.getDeclaredField("connectionBindings"); + bindings.setAccessible(true); + return ((Map) bindings.get(prepared)).size() > 1; + } + } + + @Test + public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { + compareNumericRowWritePerformance(false); + } + + @Test + public void testPreparedNumericRowAndTabletSparseWritePerformance() throws Exception { + compareNumericRowWritePerformance(true); + } + + @Test + public void testPreparedNumericRowAndTabletWriteRatioPerformance() throws Exception { + Assume.assumeTrue( + "Performance benchmark disabled; set iotdb.numeric.row.insert.perf.enabled=true to run", + Boolean.getBoolean("iotdb.numeric.row.insert.perf.enabled")); + final int fieldCount = Integer.getInteger("iotdb.numeric.row.insert.perf.fields", 99); + boolean first = true; + for (int ratio : configuredSparseWriteRatios()) { + if (!first) { + // Recreate the instance so earlier ratios cannot contribute flush or compaction work. + EnvFactory.getEnv().cleanClusterEnvironment(); + EnvFactory.getEnv() + .setTestMethodName("testPreparedNumericRowAndTabletWriteRatioPerformance_" + ratio); + setUp(); + } + first = false; + int writtenFields = Math.max(1, (fieldCount * ratio + 99) / 100); + compareNumericRowWritePerformance(true, writtenFields, "sparse-" + ratio + "%"); + } + } + + private void compareNumericRowWritePerformance(boolean sparse) throws Exception { + int writtenFields = + sparse ? Integer.getInteger("iotdb.numeric.row.insert.perf.sparse.fields", 10) : 99; + compareNumericRowWritePerformance(sparse, writtenFields, sparse ? "sparse" : "full-row"); + } + + private void compareNumericRowWritePerformance( + boolean sparse, int writtenFields, String scenarioName) throws Exception { + // Compare synchronous single-row paths and optionally account for their Linux TCP/storage I/O. + Assume.assumeTrue( + "Performance benchmark disabled; set iotdb.numeric.row.insert.perf.enabled=true to run", + Boolean.getBoolean("iotdb.numeric.row.insert.perf.enabled")); + final int concurrency = Integer.getInteger("iotdb.numeric.row.insert.perf.concurrency", 16); + // Compare sustained single-row writes after warming both paths; verify all acknowledged + // rows. + final int warmupRounds = 2; + final int measuredRounds = 6; + final int fieldCount = Integer.getInteger("iotdb.numeric.row.insert.perf.fields", 99); + final int deviceCount = + sparse ? Integer.getInteger("iotdb.numeric.row.insert.perf.sparse.devices", 100) : 1; + final long seed = Long.getLong("iotdb.numeric.row.insert.perf.sparse.seed", 20261008L); + assertTrue("Concurrency and device count must be positive", concurrency > 0 && deviceCount > 0); + assertTrue( + "Written FIELD count must be positive and within the table schema", + writtenFields > 0 && writtenFields <= fieldCount); + final String databasePrefix = + (sparse ? "numeric_sparse_perf_" : "numeric_perf_") + + scenarioName.replace('%', 'p').replace('-', '_') + + "_"; + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + StringBuilder ddl = new StringBuilder("CREATE TABLE prepared_perf (tag STRING TAG"); + for (int i = 0; i < fieldCount; i++) { + ddl.append(", f").append(i).append(" DOUBLE FIELD"); + } + ddl.append(")"); + // Give each writer its own database so concurrent writes use independent DataRegions. + for (int i = 0; i < concurrency; i++) { + session.executeNonQueryStatement("CREATE DATABASE " + databasePrefix + i); + session.executeNonQueryStatement("USE " + databasePrefix + i); + session.executeNonQueryStatement(ddl.toString()); + session.executeNonQueryStatement(ddl.toString().replace("prepared_perf", "tablet_perf")); + } + } + + List names = new ArrayList<>(); + List types = new ArrayList<>(); + List categories = new ArrayList<>(); + names.add("tag"); + types.add(TSDataType.STRING); + categories.add(ColumnCategory.TAG); + for (int i = 0; i < fieldCount; i++) { + names.add("f" + i); + types.add(TSDataType.DOUBLE); + categories.add(ColumnCategory.FIELD); + } + ITableSession[] sessions = new ITableSession[concurrency]; + NumericRowPreparedStatement[] prepared = new NumericRowPreparedStatement[concurrency]; + Tablet[] tablets = new Tablet[concurrency]; + String[][] tags = new String[concurrency][]; + long[][] numericValues = new long[concurrency][fieldCount]; + long[][] pathRows = new long[2][concurrency]; + SparseNumericRowInput[][] sparseRows = new SparseNumericRowInput[2][concurrency]; + ExecutorService executor = Executors.newFixedThreadPool(concurrency); + WritePerformanceResourceTracker resourceTracker = null; + try { + // Connections, USE, prepare and reusable input allocation are outside every timed round. + for (int i = 0; i < concurrency; i++) { + sessions[i] = EnvFactory.getEnv().getTableSessionConnection(); + sessions[i].executeNonQueryStatement("USE " + databasePrefix + i); + prepared[i] = + sessions[i].prepareInsertNumericRow("prepared_perf", names, types, categories); + tablets[i] = new Tablet("tablet_perf", names, types, categories, 1); + tags[i] = new String[] {"device-" + i}; + Arrays.fill(numericValues[i], Double.doubleToRawLongBits(1.0)); + if (sparse) { + // Independent generators replay the same device/column sequence for each path. + for (int path = 0; path < sparseRows.length; path++) { + sparseRows[path][i] = + new SparseNumericRowInput(deviceCount, fieldCount, writtenFields, seed + i); + } + } + } + IntConsumer preparedWrite = + threadId -> { + try { + if (sparse) { + SparseNumericRowInput input = sparseRows[0][threadId]; + sessions[threadId].executePreparedInsertNumericRow( + prepared[threadId], + pathRows[0][threadId], + numericValues[threadId], + input.tagValues, + input.nulls); + } else { + sessions[threadId].executePreparedInsertNumericRow( + prepared[threadId], + pathRows[0][threadId], + numericValues[threadId], + tags[threadId]); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + IntConsumer tabletPrepare = + threadId -> { + Tablet tablet = tablets[threadId]; + tablet.addTimestamp(0, pathRows[1][threadId]); + SparseNumericRowInput input = sparseRows[1][threadId]; + if (sparse) { + input.next(); + } + tablet.addValue("tag", 0, sparse ? input.tagValues[0] : tags[threadId][0]); + for (int i = 0; i < fieldCount; i++) { + if (sparse && input.nulls.isMarked(i)) { + tablet.addValue(names.get(i + 1), 0, null); + } else { + tablet.addValue(names.get(i + 1), 0, 1.0D); + } + } + }; + IntConsumer[] paths = { + preparedWrite, + threadId -> { + try { + sessions[threadId].insert(tablets[threadId]); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }; + IntConsumer[] prepareWrites = { + sparse ? threadId -> sparseRows[0][threadId].next() : null, tabletPrepare + }; + IntConsumer[] completeWrites = { + threadId -> { + if (sparse) sparseRows[0][threadId].acknowledge(); + pathRows[0][threadId]++; + }, + threadId -> { + if (sparse) sparseRows[1][threadId].acknowledge(); + tablets[threadId].reset(); + pathRows[1][threadId]++; + } + }; + String[] pathNames = {"prepared numeric row", "insertTablet batch=1"}; + System.out.printf( + "configuration: scenario=%s, concurrency=%d, tabletBatch=1, devicesPerWriter=%d, " + + "totalFields=%d, writtenFields=%d, actualWriteRatio=%.2f%%, seed=%d%n", + scenarioName, + concurrency, + deviceCount, + fieldCount, + writtenFields, + writtenFields * 100.0 / fieldCount, + seed); + WritePerformanceIoTracker ioTracker = null; + if (Boolean.getBoolean("iotdb.numeric.row.insert.perf.io.enabled")) { + assertEquals( + "I/O measurement requires one local DataNode", + 1, + EnvFactory.getEnv().getDataNodeWrapperList().size()); + assertEquals( + "I/O measurement requires one local ConfigNode", + 1, + EnvFactory.getEnv().getConfigNodeWrapperList().size()); + ioTracker = + new WritePerformanceIoTracker( + EnvFactory.getEnv().getDataNodeWrapperList().get(0).getPid(), + EnvFactory.getEnv().getConfigNodeWrapperList().get(0).getPid(), + EnvFactory.getEnv().getDataNodeWrapperList().get(0).getPort()); + } + resourceTracker = + new WritePerformanceResourceTracker( + EnvFactory.getEnv().getDataNodeWrapperList().get(0).getPid()); + double[][] rates = new double[paths.length][measuredRounds]; + long[] totalRows = new long[paths.length]; + long[] totalNanos = new long[paths.length]; + double[] peakCpuPercent = new double[paths.length]; + Arrays.fill(peakCpuPercent, Double.NaN); + long[] peakResidentBytes = new long[paths.length]; + LongSamples[] requestLatencies = new LongSamples[paths.length]; + for (int path = 0; path < paths.length; path++) { + requestLatencies[path] = new LongSamples(); + } + Map previousMetrics = + Boolean.getBoolean("iotdb.numeric.row.insert.perf.breakdown.enabled") + ? readPerformanceMetrics() + : Collections.emptyMap(); + for (int round = 0; round < warmupRounds + measuredRounds; round++) { + boolean warmup = round < warmupRounds; + int seconds = warmup ? 3 : 5; + // Rotate both paths so each occupies every position during measured rounds. + for (int position = 0; position < paths.length; position++) { + int path = (round + position) % paths.length; + WritePerformanceIoTracker.Snapshot ioBefore = + ioTracker == null ? null : ioTracker.beforeRound(sessions[0]); + resourceTracker.beginRound(); + WritePerformance result = null; + WritePerformanceResourceTracker.RoundPeak resourcePeak; + try { + result = + runConcurrentWrites( + executor, + concurrency, + seconds, + prepareWrites[path], + paths[path], + completeWrites[path]); + } finally { + resourcePeak = resourceTracker.endRound(); + } + if (ioTracker != null) { + System.out.printf( + "io %s round %d position %d %s: rows=%d, %s%n", + warmup ? "warmup" : "measured", + warmup ? round + 1 : round - warmupRounds + 1, + position + 1, + pathNames[path], + result.rows(), + ioTracker.afterRound(ioBefore, sessions[0])); + } + System.out.printf( + "%s round %d position %d %s: %d rows, %.3f s, %.1f rows/s, API latency %s%n", + warmup ? "warmup" : "measured", + warmup ? round + 1 : round - warmupRounds + 1, + position + 1, + pathNames[path], + result.rows(), + result.nanos() / 1_000_000_000.0, + result.rowsPerSecond(), + summarizeRequestLatencies(result.requestNanos())); + System.out.printf( + "resource %s round %d position %d %s: %s%n", + warmup ? "warmup" : "measured", + warmup ? round + 1 : round - warmupRounds + 1, + position + 1, + pathNames[path], + resourcePeak.format()); + if (Boolean.getBoolean("iotdb.numeric.row.insert.perf.breakdown.enabled")) { + Map currentMetrics = readPerformanceMetrics(); + printPerformanceBreakdown( + warmup ? "warmup" : "measured", + warmup ? round + 1 : round - warmupRounds + 1, + position + 1, + pathNames[path], + previousMetrics, + currentMetrics); + previousMetrics = currentMetrics; + } + if (!warmup) { + rates[path][round - warmupRounds] = result.rowsPerSecond(); + totalRows[path] += result.rows(); + totalNanos[path] += result.nanos(); + requestLatencies[path].appendAll(result.requestNanos()); + if (!Double.isNaN(resourcePeak.peakCpuPercent())) { + peakCpuPercent[path] = + Double.isNaN(peakCpuPercent[path]) + ? resourcePeak.peakCpuPercent() + : Math.max(peakCpuPercent[path], resourcePeak.peakCpuPercent()); + } + peakResidentBytes[path] = + Math.max(peakResidentBytes[path], resourcePeak.peakResidentBytes()); + } + } + } + for (int path = 0; path < paths.length; path++) { + Arrays.sort(rates[path]); + System.out.printf( + "summary %s: median %.1f rows/s, aggregate %.1f rows/s, min %.1f, max %.1f, " + + "measured API latency %s, peak DataNode CPU %s, peak DataNode RSS %s, " + + "scenario=%s, writtenFields=%d/%d, nonNullFieldsPerSecond=%.1f%n", + pathNames[path], + (rates[path][measuredRounds / 2 - 1] + rates[path][measuredRounds / 2]) / 2, + totalRows[path] * 1_000_000_000.0 / totalNanos[path], + rates[path][0], + rates[path][measuredRounds - 1], + summarizeRequestLatencies(requestLatencies[path].toArray()), + formatPeakCpu(peakCpuPercent[path]), + formatPeakMemory(peakResidentBytes[path]), + scenarioName, + writtenFields, + fieldCount, + totalRows[path] * 1_000_000_000.0 / totalNanos[path] * writtenFields); + } + // Per-device monotonic timestamps span warmup and measurement, so no rows overwrite others. + for (int path = 0; path < paths.length; path++) { + String table = + switch (path) { + case 0 -> "prepared_perf"; + default -> "tablet_perf"; + }; + long[] rowsByThread = pathRows[path]; + long expectedRows = Arrays.stream(rowsByThread).sum(); + for (int i = 0; i < concurrency; i++) { + try (SessionDataSet dataSet = + sessions[i].executeQueryStatement("SELECT count(*) FROM " + table)) { + assertTrue(dataSet.hasNext()); + // Each worker writes to its own database, including warm-up rows. + assertEquals(rowsByThread[i], dataSet.next().getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + } + if (sparse) { + // Check counts for every device and FIELD; stale Tablet bits must not fill omitted cells. + for (int i = 0; i < concurrency; i++) { + sparseRows[path][i].verify(sessions[i], table); + } + } + System.out.printf( + "verified %s: %d rows, %d non-null FIELD values%n", + table, expectedRows, expectedRows * writtenFields); + } + } finally { + executor.shutdownNow(); + try { + assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES)); + } finally { + Exception closeFailure = null; + if (resourceTracker != null) { + try { + resourceTracker.close(); + } catch (Exception e) { + closeFailure = e; + } + } + // Closing each session also releases its server-side prepared metadata. + for (ITableSession session : sessions) { + if (session != null) { + try { + session.close(); + } catch (Exception e) { + if (closeFailure == null) closeFailure = e; + else closeFailure.addSuppressed(e); + } + } + } + if (closeFailure != null) throw closeFailure; + } + } + } + + private static final Pattern PERFORMANCE_TIMER_SUM_PATTERN = + Pattern.compile( + "^(performance_overview[^\\s{]+)_sum\\{[^}]*stage=\\\"([^\\\"]+)\\\"[^}]*\\}\\s+([-+0-9.eE]+)$"); + private static final Pattern PERFORMANCE_TIMER_COUNT_PATTERN = + Pattern.compile( + "^(performance_overview[^\\s{]+)_count\\{[^}]*stage=\\\"([^\\\"]+)\\\"[^}]*\\}\\s+([-+0-9.eE]+)$"); + + private static Map readPerformanceMetrics() { + Map values = new LinkedHashMap<>(); + for (var dataNode : EnvFactory.getEnv().getDataNodeWrapperList()) { + String content = + EnvFactory.getEnv() + .getUrlContent( + Config.IOTDB_HTTP_URL_PREFIX + + UrlUtils.formatTEndPointIpv4AndIpv6Url( + dataNode.getIp(), dataNode.getMetricPort()) + + "/metrics", + null); + if (content == null) { + continue; + } + for (String line : content.split("\\n")) { + Matcher sumMatcher = PERFORMANCE_TIMER_SUM_PATTERN.matcher(line.trim()); + if (sumMatcher.matches()) { + values + .computeIfAbsent( + sumMatcher.group(1) + "." + sumMatcher.group(2), key -> new double[2])[0] += + Double.parseDouble(sumMatcher.group(3)); + continue; + } + Matcher countMatcher = PERFORMANCE_TIMER_COUNT_PATTERN.matcher(line.trim()); + if (countMatcher.matches()) { + values + .computeIfAbsent( + countMatcher.group(1) + "." + countMatcher.group(2), key -> new double[2])[ + 1] += + Double.parseDouble(countMatcher.group(3)); + } + } + } + Map result = new LinkedHashMap<>(); + values.forEach((key, value) -> result.put(key, new MetricSample(value[0], value[1]))); + return result; + } + + private static void printPerformanceBreakdown( + String phase, + int round, + int position, + String path, + Map previous, + Map current) { + StringBuilder output = + new StringBuilder( + String.format("breakdown %s round %d position %d %s:", phase, round, position, path)); + for (Map.Entry entry : current.entrySet()) { + MetricSample before = previous.getOrDefault(entry.getKey(), MetricSample.ZERO); + double count = entry.getValue().count() - before.count(); + double seconds = entry.getValue().seconds() - before.seconds(); + if (count > 0 && seconds >= 0) { + output.append( + String.format( + " %s count=%.0f total=%.3f ms avg=%.3f us;", + entry.getKey(), count, seconds * 1000, seconds * 1_000_000 / count)); + } + } + System.out.println(output); + } + + private record MetricSample(double seconds, double count) { + private static final MetricSample ZERO = new MetricSample(0, 0); + } + + private static int[] configuredSparseWriteRatios() { + String configured = + System.getProperty("iotdb.numeric.row.insert.perf.sparse.ratios", "10,50,100"); + String[] tokens = configured.split(",", -1); + int[] ratios = new int[tokens.length]; + for (int i = 0; i < tokens.length; i++) { + ratios[i] = Integer.parseInt(tokens[i].trim()); + assertTrue( + "Sparse write ratios must be between 1 and 100", ratios[i] > 0 && ratios[i] <= 100); + } + return ratios; + } + + /** Reproducible random device and distinct FIELD selection, with no per-request allocations. */ + private static final class SparseNumericRowInput { + private final SplittableRandom random; + private final String[] devices; + private final String[] tagValues = new String[1]; + private final int[] columns; + private final BitMap nulls; + private final int writtenFields; + private final long[][] expected; + private int device; + + private SparseNumericRowInput(int deviceCount, int fieldCount, int writtenFields, long seed) { + random = new SplittableRandom(seed); + devices = new String[deviceCount]; + for (int i = 0; i < deviceCount; i++) devices[i] = "device-" + i; + columns = new int[fieldCount]; + for (int i = 0; i < fieldCount; i++) columns[i] = i; + nulls = new BitMap(fieldCount); + this.writtenFields = writtenFields; + // Slot zero counts rows, followed by one counter for each FIELD. + expected = new long[deviceCount][fieldCount + 1]; + } + + private void next() { + device = random.nextInt(devices.length); + tagValues[0] = devices[device]; + nulls.markAll(); + // Partial Fisher-Yates sampling gives exactly writtenFields distinct columns per request. + for (int i = 0; i < writtenFields; i++) { + int other = random.nextInt(i, columns.length); + int column = columns[other]; + columns[other] = columns[i]; + columns[i] = column; + nulls.unmark(column); + } + } + + private void acknowledge() { + expected[device][0]++; + for (int i = 0; i < writtenFields; i++) expected[device][columns[i] + 1]++; + } + + private void verify(ITableSession session, String table) throws Exception { + StringBuilder query = new StringBuilder("SELECT tag, count(*)"); + for (int i = 0; i < columns.length; i++) query.append(", count(f").append(i).append(')'); + query.append(" FROM ").append(table).append(" GROUP BY tag"); + boolean[] seen = new boolean[devices.length]; + try (SessionDataSet result = session.executeQueryStatement(query.toString())) { + while (result.hasNext()) { + RowRecord row = result.next(); + int index = + Integer.parseInt( + row.getFields().get(0).getStringValue().substring("device-".length())); + assertTrue("Unexpected device", index >= 0 && index < devices.length); + assertEquals(devices[index], row.getFields().get(0).getStringValue()); + assertFalse("Duplicate device", seen[index]); + seen[index] = true; + for (int i = 0; i <= columns.length; i++) { + assertEquals( + table + "/" + devices[index] + "/count column " + i, + expected[index][i], + row.getFields().get(i + 1).getLongV()); + } + } + } + for (int i = 0; i < devices.length; i++) assertEquals(expected[i][0] > 0, seen[i]); + } + } + + private static String formatPeakCpu(double peakCpuPercent) { + return Double.isNaN(peakCpuPercent) ? "unavailable" : String.format("%.1f%%", peakCpuPercent); + } + + private static String formatPeakMemory(long peakResidentBytes) { + return peakResidentBytes <= 0 + ? "unavailable" + : String.format("%.1f MiB", peakResidentBytes / 1024.0 / 1024.0); + } + + private record WritePerformance(long rows, long nanos, long[] requestNanos) { + double rowsPerSecond() { + return rows * 1_000_000_000.0 / nanos; + } + } + + private record RequestLatency( + long count, double averageNanos, long p50Nanos, long p95Nanos, long p99Nanos) { + @Override + public String toString() { + return String.format( + "n=%d, avg=%.3f us, p50=%.3f us, p95=%.3f us, p99=%.3f us", + count, + averageNanos / 1_000.0, + p50Nanos / 1_000.0, + p95Nanos / 1_000.0, + p99Nanos / 1_000.0); + } + } + + private static class LongSamples { + private long[] values = new long[1024]; + private int size; + + void add(long value) { + if (size == values.length) { + values = Arrays.copyOf(values, values.length * 2); + } + values[size++] = value; + } + + void appendAll(long[] samples) { + while (values.length - size < samples.length) { + values = Arrays.copyOf(values, values.length * 2); + } + System.arraycopy(samples, 0, values, size, samples.length); + size += samples.length; + } + + long[] toArray() { + return Arrays.copyOf(values, size); + } + } + + private static RequestLatency summarizeRequestLatencies(long[] requestNanos) { + if (requestNanos.length == 0) { + return new RequestLatency(0, 0, 0, 0, 0); + } + long[] sorted = Arrays.copyOf(requestNanos, requestNanos.length); + Arrays.sort(sorted); + double sum = 0; + for (long latency : sorted) { + sum += latency; + } + return new RequestLatency( + sorted.length, + sum / sorted.length, + percentile(sorted, 0.50), + percentile(sorted, 0.95), + percentile(sorted, 0.99)); + } + + private static long percentile(long[] sorted, double quantile) { + return sorted[Math.max(0, (int) Math.ceil(quantile * sorted.length) - 1)]; + } + + private static WritePerformance runConcurrentWrites( + ExecutorService executor, + int concurrency, + int seconds, + IntConsumer prepare, + IntConsumer write, + IntConsumer complete) + throws Exception { + CountDownLatch ready = new CountDownLatch(concurrency); + CountDownLatch start = new CountDownLatch(1); + AtomicLong deadline = new AtomicLong(); + List> futures = new ArrayList<>(); + try { + for (int i = 0; i < concurrency; i++) { + final int threadId = i; + futures.add( + executor.submit( + () -> { + ready.countDown(); + start.await(); + long rows = 0; + LongSamples requestNanos = new LongSamples(); + while (!Thread.currentThread().isInterrupted() + && System.nanoTime() < deadline.get()) { + if (prepare != null) { + prepare.accept(threadId); + } + long requestStart = System.nanoTime(); + write.accept(threadId); + requestNanos.add(System.nanoTime() - requestStart); + complete.accept(threadId); + rows++; + } + return new WritePerformance(rows, System.nanoTime(), requestNanos.toArray()); + })); + } + assertTrue(ready.await(30, TimeUnit.SECONDS)); + long startNanos = System.nanoTime(); + deadline.set(startNanos + TimeUnit.SECONDS.toNanos(seconds)); + start.countDown(); + long rows = 0; + long endNanos = startNanos; + LongSamples requestNanos = new LongSamples(); + for (Future future : futures) { + WritePerformance result = future.get(seconds + 60, TimeUnit.SECONDS); + rows += result.rows(); + endNanos = Math.max(endNanos, result.nanos()); + requestNanos.appendAll(result.requestNanos()); + } + // Include completion of in-flight RPCs, but exclude main-thread collection/printing time. + return new WritePerformance(rows, endNanos - startNanos, requestNanos.toArray()); + } finally { + start.countDown(); + for (Future future : futures) { + if (!future.isDone()) future.cancel(true); + } + } + } + + private void assertNumericTagRows(ITableSession session) throws Exception { + String[] devices = {"设备甲😀", "设备乙", "", null}; + try (SessionDataSet dataSet = + session.executeQueryStatement( + "SELECT region,device,missing,v,enabled FROM readings ORDER BY v")) { + for (int i = 0; i < devices.length; i++) { + assertTrue(dataSet.hasNext()); + RowRecord row = dataSet.next(); + if (i < 2) { + assertEquals("华北", row.getFields().get(0).getStringValue()); + } else { + assertNull(row.getFields().get(0).getDataType()); + } + if (devices[i] == null) { + assertNull(row.getFields().get(1).getDataType()); + } else { + assertEquals(devices[i], row.getFields().get(1).getStringValue()); + } + assertNull(row.getFields().get(2).getDataType()); + assertEquals((i + 1) * 10, row.getFields().get(3).getLongV()); + assertEquals(i % 2 == 0, row.getFields().get(4).getBoolV()); + } + assertFalse(dataSet.hasNext()); + } + } + + private void assertNumericRows(ITableSession session) throws Exception { + try (SessionDataSet dataSet = + session.executeQueryStatement( + "SELECT time,b,i,l,f,d,event_date,event_time,tag,attr,missing FROM readings ORDER BY time")) { + for (long time = 1; time <= 2; time++) { + assertTrue(dataSet.hasNext()); + RowRecord row = dataSet.next(); + assertEquals(time, row.getFields().get(0).getLongV()); + assertTrue(row.getFields().get(1).getBoolV()); + assertEquals(Integer.MIN_VALUE, row.getFields().get(2).getIntV()); + assertEquals(Long.MIN_VALUE, row.getFields().get(3).getLongV()); + assertEquals(-1.25f, row.getFields().get(4).getFloatV(), 0); + assertEquals(2.5, row.getFields().get(5).getDoubleV(), 0); + assertEquals(TSDataType.DATE, row.getFields().get(6).getDataType()); + assertEquals(LocalDate.of(2026, 9, 10), row.getFields().get(6).getDateV()); + assertEquals(1234, row.getFields().get(7).getLongV()); + for (int column = 8; column <= 10; column++) { + assertNull(row.getFields().get(column).getDataType()); + } + } + assertFalse(dataSet.hasNext()); + } + } + @Test public void testUseDatabase() { final String[] table1Names = new String[] {"table1"}; diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/WritePerformanceIoTracker.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/WritePerformanceIoTracker.java new file mode 100644 index 0000000000000..bab6f0ad710c3 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/WritePerformanceIoTracker.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.session; + +import org.apache.iotdb.isession.ITableSession; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** Linux-only, opt-in counters sampled outside the single-row benchmark's timed write loops. */ +final class WritePerformanceIoTracker { + private static final Pattern TCP_COUNTER = + Pattern.compile("\\b(bytes_received|bytes_sent|bytes_retrans):(\\d+)\\b"); + + private final long dataNodePid; + private final long configNodePid; + private final int rpcPort; + + WritePerformanceIoTracker(long dataNodePid, long configNodePid, int rpcPort) throws IOException { + assertTrue("I/O measurement requires Linux /proc", Files.isDirectory(Path.of("/proc"))); + assertTrue("DataNode PID unavailable", dataNodePid > 0); + assertTrue("ConfigNode PID unavailable", configNodePid > 0); + this.dataNodePid = dataNodePid; + this.configNodePid = configNodePid; + this.rpcPort = rpcPort; + // Check readability early; unsupported counters must fail rather than appear as zero I/O. + readDisk(dataNodePid); + readDisk(configNodePid); + System.out.printf( + "io configuration: dataNodePid=%d, configNodePid=%d, rpcPort=%d, " + + "network=server TCP payload counters, disk=/proc/pid/io, " + + "flushBeforeAndAfter=true, compaction=false%n", + dataNodePid, configNodePid, rpcPort); + } + + Snapshot beforeRound(ITableSession session) throws Exception { + // Drain previous MemTables/WAL before the baseline; the following flush is charged to this + // round. + session.executeNonQueryStatement("FLUSH"); + return new Snapshot(readNetwork(), readDisk(dataNodePid), readDisk(configNodePid)); + } + + Result afterRound(Snapshot before, ITableSession session) throws Exception { + // Snapshot TCP before FLUSH so SQL/control traffic is excluded from write-request bytes. + Map network = readNetwork(); + assertEquals( + "RPC connections changed during I/O measurement", + before.network.keySet(), + network.keySet()); + Traffic traffic = new Traffic(0, 0, 0); + for (Map.Entry entry : network.entrySet()) { + traffic = traffic.plus(entry.getValue().minus(before.network.get(entry.getKey()))); + } + Disk activeDataNode = readDisk(dataNodePid).minus(before.dataNode); + long flushStart = System.nanoTime(); + session.executeNonQueryStatement("FLUSH"); + long flushNanos = System.nanoTime() - flushStart; + Disk dataNode = readDisk(dataNodePid).minus(before.dataNode); + Disk configNode = readDisk(configNodePid).minus(before.configNode); + return new Result(traffic, activeDataNode, dataNode, configNode, flushNanos); + } + + private Map readNetwork() throws Exception { + // Count only server endpoints on this test DataNode's RPC port. Loopback RX/TX are not doubled. + Process process = + new ProcessBuilder("ss", "-tinHO", "state", "established", "( sport = :" + rpcPort + " )") + .redirectErrorStream(true) + .start(); + CompletableFuture output = + CompletableFuture.supplyAsync( + () -> { + try { + return new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new CompletionException(e); + } + }); + try { + assertTrue("ss did not finish", process.waitFor(10, TimeUnit.SECONDS)); + String text = output.get(10, TimeUnit.SECONDS); + assertEquals(text, 0, process.exitValue()); + Map result = new HashMap<>(); + for (String line : text.split("\\R")) { + if (line.isBlank()) continue; + String[] fields = line.trim().split("\\s+", 5); + assertTrue("Unexpected ss output: " + line, fields.length == 5); + Map counters = new HashMap<>(); + Matcher matcher = TCP_COUNTER.matcher(fields[4]); + while (matcher.find()) counters.put(matcher.group(1), Long.parseLong(matcher.group(2))); + assertTrue( + "TCP byte counters unavailable: " + line, + counters.containsKey("bytes_received") && counters.containsKey("bytes_sent")); + String connection = fields[2] + "->" + fields[3]; + assertFalse("Duplicate server socket: " + connection, result.containsKey(connection)); + // ss omits zero-valued bytes_retrans; the received/sent counters above are mandatory. + result.put( + connection, + new Traffic( + counters.get("bytes_received"), + counters.get("bytes_sent"), + counters.getOrDefault("bytes_retrans", 0L))); + } + assertFalse("No established test RPC connections", result.isEmpty()); + return result; + } finally { + process.destroyForcibly(); + output.cancel(true); + } + } + + private static Disk readDisk(long pid) throws IOException { + Map counters = new HashMap<>(); + for (String line : Files.readAllLines(Path.of("/proc", Long.toString(pid), "io"))) { + String[] fields = line.split(":\\s*", 2); + counters.put(fields[0], Long.parseLong(fields[1].trim())); + } + assertTrue( + "Process storage byte counters unavailable for PID " + pid, + counters.containsKey("read_bytes") + && counters.containsKey("write_bytes") + && counters.containsKey("cancelled_write_bytes")); + // These are kernel-accounted process storage bytes, not rchar/wchar (which include socket I/O), + // nor physical NAND bytes. Preserve cancelled writes separately instead of hiding them. + return new Disk( + counters.get("read_bytes"), + counters.get("write_bytes"), + counters.get("cancelled_write_bytes")); + } + + record Snapshot(Map network, Disk dataNode, Disk configNode) {} + + private record Traffic(long received, long sent, long retransmitted) { + Traffic minus(Traffic previous) { + assertTrue( + "TCP counters decreased", + received >= previous.received + && sent >= previous.sent + && retransmitted >= previous.retransmitted); + return new Traffic( + received - previous.received, + sent - previous.sent, + retransmitted - previous.retransmitted); + } + + Traffic plus(Traffic other) { + return new Traffic( + received + other.received, sent + other.sent, retransmitted + other.retransmitted); + } + } + + private record Disk(long read, long written, long cancelled) { + Disk minus(Disk previous) { + assertTrue( + "Process I/O counters decreased", + read >= previous.read && written >= previous.written && cancelled >= previous.cancelled); + return new Disk( + read - previous.read, written - previous.written, cancelled - previous.cancelled); + } + } + + record Result( + Traffic traffic, Disk activeDataNode, Disk dataNode, Disk configNode, long flushNanos) { + @Override + public String toString() { + return String.format( + "requestBytes=%d, responseBytes=%d, responseRetransmitBytes=%d, " + + "dnActiveReadBytes=%d, dnActiveWriteBytes=%d, dnActiveCancelledBytes=%d, " + + "dnReadBytes=%d, dnWriteBytes=%d, dnCancelledBytes=%d, " + + "cnReadBytes=%d, cnWriteBytes=%d, cnCancelledBytes=%d, flushNanos=%d", + traffic.received, + traffic.sent, + traffic.retransmitted, + activeDataNode.read, + activeDataNode.written, + activeDataNode.cancelled, + dataNode.read, + dataNode.written, + dataNode.cancelled, + configNode.read, + configNode.written, + configNode.cancelled, + flushNanos); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/WritePerformanceResourceTracker.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/WritePerformanceResourceTracker.java new file mode 100644 index 0000000000000..64a680875af7c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/WritePerformanceResourceTracker.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iotdb.relational.it.session; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +/** Samples the DataNode process resource usage while one benchmark round is running. */ +final class WritePerformanceResourceTracker implements AutoCloseable { + private static final Path PROC = Path.of("/proc"); + private static final long SAMPLE_INTERVAL_MILLIS = 100; + + private final long dataNodePid; + private final long clockTicksPerSecond; + private final Thread sampler; + private final AtomicReference currentRound = new AtomicReference<>(); + private final AtomicReference samplingFailure = new AtomicReference<>(); + private final CountDownLatch stopped = new CountDownLatch(1); + + WritePerformanceResourceTracker(long dataNodePid) throws IOException { + this.dataNodePid = dataNodePid; + if (!isSupported()) { + clockTicksPerSecond = 0; + sampler = null; + System.out.println("resource configuration: DataNode CPU/memory sampling unavailable"); + return; + } + if (dataNodePid <= 0) { + throw new IOException("DataNode PID unavailable: " + dataNodePid); + } + clockTicksPerSecond = readClockTicksPerSecond(); + readProcessSample(); + sampler = new Thread(this::sampleLoop, "numeric-row-resource-sampler"); + sampler.setDaemon(true); + sampler.start(); + System.out.printf( + "resource configuration: dataNodePid=%d, sampleIntervalMs=%d, cpu=process CPU, " + + "memory=resident set size%n", + dataNodePid, SAMPLE_INTERVAL_MILLIS); + } + + boolean isSupported() { + return Files.isDirectory(PROC); + } + + void beginRound() throws IOException { + checkFailure(); + if (!isSupported()) { + return; + } + Round round = new Round(); + round.sample(); + currentRound.set(round); + } + + RoundPeak endRound() throws IOException { + if (!isSupported()) { + return RoundPeak.UNAVAILABLE; + } + Round round = currentRound.getAndSet(null); + if (round == null) { + throw new IllegalStateException("Resource round was not started"); + } + RoundPeak peak = round.finish(); + checkFailure(); + return peak; + } + + @Override + public void close() throws Exception { + if (sampler == null) { + return; + } + sampler.interrupt(); + if (!stopped.await(5, TimeUnit.SECONDS)) { + throw new IllegalStateException("Resource sampler did not stop"); + } + checkFailure(); + } + + private void sampleLoop() { + try { + while (!Thread.currentThread().isInterrupted()) { + Round round = currentRound.get(); + if (round != null) { + round.sample(); + } + Thread.sleep(SAMPLE_INTERVAL_MILLIS); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Throwable t) { + samplingFailure.compareAndSet(null, t); + } finally { + stopped.countDown(); + } + } + + private void checkFailure() throws IOException { + Throwable failure = samplingFailure.get(); + if (failure == null) { + return; + } + if (failure instanceof IOException ioException) { + throw ioException; + } + throw new IOException("Failed to sample DataNode resources", failure); + } + + private ProcessSample readProcessSample() throws IOException { + Path processDirectory = PROC.resolve(Long.toString(dataNodePid)); + String stat = Files.readString(processDirectory.resolve("stat")); + int closingParenthesis = stat.lastIndexOf(") "); + if (closingParenthesis < 0) { + throw new IOException("Malformed /proc stat for DataNode PID " + dataNodePid); + } + String[] fields = stat.substring(closingParenthesis + 2).trim().split("\\s+"); + if (fields.length <= 12) { + throw new IOException("Incomplete /proc stat for DataNode PID " + dataNodePid); + } + long cpuTicks = Long.parseLong(fields[11]) + Long.parseLong(fields[12]); + long residentBytes = 0; + for (String line : Files.readAllLines(processDirectory.resolve("status"))) { + if (line.startsWith("VmRSS:")) { + residentBytes = Long.parseLong(line.replaceAll("[^0-9]", "").trim()) * 1024; + break; + } + } + if (residentBytes <= 0) { + throw new IOException("DataNode RSS unavailable for PID " + dataNodePid); + } + return new ProcessSample(System.nanoTime(), cpuTicks, residentBytes); + } + + private static long readClockTicksPerSecond() throws IOException { + Process process = new ProcessBuilder("getconf", "CLK_TCK").redirectErrorStream(true).start(); + try { + if (!process.waitFor(5, TimeUnit.SECONDS)) { + throw new IOException("getconf CLK_TCK timed out"); + } + String output = + new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8).trim(); + if (process.exitValue() != 0) { + throw new IOException("getconf CLK_TCK failed: " + output); + } + long result = Long.parseLong(output); + if (result <= 0) { + throw new IOException("Invalid CLK_TCK: " + result); + } + return result; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while reading CLK_TCK", e); + } finally { + process.destroyForcibly(); + } + } + + private final class Round { + private long lastCpuTicks; + private long lastSampleNanos; + private double peakCpuPercent = Double.NaN; + private long peakResidentBytes; + private boolean active = true; + + private synchronized void sample() throws IOException { + if (!active) { + return; + } + // Read and apply samples under the same lock so the final sample cannot overtake the sampler. + ProcessSample sample = readProcessSample(); + peakResidentBytes = Math.max(peakResidentBytes, sample.residentBytes); + long elapsedNanos = sample.nanoTime - lastSampleNanos; + if (lastSampleNanos != 0) { + // Keep the CPU baseline for short intervals: tick quantization can inflate a tail sample. + if (elapsedNanos < TimeUnit.MILLISECONDS.toNanos(SAMPLE_INTERVAL_MILLIS)) { + return; + } + long cpuTicks = sample.cpuTicks - lastCpuTicks; + double cpuPercent = + cpuTicks * 100.0 / clockTicksPerSecond / (elapsedNanos / 1_000_000_000.0); + peakCpuPercent = + Double.isNaN(peakCpuPercent) ? cpuPercent : Math.max(peakCpuPercent, cpuPercent); + } + lastCpuTicks = sample.cpuTicks; + lastSampleNanos = sample.nanoTime; + } + + private synchronized RoundPeak finish() throws IOException { + try { + sample(); + return new RoundPeak(peakCpuPercent, peakResidentBytes); + } finally { + active = false; + } + } + } + + private record ProcessSample(long nanoTime, long cpuTicks, long residentBytes) {} + + record RoundPeak(double peakCpuPercent, long peakResidentBytes) { + private static final RoundPeak UNAVAILABLE = new RoundPeak(Double.NaN, -1); + + String format() { + if (this == UNAVAILABLE) { + return "peakDataNodeCpu=unavailable, peakDataNodeRss=unavailable"; + } + return String.format( + "peakDataNodeCpu=%s, peakDataNodeRss=%.1f MiB", + Double.isNaN(peakCpuPercent) ? "unavailable" : String.format("%.1f%%", peakCpuPercent), + peakResidentBytes / 1024.0 / 1024.0); + } + } +} diff --git a/iotdb-client/isession/src/main/i18n/en/org/apache/iotdb/isession/i18n/ISessionMessages.java b/iotdb-client/isession/src/main/i18n/en/org/apache/iotdb/isession/i18n/ISessionMessages.java index bc593534c60d2..3a15370611a4c 100644 --- a/iotdb-client/isession/src/main/i18n/en/org/apache/iotdb/isession/i18n/ISessionMessages.java +++ b/iotdb-client/isession/src/main/i18n/en/org/apache/iotdb/isession/i18n/ISessionMessages.java @@ -31,6 +31,8 @@ public final class ISessionMessages { public static final String NOT_DIRECT_CHILD_OF_TEMPLATE = "It is not a direct child of the template: "; + public static final String EXCEPTION_UNSUPPORTED_NUMERIC_TYPE_06DD3044 = "Unsupported numeric type"; + private ISessionMessages() {} // --------------------------------------------------------------------------- // Additional auto-collected messages diff --git a/iotdb-client/isession/src/main/i18n/zh/org/apache/iotdb/isession/i18n/ISessionMessages.java b/iotdb-client/isession/src/main/i18n/zh/org/apache/iotdb/isession/i18n/ISessionMessages.java index 5514a76c0f71e..a6316202ebc84 100644 --- a/iotdb-client/isession/src/main/i18n/zh/org/apache/iotdb/isession/i18n/ISessionMessages.java +++ b/iotdb-client/isession/src/main/i18n/zh/org/apache/iotdb/isession/i18n/ISessionMessages.java @@ -31,6 +31,8 @@ public final class ISessionMessages { public static final String NOT_DIRECT_CHILD_OF_TEMPLATE = "这不是该模板的直接子节点："; + public static final String EXCEPTION_UNSUPPORTED_NUMERIC_TYPE_06DD3044 = "不支持的数值类型"; + private ISessionMessages() {} // --------------------------------------------------------------------------- // Additional auto-collected messages diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java index 75a37f8384719..7fa1df842f8ad 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java @@ -22,8 +22,13 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.record.Tablet; +import java.util.List; + /** * This interface defines a session for interacting with IoTDB tables. It supports operations such * as data insertion, executing queries, and closing the session. Implementations of this interface @@ -46,6 +51,51 @@ public interface ITableSession extends AutoCloseable { */ void insert(Tablet tablet) throws StatementExecutionException, IoTDBConnectionException; + /** + * Prepares repeated single-row writes with STRING TAGs and BOOLEAN, INT32, INT64, FLOAT, DOUBLE, + * DATE or TIMESTAMP FIELDs. ATTRIBUTE columns and other FIELD types are unsupported. Column + * names, types and categories must have matching lengths and order. + * + *

The returned statement belongs to this session and is not thread-safe. Close it after use. + */ + NumericRowPreparedStatement prepareInsertNumericRow( + String tableName, + List columnNames, + List dataTypes, + List columnCategories) + throws StatementExecutionException, IoTDBConnectionException; + + /** + * Executes one row using FIELD and TAG arrays in their respective occurrence order in the + * prepared column list. TAGs may be null or empty. + * + *

BOOLEAN uses zero for false and nonzero for true. INT32 and DATE use the low 32 bits (DATE + * is yyyyMMdd); INT64 and TIMESTAMP use the long value. FLOAT uses {@link + * Float#floatToRawIntBits(float)} in the low 32 bits and DOUBLE uses {@link + * Double#doubleToRawLongBits(double)}. Do not modify input arrays during execution. + * + *

Follows write redirection recommendations for subsequent rows. Destination handles are + * prepared lazily and released when the statement is closed. An execution error reported by the + * server closes the statement; prepare it again before reuse. Transport failures are propagated + * without replaying a write whose outcome is unknown. + */ + void executePreparedInsertNumericRow( + NumericRowPreparedStatement prepared, long timestamp, long[] values, String[] tagValues) + throws StatementExecutionException, IoTDBConnectionException; + + /** + * Executes a prepared row with an optional FIELD-only null bitmap sized to {@code values.length}. + * Marked slots are null and their values are ignored. Do not mutate the bitmap during execution. + * Requires a server supporting numeric row null bitmaps. + */ + void executePreparedInsertNumericRow( + NumericRowPreparedStatement prepared, + long timestamp, + long[] values, + String[] tagValues, + BitMap nullBitmap) + throws StatementExecutionException, IoTDBConnectionException; + /** * Executes a non-query SQL statement, such as a DDL or DML command. * diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java new file mode 100644 index 0000000000000..077cc946f249d --- /dev/null +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.isession; + +import org.apache.iotdb.isession.i18n.ISessionMessages; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; + +import java.nio.ByteBuffer; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; + +/** + * Immutable metadata and reusable serialization buffers for repeated numeric table row inserts. + * + *

This statement is not thread-safe. An execution must return before the statement is reused or + * closed, because its buffers are retained until the synchronous RPC completes. + */ +public final class NumericRowPreparedStatement implements AutoCloseable { + private final String tableName; + private final String database; + private final List columnNames; + private final List dataTypes; + private final List columnCategories; + private final int tagCount; + private final int valueBytes; + private ByteBuffer tagBuffer; + private ByteBuffer valueBuffer; + private ByteBuffer nullBitmapBuffer; + private Object owner; + private final Map connectionBindings = new IdentityHashMap<>(); + private boolean closed; + + public NumericRowPreparedStatement( + String tableName, + List columnNames, + List dataTypes, + List columnCategories, + String database) { + this.tableName = tableName; + this.database = database; + this.columnNames = List.copyOf(columnNames); + this.dataTypes = List.copyOf(dataTypes); + this.columnCategories = List.copyOf(columnCategories); + // Column categories are immutable, so repeated inserts can reuse this count. + this.tagCount = + (int) this.columnCategories.stream().filter(c -> c == ColumnCategory.TAG).count(); + // FIELD encodings have fixed widths for the lifetime of the prepared schema. + int bytes = 0; + for (int i = 0; i < this.dataTypes.size(); i++) { + if (this.columnCategories.get(i) == ColumnCategory.FIELD) { + bytes += + switch (this.dataTypes.get(i)) { + case BOOLEAN -> 1; + case INT32, DATE, FLOAT -> Integer.BYTES; + case INT64, TIMESTAMP, DOUBLE -> Long.BYTES; + default -> + throw new IllegalArgumentException( + ISessionMessages.EXCEPTION_UNSUPPORTED_NUMERIC_TYPE_06DD3044); + }; + } + } + this.valueBytes = bytes; + } + + public String getTableName() { + return tableName; + } + + public String getDatabase() { + return database; + } + + public List getColumnNames() { + return columnNames; + } + + public List getDataTypes() { + return dataTypes; + } + + public List getColumnCategories() { + return columnCategories; + } + + public int getTagCount() { + return tagCount; + } + + /** + * Returns the TAG buffer cleared for writing, growing it when the encoded tags need more space. + */ + public ByteBuffer prepareTagBuffer(int requiredCapacity) { + if (tagBuffer == null || tagBuffer.capacity() < requiredCapacity) { + tagBuffer = ByteBuffer.allocate(requiredCapacity); + } + tagBuffer.clear(); + return tagBuffer; + } + + /** Returns the FIELD buffer cleared for writing; a prepared schema has a fixed encoded size. */ + public ByteBuffer prepareValueBuffer() { + if (valueBuffer == null) { + valueBuffer = ByteBuffer.allocate(valueBytes); + } + valueBuffer.clear(); + return valueBuffer; + } + + /** Returns a reusable FIELD null bitmap buffer, allocated only for nullable executions. */ + public ByteBuffer prepareNullBitmapBuffer() { + if (nullBitmapBuffer == null) { + nullBitmapBuffer = ByteBuffer.allocate((dataTypes.size() - tagCount + 7) / 8); + } + nullBitmapBuffer.clear(); + return nullBitmapBuffer; + } + + public Object getOwner() { + return owner; + } + + public void bind(long id, Object owner, long sessionId, Runnable closeAction) { + this.owner = owner; + bindConnection(owner, sessionId, id, closeAction); + this.closed = false; + } + + /** Handles belong to a physical session, including its generation after a reconnect. */ + public Long getPreparedId(Object connection, long sessionId) { + ConnectionBinding binding = connectionBindings.get(connection); + return binding != null && binding.sessionId == sessionId ? binding.preparedId : null; + } + + public void bindConnection(Object connection, long sessionId, long id, Runnable closeAction) { + connectionBindings.put(connection, new ConnectionBinding(sessionId, id, closeAction)); + } + + private record ConnectionBinding(long sessionId, long preparedId, Runnable closeAction) {} + + public boolean isClosed() { + return closed; + } + + @Override + public void close() { + if (closed) { + return; + } + closed = true; + try { + for (ConnectionBinding binding : connectionBindings.values()) { + binding.closeAction.run(); + } + } finally { + connectionBindings.clear(); + } + } +} diff --git a/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java b/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java index 69b87b2db1653..2bcac9214ce50 100644 --- a/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java +++ b/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java @@ -20,6 +20,10 @@ package org.apache.iotdb.session.i18n; public final class SessionMessages { + public static final String EXCEPTION_NUMERIC_ROW_NULL_BITMAP_SIZE_MISMATCH_EXPECTED_ARG_FIELDS_BUT_GOT_ARG_8D30F6A0 = + "Numeric row null bitmap size mismatch: expected %d FIELDs but got %d"; + public static final String EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233 = "Numeric row arguments must be non-null; column metadata lengths and per-category value counts must match."; + public static final String EXCEPTION_NUMERIC_ROWS_SUPPORT_NUMERIC_FIELD_COLUMNS_AND_STRING_TAG_COLUMNS_ONLY_87159FFA = "Numeric rows support numeric FIELD columns and STRING TAG columns only."; // Session public static final String NODE_URLS_EMPTY = "nodeUrls shouldn't be empty."; @@ -286,4 +290,11 @@ private SessionMessages() {} public static final String EXCEPTION_RETRY_EXECUTE_STATEMENT_ARG_FAILED_ARG_TIMES_ARG_216C6873 = "retry to execute statement on %s failed %d times: %s"; public static final String EXCEPTION_SESSIONPOOL_DOESN_T_SUPPORT_EXECUTING_ARG_DIRECTLY_B778F701 = "SessionPool doesn't support executing %s directly"; + public static final String EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_AND_COLUMN_NAMES_TYPES_CATEGORIES_AND_VALUES_MUST_HAVE_EQUAL_LENGTHS_AB102B01 = "Numeric row arguments must be non-null and column names, types, categories and values must have equal lengths."; + public static final String EXCEPTION_NUMERIC_ROWS_SUPPORT_ONLY_FIELD_COLUMNS_DA7A8EE7 = "Numeric rows support only FIELD columns."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_BELONGS_TO_ANOTHER_CONNECTION_PREPARE_IT_AGAIN_055CC7BC = "Prepared numeric row is closed or belongs to another connection; prepare it again."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_PREPARE_RETURNED_NO_STATUS_6FF7BC24 = "Prepared numeric row prepare returned no status"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_COUNT_MISMATCH_9ACD3A0C = "Prepared numeric row value count mismatch"; + public static final String EXCEPTION_UNSUPPORTED_NUMERIC_TYPE_06DD3044 = "Unsupported numeric type"; + public static final String EXCEPTION_TABLE_SESSION_IS_CLOSED_77FE359B = "Table session is closed."; } diff --git a/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java b/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java index 2f28590ac8840..aa9c1055fd558 100644 --- a/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java +++ b/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java @@ -20,6 +20,10 @@ package org.apache.iotdb.session.i18n; public final class SessionMessages { + public static final String EXCEPTION_NUMERIC_ROW_NULL_BITMAP_SIZE_MISMATCH_EXPECTED_ARG_FIELDS_BUT_GOT_ARG_8D30F6A0 = + "Numeric row 的 null bitmap 大小不匹配：预期 %d 个 FIELD，实际为 %d"; + public static final String EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233 = "数值行参数不能为 null；列元数据长度必须一致，FIELD 和 TAG 的数量必须分别与对应的值数组长度一致。"; + public static final String EXCEPTION_NUMERIC_ROWS_SUPPORT_NUMERIC_FIELD_COLUMNS_AND_STRING_TAG_COLUMNS_ONLY_87159FFA = "数值行仅支持数值类型的 FIELD 列和 STRING 类型的 TAG 列。"; // Session public static final String NODE_URLS_EMPTY = "nodeUrls 不能为空。"; @@ -284,4 +288,12 @@ private SessionMessages() {} public static final String EXCEPTION_RETRY_EXECUTE_STATEMENT_ARG_FAILED_ARG_TIMES_ARG_216C6873 = "在 %s 上重试执行语句失败 %d 次：%s"; public static final String EXCEPTION_SESSIONPOOL_DOESN_T_SUPPORT_EXECUTING_ARG_DIRECTLY_B778F701 = "SessionPool 不支持直接执行 %s"; + public static final String EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_AND_COLUMN_NAMES_TYPES_CATEGORIES_AND_VALUES_MUST_HAVE_EQUAL_LENGTHS_AB102B01 = "数值行参数不能为 null，列名、类型、类别和值的数量必须相同。"; + public static final String EXCEPTION_NUMERIC_ROWS_SUPPORT_ONLY_FIELD_COLUMNS_DA7A8EE7 = "数值行仅支持 FIELD 列。"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_BELONGS_TO_ANOTHER_CONNECTION_PREPARE_IT_AGAIN_055CC7BC = + "预备数值行已关闭或属于其他连接，请重新准备"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_PREPARE_RETURNED_NO_STATUS_6FF7BC24 = "预备数值行准备请求未返回状态"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_COUNT_MISMATCH_9ACD3A0C = "预备数值行的值数量不匹配"; + public static final String EXCEPTION_UNSUPPORTED_NUMERIC_TYPE_06DD3044 = "不支持的数值类型"; + public static final String EXCEPTION_TABLE_SESSION_IS_CLOSED_77FE359B = "表会话已关闭。"; } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index 5d20754fff4be..4ed368b446aa3 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -1352,8 +1352,7 @@ private SessionConnection getSessionConnection(String deviceId) throws IoTDBConn } } - private SessionConnection getSessionConnection(IDeviceID deviceId) - throws IoTDBConnectionException { + SessionConnection getSessionConnection(IDeviceID deviceId) throws IoTDBConnectionException { TEndPoint endPoint; if (enableRedirection && tableModelDeviceIdToEndpoint != null @@ -1435,7 +1434,7 @@ private void handleRedirection(String deviceId, TEndPoint endpoint) { } } - private void handleRedirection(IDeviceID deviceId, TEndPoint endpoint) { + void handleRedirection(IDeviceID deviceId, TEndPoint endpoint) { if (enableRedirection) { // no need to redirection if (UrlUtils.isWildcardAddress(endpoint.ip)) { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java index 6bcc41859a8c4..b66f7dd5c0612 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java @@ -45,6 +45,7 @@ import org.apache.iotdb.service.rpc.thrift.TSCreateTimeseriesReq; import org.apache.iotdb.service.rpc.thrift.TSDeleteDataReq; import org.apache.iotdb.service.rpc.thrift.TSDropSchemaTemplateReq; +import org.apache.iotdb.service.rpc.thrift.TSExecutePreparedNumericRowReq; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; import org.apache.iotdb.service.rpc.thrift.TSFastLastDataQueryForOneDeviceReq; @@ -60,6 +61,8 @@ import org.apache.iotdb.service.rpc.thrift.TSLastDataQueryReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowResp; import org.apache.iotdb.service.rpc.thrift.TSPruneSchemaTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSQueryTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSQueryTemplateResp; @@ -80,6 +83,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.ByteBuffer; import java.security.SecureRandom; import java.time.ZoneId; import java.util.ArrayList; @@ -785,6 +789,59 @@ protected void insertRecord(TSInsertRecordReq request) callWithRetryAndVerifyWithRedirection(() -> insertRecordInternal(request)); } + protected long prepareNumericRow(TSPrepareNumericRowReq request) + throws IoTDBConnectionException, StatementExecutionException { + request.setSessionId(sessionId); + try { + TSPrepareNumericRowResp resp = client.prepareNumericRow(request); + if (resp.getStatus() == null) { + throw new IoTDBConnectionException( + SessionMessages.EXCEPTION_PREPARED_NUMERIC_ROW_PREPARE_RETURNED_NO_STATUS_6FF7BC24); + } + RpcUtils.verifySuccess(resp.getStatus()); + return resp.getPreparedId(); + } catch (TException e) { + throw new IoTDBConnectionException(e); + } + } + + protected void executePreparedNumericRow( + long id, long timestamp, ByteBuffer values, ByteBuffer tags) + throws IoTDBConnectionException, StatementExecutionException, RedirectException { + executePreparedNumericRow(id, timestamp, values, tags, null); + } + + protected void executePreparedNumericRow( + long id, long timestamp, ByteBuffer values, ByteBuffer tags, ByteBuffer nullBitmap) + throws IoTDBConnectionException, StatementExecutionException, RedirectException { + TSExecutePreparedNumericRowReq request = + new TSExecutePreparedNumericRowReq(sessionId, id, timestamp, values, tags); + if (nullBitmap != null) { + request.setNullBitmap(nullBitmap); + } + try { + // A redirect is a recommendation for later writes: this row has already been accepted. + // Never reconnect/replay a session-local prepared ID after an ambiguous transport failure. + RpcUtils.verifySuccessWithRedirection(client.executePreparedNumericRow(request)); + } catch (TException e) { + throw new IoTDBConnectionException(e); + } + } + + protected void closePreparedNumericRow(long id) throws IoTDBConnectionException { + try { + RpcUtils.verifySuccess(client.closePreparedNumericRow(sessionId, id)); + } catch (StatementExecutionException e) { + throw new IoTDBConnectionException(e); + } catch (TException e) { + throw new IoTDBConnectionException(e); + } + } + + long getSessionId() { + return sessionId; + } + private TSStatus insertRecordInternal(TSInsertRecordReq request) throws TException { request.setSessionId(sessionId); return client.insertRecord(request); diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java index 059fdea346948..6d7ddc1f64c07 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java @@ -20,17 +20,32 @@ package org.apache.iotdb.session; import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.NumericRowPreparedStatement; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.RedirectException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; +import org.apache.iotdb.session.i18n.SessionMessages; +import org.apache.iotdb.session.util.SessionUtils; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.record.Tablet; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + public class TableSession implements ITableSession { private final Session session; - TableSession(Session session) { + public TableSession(Session session) { this.session = session; } @@ -39,6 +54,201 @@ public void insert(Tablet tablet) throws StatementExecutionException, IoTDBConne session.insertRelationalTablet(tablet); } + @Override + public NumericRowPreparedStatement prepareInsertNumericRow( + String tableName, + List columnNames, + List dataTypes, + List columnCategories) + throws StatementExecutionException, IoTDBConnectionException { + // Validate before regrouping, which would otherwise discard unsupported categories or + // surplus metadata entries. + if (tableName == null + || columnNames == null + || dataTypes == null + || columnCategories == null + || columnNames.size() != dataTypes.size() + || columnNames.size() != columnCategories.size()) { + throw new IllegalArgumentException( + SessionMessages + .EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233); + } + for (int i = 0; i < columnNames.size(); i++) { + if (columnNames.get(i) == null + || dataTypes.get(i) == null + || columnCategories.get(i) == null) { + throw new IllegalArgumentException( + SessionMessages + .EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233); + } + if ((columnCategories.get(i) != ColumnCategory.TAG + && columnCategories.get(i) != ColumnCategory.FIELD) + || (columnCategories.get(i) == ColumnCategory.TAG + && dataTypes.get(i) != TSDataType.STRING)) { + throw new IllegalArgumentException( + SessionMessages + .EXCEPTION_NUMERIC_ROWS_SUPPORT_NUMERIC_FIELD_COLUMNS_AND_STRING_TAG_COLUMNS_ONLY_87159FFA); + } + } + List names = new ArrayList<>(); + List types = new ArrayList<>(); + List categories = new ArrayList<>(); + for (ColumnCategory category : + new ColumnCategory[] {ColumnCategory.TAG, ColumnCategory.FIELD}) { + for (int i = 0; i < columnNames.size(); i++) { + if (columnCategories.get(i) == category) { + names.add(columnNames.get(i)); + types.add(dataTypes.get(i)); + categories.add(category); + } + } + } + TSPrepareNumericRowReq request = new TSPrepareNumericRowReq(); + request + .setTableName(tableName) + .setMeasurements(names) + .setDataTypes(types.stream().map(TSDataType::ordinal).toList()) + .setColumnCategories(categories.stream().map(c -> (byte) c.ordinal()).toList()); + NumericRowPreparedStatement prepared = + new NumericRowPreparedStatement(tableName, names, types, categories, session.getDatabase()); + SessionConnection connection = session.getDefaultSessionConnection(); + long id = connection.prepareNumericRow(request); + prepared.bind( + id, + connection, + connection.getSessionId(), + closePreparedRow(connection, connection.getSessionId(), id)); + return prepared; + } + + @Override + public void executePreparedInsertNumericRow( + NumericRowPreparedStatement prepared, long timestamp, long[] values, String[] tagValues) + throws StatementExecutionException, IoTDBConnectionException { + executePreparedInsertNumericRow(prepared, timestamp, values, tagValues, null); + } + + @Override + public void executePreparedInsertNumericRow( + NumericRowPreparedStatement prepared, + long timestamp, + long[] values, + String[] tagValues, + BitMap nullBitmap) + throws StatementExecutionException, IoTDBConnectionException { + SessionConnection connection = session.getDefaultSessionConnection(); + if (prepared == null + || prepared.isClosed() + || prepared.getOwner() != connection + || prepared.getPreparedId(connection, connection.getSessionId()) == null) { + throw new IoTDBConnectionException( + SessionMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_BELONGS_TO_ANOTHER_CONNECTION_PREPARE_IT_AGAIN_055CC7BC); + } + int tagCount = prepared.getTagCount(); + int fieldCount = prepared.getDataTypes().size() - tagCount; + if (values == null + || tagValues == null + || values.length != fieldCount + || tagValues.length != tagCount) { + throw new IllegalArgumentException( + SessionMessages.EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_COUNT_MISMATCH_9ACD3A0C); + } + SessionUtils.validateNumericRowNullBitmap(nullBitmap, fieldCount); + // A prepared handle remains bound to its original database after USE. Do not prepare it on + // a redirected session whose database has changed in the meantime. + IDeviceID deviceId = null; + if (session.isEnableRedirection() + && Objects.equals(prepared.getDatabase(), session.getDatabase())) { + String[] segments = new String[tagValues.length + 1]; + segments[0] = prepared.getTableName(); + System.arraycopy(tagValues, 0, segments, 1, tagValues.length); + deviceId = IDeviceID.Factory.DEFAULT_FACTORY.create(segments); + connection = session.getSessionConnection(deviceId); + } + Long id = prepared.getPreparedId(connection, connection.getSessionId()); + if (id == null) { + TSPrepareNumericRowReq request = new TSPrepareNumericRowReq(); + request + .setTableName(prepared.getTableName()) + .setMeasurements(prepared.getColumnNames()) + .setDataTypes(prepared.getDataTypes().stream().map(TSDataType::ordinal).toList()) + .setColumnCategories( + prepared.getColumnCategories().stream().map(c -> (byte) c.ordinal()).toList()); + // Prepare lazily on each destination; IDs must never cross physical sessions. + id = connection.prepareNumericRow(request); + prepared.bindConnection( + connection, + connection.getSessionId(), + id, + closePreparedRow(connection, connection.getSessionId(), id)); + } + int tagBytes = 0; + for (String tag : tagValues) { + tagBytes += ReadWriteIOUtils.sizeToWrite(tag); + } + ByteBuffer tagBuffer = prepared.prepareTagBuffer(tagBytes); + for (String tag : tagValues) { + ReadWriteIOUtils.write(tag, tagBuffer); + } + tagBuffer.flip(); + ByteBuffer valueBuffer = prepared.prepareValueBuffer(); + int valueIndex = 0; + for (int i = 0; i < prepared.getDataTypes().size(); i++) { + if (prepared.getColumnCategories().get(i) == ColumnCategory.FIELD) { + long value = values[valueIndex++]; + if (nullBitmap != null && nullBitmap.isMarked(valueIndex - 1)) { + continue; + } + switch (prepared.getDataTypes().get(i)) { + case BOOLEAN -> valueBuffer.put((byte) (value == 0 ? 0 : 1)); + case INT32, DATE, FLOAT -> valueBuffer.putInt((int) value); + case INT64, TIMESTAMP, DOUBLE -> valueBuffer.putLong(value); + default -> + throw new IllegalArgumentException( + SessionMessages.EXCEPTION_UNSUPPORTED_NUMERIC_TYPE_06DD3044); + } + } + } + valueBuffer.flip(); + try { + if (nullBitmap == null) { + connection.executePreparedNumericRow(id, timestamp, valueBuffer, tagBuffer); + } else { + connection.executePreparedNumericRow( + id, + timestamp, + valueBuffer, + tagBuffer, + SessionUtils.encodeNumericRowNullBitmap( + nullBitmap, prepared.prepareNullBitmapBuffer())); + } + } catch (RedirectException e) { + // This row succeeded. Update only the route for subsequent rows, without replaying this one. + if (deviceId != null && e.getEndPoint() != null) { + session.handleRedirection(deviceId, e.getEndPoint()); + } + } catch (StatementExecutionException e) { + // A server may have invalidated its handle on schema failure. Do not let another cached + // destination silently revive that logical statement; require an explicit new prepare. + prepared.close(); + throw e; + } + } + + private static Runnable closePreparedRow(SessionConnection connection, long sessionId, long id) { + return () -> { + if (connection.getSessionId() != sessionId) { + return; + } + try { + connection.closePreparedNumericRow(id); + } catch (IoTDBConnectionException ignored) { + // Closing the physical session releases any handles left by a failed close RPC. + } + }; + } + @Override public void executeNonQueryStatement(String sql) throws IoTDBConnectionException, StatementExecutionException { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java index 4cf7ca980f347..331a078ae9e54 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java @@ -20,16 +20,22 @@ package org.apache.iotdb.session.pool; import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.NumericRowPreparedStatement; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.TableSession; import org.apache.iotdb.session.i18n.SessionMessages; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.record.Tablet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; @@ -46,12 +52,15 @@ public class TableSessionWrapper implements ITableSession { private Session session; + private final TableSession tableSession; + private final SessionPool sessionPool; private final AtomicBoolean closed; protected TableSessionWrapper(Session session, SessionPool sessionPool) { this.session = session; + this.tableSession = new TableSession(session); this.sessionPool = sessionPool; this.closed = new AtomicBoolean(false); } @@ -68,6 +77,64 @@ public void insert(Tablet tablet) throws StatementExecutionException, IoTDBConne } } + @Override + public NumericRowPreparedStatement prepareInsertNumericRow( + String tableName, + List columnNames, + List dataTypes, + List columnCategories) + throws StatementExecutionException, IoTDBConnectionException { + checkOpen(); + try { + return tableSession.prepareInsertNumericRow( + tableName, columnNames, dataTypes, columnCategories); + } catch (IoTDBConnectionException e) { + invalidateSession(); + throw e; + } + } + + @Override + public void executePreparedInsertNumericRow( + NumericRowPreparedStatement prepared, long timestamp, long[] values, String[] tagValues) + throws StatementExecutionException, IoTDBConnectionException { + executePreparedInsertNumericRow(prepared, timestamp, values, tagValues, null); + } + + @Override + public void executePreparedInsertNumericRow( + NumericRowPreparedStatement prepared, + long timestamp, + long[] values, + String[] tagValues, + BitMap nullBitmap) + throws StatementExecutionException, IoTDBConnectionException { + checkOpen(); + try { + tableSession.executePreparedInsertNumericRow( + prepared, timestamp, values, tagValues, nullBitmap); + } catch (IoTDBConnectionException e) { + invalidateSession(); + throw e; + } + } + + private void checkOpen() throws IoTDBConnectionException { + // The cached delegate still references the physical session after it returns to the pool. + if (closed.get() || session == null) { + throw new IoTDBConnectionException( + SessionMessages.EXCEPTION_TABLE_SESSION_IS_CLOSED_77FE359B); + } + } + + private void invalidateSession() { + Session failedSession = session; + // Invalidate before cleanup so close() cannot return a failed connection to the pool. + closed.set(true); + session = null; + sessionPool.cleanSessionAndMayThrowConnectionException(failedSession); + } + @Override public SessionDataSet executeQueryStatement(String sql) throws StatementExecutionException, IoTDBConnectionException { @@ -109,6 +176,9 @@ public void executeNonQueryStatement(String sql) @Override public void close() throws IoTDBConnectionException { + if (closed.get() || session == null) { + return; + } if (!Objects.equals(session.getSqlDialect(), sessionPool.sqlDialect)) { try { session.executeNonQueryStatement("set sql_dialect=" + sessionPool.sqlDialect); diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java index d6ff4d735ae50..1f6c6a3d0aaba 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java @@ -123,6 +123,33 @@ public static ByteBuffer getValueBuffer( return buffer; } + public static void validateNumericRowNullBitmap(BitMap nullBitmap, int fieldCount) { + if (nullBitmap != null && nullBitmap.getSize() != fieldCount) { + throw new IllegalArgumentException( + String.format( + SessionMessages + .EXCEPTION_NUMERIC_ROW_NULL_BITMAP_SIZE_MISMATCH_EXPECTED_ARG_FIELDS_BUT_GOT_ARG_8D30F6A0, + fieldCount, + nullBitmap.getSize())); + } + } + + /** Encodes FIELD bits LSB first with zero padding, including when BitMap.markAll() was used. */ + public static ByteBuffer encodeNumericRowNullBitmap(BitMap nullBitmap, ByteBuffer buffer) { + int fieldCount = nullBitmap.getSize(); + int length = (fieldCount + 7) / 8; + if (buffer == null) { + buffer = ByteBuffer.allocate(length); + } + buffer.clear(); + buffer.put(nullBitmap.getByteArray(), 0, length); + if (fieldCount % 8 != 0) { + buffer.put(length - 1, (byte) (buffer.get(length - 1) & ((1 << (fieldCount % 8)) - 1))); + } + buffer.flip(); + return buffer; + } + public static Object sortValueList(Object valueList, TSDataType dataType, int[] index) { return SessionTypeServices.valueListSorter(dataType).sort(valueList, index); } diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java new file mode 100644 index 0000000000000..6e889f56875f0 --- /dev/null +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session; + +import org.apache.iotdb.isession.NumericRowPreparedStatement; +import org.apache.iotdb.rpc.IoTDBConnectionException; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class PreparedNumericRowBufferTest { + @Test + public void testAllNumericTypesPreserveRawBits() throws Exception { + Session session = mock(Session.class); + SessionConnection connection = mock(SessionConnection.class); + when(session.getDefaultSessionConnection()).thenReturn(connection); + when(connection.prepareNumericRow(any())).thenReturn(1L); + TableSession tableSession = new TableSession(session); + NumericRowPreparedStatement prepared = + tableSession.prepareInsertNumericRow( + "t", + List.of("b", "i", "l", "f", "d", "date", "ts"), + List.of( + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.DATE, + TSDataType.TIMESTAMP), + Collections.nCopies(7, ColumnCategory.FIELD)); + doAnswer( + invocation -> { + ByteBuffer values = invocation.getArgument(2); + assertEquals(37, values.remaining()); + assertEquals(1, values.get()); + assertEquals(Integer.MIN_VALUE, values.getInt()); + assertEquals(Long.MIN_VALUE, values.getLong()); + assertEquals(0x7fc01234, values.getInt()); + assertEquals(0x7ff8000000001234L, values.getLong()); + assertEquals(20260910, values.getInt()); + assertEquals(Long.MAX_VALUE, values.getLong()); + assertFalse(values.hasRemaining()); + return null; + }) + .when(connection) + .executePreparedNumericRow(eq(1L), anyLong(), any(ByteBuffer.class), any(ByteBuffer.class)); + // High bits of 32-bit fields are ignored; noncanonical NaNs must survive without normalization. + tableSession.executePreparedInsertNumericRow( + prepared, + 42, + new long[] { + -1, + 0x1234567880000000L, + Long.MIN_VALUE, + 0x123456787fc01234L, + 0x7ff8000000001234L, + 0x1234567800000000L | 20260910, + Long.MAX_VALUE + }, + new String[0]); + verify(connection) + .executePreparedNumericRow(eq(1L), eq(42L), any(ByteBuffer.class), any(ByteBuffer.class)); + } + + // Reusing the statement after sparse/all-null rows must reset both payload and bitmap cursors. + @Test + public void testNullBitmapBufferReuse() throws Exception { + Session session = mock(Session.class); + SessionConnection connection = mock(SessionConnection.class); + when(session.getDefaultSessionConnection()).thenReturn(connection); + when(connection.prepareNumericRow(any())).thenReturn(1L); + TableSession tableSession = new TableSession(session); + NumericRowPreparedStatement prepared = + tableSession.prepareInsertNumericRow( + "t", + List.of("a", "b", "c"), + List.of(TSDataType.INT64, TSDataType.INT64, TSDataType.INT64), + List.of(ColumnCategory.FIELD, ColumnCategory.FIELD, ColumnCategory.FIELD)); + List bitmaps = new ArrayList<>(); + List payloads = new ArrayList<>(); + int[] masks = {2, 7, 0}; + doAnswer( + invocation -> { + int index = bitmaps.size(); + ByteBuffer values = invocation.getArgument(2); + ByteBuffer bitmap = invocation.getArgument(4); + bitmaps.add(bitmap); + payloads.add(values); + assertEquals(masks[index], bitmap.get()); + assertFalse(bitmap.hasRemaining()); + for (int field = 0; field < 3; field++) { + if ((masks[index] & (1 << field)) == 0) { + assertEquals(field + 10, values.getLong()); + } + } + assertFalse(values.hasRemaining()); + return null; + }) + .when(connection) + .executePreparedNumericRow( + eq(1L), anyLong(), any(ByteBuffer.class), any(ByteBuffer.class), any(ByteBuffer.class)); + BitMap nulls = new BitMap(3); + for (int mask : masks) { + nulls.reset(); + for (int i = 0; i < 3; i++) { + if ((mask & (1 << i)) != 0) { + nulls.mark(i); + } + } + tableSession.executePreparedInsertNumericRow( + prepared, 1, new long[] {10, 11, 12}, new String[0], nulls); + } + assertEquals(3, bitmaps.size()); + for (int i = 1; i < 3; i++) { + assertSame(bitmaps.get(0), bitmaps.get(i)); + assertSame(payloads.get(0), payloads.get(i)); + } + } + + @Test + public void testBuffersAcrossTagGrowthShrinkAndRpcFailure() throws Exception { + Session session = mock(Session.class); + SessionConnection connection = mock(SessionConnection.class); + when(session.getDefaultSessionConnection()).thenReturn(connection); + when(connection.prepareNumericRow(any())).thenReturn(1L); + TableSession tableSession = new TableSession(session); + NumericRowPreparedStatement prepared = + tableSession.prepareInsertNumericRow( + "t", + List.of("tag", "flag", "count", "value"), + List.of(TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.DOUBLE), + List.of( + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD)); + List tagBuffers = new ArrayList<>(); + List valueBuffers = new ArrayList<>(); + String[] tags = {"x", "long-tag-\u4e2d\u6587", "", null, "retry"}; + doAnswer( + invocation -> { + int index = tagBuffers.size(); + ByteBuffer values = invocation.getArgument(2); + ByteBuffer encodedTags = invocation.getArgument(3); + tagBuffers.add(encodedTags); + valueBuffers.add(values); + // Consume both buffers as a transport may do, and verify no previous row bytes + // remain. + assertEquals(tags[index], ReadWriteIOUtils.readString(encodedTags)); + assertFalse(encodedTags.hasRemaining()); + assertEquals(13, values.remaining()); + assertEquals((byte) (index % 2), values.get()); + assertEquals(index, values.getInt()); + assertEquals(Double.doubleToRawLongBits(index + 0.5), values.getLong()); + assertFalse(values.hasRemaining()); + if (index == 3) { + throw new IoTDBConnectionException("simulated transport failure"); + } + return null; + }) + .when(connection) + .executePreparedNumericRow(eq(1L), anyLong(), any(ByteBuffer.class), any(ByteBuffer.class)); + for (int i = 0; i < tags.length; i++) { + try { + tableSession.executePreparedInsertNumericRow( + prepared, + i, + new long[] {i % 2, i, Double.doubleToRawLongBits(i + 0.5)}, + new String[] {tags[i]}); + assertNotEquals(3, i); + } catch (IoTDBConnectionException e) { + assertEquals(3, i); + } + } + assertEquals(tags.length, tagBuffers.size()); + assertNotSame(tagBuffers.get(0), tagBuffers.get(1)); + for (int i = 1; i < tags.length; i++) { + assertSame(valueBuffers.get(0), valueBuffers.get(i)); + assertSame(tagBuffers.get(1), tagBuffers.get(i)); + } + } + + @Test + public void testBuffersAreOwnedByEachStatementWithoutTags() throws Exception { + Session session = mock(Session.class); + SessionConnection connection = mock(SessionConnection.class); + when(session.getDefaultSessionConnection()).thenReturn(connection); + when(connection.prepareNumericRow(any())).thenReturn(0L, 1L); + TableSession tableSession = new TableSession(session); + List buffers = new ArrayList<>(); + doAnswer( + invocation -> { + ByteBuffer values = invocation.getArgument(2); + ByteBuffer tags = invocation.getArgument(3); + assertEquals(0, tags.remaining()); + assertEquals(42L, values.getLong()); + assertFalse(values.hasRemaining()); + buffers.add(values); + buffers.add(tags); + return null; + }) + .when(connection) + .executePreparedNumericRow( + anyLong(), anyLong(), any(ByteBuffer.class), any(ByteBuffer.class)); + for (int i = 0; i < 2; i++) { + NumericRowPreparedStatement prepared = + tableSession.prepareInsertNumericRow( + "t", List.of("v"), List.of(TSDataType.INT64), List.of(ColumnCategory.FIELD)); + tableSession.executePreparedInsertNumericRow(prepared, 0, new long[] {42}, new String[0]); + } + assertNotSame(buffers.get(0), buffers.get(2)); + assertNotSame(buffers.get(1), buffers.get(3)); + } +} diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowMetadataTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowMetadataTest.java new file mode 100644 index 0000000000000..b47b71a961d03 --- /dev/null +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowMetadataTest.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session; + +import org.apache.iotdb.isession.NumericRowPreparedStatement; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; +import org.apache.iotdb.session.i18n.SessionMessages; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class PreparedNumericRowMetadataTest { + private static final String INVALID_ARGUMENTS = + SessionMessages + .EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233; + private static final String INVALID_CATEGORIES = + SessionMessages + .EXCEPTION_NUMERIC_ROWS_SUPPORT_NUMERIC_FIELD_COLUMNS_AND_STRING_TAG_COLUMNS_ONLY_87159FFA; + + @Test + public void testNullMetadataRejectedBeforeRpc() throws Exception { + assertInvalid( + null, + List.of("v"), + List.of(TSDataType.DOUBLE), + List.of(ColumnCategory.FIELD), + INVALID_ARGUMENTS); + assertInvalid( + "t", null, List.of(TSDataType.DOUBLE), List.of(ColumnCategory.FIELD), INVALID_ARGUMENTS); + assertInvalid("t", List.of("v"), null, List.of(ColumnCategory.FIELD), INVALID_ARGUMENTS); + assertInvalid("t", List.of("v"), List.of(TSDataType.DOUBLE), null, INVALID_ARGUMENTS); + } + + @Test + public void testNullMetadataElementsRejectedBeforeRpc() throws Exception { + assertInvalid( + "t", + Arrays.asList("tag", null), + List.of(TSDataType.STRING, TSDataType.DOUBLE), + List.of(ColumnCategory.TAG, ColumnCategory.FIELD), + INVALID_ARGUMENTS); + assertInvalid( + "t", + List.of("tag", "v"), + Arrays.asList(TSDataType.STRING, null), + List.of(ColumnCategory.TAG, ColumnCategory.FIELD), + INVALID_ARGUMENTS); + assertInvalid( + "t", + List.of("tag", "v"), + List.of(TSDataType.STRING, TSDataType.DOUBLE), + Arrays.asList(ColumnCategory.TAG, null), + INVALID_ARGUMENTS); + } + + @Test + public void testMetadataLengthMismatchRejectedBeforeRpc() throws Exception { + final List names = List.of("a", "b"); + final List types = List.of(TSDataType.DOUBLE, TSDataType.DOUBLE); + final List categories = List.of(ColumnCategory.FIELD, ColumnCategory.FIELD); + assertInvalid("t", names.subList(0, 1), types, categories, INVALID_ARGUMENTS); + assertInvalid("t", names, types.subList(0, 1), categories, INVALID_ARGUMENTS); + assertInvalid("t", names, types, categories.subList(0, 1), INVALID_ARGUMENTS); + assertInvalid( + "t", + names, + List.of(TSDataType.DOUBLE, TSDataType.DOUBLE, TSDataType.DOUBLE), + categories, + INVALID_ARGUMENTS); + assertInvalid( + "t", + names, + types, + List.of(ColumnCategory.FIELD, ColumnCategory.FIELD, ColumnCategory.FIELD), + INVALID_ARGUMENTS); + } + + @Test + public void testAttributeColumnRejectedBeforeRpc() throws Exception { + assertInvalid( + "t", + List.of("tag", "attr", "v"), + List.of(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE), + List.of(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD), + INVALID_CATEGORIES); + } + + @Test + public void testNonStringTagRejectedBeforeRpc() throws Exception { + for (TSDataType type : List.of(TSDataType.INT64, TSDataType.TEXT)) { + assertInvalid( + "t", + List.of("tag", "v"), + List.of(type, TSDataType.DOUBLE), + List.of(ColumnCategory.TAG, ColumnCategory.FIELD), + INVALID_CATEGORIES); + } + } + + @Test + public void testMixedColumnOrderPreservesMetadataAndValues() throws Exception { + final Session session = mock(Session.class); + final SessionConnection connection = mock(SessionConnection.class); + when(session.getDefaultSessionConnection()).thenReturn(connection); + when(session.getDatabase()).thenReturn("db"); + when(connection.getSessionId()).thenReturn(10L); + when(connection.prepareNumericRow(any())).thenReturn(42L); + final TableSession tableSession = new TableSession(session); + final List names = List.of("v1", "tag1", "v2", "tag2"); + final List types = + List.of(TSDataType.DOUBLE, TSDataType.STRING, TSDataType.INT32, TSDataType.STRING); + final List categories = + List.of(ColumnCategory.FIELD, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.TAG); + final NumericRowPreparedStatement prepared = + tableSession.prepareInsertNumericRow("t", names, types, categories); + final ArgumentCaptor request = + ArgumentCaptor.forClass(TSPrepareNumericRowReq.class); + verify(connection).prepareNumericRow(request.capture()); + assertMetadata(prepared, request.getValue()); + doAnswer( + invocation -> { + final ByteBuffer values = invocation.getArgument(2); + final ByteBuffer tags = invocation.getArgument(3); + assertEquals("a", ReadWriteIOUtils.readString(tags)); + assertEquals("b", ReadWriteIOUtils.readString(tags)); + assertFalse(tags.hasRemaining()); + assertEquals(12.5, values.getDouble(), 0.0); + assertEquals(7, values.getInt()); + assertFalse(values.hasRemaining()); + return null; + }) + .when(connection) + .executePreparedNumericRow(eq(42L), eq(123L), any(ByteBuffer.class), any(ByteBuffer.class)); + tableSession.executePreparedInsertNumericRow( + prepared, 123L, new long[] {Double.doubleToRawLongBits(12.5), 7}, new String[] {"a", "b"}); + verify(connection) + .executePreparedNumericRow(eq(42L), eq(123L), any(ByteBuffer.class), any(ByteBuffer.class)); + assertEquals(List.of("v1", "tag1", "v2", "tag2"), names); + assertEquals( + List.of(TSDataType.DOUBLE, TSDataType.STRING, TSDataType.INT32, TSDataType.STRING), types); + assertEquals( + List.of(ColumnCategory.FIELD, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.TAG), + categories); + } + + private static void assertMetadata( + NumericRowPreparedStatement prepared, TSPrepareNumericRowReq request) { + assertEquals("t", prepared.getTableName()); + assertEquals("db", prepared.getDatabase()); + assertEquals(2, prepared.getTagCount()); + assertEquals(List.of("tag1", "tag2", "v1", "v2"), prepared.getColumnNames()); + assertEquals( + List.of(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE, TSDataType.INT32), + prepared.getDataTypes()); + assertEquals( + List.of(ColumnCategory.TAG, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD), + prepared.getColumnCategories()); + assertEquals("t", request.getTableName()); + assertEquals(prepared.getColumnNames(), request.getMeasurements()); + assertEquals( + prepared.getDataTypes().stream().map(TSDataType::ordinal).toList(), request.getDataTypes()); + assertEquals( + prepared.getColumnCategories().stream().map(c -> (byte) c.ordinal()).toList(), + request.getColumnCategories()); + } + + private static void assertInvalid( + String table, + List names, + List types, + List categories, + String message) + throws Exception { + final Session session = mock(Session.class); + try { + new TableSession(session).prepareInsertNumericRow(table, names, types, categories); + fail("Invalid metadata must be rejected before preparing a statement"); + } catch (IllegalArgumentException e) { + assertEquals(message, e.getMessage()); + } + verifyNoMoreInteractions(session); + } +} diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowRedirectionTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowRedirectionTest.java new file mode 100644 index 0000000000000..cdbd984aaf74a --- /dev/null +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowRedirectionTest.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.session; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.isession.NumericRowPreparedStatement; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.IClientRPCService; +import org.apache.iotdb.service.rpc.thrift.TSExecutePreparedNumericRowReq; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowResp; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.powermock.reflect.Whitebox; + +import java.util.HashMap; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class PreparedNumericRowRedirectionTest { + private final TEndPoint leader = new TEndPoint("127.0.0.1", 6668); + private Session session; + private TableSession tableSession; + private SessionConnection initial; + private SessionConnection destination; + private IClientRPCService.Iface initialClient; + private IClientRPCService.Iface destinationClient; + + @Before + public void setUp() throws Exception { + session = new Session.Builder().enableRedirection(true).enableAutoFetch(false).build(); + initialClient = mock(IClientRPCService.Iface.class); + destinationClient = mock(IClientRPCService.Iface.class); + initial = connection(initialClient, 11); + destination = connection(destinationClient, 22); + session.setDefaultSessionConnection(initial); + session.tableModelDeviceIdToEndpoint = new HashMap<>(); + session.endPointToSessionConnection = new HashMap<>(); + session.endPointToSessionConnection.put(leader, destination); + tableSession = new TableSession(session); + when(initialClient.prepareNumericRow(any())) + .thenReturn(new TSPrepareNumericRowResp(success()).setPreparedId(101)); + when(destinationClient.prepareNumericRow(any())) + .thenReturn(new TSPrepareNumericRowResp(success()).setPreparedId(202)); + when(initialClient.executePreparedNumericRow(any())) + .thenReturn( + new TSStatus(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) + .setRedirectNode(leader)); + when(destinationClient.executePreparedNumericRow(any())).thenReturn(success()); + when(initialClient.closePreparedNumericRow(anyLong(), anyLong())).thenReturn(success()); + when(destinationClient.closePreparedNumericRow(anyLong(), anyLong())).thenReturn(success()); + } + + // Recommendations acknowledge this row; later rows prepare once on the leader with intact nulls. + @Test + public void testRedirectPreparesPerSessionWithoutReplaying() throws Exception { + NumericRowPreparedStatement prepared = prepare(); + write(prepared, 1, "a"); + verify(destinationClient, never()).executePreparedNumericRow(any()); + write(prepared, 2, "a"); + write(prepared, 3, "a"); + // A different device has no route yet, so it still uses the original handle. + write(prepared, 4, "b"); + verify(initialClient, times(2)).executePreparedNumericRow(any()); + verify(destinationClient, times(1)).prepareNumericRow(any()); + ArgumentCaptor requests = + ArgumentCaptor.forClass(TSExecutePreparedNumericRowReq.class); + verify(destinationClient, times(2)).executePreparedNumericRow(requests.capture()); + for (TSExecutePreparedNumericRowReq request : requests.getAllValues()) { + assertEquals(22, request.getSessionId()); + assertEquals(202, request.getPreparedId()); + assertEquals(2, request.bufferForNullBitmap().get()); + assertEquals(8, request.bufferForValues().remaining()); + assertEquals(42, request.bufferForValues().getLong()); + assertEquals("a", ReadWriteIOUtils.readString(request.bufferForTagValues())); + } + prepared.close(); + prepared.close(); + verify(initialClient).closePreparedNumericRow(11, 101); + verify(destinationClient).closePreparedNumericRow(22, 202); + } + + // Disabling redirection ignores recommendations; USE must not prepare an old row in a new DB. + @Test + public void testDisabledRedirectionAndDatabaseChange() throws Exception { + try (NumericRowPreparedStatement prepared = prepare()) { + session.setEnableRedirection(false); + write(prepared, 1, "a"); + write(prepared, 2, "a"); + assertTrue(session.tableModelDeviceIdToEndpoint.isEmpty()); + session.setEnableRedirection(true); + write(prepared, 3, "a"); + session.changeDatabase("another_database"); + write(prepared, 4, "a"); + verify(initialClient, times(4)).executePreparedNumericRow(any()); + verify(destinationClient, never()).prepareNumericRow(any()); + } + } + + // Reconnecting a destination requires a new ID; reconnecting the owner invalidates the statement. + @Test + public void testReconnectedSessionsNeverReuseOldIds() throws Exception { + NumericRowPreparedStatement prepared = prepare(); + write(prepared, 1, "a"); + write(prepared, 2, "a"); + Whitebox.setInternalState(destination, "sessionId", 23L); + when(destinationClient.prepareNumericRow(any())) + .thenReturn(new TSPrepareNumericRowResp(success()).setPreparedId(303)); + write(prepared, 3, "a"); + assertEquals(Long.valueOf(303), prepared.getPreparedId(destination, 23)); + verify(destinationClient, times(2)).prepareNumericRow(any()); + Whitebox.setInternalState(initial, "sessionId", 12L); + assertThrows(IoTDBConnectionException.class, () -> write(prepared, 4, "a")); + prepared.close(); + verify(initialClient, never()).closePreparedNumericRow(anyLong(), anyLong()); + verify(destinationClient).closePreparedNumericRow(23, 303); + verify(destinationClient, never()).closePreparedNumericRow(22, 202); + } + + // A failed redirected execution is propagated once, and all handles close on a server error. + @Test + public void testFailureDoesNotReplayOrReviveStatement() throws Exception { + NumericRowPreparedStatement prepared = prepare(); + write(prepared, 1, "a"); + when(destinationClient.executePreparedNumericRow(any())) + .thenThrow(new TException("lost response")); + assertThrows(IoTDBConnectionException.class, () -> write(prepared, 2, "a")); + verify(initialClient).executePreparedNumericRow(any()); + verify(destinationClient).executePreparedNumericRow(any()); + assertFalse(prepared.isClosed()); + doReturn( + new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()) + .setMessage("schema error")) + .when(destinationClient) + .executePreparedNumericRow(any()); + assertThrows(StatementExecutionException.class, () -> write(prepared, 3, "a")); + assertTrue(prepared.isClosed()); + verify(initialClient).closePreparedNumericRow(11, 101); + verify(destinationClient).closePreparedNumericRow(22, 202); + assertThrows(IoTDBConnectionException.class, () -> write(prepared, 4, "b")); + verify(initialClient).executePreparedNumericRow(any()); + } + + // A preexisting device route is used immediately, including a device with a null TAG. + @Test + public void testExistingRouteAndNullTag() throws Exception { + session.tableModelDeviceIdToEndpoint.put( + IDeviceID.Factory.DEFAULT_FACTORY.create(new String[] {"t", null}), leader); + try (NumericRowPreparedStatement prepared = prepare()) { + write(prepared, 1, null); + verify(initialClient, never()).executePreparedNumericRow(any()); + verify(destinationClient).executePreparedNumericRow(any()); + } + } + + private NumericRowPreparedStatement prepare() throws Exception { + return tableSession.prepareInsertNumericRow( + "t", + List.of("tag", "v", "n"), + List.of(TSDataType.STRING, TSDataType.INT64, TSDataType.INT64), + List.of(ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD)); + } + + private void write(NumericRowPreparedStatement prepared, long time, String tag) throws Exception { + BitMap nulls = new BitMap(2); + nulls.mark(1); + tableSession.executePreparedInsertNumericRow( + prepared, time, new long[] {42, 99}, new String[] {tag}, nulls); + } + + private static SessionConnection connection(IClientRPCService.Iface client, long sessionId) { + SessionConnection connection = new SessionConnection("table"); + Whitebox.setInternalState(connection, "client", client); + Whitebox.setInternalState(connection, "sessionId", sessionId); + return connection; + } + + private static TSStatus success() { + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } +} diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/PreparedNumericRowPoolTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/PreparedNumericRowPoolTest.java new file mode 100644 index 0000000000000..7a356747001b7 --- /dev/null +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/PreparedNumericRowPoolTest.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.pool; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.isession.NumericRowPreparedStatement; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.IClientRPCService; +import org.apache.iotdb.service.rpc.thrift.TSExecutePreparedNumericRowReq; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowResp; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.SessionConnection; +import org.apache.iotdb.session.i18n.SessionMessages; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.powermock.reflect.Whitebox; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +public class PreparedNumericRowPoolTest { + private Session session; + private SessionPool pool; + private TableSessionWrapper wrapper; + private IClientRPCService.Iface client; + + @Before + public void setUp() throws Exception { + session = new Session.Builder().enableRedirection(false).enableAutoFetch(false).build(); + client = mock(IClientRPCService.Iface.class); + SessionConnection connection = new SessionConnection("table"); + Whitebox.setInternalState(connection, "client", client); + Whitebox.setInternalState(connection, "sessionId", 11L); + Whitebox.setInternalState(session, "defaultSessionConnection", connection); + pool = mock(SessionPool.class); + pool.sqlDialect = session.getSqlDialect(); + wrapper = new TableSessionWrapper(session, pool); + when(client.prepareNumericRow(any())) + .thenReturn(new TSPrepareNumericRowResp(success()).setPreparedId(101)) + .thenReturn(new TSPrepareNumericRowResp(success()).setPreparedId(102)); + when(client.executePreparedNumericRow(any())).thenReturn(success()); + when(client.closePreparedNumericRow(anyLong(), anyLong())).thenReturn(success()); + } + + @Test + public void testClosedWrapperCannotUseReborrowedSession() throws Exception { + NumericRowPreparedStatement oldPrepared = prepare(wrapper); + wrapper.close(); + // A new borrower owns the same physical connection; the previous wrapper must stay closed. + try (TableSessionWrapper next = new TableSessionWrapper(session, pool); + NumericRowPreparedStatement nextPrepared = prepare(next)) { + clearInvocations(client); + IoTDBConnectionException error = + assertThrows(IoTDBConnectionException.class, () -> prepare(wrapper)); + assertEquals(SessionMessages.EXCEPTION_TABLE_SESSION_IS_CLOSED_77FE359B, error.getMessage()); + assertThrows( + IoTDBConnectionException.class, + () -> + wrapper.executePreparedInsertNumericRow( + oldPrepared, 1L, new long[] {42}, new String[] {"a"})); + assertThrows( + IoTDBConnectionException.class, + () -> + wrapper.executePreparedInsertNumericRow( + oldPrepared, 1L, new long[] {42}, new String[] {"a"}, new BitMap(1))); + verifyZeroInteractions(client); + verify(pool, never()).cleanSessionAndMayThrowConnectionException(any()); + + next.executePreparedInsertNumericRow(nextPrepared, 2L, new long[] {43}, new String[] {"a"}); + ArgumentCaptor request = + ArgumentCaptor.forClass(TSExecutePreparedNumericRowReq.class); + verify(client).executePreparedNumericRow(request.capture()); + assertEquals(102L, request.getValue().getPreparedId()); + } + wrapper.close(); + verify(pool, times(2)).putBack(session); + } + + @Test + public void testPrepareTransportFailureDiscardsSession() throws Exception { + TException transportFailure = new TException("transport closed"); + when(client.prepareNumericRow(any())).thenThrow(transportFailure); + IoTDBConnectionException error = + assertThrows( + IoTDBConnectionException.class, + () -> { + try (TableSessionWrapper borrowed = wrapper) { + prepare(borrowed); + } + }); + assertSame(transportFailure, error.getCause()); + // Automatic close must neither mask the transport failure nor return the broken session. + assertEquals(0, error.getSuppressed().length); + assertThrows(IoTDBConnectionException.class, () -> prepare(wrapper)); + wrapper.close(); + verify(pool).cleanSessionAndMayThrowConnectionException(session); + verify(pool, never()).putBack(any()); + verify(client).prepareNumericRow(any()); + } + + @Test + public void testExecuteTransportFailureDiscardsSession() throws Exception { + NumericRowPreparedStatement prepared = prepare(wrapper); + TException transportFailure = new TException("transport closed"); + when(client.executePreparedNumericRow(any())).thenThrow(transportFailure); + IoTDBConnectionException error = + assertThrows( + IoTDBConnectionException.class, + () -> { + try (TableSessionWrapper borrowed = wrapper) { + borrowed.executePreparedInsertNumericRow( + prepared, 1L, new long[] {42}, new String[] {"a"}); + } + }); + assertSame(transportFailure, error.getCause()); + assertEquals(0, error.getSuppressed().length); + assertThrows( + IoTDBConnectionException.class, + () -> + wrapper.executePreparedInsertNumericRow( + prepared, 2L, new long[] {43}, new String[] {"a"})); + verify(pool).cleanSessionAndMayThrowConnectionException(session); + verify(pool, never()).putBack(any()); + verify(client).executePreparedNumericRow(any()); + } + + @Test + public void testPrepareStatementErrorPreservesHealthySession() throws Exception { + when(client.prepareNumericRow(any())) + .thenReturn( + new TSPrepareNumericRowResp( + new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()) + .setMessage("invalid prepared metadata"))); + try (TableSessionWrapper borrowed = wrapper) { + assertThrows(StatementExecutionException.class, () -> prepare(borrowed)); + } + verify(pool, never()).cleanSessionAndMayThrowConnectionException(any()); + verify(pool).putBack(session); + } + + @Test + public void testCloseReturnsSessionOnlyOnce() throws Exception { + wrapper.close(); + wrapper.close(); + verify(pool).putBack(session); + verify(pool, never()).cleanSessionAndMayThrowConnectionException(any()); + verifyZeroInteractions(client); + } + + private static NumericRowPreparedStatement prepare(TableSessionWrapper borrowed) + throws Exception { + return borrowed.prepareInsertNumericRow( + "t", + List.of("tag", "v"), + List.of(TSDataType.STRING, TSDataType.INT64), + List.of(ColumnCategory.TAG, ColumnCategory.FIELD)); + } + + private static TSStatus success() { + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } +} diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java index 88b102642620b..fbfd1a426ae0d 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java @@ -45,6 +45,23 @@ public class SessionUtilsTest { + @Test + public void testNumericRowNullBitmapBoundaries() { + for (int size : new int[] {1, 7, 8, 9, 16}) { + BitMap nulls = new BitMap(size); + nulls.markAll(); + byte[] original = nulls.getByteArray().clone(); + ByteBuffer bitmap = SessionUtils.encodeNumericRowNullBitmap(nulls, null); + Assert.assertEquals((size + 7) / 8, bitmap.remaining()); + Assert.assertEquals( + size % 8 == 0 ? 255 : (1 << (size % 8)) - 1, bitmap.get(bitmap.limit() - 1) & 255); + Assert.assertArrayEquals(original, nulls.getByteArray()); + } + assertThrows( + IllegalArgumentException.class, + () -> SessionUtils.validateNumericRowNullBitmap(new BitMap(2), 1)); + } + @Test public void testGetTimeBuffer() { List schemas = new ArrayList<>(); diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml index 48bc0490611dd..658861c66a9bf 100644 --- a/iotdb-core/datanode/pom.xml +++ b/iotdb-core/datanode/pom.xml @@ -279,6 +279,11 @@ powermock-core test + + org.powermock + powermock-reflect + test + org.powermock powermock-module-junit4 diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index fc241312e51c2..1a7a5996fc570 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1507,4 +1507,28 @@ private DataNodeMiscMessages() {} LOG_MPP_DATA_EXCHANGE_MAX_PAYLOAD_SIZE_ARG_IS_BELOW_MINIMUM_ALLOWED_VALUE_ARG_USING_ARG_794ABC76 = "mpp_data_exchange_max_payload_size_in_bytes (%d) is below the minimum allowed value %d, using %d"; + public static final String EXCEPTION_INVALID_PREPARED_NUMERIC_ROW_METADATA_OR_PAYLOAD_AECCCEAA = "Invalid prepared numeric row metadata or payload."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_REQUEST_IS_NULL_5A9F9BA9 = "Prepared numeric row request is null."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATABASE_IS_NULL_F2A2E0AC = "Prepared numeric row database is null."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TABLE_NAME_IS_NULL_OR_EMPTY_F2D38BE5 = "Prepared numeric row table name is null or empty."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_MEASUREMENTS_ARE_NULL_830DE646 = "Prepared numeric row measurements are null."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPES_ARE_NULL_7C15F477 = "Prepared numeric row data types are null."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_CATEGORIES_ARE_NULL_6F2EE1BE = "Prepared numeric row column categories are null."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_HAS_NO_COLUMNS_D4E40365 = "Prepared numeric row has no columns."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_METADATA_SIZE_MISMATCH_MEASUREMENTS_ARG_DATA_TYPES_ARG_CATEGORIES_ARG_23DAB34B = "Prepared numeric row column metadata size mismatch: measurements %d, data types %d, categories %d."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_NAME_IS_NULL_OR_EMPTY_AT_INDEX_ARG_DEBDCFF8 = "Prepared numeric row column name is null or empty at index %d."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_NAME_IS_DUPLICATED_AT_INDEX_ARG_ARG_73AAFECC = "Prepared numeric row column name is duplicated at index %d: %s."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPE_IS_NULL_AT_INDEX_ARG_FB2A1FD6 = "Prepared numeric row data type is null at index %d."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPE_ORDINAL_IS_INVALID_AT_INDEX_ARG_ARG_BB5059BB = "Prepared numeric row data type ordinal is invalid at index %d: %d."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_CATEGORY_IS_NULL_AT_INDEX_ARG_3ED8315E = "Prepared numeric row column category is null at index %d."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_IS_INVALID_AT_INDEX_ARG_CATEGORY_ARG_AND_TYPE_ARG_9DAEEA10 = "Prepared numeric row column is invalid at index %d: category %d and type %s."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_HAS_NO_FIELD_COLUMNS_A76209AE = "Prepared numeric row has no FIELD columns."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_BUFFER_IS_NULL_3FF27BF7 = "Prepared numeric row tag buffer is null."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_BUFFER_IS_NULL_A928F919 = "Prepared numeric row value buffer is null."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_BUFFER_SIZE_MISMATCH_EXPECTED_ARG_BYTES_BUT_GOT_ARG_B75D48C8 = "Prepared numeric row value buffer size mismatch: expected %d bytes but got %d."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_LENGTH_PREFIX_IS_INCOMPLETE_AT_INDEX_ARG_ARG_BYTES_REMAINING_6D342F3A = "Prepared numeric row tag length prefix is incomplete at index %d: %d bytes remaining."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_LENGTH_IS_INVALID_AT_INDEX_ARG_LENGTH_ARG_ARG_BYTES_REMAINING_DEDD1E8F = "Prepared numeric row tag length is invalid at index %d: length %d, %d bytes remaining."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_BUFFER_HAS_ARG_TRAILING_BYTES_042CF4D2 = "Prepared numeric row tag buffer has %d trailing bytes."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_PAYLOAD_DECODING_FAILED_ARG_F87689DD = "Prepared numeric row payload decoding failed: %s."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_DOES_NOT_BELONG_TO_THIS_SESSION_5CC5C50A = "Prepared numeric row is closed or does not belong to this session."; } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 1672c9024f573..fbc9fae8b17ee 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -20,6 +20,12 @@ package org.apache.iotdb.db.i18n; public final class DataNodeQueryMessages { + public static final String EXCEPTION_NUMERIC_ROW_NULL_BITMAP_LENGTH_MISMATCH_EXPECTED_ARG_BYTES_FOR_ARG_FIELDS_BUT_GOT_ARG_31F1DB42 = + "Numeric row null bitmap length mismatch: expected %d bytes for %d FIELDs but got %d"; + public static final String EXCEPTION_NUMERIC_ROW_NULL_BITMAP_HAS_SET_BITS_OUTSIDE_ARG_FIELDS_AD6066D1 = + "Numeric row null bitmap has set bits outside %d FIELDs"; + public static final String EXCEPTION_NUMERIC_ROW_PAYLOAD_HAS_ARG_TRAILING_BYTES_89E46A34 = + "Numeric row payload has %d trailing bytes"; public static final String EXCEPTION_FRAGMENT_INSTANCE_ARG_IS_ALREADY_ARG_B44984B4 = "Fragment instance %s is already %s"; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index b0266763747a8..a671ebe22435e 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1488,4 +1488,30 @@ private DataNodeMiscMessages() {} LOG_MPP_DATA_EXCHANGE_MAX_PAYLOAD_SIZE_ARG_IS_BELOW_MINIMUM_ALLOWED_VALUE_ARG_USING_ARG_794ABC76 = "mpp_data_exchange_max_payload_size_in_bytes（%d）低于允许的最小值 %d，将使用 %d"; + public static final String EXCEPTION_INVALID_PREPARED_NUMERIC_ROW_METADATA_OR_PAYLOAD_AECCCEAA = + "预备数值行元数据或数据载荷无效"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_REQUEST_IS_NULL_5A9F9BA9 = "预备数值行请求为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATABASE_IS_NULL_F2A2E0AC = "预备数值行数据库为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TABLE_NAME_IS_NULL_OR_EMPTY_F2D38BE5 = "预备数值行表名为 null 或为空"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_MEASUREMENTS_ARE_NULL_830DE646 = "预备数值行 measurements 为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPES_ARE_NULL_7C15F477 = "预备数值行数据类型为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_CATEGORIES_ARE_NULL_6F2EE1BE = "预备数值行列类别为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_HAS_NO_COLUMNS_D4E40365 = "预备数值行没有列"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_METADATA_SIZE_MISMATCH_MEASUREMENTS_ARG_DATA_TYPES_ARG_CATEGORIES_ARG_23DAB34B = "预备数值行列元数据数量不匹配：measurements %d，数据类型 %d，列类别 %d"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_NAME_IS_NULL_OR_EMPTY_AT_INDEX_ARG_DEBDCFF8 = "预备数值行列名在下标 %d 处为 null 或为空"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_NAME_IS_DUPLICATED_AT_INDEX_ARG_ARG_73AAFECC = "预备数值行列名在下标 %d 处重复：%s"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPE_IS_NULL_AT_INDEX_ARG_FB2A1FD6 = "预备数值行数据类型在下标 %d 处为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPE_ORDINAL_IS_INVALID_AT_INDEX_ARG_ARG_BB5059BB = "预备数值行数据类型序号在下标 %d 处无效：%d"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_CATEGORY_IS_NULL_AT_INDEX_ARG_3ED8315E = "预备数值行列类别在下标 %d 处为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_IS_INVALID_AT_INDEX_ARG_CATEGORY_ARG_AND_TYPE_ARG_9DAEEA10 = "预备数值行列在下标 %d 处无效：类别 %d，类型 %s"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_HAS_NO_FIELD_COLUMNS_A76209AE = "预备数值行没有 FIELD 列"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_BUFFER_IS_NULL_3FF27BF7 = "预备数值行 TAG 缓冲区为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_BUFFER_IS_NULL_A928F919 = "预备数值行 FIELD 缓冲区为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_BUFFER_SIZE_MISMATCH_EXPECTED_ARG_BYTES_BUT_GOT_ARG_B75D48C8 = "预备数值行 FIELD 缓冲区大小不匹配：期望 %d 字节，实际 %d 字节"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_LENGTH_PREFIX_IS_INCOMPLETE_AT_INDEX_ARG_ARG_BYTES_REMAINING_6D342F3A = "预备数值行 TAG 长度前缀在下标 %d 处不完整：剩余 %d 字节"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_LENGTH_IS_INVALID_AT_INDEX_ARG_LENGTH_ARG_ARG_BYTES_REMAINING_DEDD1E8F = "预备数值行 TAG 长度在下标 %d 处无效：长度 %d，剩余 %d 字节"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_BUFFER_HAS_ARG_TRAILING_BYTES_042CF4D2 = "预备数值行 TAG 缓冲区有 %d 个尾随字节"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_PAYLOAD_DECODING_FAILED_ARG_F87689DD = "预备数值行数据载荷解码失败：%s"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_DOES_NOT_BELONG_TO_THIS_SESSION_5CC5C50A = + "预备数值行已关闭或不属于当前会话"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 114fc1240ffda..4066f96520374 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -20,6 +20,12 @@ package org.apache.iotdb.db.i18n; public final class DataNodeQueryMessages { + public static final String EXCEPTION_NUMERIC_ROW_NULL_BITMAP_LENGTH_MISMATCH_EXPECTED_ARG_BYTES_FOR_ARG_FIELDS_BUT_GOT_ARG_31F1DB42 = + "Numeric row 的 null bitmap 长度不匹配：预期 %d 字节（对应 %d 个 FIELD），实际为 %d"; + public static final String EXCEPTION_NUMERIC_ROW_NULL_BITMAP_HAS_SET_BITS_OUTSIDE_ARG_FIELDS_AD6066D1 = + "Numeric row 的 null bitmap 在 %d 个 FIELD 范围外存在置位"; + public static final String EXCEPTION_NUMERIC_ROW_PAYLOAD_HAS_ARG_TRAILING_BYTES_89E46A34 = + "Numeric row 数据存在 %d 个多余的尾随字节"; public static final String EXCEPTION_FRAGMENT_INSTANCE_ARG_IS_ALREADY_ARG_B44984B4 = "Fragment instance %s 已处于 %s 状态"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java index 96bbb6ccc16ba..d5d48dcf38ab4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedInsertNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiTabletsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; @@ -44,6 +45,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; @@ -76,6 +78,16 @@ public TSStatus visitRelationalInsertRows(RelationalInsertRowsNode node, DataReg @Override public TSStatus visitInsertRow(InsertRowNode node, DataRegion dataRegion) { + return executeInsertRow(node, dataRegion); + } + + @Override + public TSStatus visitRelationalInsertNumericRow( + RelationalInsertNumericRowNode node, DataRegion dataRegion) { + return executeInsertRow(node, dataRegion); + } + + private TSStatus executeInsertRow(AbstractInsertRowNode node, DataRegion dataRegion) { try { dataRegion.insert(node); dataRegion.insertSeparatorToWAL(node); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java index 3bebbc9115305..dc8a8877b6d7c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java @@ -46,10 +46,12 @@ import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; import org.apache.iotdb.db.pipe.resource.memory.PipeTabletMemoryBlock; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; @@ -366,8 +368,8 @@ public boolean mayEventTimeOverlappedWithTimeRange() { return true; } - if (insertNode instanceof InsertRowNode) { - final long timestamp = ((InsertRowNode) insertNode).getTime(); + if (insertNode instanceof AbstractInsertRowNode row) { + final long timestamp = row.getTime(); return startTime <= timestamp && timestamp <= endTime; } @@ -409,13 +411,14 @@ public boolean mayEventPathsOverlappedWithPattern() { try { final InsertNode insertNode = getInsertNode(); - if (insertNode instanceof RelationalInsertRowNode + if (insertNode instanceof RelationalInsertNumericRowNode + || insertNode instanceof RelationalInsertRowNode || insertNode instanceof RelationalInsertTabletNode || insertNode instanceof RelationalInsertRowsNode) { return true; } - if (insertNode instanceof InsertRowNode || insertNode instanceof InsertTabletNode) { + if (insertNode instanceof AbstractInsertRowNode || insertNode instanceof InsertTabletNode) { final PartialPath devicePartialPath = insertNode.getTargetPath(); return Objects.isNull(devicePartialPath) || treePattern.mayOverlapWithDevice(devicePartialPath.getIDeviceIDAsFullDevice()); @@ -539,6 +542,7 @@ private List initEventParsers() { pipeTaskMeta, this, insertRowNode, treePattern, userEntity)); } break; + case RELATIONAL_INSERT_NUMERIC_ROW: case RELATIONAL_INSERT_ROW: case RELATIONAL_INSERT_TABLET: eventParsers.add( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java index 7a60e99678a64..08f1ce24332d5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java @@ -25,7 +25,7 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.pipe.api.access.Row; @@ -111,7 +111,7 @@ public void markAsNeedToReport() { //////////////////////////// parse //////////////////////////// - protected void parse(final InsertRowNode insertRowNode) { + protected void parse(final AbstractInsertRowNode insertRowNode) { final String[] originColumnNameStringList = insertRowNode.getMeasurements(); final int originColumnSize = originColumnNameStringList == null ? 0 : originColumnNameStringList.length; @@ -129,7 +129,8 @@ protected void parse(final InsertRowNode insertRowNode) { final MeasurementSchema[] originMeasurementSchemaList = insertRowNode.getMeasurementSchemas(); final TsTableColumnCategory[] originColumnCategories = insertRowNode.getColumnCategories(); final TSDataType[] originValueDataTypes = insertRowNode.getDataTypes(); - final Object[] originValues = insertRowNode.getValues(); + // Read individual columns so numeric rows never need an Object[] row snapshot. + final int originValueCount = insertRowNode.getValueCount(); generateColumnIndexMapper( originColumnNameStringList, originColumnIndex2FilteredColumnIndexMapperList); @@ -143,8 +144,7 @@ protected void parse(final InsertRowNode insertRowNode) { originMeasurementSchemaList, originValueDataTypes, i) - && originValues != null - && i < originValues.length); + && i < originValueCount); this.measurementSchemaList = new MeasurementSchema[filteredColumnSize]; this.columnNameStringList = new String[filteredColumnSize]; @@ -166,7 +166,7 @@ protected void parse(final InsertRowNode insertRowNode) { : ColumnCategory.FIELD; this.valueColumnDataTypes[filteredColumnIndex] = originValueDataTypes[i]; final BitMap bitMap = new BitMap(this.timestampColumn.length); - if (Objects.isNull(originValues[i]) || Objects.isNull(originValueDataTypes[i])) { + if (insertRowNode.isNull(i) || Objects.isNull(originValueDataTypes[i])) { fillNullValue( originValueDataTypes[i], this.valueColumns, @@ -176,7 +176,12 @@ protected void parse(final InsertRowNode insertRowNode) { } else { this.valueColumns[filteredColumnIndex] = filterValueColumnsByRowIndexList( - originValueDataTypes[i], originValues[i], rowIndexList, true, null, bitMap); + originValueDataTypes[i], + insertRowNode.getValue(i), + rowIndexList, + true, + null, + bitMap); } this.nullValueColumnBitmaps[filteredColumnIndex] = bitMap; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTablePatternParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTablePatternParser.java index 14708aa4be95e..84dfb70baccbb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTablePatternParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTablePatternParser.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.pipe.api.access.Row; @@ -60,7 +61,9 @@ public TabletInsertionEventTablePatternParser( super(pipeTaskMeta, sourceEvent); this.pattern = pattern; - if (insertNode instanceof RelationalInsertRowNode) { + if (insertNode instanceof RelationalInsertNumericRowNode numeric) { + parse(numeric); + } else if (insertNode instanceof RelationalInsertRowNode) { parse((RelationalInsertRowNode) insertNode); } else if (insertNode instanceof RelationalInsertTabletNode) { parse((RelationalInsertTabletNode) insertNode); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java index 99edef690b5cf..e83d2afc53b1c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java @@ -26,6 +26,8 @@ import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.receiver.transform.converter.ValueConverter; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.tsfile.annotations.TableModel; @@ -45,6 +47,31 @@ public class PipeConvertedInsertRowStatement extends InsertRowStatement { LoggerFactory.getLogger(PipeConvertedInsertRowStatement.class); public PipeConvertedInsertRowStatement(final InsertRowStatement insertRowStatement) { + this((InsertBaseStatement) insertRowStatement); + time = insertRowStatement.getTime(); + values = insertRowStatement.getValues(); + isNeedInferType = insertRowStatement.isNeedInferType(); + deviceID = insertRowStatement.getRawTableDeviceID(); + removeAllFailedMeasurementMarks(); + } + + public PipeConvertedInsertRowStatement(final InsertNumericRowStatement insertRowStatement) { + this((InsertBaseStatement) insertRowStatement); + time = insertRowStatement.getTime(); + // Box only at the Pipe conversion boundary: the receiver may require non-numeric FIELDs. + values = new Object[insertRowStatement.getValueCount()]; + for (int i = 0; i < values.length; i++) { + values[i] = insertRowStatement.getFirstValueOfIndex(i); + } + isNeedInferType = false; + // Schema analysis and failed-column recovery must not mutate the original numeric row. + columnCategories = columnCategories == null ? null : columnCategories.clone(); + tagColumnIndices = null; + attrColumnIndices = null; + removeAllFailedMeasurementMarks(); + } + + private PipeConvertedInsertRowStatement(final InsertBaseStatement insertRowStatement) { super(); // Statement isDebug = insertRowStatement.isDebug(); @@ -59,12 +86,6 @@ public PipeConvertedInsertRowStatement(final InsertRowStatement insertRowStateme attrColumnIndices = insertRowStatement.getAttrColumnIndices(); writeToTable = insertRowStatement.isWriteToTable(); databaseName = insertRowStatement.getDatabaseName().orElse(null); - // InsertRowStatement - time = insertRowStatement.getTime(); - values = insertRowStatement.getValues(); - isNeedInferType = insertRowStatement.isNeedInferType(); - deviceID = insertRowStatement.getRawTableDeviceID(); - // To ensure that the measurement remains unchanged during the WAL writing process, the array // needs to be copied before the failed Measurement mark can be deleted. final MeasurementSchema[] measurementSchemas = insertRowStatement.getMeasurementSchemas(); @@ -87,8 +108,6 @@ public PipeConvertedInsertRowStatement(final InsertRowStatement insertRowStateme if (failedMeasurementIndex2Info != null) { this.failedMeasurementIndex2Info = new HashMap<>(failedMeasurementIndex2Info); } - - removeAllFailedMeasurementMarks(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java index 6158c19bb9950..210defbeb7013 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java @@ -42,6 +42,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; @@ -50,6 +51,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Update; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.crud.DeleteDataStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; @@ -84,6 +86,19 @@ public Statement visitPlan(final PlanNode node, final Void context) { node)); } + @Override + public InsertNumericRowStatement visitRelationalInsertNumericRow( + final RelationalInsertNumericRowNode node, final Void context) { + InsertNumericRowStatement statement = new InsertNumericRowStatement(); + statement.setDevicePath(node.getTargetPath()); + statement.setTime(node.getTime()); + statement.setMeasurements(node.getMeasurements()); + statement.setDataTypes(node.getDataTypes()); + statement.setNumericValues(node.getNumericValues()); + statement.setColumnCategories(node.getColumnCategories()); + return statement; + } + @Override public InsertRowStatement visitInsertRow(final InsertRowNode node, final Void context) { // MeasurementSchema is only allowed to be set at the analysis type stage. However, InsertNode diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java index 5a80da6f6e6e1..d76428d481382 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.StatementNode; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; @@ -221,6 +222,15 @@ public Optional visitInsertRow( new PipeConvertedInsertRowStatement(insertRowStatement), statusAndDatabaseName.getRight()); } + @Override + public Optional visitInsertNumericRow( + final InsertNumericRowStatement insertNumericRowStatement, + final Pair statusAndDatabaseName) { + return tryExecute( + new PipeConvertedInsertRowStatement(insertNumericRowStatement), + statusAndDatabaseName.getRight()); + } + @Override public Optional visitInsertRows( final InsertRowsStatement insertRowsStatement, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java index 39cde6ea4db82..72ad9a2370863 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; @@ -74,6 +75,7 @@ public class InsertNodeMemoryEstimator { private static final String INSERT_MULTI_TABLETS_NODE = "InsertMultiTabletsNode"; private static final String RELATIONAL_INSERT_ROWS_NODE = "RelationalInsertRowsNode"; private static final String RELATIONAL_INSERT_ROW_NODE = "RelationalInsertRowNode"; + private static final String RELATIONAL_INSERT_NUMERIC_ROW_NODE = "RelationalInsertNumericRowNode"; private static final String RELATIONAL_INSERT_TABLET_NODE = "RelationalInsertTabletNode"; private static final long NUM_BYTES_OBJECT_REF = RamUsageEstimator.NUM_BYTES_OBJECT_REF; @@ -106,6 +108,9 @@ public class InsertNodeMemoryEstimator { private static final long RELATIONAL_INSERT_ROW_NODE_SIZE = RamUsageEstimator.shallowSizeOfInstance(RelationalInsertRowNode.class); + private static final long RELATIONAL_INSERT_NUMERIC_ROW_NODE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(RelationalInsertNumericRowNode.class); + private static final long RELATIONAL_INSERT_TABLET_NODE_SIZE = RamUsageEstimator.shallowSizeOfInstance(RelationalInsertTabletNode.class); @@ -176,6 +181,8 @@ public static long sizeOf(final InsertNode insertNode) { return sizeOfRelationalInsertRowsNode((RelationalInsertRowsNode) insertNode); case RELATIONAL_INSERT_ROW_NODE: return sizeOfRelationalInsertRowNode((RelationalInsertRowNode) insertNode); + case RELATIONAL_INSERT_NUMERIC_ROW_NODE: + return sizeOfRelationalInsertNumericRowNode((RelationalInsertNumericRowNode) insertNode); case RELATIONAL_INSERT_TABLET_NODE: return sizeOfRelationalInsertTabletNode((RelationalInsertTabletNode) insertNode); default: @@ -320,6 +327,13 @@ private static long sizeOfRelationalInsertRowNode( return size; } + private static long sizeOfRelationalInsertNumericRowNode( + final RelationalInsertNumericRowNode node) { + return RELATIONAL_INSERT_NUMERIC_ROW_NODE_SIZE + + calculateFullInsertNodeSize(node) + + node.getNumericValues().ramBytesUsed(); + } + private static long sizeOfRelationalInsertTabletNode(final RelationalInsertTabletNode node) { return sizeOfRelationalInsertTabletNode(node, null); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBatchReqV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBatchReqV2.java index 62cc86a44ca7c..5943ef650030f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBatchReqV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBatchReqV2.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; @@ -78,9 +79,12 @@ public List constructStatements() { continue; } if (statement.isWriteToTable()) { - if (statement instanceof InsertRowStatement) { - addTableModelInsertRowStatement( - tableModelDatabaseInsertRowStatementMap, (InsertRowStatement) statement); + if (statement instanceof InsertNumericRowStatement) { + // Keep numeric rows independent: grouping into InsertRowsStatement would box their + // values. + statements.add(statement); + } else if (statement instanceof InsertRowStatement row) { + addTableModelInsertRowStatement(tableModelDatabaseInsertRowStatementMap, row); } else if (statement instanceof InsertTabletStatement) { statements.add(statement); } else if (statement instanceof InsertRowsStatement) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReq.java index cf4a746cb0eaf..cf89d64461ab7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReq.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReq.java @@ -24,8 +24,8 @@ import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.receiver.protocol.thrift.IoTDBDataNodeReceiver; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; @@ -55,7 +55,7 @@ public ByteBuffer getByteBuffer() { public InsertBaseStatement constructStatement() { final InsertNode insertNode = parseByteBuffer(); - if (!(insertNode instanceof InsertRowNode + if (!(insertNode instanceof AbstractInsertRowNode || insertNode instanceof InsertTabletNode || insertNode instanceof InsertRowsNode)) { throw new UnsupportedOperationException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReqV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReqV2.java index 196af0b16d2d1..d764dc7ce9847 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReqV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReqV2.java @@ -24,8 +24,8 @@ import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.receiver.protocol.thrift.IoTDBDataNodeReceiver; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; @@ -56,7 +56,7 @@ public String getDataBaseName() { public InsertBaseStatement constructStatement() { final InsertNode insertNode = parseByteBuffer(); - if (!(insertNode instanceof InsertRowNode + if (!(insertNode instanceof AbstractInsertRowNode || insertNode instanceof InsertTabletNode || insertNode instanceof InsertRowsNode)) { throw new UnsupportedOperationException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java index 42f353c250667..40fd29ad98fb4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java @@ -24,8 +24,8 @@ import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.receiver.protocol.thrift.IoTDBDataNodeReceiver; import org.apache.iotdb.db.queryengine.plan.planner.plan.PlanFragment; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; @@ -51,7 +51,7 @@ public InsertNode getInsertNode() { } public InsertBaseStatement constructStatement() { - if (!(insertNode instanceof InsertRowNode + if (!(insertNode instanceof AbstractInsertRowNode || insertNode instanceof InsertTabletNode || insertNode instanceof InsertRowsNode)) { throw new UnsupportedOperationException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReqV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReqV2.java index 4fc18398dda75..45dd2607ef625 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReqV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReqV2.java @@ -25,8 +25,8 @@ import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.receiver.protocol.thrift.IoTDBDataNodeReceiver; import org.apache.iotdb.db.queryengine.plan.planner.plan.PlanFragment; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; @@ -56,7 +56,7 @@ public String getDataBaseName() { @Override public InsertBaseStatement constructStatement() { - if (!(insertNode instanceof InsertRowNode + if (!(insertNode instanceof AbstractInsertRowNode || insertNode instanceof InsertTabletNode || insertNode instanceof InsertRowsNode)) { throw new UnsupportedOperationException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/IClientSession.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/IClientSession.java index 5114ecb0d78a9..8065dc2f5e61d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/IClientSession.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/IClientSession.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; +import org.apache.iotdb.db.queryengine.plan.execution.config.session.PreparedStatementMemoryManager; import org.apache.iotdb.rpc.subscription.annotation.TableModel; import org.apache.iotdb.service.rpc.thrift.TSConnectionInfo; import org.apache.iotdb.service.rpc.thrift.TSConnectionType; @@ -33,12 +34,64 @@ import javax.annotation.Nullable; import java.time.ZoneId; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicLong; public abstract class IClientSession { + private static final AtomicLong NUMERIC_ROW_IDS = new AtomicLong(); + private Map preparedNumericRows; + private boolean numericRowsClosed; + + public synchronized long addPreparedNumericRow(PreparedNumericRowInfo info) { + if (numericRowsClosed) { + throw new SemanticException( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_DOES_NOT_BELONG_TO_THIS_SESSION_5CC5C50A); + } + long preparedId = NUMERIC_ROW_IDS.incrementAndGet(); + if (preparedNumericRows == null) { + preparedNumericRows = new HashMap<>(); + } + PreparedStatementMemoryManager.getInstance() + .allocate(Long.toString(preparedId), info.ramBytesUsed()); + preparedNumericRows.put(preparedId, info); + return preparedId; + } + + public synchronized PreparedNumericRowInfo getPreparedNumericRow(long preparedId) { + PreparedNumericRowInfo info = + preparedNumericRows == null ? null : preparedNumericRows.get(preparedId); + if (info == null) { + throw new SemanticException( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_DOES_NOT_BELONG_TO_THIS_SESSION_5CC5C50A); + } + return info; + } + + public synchronized void removePreparedNumericRow(long preparedId) { + PreparedNumericRowInfo info = + preparedNumericRows == null ? null : preparedNumericRows.remove(preparedId); + if (info != null) { + PreparedStatementMemoryManager.getInstance().release(info.ramBytesUsed()); + } + } + + public synchronized void releasePreparedNumericRows() { + numericRowsClosed = true; + if (preparedNumericRows != null) { + for (PreparedNumericRowInfo info : preparedNumericRows.values()) { + PreparedStatementMemoryManager.getInstance().release(info.ramBytesUsed()); + } + preparedNumericRows = null; + } + } + private long id; private ClientVersion clientVersion; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java new file mode 100644 index 0000000000000..0b4259f2fa96a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java @@ -0,0 +1,345 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.protocol.session; + +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.utils.TimestampPrecisionUtils; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.i18n.DataNodeMiscMessages; +import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.db.schemaengine.table.DataNodeTreeViewSchemaUtils; +import org.apache.iotdb.db.utils.NumericRowValues; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; + +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; + +import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; + +/** + * Session-owned metadata. Each execution gets fresh arrays because schema analysis mutates them. + */ +public final class PreparedNumericRowInfo { + private final String database; + private final String table; + private final String[] measurements; + private final TSDataType[] types; + private final TsTableColumnCategory[] categories; + private final int tagCount; + private final int valueBytes; + private volatile Pair tableVersion; + private final MeasurementSchema[] measurementSchemas; + + public PreparedNumericRowInfo(String database, TSPrepareNumericRowReq req) { + if (req == null) { + throw invalid(DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_REQUEST_IS_NULL_5A9F9BA9); + } + if (database == null) { + throw invalid(DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_DATABASE_IS_NULL_F2A2E0AC); + } + if (req.getTableName() == null || req.getTableName().isEmpty()) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_TABLE_NAME_IS_NULL_OR_EMPTY_F2D38BE5); + } + if (req.getMeasurements() == null) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_MEASUREMENTS_ARE_NULL_830DE646); + } + if (req.getDataTypes() == null) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPES_ARE_NULL_7C15F477); + } + if (req.getColumnCategories() == null) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_CATEGORIES_ARE_NULL_6F2EE1BE); + } + if (req.getMeasurementsSize() == 0) { + throw invalid(DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_HAS_NO_COLUMNS_D4E40365); + } + if (req.getMeasurementsSize() != req.getDataTypesSize() + || req.getMeasurementsSize() != req.getColumnCategoriesSize()) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_METADATA_SIZE_MISMATCH_MEASUREMENTS_ARG_DATA_TYPES_ARG_CATEGORIES_ARG_23DAB34B, + req.getMeasurementsSize(), + req.getDataTypesSize(), + req.getColumnCategoriesSize()); + } + this.database = database; + table = req.getTableName(); + measurements = req.getMeasurements().toArray(new String[0]); + types = new TSDataType[measurements.length]; + categories = new TsTableColumnCategory[measurements.length]; + int tags = 0; + int bytes = 0; + Set names = new HashSet<>(); + for (int i = 0; i < measurements.length; i++) { + Integer type = req.getDataTypes().get(i); + Byte category = req.getColumnCategories().get(i); + if (measurements[i] == null || measurements[i].isEmpty()) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_NAME_IS_NULL_OR_EMPTY_AT_INDEX_ARG_DEBDCFF8, + i); + } + if (!names.add(measurements[i].toLowerCase(Locale.ROOT))) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_NAME_IS_DUPLICATED_AT_INDEX_ARG_ARG_73AAFECC, + i, + measurements[i]); + } + if (type == null) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPE_IS_NULL_AT_INDEX_ARG_FB2A1FD6, + i); + } + if (type < 0 || type >= TSDataType.values().length) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPE_ORDINAL_IS_INVALID_AT_INDEX_ARG_ARG_BB5059BB, + i, + type); + } + if (category == null) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_CATEGORY_IS_NULL_AT_INDEX_ARG_3ED8315E, + i); + } + types[i] = TSDataType.values()[type]; + if (category == ColumnCategory.TAG.ordinal() && types[i] == TSDataType.STRING && tags == i) { + tags++; + categories[i] = TsTableColumnCategory.TAG; + } else if (category == ColumnCategory.FIELD.ordinal() + && NumericRowValues.supports(types[i])) { + categories[i] = TsTableColumnCategory.FIELD; + bytes = Math.addExact(bytes, types[i].getDataTypeSize()); + } else { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_IS_INVALID_AT_INDEX_ARG_CATEGORY_ARG_AND_TYPE_ARG_9DAEEA10, + i, + category, + types[i]); + } + } + tagCount = tags; + valueBytes = bytes; + if (tagCount == measurements.length) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_HAS_NO_FIELD_COLUMNS_A76209AE); + } + TsTable currentTable = DataNodeTableCache.getInstance().getTableInWrite(database, table); + Pair version = currentTable == null ? null : currentTable.getInstanceVersion(); + MeasurementSchema[] schemas = prepareMeasurementSchemas(currentTable); + // Never retain a snapshot assembled across concurrent schema revisions. + if (schemas != null && currentTable.matchesInstanceVersion(version)) { + tableVersion = version; + measurementSchemas = schemas; + } else { + tableVersion = null; + measurementSchemas = null; + } + } + + private MeasurementSchema[] prepareMeasurementSchemas(TsTable currentTable) { + if (!matchesTableSchema(currentTable)) { + return null; + } + MeasurementSchema[] schemas = new MeasurementSchema[measurements.length]; + for (int i = 0; i < measurements.length; i++) { + TsTableColumnSchema column = currentTable.getColumnSchema(measurements[i]); + if (column == null) { + return null; + } + schemas[i] = + new MeasurementSchema( + column.getColumnName(), + types[i], + getDefaultEncoding(types[i]), + TSFileDescriptor.getInstance().getConfig().getCompressor(types[i])); + } + return schemas; + } + + private boolean matchesTableSchema(TsTable currentTable) { + // Views and requests needing TAG padding/reordering or type conversion use normal analysis. + if (currentTable == null + || currentTable.getClass() != TsTable.class + || DataNodeTreeViewSchemaUtils.isTreeViewTable(currentTable) + || currentTable.getTagColumnSchemaList().size() != tagCount) { + return false; + } + for (int i = 0; i < measurements.length; i++) { + TsTableColumnSchema column = currentTable.getColumnSchema(measurements[i]); + if (column == null + || column.getDataType() != types[i] + || column.getColumnCategory() != categories[i] + || (i < tagCount && currentTable.getTagColumnOrdinal(measurements[i]) != i)) { + return false; + } + } + return true; + } + + /** Returns a stable compatible version, retaining the cached schemas and wire layout. */ + public Pair refreshTableVersionIfCompatible(TsTable currentTable) { + if (measurementSchemas == null || currentTable == null) { + return null; + } + Pair version = currentTable.getInstanceVersion(); + if (!matchesTableSchema(currentTable) || !currentTable.matchesInstanceVersion(version)) { + return null; + } + // Publish only a compatible candidate. Each execution rechecks it under the schema lock; + // retaining the old candidate on failure also keeps the session's memory reservation stable. + tableVersion = version; + return version; + } + + public InsertNumericRowStatement bind( + long timestamp, ByteBuffer tags, ByteBuffer values, ByteBuffer nullBitmap) { + if (tags == null) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_TAG_BUFFER_IS_NULL_3FF27BF7); + } + if (values == null) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_BUFFER_IS_NULL_A928F919); + } + NumericRowValues.validateNullBitmap(nullBitmap, types.length - tagCount); + // Preserve the cached fixed-width length for the common non-null path. + int expectedBytes = valueBytes; + if (nullBitmap != null) { + for (int i = tagCount; i < types.length; i++) { + if (NumericRowValues.isNullField(nullBitmap, i - tagCount)) { + expectedBytes -= types[i].getDataTypeSize(); + } + } + } + if (values.remaining() != expectedBytes) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_BUFFER_SIZE_MISMATCH_EXPECTED_ARG_BYTES_BUT_GOT_ARG_B75D48C8, + expectedBytes, + values.remaining()); + } + TimestampPrecisionUtils.checkTimestampPrecision(timestamp); + NumericRowValues row = new NumericRowValues(tagCount, types.length - tagCount); + // Duplicate cursors so validation never changes a caller's buffers, even on failure. + tags = tags.duplicate(); + values = values.duplicate(); + try { + for (int i = 0; i < tagCount; i++) { + if (tags.remaining() < Integer.BYTES) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_TAG_LENGTH_PREFIX_IS_INCOMPLETE_AT_INDEX_ARG_ARG_BYTES_REMAINING_6D342F3A, + i, + tags.remaining()); + } + int length = tags.getInt(tags.position()); + if (length < -1 || length > tags.remaining() - Integer.BYTES) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_TAG_LENGTH_IS_INVALID_AT_INDEX_ARG_LENGTH_ARG_ARG_BYTES_REMAINING_DEDD1E8F, + i, + length, + tags.remaining()); + } + row.setTag(i, ReadWriteIOUtils.readString(tags)); + } + if (tags.hasRemaining()) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_TAG_BUFFER_HAS_ARG_TRAILING_BYTES_042CF4D2, + tags.remaining()); + } + for (int i = tagCount; i < types.length; i++) { + if (NumericRowValues.isNullField(nullBitmap, i - tagCount)) { + row.setNull(i); + } else { + row.read(i, types[i], values); + } + } + } catch (BufferUnderflowException e) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_PAYLOAD_DECODING_FAILED_ARG_F87689DD, + e.getMessage()); + } + InsertNumericRowStatement statement = new InsertNumericRowStatement(); + statement.setDevicePath(new PartialPath(table, false)); + statement.setDatabaseName(database); + statement.setMeasurements(measurements.clone()); + statement.setDataTypes(types.clone()); + statement.setColumnCategories(categories.clone()); + statement.setTime(timestamp); + statement.setNumericValues(row); + if (measurementSchemas != null) { + statement.setPreparedNumericRowInfo(this); + statement.setPreparedTableVersion(tableVersion); + statement.setMeasurementSchemas(measurementSchemas.clone()); + } + return statement; + } + + public long ramBytesUsed() { + return RamUsageEstimator.shallowSizeOfInstance(PreparedNumericRowInfo.class) + + RamUsageEstimator.sizeOf(database) + + RamUsageEstimator.sizeOf(table) + + RamUsageEstimator.sizeOf(measurements) + + RamUsageEstimator.shallowSizeOf(types) + + RamUsageEstimator.shallowSizeOf(categories) + + InsertNodeMemoryEstimator.sizeOfMeasurementSchemas(measurementSchemas) + + (tableVersion == null + ? 0 + : RamUsageEstimator.shallowSizeOf(tableVersion) + + 2 * RamUsageEstimator.shallowSizeOfInstance(Long.class)) + // Account for the map entry and boxed ID retained by the session. + + 64; + } + + private static SemanticException invalid() { + return new SemanticException( + DataNodeMiscMessages.EXCEPTION_INVALID_PREPARED_NUMERIC_ROW_METADATA_OR_PAYLOAD_AECCCEAA); + } + + private static SemanticException invalid(String message, Object... args) { + return new SemanticException(String.format(message, args)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java index 21dbffb606ebd..fa1da5d8ea56e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java @@ -260,6 +260,8 @@ private void releaseSessionResource(IClientSession session, LongConsumer release } } + session.releasePreparedNumericRows(); + // Release PreparedStatement memory resources try { PreparedStatementMemoryManager.getInstance().releaseAllForSession(session); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 692b5bedd2d66..14ea90cca0324 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -64,6 +64,7 @@ import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.protocol.session.PreparedNumericRowInfo; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.protocol.thrift.OperationType; import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; @@ -107,8 +108,9 @@ import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; import org.apache.iotdb.db.queryengine.plan.statement.crud.DeleteDataStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; -import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; @@ -166,6 +168,7 @@ import org.apache.iotdb.service.rpc.thrift.TSDeleteDataReq; import org.apache.iotdb.service.rpc.thrift.TSDropSchemaTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementReq; +import org.apache.iotdb.service.rpc.thrift.TSExecutePreparedNumericRowReq; import org.apache.iotdb.service.rpc.thrift.TSExecutePreparedReq; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; @@ -187,6 +190,8 @@ import org.apache.iotdb.service.rpc.thrift.TSLastDataQueryReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowResp; import org.apache.iotdb.service.rpc.thrift.TSPrepareReq; import org.apache.iotdb.service.rpc.thrift.TSPrepareResp; import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion; @@ -2599,6 +2604,82 @@ public TSStatus insertStringRecordsOfOneDevice(TSInsertStringRecordsOfOneDeviceR } } + @Override + public TSPrepareNumericRowResp prepareNumericRow(TSPrepareNumericRowReq req) { + IClientSession session = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); + if (!SESSION_MANAGER.checkLogin(session)) { + return new TSPrepareNumericRowResp(getNotLoggedInStatus()); + } + try { + if (req.getSessionId() != session.getId()) { + return new TSPrepareNumericRowResp(RpcUtils.getStatus(TSStatusCode.NOT_LOGIN)); + } + PreparedNumericRowInfo info = new PreparedNumericRowInfo(session.getDatabaseName(), req); + TSPrepareNumericRowResp response = new TSPrepareNumericRowResp(); + response.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + response.setPreparedId(session.addPreparedNumericRow(info)); + return response; + } catch (Exception e) { + return new TSPrepareNumericRowResp( + onQueryException( + e, OperationType.INSERT_RECORD.getName(), TSStatusCode.EXECUTE_STATEMENT_ERROR)); + } + } + + @Override + public TSStatus executePreparedNumericRow(TSExecutePreparedNumericRowReq req) { + IClientSession session = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); + if (!SESSION_MANAGER.checkLogin(session)) return getNotLoggedInStatus(); + InsertNumericRowStatement statement = null; + OperationQuota quota = null; + try { + if (req.getSessionId() != session.getId()) return RpcUtils.getStatus(TSStatusCode.NOT_LOGIN); + statement = + session + .getPreparedNumericRow(req.getPreparedId()) + .bind( + req.getTimestamp(), + ByteBuffer.wrap(req.getTagValues()), + ByteBuffer.wrap(req.getValues()), + req.bufferForNullBitmap()); + quota = + DataNodeThrottleQuotaManager.getInstance().checkQuota(session.getUsername(), statement); + long queryId = SESSION_MANAGER.requestQueryId(); + return COORDINATOR.executeForTableModel( + statement, + relationSqlParser, + session, + queryId, + SESSION_MANAGER.getSessionInfo(session), + "", + metadata, + config.getConnectionTimeoutInMS()) + .status; + } catch (IoTDBException e) { + return onIoTDBException(e, OperationType.INSERT_RECORD, e.getErrorCode()); + } catch (Exception e) { + return onQueryException( + e, OperationType.INSERT_RECORD.getName(), TSStatusCode.EXECUTE_STATEMENT_ERROR); + } finally { + if (statement != null && statement.isTableSchemaValidationFailed()) { + // Remove only this session's invalid handle, releasing its reservation exactly once. + session.removePreparedNumericRow(req.getPreparedId()); + } + if (quota != null) { + quota.close(); + } + } + } + + @Override + public TSStatus closePreparedNumericRow(long sessionId, long preparedId) { + IClientSession session = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); + if (!SESSION_MANAGER.checkLogin(session) || session.getId() != sessionId) + return getNotLoggedInStatus(); + session.removePreparedNumericRow(preparedId); + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } + @Override public TSStatus insertRecord(TSInsertRecordReq req) { long t1 = System.nanoTime(); @@ -2615,22 +2696,24 @@ public TSStatus insertRecord(TSInsertRecordReq req) { PathUtils.checkIsLegalSingleMeasurementsAndUpdate(req.getMeasurements())); } - InsertRowStatement statement = StatementGenerator.createStatement(req); + InsertBaseStatement statement = StatementGenerator.createStatement(req); // return success when this statement is empty because server doesn't need to execute it if (statement.isEmpty()) { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } - // permission check - TSStatus status = - AuthorityChecker.checkAuthority( - statement, - new TreeAccessCheckContext( - clientSession.getUserId(), - clientSession.getUsername(), - clientSession.getClientAddress())); - if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return status; + // Table-model privileges are checked during relational analysis. + if (!statement.isWriteToTable()) { + TSStatus status = + AuthorityChecker.checkAuthority( + statement, + new TreeAccessCheckContext( + clientSession.getUserId(), + clientSession.getUsername(), + clientSession.getClientAddress())); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return status; + } } quota = @@ -3561,7 +3644,7 @@ public TSStatus insertStringRecord(final TSInsertStringRecordReq req) { // check whether measurement is legal according to syntax convention req.setMeasurements(PathUtils.checkIsLegalSingleMeasurementsAndUpdate(req.getMeasurements())); - final InsertRowStatement statement = StatementGenerator.createStatement(req); + final InsertBaseStatement statement = StatementGenerator.createStatement(req); // Permission check final TSStatus status = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index adce7c280fcd6..732743ef18716 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -191,7 +191,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DeleteDevice; import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; -import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; import org.apache.iotdb.db.schemaengine.SchemaEngine; import org.apache.iotdb.db.schemaengine.lease.MetadataLeaseManager; @@ -3452,7 +3452,7 @@ public TSStatus stopAndClearDataNode() { @Override public TSStatus insertRecord(TSInsertRecordReq req) throws TException { try { - InsertRowStatement statement = StatementGenerator.createStatement(req); + InsertBaseStatement statement = StatementGenerator.createStatement(req); SessionInfo sessionInfo = new SessionInfo( 0, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java index 6fc521a7f724c..a49fd33161c78 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java @@ -71,6 +71,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; @@ -231,6 +232,12 @@ private TSStatus executePlanNodeInConsensusLayer( } } + @Override + public RegionExecutionResult visitRelationalInsertNumericRow( + final RelationalInsertNumericRowNode node, final WritePlanNodeExecutionContext context) { + return executeDataInsert(node, context); + } + @Override public RegionExecutionResult visitInsertRow( final InsertRowNode node, final WritePlanNodeExecutionContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java index 5da7306656778..9089a9d0242c2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java @@ -60,6 +60,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TimeRange; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; @@ -217,6 +218,11 @@ public static List computeTableDataPartitionParams( } } return computeDataPartitionParams(timePartitionSlotMap, getDatabaseName(statement, context)); + } else if (statement instanceof InsertNumericRowStatement row) { + return computeDataPartitionParams( + Collections.singletonMap( + row.getTableDeviceID(), Collections.singleton(row.getTimePartitionSlot())), + getDatabaseName(statement, context)); } else if (statement instanceof InsertRowStatement) { final InsertRowStatement insertRowStatement = (InsertRowStatement) statement; return computeDataPartitionParams( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java index 7464e6d99fbfc..874f8ffa02e9a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java @@ -47,6 +47,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.component.SelectComponent; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; import org.apache.iotdb.db.queryengine.plan.statement.crud.DeleteDataStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; @@ -278,19 +279,22 @@ public static Statement createStatement(TSAggregationQueryReq req) throws Illega return queryStatement; } - public static InsertRowStatement createStatement(TSInsertRecordReq insertRecordReq) + public static InsertBaseStatement createStatement(TSInsertRecordReq insertRecordReq) throws IllegalPathException, QueryProcessException { final long startTime = System.nanoTime(); // construct insert statement InsertRowStatement insertStatement = new InsertRowStatement(); insertStatement.setDevicePath( - DEVICE_PATH_CACHE.getPartialPath(insertRecordReq.getPrefixPath())); + insertRecordReq.isIsWriteToTable() + ? new PartialPath(insertRecordReq.getPrefixPath(), false) + : DEVICE_PATH_CACHE.getPartialPath(insertRecordReq.getPrefixPath())); TimestampPrecisionUtils.checkTimestampPrecision(insertRecordReq.getTimestamp()); - insertStatement.setTime(insertRecordReq.getTimestamp()); + insertStatement.setMeasurements(insertRecordReq.getMeasurements().toArray(new String[0])); insertStatement.setAligned(insertRecordReq.isAligned); + insertStatement.setTime(insertRecordReq.getTimestamp()); insertStatement.fillValues(insertRecordReq.values); - if (insertRecordReq.isSetIsWriteToTable()) { + if (insertRecordReq.isIsWriteToTable()) { insertStatement.setWriteToTable(insertRecordReq.isIsWriteToTable()); if (!insertRecordReq.isSetColumnCategoryies() || insertRecordReq.getColumnCategoryiesSize() != insertRecordReq.getMeasurementsSize()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/DataNodePlanNodeDeserializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/DataNodePlanNodeDeserializer.java index 4ea0757014908..ab693079ef9d9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/DataNodePlanNodeDeserializer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/DataNodePlanNodeDeserializer.java @@ -120,6 +120,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; @@ -165,6 +166,8 @@ public PlanNode deserializeFromWAL(DataInputStream stream) throws IOException { return ContinuousSameSearchIndexSeparatorNode.deserializeFromWAL(stream); case 2000: return RelationalInsertTabletNode.deserializeFromWAL(stream); + case 2005: + return RelationalInsertNumericRowNode.deserializeFromWAL(stream); case 2001: return RelationalInsertRowNode.deserializeFromWAL(stream); case 2002: @@ -194,6 +197,8 @@ public PlanNode deserializeFromWAL(ByteBuffer buffer) { return ContinuousSameSearchIndexSeparatorNode.deserializeFromWAL(buffer); case 2000: return RelationalInsertTabletNode.deserializeFromWAL(buffer); + case 2005: + return RelationalInsertNumericRowNode.deserializeFromWAL(buffer); case 2001: return RelationalInsertRowNode.deserializeFromWAL(buffer); case 2002: @@ -476,6 +481,8 @@ public PlanNode deserialize(ByteBuffer buffer, short nodeType) { return NonAlignedAggregationTreeDeviceViewScanNode.deserialize(buffer); case 2000: return RelationalInsertTabletNode.deserialize(buffer); + case 2005: + return RelationalInsertNumericRowNode.deserialize(buffer); case 2001: return RelationalInsertRowNode.deserialize(buffer); case 2002: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 43afdc5077f03..ca3ae680fe3be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -118,6 +118,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; @@ -586,6 +587,10 @@ default R visitInsertRow(InsertRowNode node, C context) { return visitPlan(node, context); } + default R visitRelationalInsertNumericRow(RelationalInsertNumericRowNode node, C context) { + return visitPlan(node, context); + } + default R visitRelationalInsertRow(RelationalInsertRowNode node, C context) { return visitInsertRow(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/AbstractInsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/AbstractInsertRowNode.java new file mode 100644 index 0000000000000..f1c0785afa3aa --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/AbstractInsertRowNode.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.LastCacheUpdateSource; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; + +import org.apache.tsfile.enums.TSDataType; + +/** + * Storage-facing contract for single-row writes. Concrete nodes own their value representation and + * wire format; sharing this contract never converts a primitive row to an object row. + */ +public abstract class AbstractInsertRowNode extends InsertNode + implements WALEntryValue, LastCacheUpdateSource { + protected AbstractInsertRowNode(PlanNodeId id) { + super(id); + } + + protected AbstractInsertRowNode( + PlanNodeId id, + PartialPath path, + boolean aligned, + String[] measurements, + TSDataType[] dataTypes) { + super(id, path, aligned, measurements, dataTypes); + } + + public abstract long getTime(); + + public abstract int getValueCount(); + + public abstract boolean isNull(int index); + + /** Boxes only the requested cell for cache and Tablet consumers. */ + public abstract Object getValue(int index); + + /** Returns the original backing storage for allocation accounting, without copying it. */ + public abstract Object getValuesForMemoryAccounting(); + + public abstract void updateLastCache(String databaseName); + + public abstract void serializeToWAL(IWALByteBufferView buffer, long encodedSearchIndex); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index ecf76892e4cd9..ba3f285d7f3d7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -33,12 +33,10 @@ import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; -import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.LastCacheUpdateSource; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunkGroup; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; -import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; import org.apache.iotdb.db.utils.TypeInferenceUtils; import org.apache.iotdb.db.utils.TypeServices; @@ -61,7 +59,7 @@ import java.util.List; import java.util.Objects; -public class InsertRowNode extends InsertNode implements WALEntryValue, LastCacheUpdateSource { +public class InsertRowNode extends AbstractInsertRowNode { private static final byte TYPE_RAW_STRING = -1; @@ -213,6 +211,26 @@ public Object[] getValues() { return values; } + @Override + public int getValueCount() { + return values == null ? 0 : values.length; + } + + @Override + public boolean isNull(int index) { + return index < 0 || index >= getValueCount() || values[index] == null; + } + + @Override + public Object getValue(int index) { + return isNull(index) ? null : values[index]; + } + + @Override + public Object getValuesForMemoryAccounting() { + return values; + } + public void setValues(Object[] values) { this.values = values; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertNumericRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertNumericRowNode.java new file mode 100644 index 0000000000000..0f8a78ed91447 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertNumericRowNode.java @@ -0,0 +1,590 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write; + +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.IPlanVisitor; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; +import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; +import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunkGroup; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; +import org.apache.iotdb.db.utils.NumericRowValues; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * A numeric table row with its own plan type and wire format. FIELDs retain raw long bits and TAGs + * retain strings through consensus and WAL recovery. + */ +public class RelationalInsertNumericRowNode extends AbstractInsertRowNode { + private long time; + private NumericRowValues numericValues; + + public RelationalInsertNumericRowNode(PlanNodeId id) { + super(id); + isAligned = true; + } + + public RelationalInsertNumericRowNode( + PlanNodeId id, + PartialPath table, + String[] measurements, + TSDataType[] dataTypes, + MeasurementSchema[] schemas, + long time, + NumericRowValues values, + TsTableColumnCategory[] categories) { + super(id, table, true, measurements, dataTypes); + this.measurementSchemas = schemas; + this.time = time; + this.numericValues = values; + setColumnCategories(categories); + } + + @Override + public PlanNodeType getType() { + return PlanNodeType.RELATIONAL_INSERT_NUMERIC_ROW; + } + + @Override + public R accept(IPlanVisitor visitor, C context) { + return ((PlanVisitor) visitor).visitRelationalInsertNumericRow(this, context); + } + + @Override + public InsertNode mergeInsertNode(List nodes) { + // One numeric row has exactly one device and timestamp, so it cannot be split into fragments. + // InsertNode.merge returns the sole node before reaching this method. + throw new UnsupportedOperationException(); + } + + @Override + public long getTime() { + return time; + } + + public void setTime(long time) { + this.time = time; + } + + @Override + public long getMinTime() { + return time; + } + + public NumericRowValues getNumericValues() { + return numericValues; + } + + public long[] getValues() { + return numericValues.getValues(); + } + + public void setNumericValues(NumericRowValues values) { + numericValues = values; + deviceID = null; + } + + @Override + public Object getValuesForMemoryAccounting() { + return numericValues; + } + + @Override + public int getValueCount() { + return numericValues == null ? 0 : numericValues.size(); + } + + @Override + public boolean isNull(int index) { + return index < 0 || index >= getValueCount() || numericValues.isNull(index); + } + + @Override + public Object getValue(int index) { + return isNull(index) ? null : numericValues.getObject(index, getDataType(index)); + } + + public String getTableName() { + return targetPath.getFullPath(); + } + + @Override + public IDeviceID getDeviceID() { + if (deviceID == null) { + int tagCount = 0; + for (int i = 0; columnCategories != null && i < columnCategories.length; i++) { + if (columnCategories[i] == TsTableColumnCategory.TAG && isPresent(i)) { + tagCount++; + } + } + String[] segments = new String[tagCount + 1]; + segments[0] = getTableName(); + int tagIndex = 1; + for (int i = 0; columnCategories != null && i < columnCategories.length; i++) { + if (columnCategories[i] == TsTableColumnCategory.TAG && isPresent(i)) { + segments[tagIndex++] = numericValues.getTag(i); + } + } + deviceID = IDeviceID.Factory.DEFAULT_FACTORY.create(segments); + } + return deviceID; + } + + private boolean isPresent(int index) { + return measurements != null + && index >= 0 + && index < measurements.length + && measurements[index] != null + && index < getValueCount() + && columnCategories != null + && index < columnCategories.length + && columnCategories[index] != null + && (measurementSchemas == null + || index < measurementSchemas.length && measurementSchemas[index] != null); + } + + @Override + protected int getValidMeasurementNumber() { + int count = 0; + for (int i = 0; i < getValueCount(); i++) { + if (isPresent(i)) { + count++; + } + } + return count; + } + + @Override + public int getValidMeasurementNumber(boolean fieldOnly) { + int count = 0; + for (int i = 0; i < getValueCount(); i++) { + if (isPresent(i) && isValidMeasurement(i, fieldOnly)) { + count++; + } + } + return count; + } + + @Override + public void markFailedMeasurement(int index) { + if (!isPresent(index)) { + return; + } + measurements[index] = null; + if (dataTypes != null && index < dataTypes.length) { + dataTypes[index] = null; + } + numericValues.setNull(index); + deviceID = null; + measurementColumnCnt = -1; + } + + @Override + public List splitByPartition(IAnalysis analysis) { + TTimePartitionSlot slot = TimePartitionUtils.getTimePartitionSlot(time); + dataRegionReplicaSet = + analysis + .getDataPartitionInfo() + .getDataRegionReplicaSetForWriting(getDeviceID(), slot, analysis.getDatabaseName()); + analysis.setRedirectNodeList( + Collections.singletonList( + dataRegionReplicaSet.getDataNodeLocations().get(0).getClientRpcEndPoint())); + return Collections.singletonList(this); + } + + @Override + public void addChild(PlanNode child) { + /* A single-row write has no children. */ + } + + @Override + public int allowedChildCount() { + return NO_CHILD_ALLOWED; + } + + @Override + public List getOutputColumnNames() { + return null; + } + + @Override + public PlanNode clone() { + throw new UnsupportedOperationException(); + } + + @Override + public long getLastCacheTimestamp() { + return time; + } + + @Override + public boolean hasLastCacheValue(int index) { + return isPresent(index) && !isNull(index) && isWritableFieldMeasurement(index); + } + + @Override + public TimeValuePair getLastCacheValue(int index) { + return hasLastCacheValue(index) + ? new TimeValuePair(time, TsPrimitiveType.getByType(getDataType(index), getValue(index))) + : null; + } + + @Override + public void updateLastCache(String database) { + TableDeviceSchemaCache.getInstance() + .updateLastCacheIfExists(database, getDeviceID(), measurements, measurementSchemas, this); + } + + @Override + public void checkDataType(AbstractMemTable memTable) throws DataTypeInconsistentException { + IWritableMemChunkGroup group = memTable.getWritableMemChunkGroup(getDeviceID()); + if (group != null) { + group.checkDataType(this); + } + } + + private int bodySize() { + // Fixed body fields: timestamp, table name, number of valid columns, and schema-present flag. + int size = + Long.BYTES + ReadWriteIOUtils.sizeToWrite(getTableName()) + Integer.BYTES + Byte.BYTES; + for (int i = 0; i < getValueCount(); i++) { + if (!isPresent(i)) { + continue; + } + // Column identity: either the measurement name or the serialized measurement schema. + size += + measurementSchemas == null + ? ReadWriteIOUtils.sizeToWrite(measurements[i]) + : measurementSchemas[i].serializedSize(); + // Column data type, column category, and null marker. + size += 3 * Byte.BYTES; + if (!isNull(i)) { + // Non-null TAGs carry a length-prefixed String; numeric FIELDs carry one long value. + size += + columnCategories[i] == TsTableColumnCategory.TAG + ? ReadWriteIOUtils.sizeToWrite(numericValues.getTag(i)) + : Long.BYTES; + } + } + return size; + } + + @Override + protected int serializedAttributesSize() { + return PlanNodeType.BYTES + bodySize(); + } + + @Override + public int serializedSize() { + return PlanNodeType.BYTES + Long.BYTES + bodySize(); + } + + @Override + protected void serializeAttributes(ByteBuffer buffer) { + getType().serialize(buffer); + serializeBody(buffer); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + getType().serialize(stream); + serializeBody(stream); + } + + @Override + public void serializeToWAL(IWALByteBufferView buffer) { + serializeToWAL(buffer, getEncodedSearchIndex()); + } + + @Override + public void serializeToWAL(IWALByteBufferView buffer, long encodedSearchIndex) { + buffer.putShort(getType().getNodeType()); + buffer.putLong(encodedSearchIndex); + serializeBody(buffer); + } + + private void serializeBody(ByteBuffer buffer) { + ReadWriteIOUtils.write(time, buffer); + ReadWriteIOUtils.write(getTableName(), buffer); + ReadWriteIOUtils.write(getValidMeasurementNumber(), buffer); + ReadWriteIOUtils.write((byte) (measurementSchemas == null ? 0 : 1), buffer); + for (int i = 0; i < getValueCount(); i++) { + if (!isPresent(i)) { + continue; + } + if (measurementSchemas == null) { + ReadWriteIOUtils.write(measurements[i], buffer); + } else { + measurementSchemas[i].serializeTo(buffer); + } + ReadWriteIOUtils.write( + getDataType(i) == null ? (byte) -1 : getDataType(i).serialize(), buffer); + ReadWriteIOUtils.write(columnCategories[i].getCategory(), buffer); + ReadWriteIOUtils.write((byte) (isNull(i) ? 1 : 0), buffer); + if (!isNull(i)) { + if (columnCategories[i] == TsTableColumnCategory.TAG) { + ReadWriteIOUtils.write(numericValues.getTag(i), buffer); + } else { + ReadWriteIOUtils.write(numericValues.get(i), buffer); + } + } + } + } + + private void serializeBody(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(time, stream); + ReadWriteIOUtils.write(getTableName(), stream); + ReadWriteIOUtils.write(getValidMeasurementNumber(), stream); + ReadWriteIOUtils.write((byte) (measurementSchemas == null ? 0 : 1), stream); + for (int i = 0; i < getValueCount(); i++) { + if (!isPresent(i)) { + continue; + } + if (measurementSchemas == null) { + ReadWriteIOUtils.write(measurements[i], stream); + } else { + measurementSchemas[i].serializeTo(stream); + } + ReadWriteIOUtils.write( + getDataType(i) == null ? (byte) -1 : getDataType(i).serialize(), stream); + ReadWriteIOUtils.write(columnCategories[i].getCategory(), stream); + ReadWriteIOUtils.write((byte) (isNull(i) ? 1 : 0), stream); + if (!isNull(i)) { + if (columnCategories[i] == TsTableColumnCategory.TAG) { + ReadWriteIOUtils.write(numericValues.getTag(i), stream); + } else { + ReadWriteIOUtils.write(numericValues.get(i), stream); + } + } + } + } + + private void serializeBody(IWALByteBufferView buffer) { + WALWriteUtils.write(time, buffer); + WALWriteUtils.write(getTableName(), buffer); + WALWriteUtils.write(getValidMeasurementNumber(), buffer); + WALWriteUtils.write((byte) (measurementSchemas == null ? 0 : 1), buffer); + for (int i = 0; i < getValueCount(); i++) { + if (!isPresent(i)) { + continue; + } + if (measurementSchemas == null) { + WALWriteUtils.write(measurements[i], buffer); + } else { + WALWriteUtils.write(measurementSchemas[i], buffer); + } + WALWriteUtils.write(getDataType(i) == null ? (byte) -1 : getDataType(i).serialize(), buffer); + WALWriteUtils.write(columnCategories[i].getCategory(), buffer); + WALWriteUtils.write((byte) (isNull(i) ? 1 : 0), buffer); + if (!isNull(i)) { + if (columnCategories[i] == TsTableColumnCategory.TAG) { + WALWriteUtils.write(numericValues.getTag(i), buffer); + } else { + WALWriteUtils.write(numericValues.get(i), buffer); + } + } + } + } + + private static RelationalInsertNumericRowNode deserializeBody(ByteBuffer buffer) { + RelationalInsertNumericRowNode node = new RelationalInsertNumericRowNode(new PlanNodeId("")); + node.time = buffer.getLong(); + node.targetPath = new PartialPath(ReadWriteIOUtils.readString(buffer), false); + int count = buffer.getInt(); + boolean hasSchemas = buffer.get() != 0; + node.measurements = new String[count]; + node.dataTypes = new TSDataType[count]; + node.measurementSchemas = hasSchemas ? new MeasurementSchema[count] : null; + TsTableColumnCategory[] categories = new TsTableColumnCategory[count]; + String[] tagPayload = new String[count]; + long[] valuePayload = new long[count]; + boolean[] nonNull = new boolean[count]; + for (int i = 0; i < count; i++) { + if (hasSchemas) { + node.measurementSchemas[i] = MeasurementSchema.deserializeFrom(buffer); + node.measurements[i] = node.measurementSchemas[i].getMeasurementName(); + } else { + node.measurements[i] = ReadWriteIOUtils.readString(buffer); + } + byte type = buffer.get(); + node.dataTypes[i] = type == -1 ? null : TSDataType.deserialize(type); + categories[i] = TsTableColumnCategory.deserialize(buffer); + nonNull[i] = buffer.get() == 0; + if (nonNull[i]) { + if (categories[i] == TsTableColumnCategory.TAG && node.dataTypes[i] == TSDataType.STRING) { + tagPayload[i] = ReadWriteIOUtils.readString(buffer); + } else { + if (!NumericRowValues.supports(node.dataTypes[i]) + || categories[i] != TsTableColumnCategory.FIELD) { + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE + node.dataTypes[i]); + } + valuePayload[i] = buffer.getLong(); + } + } + } + node.numericValues = buildNumericValues(categories, tagPayload, valuePayload, nonNull); + node.setColumnCategories(categories); + return node; + } + + public static RelationalInsertNumericRowNode deserializeFromWAL(ByteBuffer buffer) { + long searchIndex = buffer.getLong(); + RelationalInsertNumericRowNode node = deserializeBody(buffer); + node.setSearchIndexFromWAL(searchIndex); + return node; + } + + private static RelationalInsertNumericRowNode deserializeBody(DataInputStream stream) + throws IOException { + RelationalInsertNumericRowNode node = new RelationalInsertNumericRowNode(new PlanNodeId("")); + node.time = stream.readLong(); + node.targetPath = new PartialPath(ReadWriteIOUtils.readString(stream), false); + int count = stream.readInt(); + boolean hasSchemas = stream.readByte() != 0; + node.measurements = new String[count]; + node.dataTypes = new TSDataType[count]; + node.measurementSchemas = hasSchemas ? new MeasurementSchema[count] : null; + TsTableColumnCategory[] categories = new TsTableColumnCategory[count]; + String[] tagPayload = new String[count]; + long[] valuePayload = new long[count]; + boolean[] nonNull = new boolean[count]; + for (int i = 0; i < count; i++) { + if (hasSchemas) { + node.measurementSchemas[i] = MeasurementSchema.deserializeFrom(stream); + node.measurements[i] = node.measurementSchemas[i].getMeasurementName(); + } else { + node.measurements[i] = ReadWriteIOUtils.readString(stream); + } + byte type = stream.readByte(); + node.dataTypes[i] = type == -1 ? null : TSDataType.deserialize(type); + categories[i] = TsTableColumnCategory.deserialize(stream); + nonNull[i] = stream.readByte() == 0; + if (nonNull[i]) { + if (categories[i] == TsTableColumnCategory.TAG && node.dataTypes[i] == TSDataType.STRING) { + tagPayload[i] = ReadWriteIOUtils.readString(stream); + } else { + if (!NumericRowValues.supports(node.dataTypes[i]) + || categories[i] != TsTableColumnCategory.FIELD) { + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE + node.dataTypes[i]); + } + valuePayload[i] = stream.readLong(); + } + } + } + node.numericValues = buildNumericValues(categories, tagPayload, valuePayload, nonNull); + node.setColumnCategories(categories); + return node; + } + + private static NumericRowValues buildNumericValues( + TsTableColumnCategory[] categories, + String[] tagPayload, + long[] valuePayload, + boolean[] nonNull) { + int tagCount = 0; + while (tagCount < categories.length && categories[tagCount] == TsTableColumnCategory.TAG) { + tagCount++; + } + for (int i = tagCount; i < categories.length; i++) { + if (categories[i] == TsTableColumnCategory.TAG) { + throw new IllegalArgumentException(); + } + } + NumericRowValues values = new NumericRowValues(tagCount, categories.length - tagCount); + for (int i = 0; i < categories.length; i++) { + if (!nonNull[i]) { + values.setNull(i); + } else if (i < tagCount) { + values.setTag(i, tagPayload[i]); + } else { + values.setRawValue(i, valuePayload[i]); + } + } + return values; + } + + public static RelationalInsertNumericRowNode deserializeFromWAL(DataInputStream stream) + throws IOException { + long searchIndex = stream.readLong(); + RelationalInsertNumericRowNode node = deserializeBody(stream); + node.setSearchIndexFromWAL(searchIndex); + return node; + } + + public static RelationalInsertNumericRowNode deserialize(ByteBuffer buffer) { + RelationalInsertNumericRowNode node = deserializeBody(buffer); + node.setPlanNodeId(PlanNodeId.deserialize(buffer)); + return node; + } + + @Override + public boolean equals(Object other) { + return other instanceof RelationalInsertNumericRowNode node + && super.equals(other) + && time == node.time + && Objects.equals(numericValues, node.numericValues); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), time, numericValues); + } + + @Override + public String toString() { + return "RelationalInsertNumericRowNode{table=" + + targetPath + + ", time=" + + time + + ", values=" + + Arrays.toString(getValues()) + + '}'; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 88f605f6dfd65..caf839d8e237c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -186,6 +186,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Insert; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertNumericRow; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRow; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRows; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertTablet; @@ -1320,6 +1321,11 @@ public Scope visitInsert(Insert insert, Optional scope) { insert, scope, Field.newUnqualified(Insert.ROWS, Insert.ROWS_TYPE, Insert.ROWS_CATEGORY)); } + @Override + public Scope visitInsertNumericRow(InsertNumericRow node, Optional context) { + return visitInsert(node, context); + } + @Override public Scope visitInsertRow(InsertRow node, Optional context) { return visitInsert(node, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 6f5f3c278ed3d..c047be7c2b466 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -100,6 +100,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; @@ -130,11 +131,13 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AsofJoinOn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Delete; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertNumericRow; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRow; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRows; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertTablet; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeEnriched; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; @@ -1370,6 +1373,26 @@ public RelationPlan visitInsertTablet(InsertTablet node, Void context) { insertNode, analysis.getRootScope(), Collections.emptyList(), outerContext); } + @Override + public RelationPlan visitInsertNumericRow(InsertNumericRow node, Void context) { + InsertNumericRowStatement statement = node.getInnerTreeStatement(); + String[] measurements = statement.getMeasurements(); + MeasurementSchema[] schemas = statement.getMeasurementSchemas(); + stayConsistent(measurements, schemas); + RelationalInsertNumericRowNode insert = + new RelationalInsertNumericRowNode( + idAllocator.genPlanNodeId(), + statement.getDevicePath(), + measurements, + statement.getDataTypes(), + schemas, + statement.getTime(), + statement.getNumericValues(), + statement.getColumnCategories()); + insert.setFailedMeasurementNumber(statement.getFailedMeasurementNumber()); + return new RelationPlan(insert, analysis.getRootScope(), Collections.emptyList(), outerContext); + } + @Override public RelationPlan visitInsertRow(InsertRow node, Void context) { InsertRowStatement insertRowStatement = node.getInnerTreeStatement(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java index cca616e0a3b17..c91eb3e63fcd1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java @@ -55,6 +55,7 @@ import org.apache.iotdb.db.queryengine.plan.scheduler.IScheduler; import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.rpc.RpcUtils; @@ -239,6 +240,19 @@ public void setRedirectInfo(IAnalysis iAnalysis, TEndPoint localEndPoint, TSStat ((WrappedInsertStatement) statementToRedirect).getInnerTreeStatement(); if (!analysis.isFinishQueryAfterAnalyze()) { + if (insertStatement instanceof InsertNumericRowStatement) { + // A numeric row uses insertRecord, whose redirect response carries one endpoint. + if (tsstatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + List endpoints = analysis.getRedirectNodeList(); + if (endpoints != null + && !endpoints.isEmpty() + && !localEndPoint.equals(endpoints.get(0))) { + tsstatus.setCode(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()); + tsstatus.setRedirectNode(endpoints.get(0)); + } + } + return; + } // Table Model Session supports insertTablet and pipe-generated insertRows statements. if (insertStatement instanceof InsertTabletStatement || insertStatement instanceof InsertRowsStatement) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index 9e62a311713ea..d829551f82f74 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -235,6 +235,10 @@ default R visitSetSystemStatus(SetSystemStatus node, C context) { return visitStatement(node, context); } + default R visitInsertNumericRow(InsertNumericRow node, C context) { + return visitNode(node, context); + } + default R visitInsertRow(InsertRow node, C context) { return visitStatement(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertNumericRow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertNumericRow.java new file mode 100644 index 0000000000000..8145485a96e9a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertNumericRow.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IAstVisitor; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.analyze.lock.DataNodeSchemaLockManager; +import org.apache.iotdb.db.queryengine.plan.analyze.lock.SchemaLockType; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; + +import org.apache.tsfile.utils.Pair; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** Relational analysis entry for a primitive numeric row. */ +public class InsertNumericRow extends WrappedInsertStatement { + public InsertNumericRow(InsertNumericRowStatement statement, MPPQueryContext context) { + super(statement, context); + } + + @Override + public R accept(IAstVisitor visitor, C context) { + return ((AstVisitor) visitor).visitInsertNumericRow(this, context); + } + + @Override + public InsertNumericRowStatement getInnerTreeStatement() { + return (InsertNumericRowStatement) super.getInnerTreeStatement(); + } + + @Override + public void validateTableSchema(Metadata metadata, MPPQueryContext context) { + try { + validatePreparedTableSchema(metadata, context); + if (getInnerTreeStatement().hasFailedMeasurements()) { + getInnerTreeStatement().markTableSchemaValidationFailed(); + } + } catch (RuntimeException e) { + // The coordinator may convert exceptions to statuses; retain the failure for RPC cleanup. + getInnerTreeStatement().markTableSchemaValidationFailed(); + throw e; + } + } + + private void validatePreparedTableSchema(Metadata metadata, MPPQueryContext context) { + InsertNumericRowStatement statement = getInnerTreeStatement(); + if (statement.getPreparedTableVersion() != null) { + // Check at analysis time under the same deletion lock and metadata lease as normal inserts. + DataNodeSchemaLockManager.getInstance() + .takeReadLock(context, SchemaLockType.VALIDATE_VS_DELETION_TABLE); + TsTable currentTable = + DataNodeTableCache.getInstance().getTableInWrite(getDatabase(), getTableName()); + boolean compatible = + currentTable != null + && currentTable.matchesInstanceVersion(statement.getPreparedTableVersion()); + if (!compatible && statement.getPreparedNumericRowInfo() != null) { + Pair version = + statement.getPreparedNumericRowInfo().refreshTableVersionIfCompatible(currentTable); + if (version != null) { + statement.setPreparedTableVersion(version); + compatible = true; + } + } + if (compatible) { + statement.setAttributeColumnsPresent(false); + // Cached schema names must not be mutated by later normalization. + statement.setToLowerCaseApplied(true); + statement.semanticCheck(); + return; + } + // Full validation may cast, reorder, or fail columns; discard all cached schema state first. + statement.setPreparedTableVersion(null); + statement.setMeasurementSchemas(null); + } + super.validateTableSchema(metadata, context); + } + + @Override + public void updateAfterSchemaValidation(MPPQueryContext context) { + // Numeric rows are already typed; schema validation casts values in their primitive storage. + } + + @Override + public String getTableName() { + return getInnerTreeStatement().getTableName(); + } + + @Override + public List getDeviceIdList() { + Object[] segments = getInnerTreeStatement().getTableDeviceID().getSegments(); + return Collections.singletonList(Arrays.copyOfRange(segments, 1, segments.length)); + } + + @Override + public List getAttributeColumnNameList() { + return Collections.emptyList(); + } + + @Override + public List getAttributeValueList() { + return Collections.singletonList(new Object[0]); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java index a4eca4ba7ae1b..a66cdcfb695fd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java @@ -204,4 +204,5 @@ public enum StatementType { SHOW_DISK_USAGE, SHOW_RECEIVERS, + INSERT_NUMERIC_ROW, } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index fdb01b9c62b95..ee5062bc70e63 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.DeleteDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; @@ -406,6 +407,10 @@ public R visitLoadFile(LoadTsFileStatement loadTsFileStatement, C context) { return visitStatement(loadTsFileStatement, context); } + public R visitInsertNumericRow(InsertNumericRowStatement statement, C context) { + return visitInsertBase(statement, context); + } + public R visitInsertRow(InsertRowStatement insertRowStatement, C context) { return visitInsertBase(insertRowStatement, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java new file mode 100644 index 0000000000000..46b70fea2c67d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.crud; + +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Statement; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator; +import org.apache.iotdb.db.protocol.session.PreparedNumericRowInfo; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaValidation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertNumericRow; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; +import org.apache.iotdb.db.utils.NumericRowValues; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +/** A table-only numeric row. Schema edits and partial failures keep the payload in long[]. */ +public class InsertNumericRowStatement extends InsertBaseStatement { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(InsertNumericRowStatement.class); + private long time; + private NumericRowValues numericValues; + private IDeviceID deviceID; + // A candidate snapshot, checked again under the schema analysis lock before reuse. + private Pair preparedTableVersion; + private PreparedNumericRowInfo preparedNumericRowInfo; + private volatile boolean tableSchemaValidationFailed; + + public InsertNumericRowStatement() { + statementType = StatementType.INSERT_NUMERIC_ROW; + writeToTable = true; + isAligned = true; + } + + public Pair getPreparedTableVersion() { + return preparedTableVersion; + } + + public void setPreparedTableVersion(Pair preparedTableVersion) { + this.preparedTableVersion = preparedTableVersion; + } + + public PreparedNumericRowInfo getPreparedNumericRowInfo() { + return preparedNumericRowInfo; + } + + public void setPreparedNumericRowInfo(PreparedNumericRowInfo preparedNumericRowInfo) { + this.preparedNumericRowInfo = preparedNumericRowInfo; + } + + public boolean isTableSchemaValidationFailed() { + return tableSchemaValidationFailed; + } + + public void markTableSchemaValidationFailed() { + tableSchemaValidationFailed = true; + } + + public long getTime() { + return time; + } + + public void setTime(long time) { + this.time = time; + } + + public NumericRowValues getNumericValues() { + return numericValues; + } + + public long[] getValues() { + return numericValues.getValues(); + } + + public void setNumericValues(NumericRowValues values) { + numericValues = values; + deviceID = null; + } + + public int getValueCount() { + return numericValues == null ? 0 : numericValues.size(); + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitInsertNumericRow(this, context); + } + + @Override + public Statement toRelationalStatement(MPPQueryContext context) { + return new InsertNumericRow(this, context); + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } + + @Override + public long getMinTime() { + return time; + } + + public TTimePartitionSlot getTimePartitionSlot() { + return TimePartitionUtils.getTimePartitionSlot(time); + } + + @Override + public boolean isEmpty() { + return getValueCount() == 0; + } + + @Override + public Object getFirstValueOfIndex(int index) { + return index < 0 || index >= getValueCount() + ? null + : numericValues.getObject(index, getDataType(index)); + } + + @Override + public boolean isColumnPresent(int index) { + return super.isColumnPresent(index) && index < getValueCount(); + } + + @Override + protected boolean checkAndCastDataType(int index, TSDataType type) { + if (index < 0 + || index >= getValueCount() + || !numericValues.cast(index, getDataType(index), type)) { + return false; + } + dataTypes[index] = type; + return true; + } + + @Override + public void semanticCheck() { + super.semanticCheck(); + if (measurements.length != getValueCount()) { + throw new SemanticException( + String.format( + DataNodeQueryMessages + .THE_MEASUREMENTLIST_S_SIZE_D_IS_NOT_CONSISTENT_WITH_THE_VALUELIST_S_SIZE_D, + measurements.length, + getValueCount())); + } + } + + @Override + public void markFailedMeasurement(int index, Exception cause) { + if (!isColumnPresent(index)) { + return; + } + if (failedMeasurementIndex2Info == null) { + failedMeasurementIndex2Info = new HashMap<>(); + } + failedMeasurementIndex2Info.putIfAbsent( + index, + new FailedMeasurementInfo( + measurements[index], getDataType(index), getFirstValueOfIndex(index), cause)); + measurements[index] = null; + if (dataTypes != null && index < dataTypes.length) { + dataTypes[index] = null; + } + numericValues.setNull(index); + tagColumnIndices = null; + deviceID = null; + } + + @Override + public void removeAllFailedMeasurementMarks() { + if (failedMeasurementIndex2Info == null) { + return; + } + failedMeasurementIndex2Info.forEach( + (index, info) -> { + measurements[index] = info.getMeasurement(); + dataTypes[index] = info.getDataType(); + numericValues.setObject(index, info.getDataType(), info.getValue()); + }); + failedMeasurementIndex2Info.clear(); + tagColumnIndices = null; + deviceID = null; + } + + @Override + public void insertColumn(int pos, ColumnSchema columnSchema) { + super.insertColumn(pos, columnSchema); + int[] mapping = new int[measurements.length]; + for (int i = 0; i < mapping.length; i++) { + mapping[i] = i < pos ? i : i == pos ? -1 : i - 1; + } + numericValues = numericValues.remap(mapping, countTagColumns(columnCategories)); + deviceID = null; + } + + @Override + public void swapColumn(int src, int target) { + super.swapColumn(src, target); + int[] newToOld = new int[measurements.length]; + for (int i = 0; i < newToOld.length; i++) { + newToOld[i] = i; + } + newToOld[src] = target; + newToOld[target] = src; + numericValues = numericValues.remap(newToOld, countTagColumns(columnCategories)); + deviceID = null; + } + + @Override + public void rebuildArraysAfterExpansion(int[] mapping, String[] newMeasurements) { + super.rebuildArraysAfterExpansion(mapping, newMeasurements); + numericValues = numericValues.remap(mapping, countTagColumns(columnCategories)); + deviceID = null; + } + + @Override + protected void subRemoveAttributeColumns(List columnsToKeep) { + numericValues = + numericValues.remap( + columnsToKeep.stream().mapToInt(Integer::intValue).toArray(), + countTagColumns(columnCategories)); + deviceID = null; + } + + @Override + public void setColumnCategories(TsTableColumnCategory[] categories) { + super.setColumnCategories(categories); + if (numericValues != null) { + ColumnCategory[] compactCategories = new ColumnCategory[categories.length]; + for (int i = 0; i < categories.length; i++) { + compactCategories[i] = + categories[i] == TsTableColumnCategory.TAG ? ColumnCategory.TAG : ColumnCategory.FIELD; + } + numericValues.setColumnCategories(compactCategories); + } + deviceID = null; + } + + private static int countTagColumns(TsTableColumnCategory[] categories) { + int count = 0; + while (count < categories.length && categories[count] == TsTableColumnCategory.TAG) { + count++; + } + return count; + } + + public IDeviceID getTableDeviceID() { + if (deviceID == null) { + String[] segments = new String[getTagColumnIndices().size() + 1]; + segments[0] = getTableName(); + for (int i = 0; i < getTagColumnIndices().size(); i++) { + segments[i + 1] = numericValues.getTag(getTagColumnIndices().get(i)); + } + deviceID = IDeviceID.Factory.DEFAULT_FACTORY.create(segments); + } + return deviceID; + } + + @Override + protected long calculateBytesUsed() { + return INSTANCE_SIZE + + (numericValues == null ? 0 : numericValues.ramBytesUsed()) + + InsertNodeMemoryEstimator.sizeOfIDeviceID(deviceID); + } + + // Tree schema validation and logical view splitting do not apply to numeric table rows. + @Override + public ISchemaValidation getSchemaValidation() { + throw new UnsupportedOperationException(); + } + + @Override + public List getSchemaValidationList() { + throw new UnsupportedOperationException(); + } + + @Override + public InsertBaseStatement removeLogicalView() { + return this; + } + + @Override + public String getPipeLoggingString() { + return "InsertNumericRowStatement{table=" + + getTableName() + + ", time=" + + time + + ", valueLength=" + + getValueCount() + + '}'; + } + + @Override + public String toString() { + return getPipeLoggingString() + Arrays.toString(getValues()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/IoTDBInternalLocalReporter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/IoTDBInternalLocalReporter.java index 1630175738f98..1575cbf482fd2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/IoTDBInternalLocalReporter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/IoTDBInternalLocalReporter.java @@ -46,7 +46,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator; -import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateMultiTimeSeriesStatement; import org.apache.iotdb.metrics.config.MetricConfigDescriptor; import org.apache.iotdb.metrics.reporter.iotdb.IoTDBInternalReporter; @@ -212,7 +212,7 @@ private TSStatus insertRecord(Map valueMap, String prefix, long request.setValues(buffer); request.setIsAligned(false); - InsertRowStatement s = StatementGenerator.createStatement(request); + InsertBaseStatement s = StatementGenerator.createStatement(request); final long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = COORDINATOR.executeForTreeModel( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index ead75db6bac6e..ffce64b21af89 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -84,6 +84,7 @@ import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ContinuousSameSearchIndexSeparatorNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiTabletsNode; @@ -1175,7 +1176,7 @@ private int compareFileName(File o1, File o2) { * * @param insertRowNode one row of data */ - public void insert(InsertRowNode insertRowNode) throws WriteProcessException { + public void insert(AbstractInsertRowNode insertRowNode) throws WriteProcessException { // reject insertions that are out of ttl long ttl = getTTL(insertRowNode); if (!CommonUtils.isAlive(insertRowNode.getTime(), ttl)) { @@ -1802,7 +1803,7 @@ private void tryToUpdateInsertTabletLastCache( } private TsFileProcessor insertToTsFileProcessor( - InsertRowNode insertRowNode, boolean sequence, long timePartitionId) + AbstractInsertRowNode insertRowNode, boolean sequence, long timePartitionId) throws WriteProcessException { if (insertRowNode.allMeasurementFailed()) { return null; @@ -1821,7 +1822,7 @@ private TsFileProcessor insertToTsFileProcessor( return tsFileProcessor; } - private void tryToUpdateInsertRowLastCache(final InsertRowNode node) { + private void tryToUpdateInsertRowLastCache(final AbstractInsertRowNode node) { node.updateLastCache(databaseName); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index 35a3696630e50..bc1db1767225c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -26,15 +26,18 @@ import org.apache.iotdb.commons.path.IFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.schemaengine.schemaregion.utils.ResourceByPathUtils; import org.apache.iotdb.db.storageengine.dataregion.flush.FlushStatus; import org.apache.iotdb.db.storageengine.dataregion.flush.NotifyFlushMemTable; @@ -180,12 +183,13 @@ private IWritableMemChunkGroup createMemChunkGroupIfNotExistAndGet( private IWritableMemChunkGroup createAlignedMemChunkGroupIfNotExistAndGet( IDeviceID deviceId, List schemaList) { - List filteredSchemaList = - schemaList.stream().filter(Objects::nonNull).collect(Collectors.toList()); IWritableMemChunkGroup memChunkGroup = memTableMap.computeIfAbsent( deviceId, k -> { + // Existing devices need no filtered copy; new chunks own this mutable schema list. + List filteredSchemaList = + schemaList.stream().filter(Objects::nonNull).collect(Collectors.toList()); seriesNumber += filteredSchemaList.size(); return new AlignedWritableMemChunkGroup(filteredSchemaList, k.isTableModel()); }); @@ -241,10 +245,13 @@ public int insert(InsertRowNode insertRowNode) { } @Override - public int insertAlignedRow(InsertRowNode insertRowNode) { + public int insertAlignedRow(AbstractInsertRowNode insertRowNode) { + if (insertRowNode instanceof RelationalInsertNumericRowNode numeric) { + return insertAlignedNumericRow(numeric); + } String[] measurements = insertRowNode.getMeasurements(); - Object[] values = insertRowNode.getValues(); + Object[] values = ((InsertRowNode) insertRowNode).getValues(); List schemaList = new ArrayList<>(); List dataTypes = new ArrayList<>(); Object[] writableValues = new Object[measurements == null ? 0 : measurements.length]; @@ -286,6 +293,56 @@ public int insertAlignedRow(InsertRowNode insertRowNode) { return pointsInserted; } + private int insertAlignedNumericRow(RelationalInsertNumericRowNode row) { + int measurementCount = row.getMeasurements().length; + int valueCount = row.getValueCount(); + TsTableColumnCategory[] categories = row.getColumnCategories(); + List schemas = new ArrayList<>(measurementCount); + int validValues = 0; + int nullPoints = 0; + long rowSize = Long.BYTES + Integer.BYTES; + for (int i = 0; i < measurementCount; i++) { + boolean valid = row.isValidMeasurement(i, true); + boolean hasValue = i < valueCount; + // Match numeric-row presence checks, counting valid NULL fields but not missing slots. + if (valid + && hasValue + && categories != null + && i < categories.length + && categories[i] != null) { + validValues++; + } + if (!valid || !hasValue || row.isNull(i)) { + if (valid && hasValue) { + nullPoints++; + } + schemas.add(null); + } else { + schemas.add(row.getMeasurementSchemas()[i]); + rowSize += row.getDataType(i).getDataTypeSize(); + } + } + if (schemas.isEmpty()) { + return 0; + } + AlignedWritableMemChunkGroup group = + (AlignedWritableMemChunkGroup) + createAlignedMemChunkGroupIfNotExistAndGet(row.getDeviceID(), schemas); + group + .getAlignedMemChunk() + .writeAlignedNumericPoints(row.getTime(), row.getNumericValues(), schemas); + memSize += rowSize; + int inserted = + validValues + - (IoTDBDescriptor.getInstance().getConfig().isIncludeNullValueInWriteThroughputMetric() + ? 0 + : nullPoints); + totalPointsNum += inserted; + totalValueCount += validValues; + nullValueCount += nullPoints; + return inserted; + } + @Override public int insertTablet(InsertTabletNode insertTabletNode, int start, int end) throws WriteProcessException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index ebdc024aac84b..4f3b54e5ca5fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; +import org.apache.iotdb.db.utils.NumericRowValues; import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.db.utils.datastructure.AlignedTVList; import org.apache.iotdb.db.utils.datastructure.BatchEncodeInfo; @@ -76,6 +77,8 @@ public class AlignedWritableMemChunk extends AbstractWritableMemChunk { private List sortedList; private long sortedRowCount = 0; private final boolean ignoreAllNullRows; + // Allocate reusable scratch space only when the numeric row path is first used. + private int[] numericColumnMapping; private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); private final int TVLIST_SORT_THRESHOLD = CONFIG.getTvListSortThreshold(); @@ -242,6 +245,38 @@ public void writeAlignedPoints( } } + /** Reorders primitive slots directly, including null placeholders for absent columns. */ + public void writeAlignedNumericPoints( + long time, NumericRowValues row, List incomingSchemas) { + int[] mapping = numericColumnMapping; + // Other write paths may have extended the schema since the last numeric row. + if (mapping == null || mapping.length < schemaList.size()) { + mapping = new int[schemaList.size()]; + } + // Missing columns must not reuse source indices from the previous row. + Arrays.fill(mapping, -1); + for (int i = 0; i < incomingSchemas.size(); i++) { + IMeasurementSchema schema = incomingSchemas.get(i); + if (schema == null) { + continue; + } + Integer index = measurementIndexMap.get(schema.getMeasurementName()); + if (index == null) { + index = list.getTsDataTypes().size(); + measurementIndexMap.put(schema.getMeasurementName(), index); + schemaList.add(schema); + list.extendColumn(schema.getType()); + mapping = Arrays.copyOf(mapping, mapping.length + 1); + } + mapping[index] = i; + } + numericColumnMapping = mapping; + list.putAlignedNumericValue(time, row, mapping); + if (TVLIST_SORT_THRESHOLD > 0 && list.rowCount() >= TVLIST_SORT_THRESHOLD) { + handoverAlignedTvList(); + } + } + @Override public void writeAlignedTablet( long[] times, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java index afa24083ff6d3..544963ef7989b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; @@ -106,7 +107,7 @@ void writeAlignedRow( */ int insert(InsertRowNode insertRowNode); - int insertAlignedRow(InsertRowNode insertRowNode); + int insertAlignedRow(AbstractInsertRowNode insertRowNode); /** * insert tablet into this memtable. The rows to be inserted are in the range [start, end). Null diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 9589dcf9a402c..a608e0c251661 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -42,6 +42,7 @@ import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; @@ -80,6 +81,7 @@ import org.apache.iotdb.db.utils.EncryptDBUtils; import org.apache.iotdb.db.utils.MemUtils; import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.iotdb.db.utils.NumericRowValues; import org.apache.iotdb.db.utils.datastructure.AlignedTVList; import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.iotdb.rpc.RpcUtils; @@ -293,7 +295,7 @@ private void ensureMemTable(long[] infoForMetrics) { } } - private static void clearDataRegionReplicaSet(final InsertRowNode insertRowNode) { + private static void clearDataRegionReplicaSet(final AbstractInsertRowNode insertRowNode) { insertRowNode.setDataRegionReplicaSet(null); } @@ -313,7 +315,7 @@ private static void clearDataRegionReplicaSet(final InsertTabletNode insertTable * * @param insertRowNode physical plan of insertion */ - public void insert(InsertRowNode insertRowNode, long[] infoForMetrics) + public void insert(AbstractInsertRowNode insertRowNode, long[] infoForMetrics) throws WriteProcessException { ensureMemTable(infoForMetrics); @@ -332,7 +334,7 @@ public void insert(InsertRowNode insertRowNode, long[] infoForMetrics) insertRowNode.getDeviceID(), insertRowNode.getMeasurements(), insertRowNode.getDataTypes(), - insertRowNode.getValues(), + insertRowNode.getValuesForMemoryAccounting(), insertRowNode.getColumnCategories()); } else { memIncrements = @@ -340,7 +342,7 @@ public void insert(InsertRowNode insertRowNode, long[] infoForMetrics) insertRowNode.getDeviceID(), insertRowNode.getMeasurements(), insertRowNode.getDataTypes(), - insertRowNode.getValues(), + insertRowNode.getValuesForMemoryAccounting(), insertRowNode.getColumnCategories()); } // recordScheduleMemoryBlockCost @@ -393,7 +395,7 @@ public void insert(InsertRowNode insertRowNode, long[] infoForMetrics) if (insertRowNode.isAligned()) { pointInserted = workMemTable.insertAlignedRow(insertRowNode); } else { - pointInserted = workMemTable.insert(insertRowNode); + pointInserted = workMemTable.insert((InsertRowNode) insertRowNode); } } finally { reconcileAlignedTVListRamCost(alignedRamCostSnapshot, memIncrements[0]); @@ -517,7 +519,7 @@ public void insertRows(InsertRowsNode insertRowsNode, long[] infoForMetrics) if (insertRowNode.isAligned()) { pointInserted += workMemTable.insertAlignedRow(insertRowNode); } else { - pointInserted += workMemTable.insert(insertRowNode); + pointInserted += workMemTable.insert((InsertRowNode) insertRowNode); } // update start time of this memtable tsFileResource.updateStartTime(insertRowNode.getDeviceID(), insertRowNode.getTime()); @@ -801,7 +803,7 @@ private long[] checkMemCostAndAddToTspInfoForRow( IDeviceID deviceId, String[] measurements, TSDataType[] dataTypes, - Object[] values, + Object values, TsTableColumnCategory[] columnCategories) throws WriteProcessException { // Fixed-size TVList structures and materialized value primitive arrays. @@ -829,8 +831,8 @@ private long[] checkMemCostAndAddToTspInfoForRow( } } // TEXT data mem size - if (dataTypes[i].isBinary() && values[i] != null) { - textDataIncrement += MemUtils.getBinarySize((Binary) values[i]); + if (dataTypes[i].isBinary() && values instanceof Object[] rowValues && rowValues[i] != null) { + textDataIncrement += MemUtils.getBinarySize((Binary) rowValues[i]); } } updateMemoryInfo(memTableIncrement, chunkMetadataIncrement, textDataIncrement); @@ -849,7 +851,7 @@ private long[] checkMemCostAndAddToTspInfoForRows(List insertRowN for (InsertRowNode insertRowNode : insertRowNodeList) { IDeviceID deviceId = insertRowNode.getDeviceID(); TSDataType[] dataTypes = insertRowNode.getDataTypes(); - Object[] values = insertRowNode.getValues(); + Object values = insertRowNode.getValuesForMemoryAccounting(); String[] measurements = insertRowNode.getMeasurements(); for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements @@ -883,8 +885,10 @@ private long[] checkMemCostAndAddToTspInfoForRows(List insertRowN increasingMemTableInfo.get(deviceId).computeIfPresent(measurements[i], (k, v) -> v + 1); } // TEXT data mem size - if (dataTypes[i].isBinary() && values[i] != null) { - textDataIncrement += MemUtils.getBinarySize((Binary) values[i]); + if (dataTypes[i].isBinary() + && values instanceof Object[] rowValues + && rowValues[i] != null) { + textDataIncrement += MemUtils.getBinarySize((Binary) rowValues[i]); } } } @@ -897,7 +901,7 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRow( IDeviceID deviceId, String[] measurements, TSDataType[] dataTypes, - Object[] values, + Object values, TsTableColumnCategory[] columnCategories) throws WriteProcessException { // Fixed-size TVList structures and materialized value primitive arrays. @@ -946,9 +950,10 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRow( } for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { if (isWritableFieldMeasurement(measurements, dataTypes, values, columnCategories, i) - && dataTypes[i].isBinary()) { + && dataTypes[i].isBinary() + && values instanceof Object[] rowValues) { // Binary values keep their variable-length content outside the fixed Binary[] array. - textDataIncrement += MemUtils.getBinarySize((Binary) values[i]); + textDataIncrement += MemUtils.getBinarySize((Binary) rowValues[i]); } } updateMemoryInfo(memTableIncrement, chunkMetadataIncrement, textDataIncrement); @@ -970,7 +975,7 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins for (InsertRowNode insertRowNode : insertRowNodeList) { IDeviceID deviceId = insertRowNode.getDeviceID(); TSDataType[] dataTypes = insertRowNode.getDataTypes(); - Object[] values = insertRowNode.getValues(); + Object values = insertRowNode.getValuesForMemoryAccounting(); String[] measurements = insertRowNode.getMeasurements(); IWritableMemChunk memChunk = @@ -1062,9 +1067,11 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins measurements, dataTypes, values, insertRowNode.getColumnCategories(), i)) { continue; } - if (dataTypes[i].isBinary() && values[i] != null) { + if (dataTypes[i].isBinary() + && values instanceof Object[] rowValues + && rowValues[i] != null) { // Binary values keep their variable-length content outside the fixed Binary[] array. - textDataIncrement += MemUtils.getBinarySize((Binary) values[i]); + textDataIncrement += MemUtils.getBinarySize((Binary) rowValues[i]); } } } @@ -1357,7 +1364,7 @@ private static boolean containsNonNullValue( private static TSDataType[] getWritableFieldDataTypes( String[] measurementIds, TSDataType[] dataTypes, - Object[] valuesOrColumns, + Object valuesOrColumns, TsTableColumnCategory[] columnCategories) { List writableFieldDataTypes = new ArrayList<>(); for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { @@ -1372,13 +1379,15 @@ private static TSDataType[] getWritableFieldDataTypes( private static boolean isWritableFieldMeasurement( String[] measurementIds, TSDataType[] dataTypes, - Object[] valuesOrColumns, + Object valuesOrColumns, TsTableColumnCategory[] columnCategories, int index) { return isFieldMeasurement(measurementIds, dataTypes, columnCategories, index) - && valuesOrColumns != null - && index < valuesOrColumns.length - && valuesOrColumns[index] != null; + && (valuesOrColumns instanceof NumericRowValues row + ? index < row.size() && !row.isNull(index) + : valuesOrColumns instanceof Object[] columns + && index < columns.length + && columns[index] != null); } private static boolean isFieldMeasurement( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java index 8c0a5e353291e..346e3007c7e5c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java @@ -22,9 +22,9 @@ import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.i18n.StorageEngineMessages; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ContinuousSameSearchIndexSeparatorNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; @@ -65,7 +65,7 @@ protected WALEntry(long memTableId, WALEntryValue value, boolean wait) { this.value = value; if (value instanceof IMemTable) { this.type = WALEntryType.MEMORY_TABLE_SNAPSHOT; - } else if (value instanceof InsertRowNode) { + } else if (value instanceof AbstractInsertRowNode) { this.type = WALEntryType.INSERT_ROW_NODE; } else if (value instanceof InsertTabletNode) { this.type = WALEntryType.INSERT_TABLET_NODE; @@ -121,7 +121,7 @@ public static WALEntry deserialize(DataInputStream stream) throws IOException { value = AbstractMemTable.Factory.createFromOldMemTableSnapshot(stream); break; case INSERT_ROW_NODE: - value = (InsertRowNode) PlanNodeType.deserializeFromWAL(stream); + value = (AbstractInsertRowNode) PlanNodeType.deserializeFromWAL(stream); break; case INSERT_TABLET_NODE: value = (InsertTabletNode) PlanNodeType.deserializeFromWAL(stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java index 957cf03d89237..37cc8aff863b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java @@ -22,9 +22,9 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; @@ -102,7 +102,7 @@ public void serialize(IWALByteBufferView buffer) { .serializeToWAL(buffer, tabletInfo.tabletRangeList, encodedSearchIndex); break; case INSERT_ROW_NODE: - ((InsertRowNode) value).serializeToWAL(buffer, encodedSearchIndex); + ((AbstractInsertRowNode) value).serializeToWAL(buffer, encodedSearchIndex); break; case INSERT_ROWS_NODE: ((InsertRowsNode) value).serializeToWAL(buffer, encodedSearchIndex); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java index a8fbbee0dc4f5..b2689a5f99ae7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java @@ -21,9 +21,9 @@ import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.consensus.iot.log.ConsensusReqReader; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ContinuousSameSearchIndexSeparatorNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; @@ -38,7 +38,7 @@ public interface IWALNode extends FlushListener, AutoCloseable, ConsensusReqReader, DataSet { /** Log InsertRowNode. */ - WALFlushListener log(long memTableId, InsertRowNode insertRowNode); + WALFlushListener log(long memTableId, AbstractInsertRowNode insertRowNode); /** Log InsertRowsNode. */ WALFlushListener log(long memTableId, InsertRowsNode insertRowsNode); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java index d24d1a1d82348..93ac79c05a982 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java @@ -20,9 +20,9 @@ package org.apache.iotdb.db.storageengine.dataregion.wal.node; import org.apache.iotdb.db.i18n.StorageEngineMessages; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ContinuousSameSearchIndexSeparatorNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; @@ -53,7 +53,7 @@ public WALFakeNode(Status status, Exception cause) { } @Override - public WALFlushListener log(long memTableId, InsertRowNode insertRowNode) { + public WALFlushListener log(long memTableId, AbstractInsertRowNode insertRowNode) { return getResult(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java index 77350cca9755d..ff2f2b3e13a77 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java @@ -29,9 +29,9 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ContinuousSameSearchIndexSeparatorNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; @@ -222,7 +222,7 @@ private void removeDeletedWalFilesFromCache(List deletedVersionIds) { } @Override - public WALFlushListener log(long memTableId, InsertRowNode insertRowNode) { + public WALFlushListener log(long memTableId, AbstractInsertRowNode insertRowNode) { logger.debug( StorageEngineMessages.STORAGE_LOG_WAL_NODE_LOGS_INSERTROWNODE_THE_SEARCH_INDEX_IS_027450AC, identifier, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java index 221d39cdfccb3..dbd26029418f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; import org.apache.iotdb.db.exception.WriteProcessException; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; @@ -96,8 +97,8 @@ void redoInsert(InsertNode node) throws WriteProcessException { // the last chunk group may contain the same data with the logs, ignore such logs in seq file Optional lastEndTime = tsFileResource.getEndTime(node.getDeviceID()); long minTimeInNode; - if (node instanceof InsertRowNode) { - minTimeInNode = ((InsertRowNode) node).getTime(); + if (node instanceof AbstractInsertRowNode row) { + minTimeInNode = row.getTime(); } else { minTimeInNode = ((InsertTabletNode) node).getTimes()[0]; } @@ -107,9 +108,9 @@ void redoInsert(InsertNode node) throws WriteProcessException { } int pointsInserted; - if (node instanceof InsertRowNode) { + if (node instanceof AbstractInsertRowNode row) { if (node.isAligned()) { - pointsInserted = recoveryMemTable.insertAlignedRow((InsertRowNode) node); + pointsInserted = recoveryMemTable.insertAlignedRow(row); } else { pointsInserted = recoveryMemTable.insert((InsertRowNode) node); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DataNodeThrottleQuotaManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DataNodeThrottleQuotaManager.java index db7ad470930ea..817a46f4259f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DataNodeThrottleQuotaManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DataNodeThrottleQuotaManager.java @@ -81,6 +81,7 @@ public OperationQuota checkQuota(String userName, Statement s) throws RpcThrottl return NoopOperationQuota.get(); } switch (s.getType()) { + case INSERT_NUMERIC_ROW: case INSERT: case BATCH_INSERT: case BATCH_INSERT_ONE_DEVICE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DefaultOperationQuota.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DefaultOperationQuota.java index 764d6763184ac..b10668d3357d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DefaultOperationQuota.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DefaultOperationQuota.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; @@ -33,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement; import org.apache.iotdb.db.utils.TypeInferenceUtils; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.BitMap; @@ -101,6 +103,21 @@ protected void updateEstimateConsumeQuota(int numWrites, int numReads, Statement } final StatementType statementType = s.getType(); switch (statementType) { + case INSERT_NUMERIC_ROW: + InsertNumericRowStatement numeric = (InsertNumericRowStatement) s; + for (int i = 0; i < numeric.getValueCount(); i++) { + if (!numeric.getNumericValues().isNull(i)) { + avgSize += + numeric.getDataType(i) == TSDataType.STRING + ? numeric + .getNumericValues() + .getTag(i) + .getBytes(TSFileConfig.STRING_CHARSET) + .length + : numeric.getDataType(i).getDataTypeSize(); + } + } + break; case INSERT: // InsertStatement InsertRowStatement if (s instanceof InsertStatement) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java index abd4c5cc381af..51bec7a53204d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java @@ -28,6 +28,7 @@ import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest; import org.apache.iotdb.consensus.common.request.IoTConsensusRequest; import org.apache.iotdb.db.i18n.DataNodePipeMessages; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiTabletsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; @@ -226,8 +227,9 @@ public List convert(final InsertNode insertNode) { return convertInsertRowsOfOneDeviceNode((InsertRowsOfOneDeviceNode) insertNode); case INSERT_MULTI_TABLET: return convertInsertMultiTabletsNode((InsertMultiTabletsNode) insertNode); + case RELATIONAL_INSERT_NUMERIC_ROW: case RELATIONAL_INSERT_ROW: - return convertRelationalInsertRowNode((RelationalInsertRowNode) insertNode); + return convertRelationalInsertRowNode((AbstractInsertRowNode) insertNode); case RELATIONAL_INSERT_TABLET: return convertRelationalInsertTabletNode((RelationalInsertTabletNode) insertNode); case RELATIONAL_INSERT_ROWS: @@ -471,8 +473,8 @@ private List convertInsertMultiTabletsNode(final InsertMultiTabletsNode // ======================== Table Model Conversion ======================== - private List convertRelationalInsertRowNode(final RelationalInsertRowNode node) { - final String tableName = node.getTableName(); + private List convertRelationalInsertRowNode(final AbstractInsertRowNode node) { + final String tableName = node.getTargetPath().getFullPath(); // Table-level pattern filtering if (tablePattern != null) { @@ -484,13 +486,7 @@ private List convertRelationalInsertRowNode(final RelationalInsertRowNod } } - final long time = node.getTime(); - final String[] measurements = node.getMeasurements(); - final TSDataType[] dataTypes = node.getDataTypes(); - final Object[] values = node.getValues(); - final Tablet tablet = - buildTableModelTabletFromRow( - tableName, time, measurements, dataTypes, values, node.getColumnCategories()); + final Tablet tablet = buildTableModelTabletFromRow(node); if (Objects.isNull(tablet)) { return Collections.emptyList(); } @@ -503,7 +499,7 @@ private List convertRelationalInsertRowNode(final RelationalInsertRowNod } private List convertRelationalInsertTabletNode(final RelationalInsertTabletNode node) { - final String tableName = node.getTableName(); + final String tableName = node.getTargetPath().getFullPath(); // Table-level pattern filtering if (tablePattern != null) { @@ -582,14 +578,12 @@ private List convertRelationalInsertRowsNode(final RelationalInsertRowsN return tablets; } - private Tablet buildTableModelTabletFromRow( - final String tableName, - final long time, - final String[] measurements, - final TSDataType[] dataTypes, - final Object[] values, - final TsTableColumnCategory[] columnCategories) { - if (Objects.isNull(measurements) || Objects.isNull(dataTypes) || Objects.isNull(values)) { + private Tablet buildTableModelTabletFromRow(final AbstractInsertRowNode node) { + final String tableName = node.getTargetPath().getFullPath(); + final String[] measurements = node.getMeasurements(); + final TSDataType[] dataTypes = node.getDataTypes(); + final TsTableColumnCategory[] columnCategories = node.getColumnCategories(); + if (Objects.isNull(measurements) || Objects.isNull(dataTypes)) { return null; } @@ -597,7 +591,9 @@ private Tablet buildTableModelTabletFromRow( final List columnDataTypes = new ArrayList<>(measurements.length); final List columnTypes = new ArrayList<>(measurements.length); final List originalColumnIndexes = new ArrayList<>(measurements.length); - for (int i = 0; i < measurements.length && i < dataTypes.length && i < values.length; i++) { + for (int i = 0; + i < measurements.length && i < dataTypes.length && i < node.getValueCount(); + i++) { if (Objects.isNull(measurements[i]) || Objects.isNull(dataTypes[i])) { continue; } @@ -617,10 +613,11 @@ private Tablet buildTableModelTabletFromRow( columnDataTypes, columnTypes, 1); - tablet.addTimestamp(0, time); + tablet.addTimestamp(0, node.getTime()); for (int i = 0; i < originalColumnIndexes.size(); i++) { final int originalColumnIndex = originalColumnIndexes.get(i); - final Object value = values[originalColumnIndex]; + // Convert one column at a time, keeping numeric rows in their primitive representation. + final Object value = node.getValue(originalColumnIndex); if (Objects.isNull(value)) { if (Objects.isNull(tablet.getBitMaps())) { tablet.initBitMaps(); @@ -718,7 +715,12 @@ private void addValueToTablet( final int columnIndex, final TSDataType dataType, final Object value) { - Type.fromTsDataType(dataType).addValue(rowIndex, value, tablet.getValues()[columnIndex]); + // Row plans encode DATE as yyyyMMdd integers; Tablet stores DATE columns as LocalDate arrays. + final Object tabletValue = + dataType == TSDataType.DATE && value instanceof Integer date + ? DateUtils.parseIntToLocalDate(date) + : value; + Type.fromTsDataType(dataType).addValue(rowIndex, tabletValue, tablet.getValues()[columnIndex]); // Unmark the bitmap position to indicate this value is NOT null. // addTimestamp() triggers initBitMapsWithApiUsage() which marks all positions as null. final BitMap[] bitMaps = tablet.getBitMaps(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java index e24748edc0b1d..d474a8312d562 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java @@ -37,6 +37,7 @@ import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiTabletsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; @@ -3898,8 +3899,8 @@ private void mergeTailProgress( */ private long extractMaxTime(final InsertNode insertNode) { try { - if (insertNode instanceof InsertRowNode) { - return ((InsertRowNode) insertNode).getTime(); + if (insertNode instanceof AbstractInsertRowNode row) { + return row.getTime(); } if (insertNode instanceof InsertTabletNode) { final InsertTabletNode tabletNode = (InsertTabletNode) insertNode; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/NumericRowValues.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/NumericRowValues.java new file mode 100644 index 0000000000000..c1b3b14bca67c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/NumericRowValues.java @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.write.UnSupportedDataTypeException; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Objects; + +/** + * Primitive storage for a numeric row. FLOAT and DOUBLE use raw IEEE 754 bits; BOOLEAN uses 0/1. + * TAGs live in a separate String[] and occupy the first logical columns; FIELDs follow them in + * {@code long[]}. Nulls need separate bits because every long bit pattern is a valid numeric value. + */ +public final class NumericRowValues { + private long[] values; + private final BitSet nulls; + private String[] tagValues; + private final int size; + private int tagCount; + + public NumericRowValues(int tagLength, int valueLength) { + this.values = new long[valueLength]; + this.nulls = new BitSet(tagLength + valueLength); + this.tagValues = tagLength == 0 ? null : new String[tagLength]; + this.size = tagLength + valueLength; + this.tagCount = tagLength; + } + + public int size() { + return size; + } + + public long[] getValues() { + return values; + } + + public String getTag(int index) { + return index < 0 || index >= size || isNull(index) || tagValues == null || index >= tagCount + ? null + : tagValues[index]; + } + + public void setTag(int index, String value) { + if (index < 0 || index >= tagCount) { + throw new IndexOutOfBoundsException(index); + } + if (value == null) { + tagValues[index] = null; + nulls.set(index); + return; + } + tagValues[index] = value; + nulls.clear(index); + } + + public boolean isNull(int index) { + return nulls.get(index); + } + + public void setNull(int index) { + if (index < 0 || index >= size) { + throw new IndexOutOfBoundsException(index); + } + // Null slots have no payload on the wire; keep their in-memory representation canonical too. + if (index >= tagCount) { + values[index - tagCount] = 0; + } + if (tagValues != null && index < tagCount) { + tagValues[index] = null; + } + nulls.set(index); + } + + public long get(int index) { + if (index < 0 || index >= size) { + throw new IndexOutOfBoundsException(index); + } + return index < tagCount ? 0 : values[index - tagCount]; + } + + public void setRawValue(int index, long value) { + if (index < tagCount) { + throw new IllegalArgumentException(); + } + if (index < 0 || index >= size) { + throw new IndexOutOfBoundsException(index); + } + values[index - tagCount] = value; + nulls.clear(index); + } + + public String[] getTagValues() { + return tagValues; + } + + public void setColumnCategories(ColumnCategory[] categories) { + if (categories == null || categories.length != size) { + throw new IllegalArgumentException(); + } + int newTagCount = 0; + while (newTagCount < size && categories[newTagCount] == ColumnCategory.TAG) { + newTagCount++; + } + for (int i = newTagCount; i < size; i++) { + if (categories[i] == ColumnCategory.TAG) { + throw new IllegalArgumentException(); + } + } + if (newTagCount != tagCount) { + throw new IllegalArgumentException(); + } + } + + public static boolean supports(TSDataType type) { + return type != null + && switch (type) { + case BOOLEAN, INT32, INT64, FLOAT, DOUBLE, DATE, TIMESTAMP -> true; + case TEXT, STRING, BLOB, OBJECT, VECTOR, UNKNOWN -> false; + }; + } + + /** Validates the optional FIELD-only RPC bitmap without changing its cursor. */ + public static void validateNullBitmap(ByteBuffer bitmap, int fieldCount) { + if (bitmap == null) { + return; + } + int length = (fieldCount + 7) / 8; + if (bitmap.remaining() != length) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages + .EXCEPTION_NUMERIC_ROW_NULL_BITMAP_LENGTH_MISMATCH_EXPECTED_ARG_BYTES_FOR_ARG_FIELDS_BUT_GOT_ARG_31F1DB42, + length, + fieldCount, + bitmap.remaining())); + } + if (fieldCount % 8 != 0 + && (bitmap.get(bitmap.limit() - 1) & ~((1 << (fieldCount % 8)) - 1)) != 0) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages + .EXCEPTION_NUMERIC_ROW_NULL_BITMAP_HAS_SET_BITS_OUTSIDE_ARG_FIELDS_AD6066D1, + fieldCount)); + } + } + + public static boolean isNullField(ByteBuffer bitmap, int fieldIndex) { + return bitmap != null + && (bitmap.get(bitmap.position() + fieldIndex / 8) & (1 << (fieldIndex % 8))) != 0; + } + + public Object getObject(int index, TSDataType type) { + if (isNull(index)) { + return null; + } + return switch (type) { + case BOOLEAN -> get(index) != 0; + case INT32, DATE -> (int) get(index); + case INT64, TIMESTAMP -> get(index); + case FLOAT -> Float.intBitsToFloat((int) get(index)); + case DOUBLE -> Double.longBitsToDouble(get(index)); + // Binary is required only at existing per-cell APIs (schema, Pipe and subscription). + case STRING -> { + if (index >= tagCount) { + throw new IllegalArgumentException(); + } + yield new Binary(getTag(index), TSFileConfig.STRING_CHARSET); + } + default -> throw unsupported(type); + }; + } + + // Used only at APIs that consume individual objects (last cache, diagnostics and type coercion). + public void setObject(int index, TSDataType type, Object value) { + if (value == null) { + setNull(index); + return; + } + if (type == TSDataType.STRING) { + setTag( + index, + value instanceof Binary binary + ? binary.getStringValue(TSFileConfig.STRING_CHARSET) + : (String) value); + return; + } + if (index < tagCount || index < 0 || index >= size) { + throw new IllegalArgumentException(); + } + values[index - tagCount] = + switch (type) { + case BOOLEAN -> (boolean) value ? 1 : 0; + case INT32, DATE -> (int) value; + case INT64, TIMESTAMP -> (long) value; + case FLOAT -> Float.floatToRawIntBits((float) value); + case DOUBLE -> Double.doubleToRawLongBits((double) value); + default -> throw unsupported(type); + }; + nulls.clear(index); + } + + public boolean cast(int index, TSDataType source, TSDataType target) { + if (isNull(index)) { + return true; + } + if (source == TSDataType.STRING && target == TSDataType.STRING) { + return true; + } + if (!supports(target) || source == null || !target.isCompatible(source)) { + return false; + } + if (source == target) { + return true; + } + setObject(index, target, target.castFromSingleValue(source, getObject(index, source))); + return true; + } + + /** Returns a new primitive row; -1 in the mapping represents an inserted null column. */ + public NumericRowValues remap(int[] newToOld, int newTagCount) { + NumericRowValues result = new NumericRowValues(newTagCount, newToOld.length - newTagCount); + for (int i = 0; i < newToOld.length; i++) { + int old = newToOld[i]; + if (old < 0 || old >= size() || isNull(old)) { + result.setNull(i); + } else if (i < newTagCount) { + if (tagValues != null && old < this.tagCount && tagValues[old] != null) { + result.setTag(i, tagValues[old]); + } + } else if (old >= this.tagCount) { + result.setRawValue(i, get(old)); + } else { + result.setNull(i); + } + } + return result; + } + + public void read(int index, TSDataType type, ByteBuffer buffer) { + if (type == TSDataType.STRING) { + setTag(index, ReadWriteIOUtils.readString(buffer)); + return; + } + if (index < tagCount || index < 0 || index >= size) { + throw new IllegalArgumentException(); + } + values[index - tagCount] = + switch (type) { + case BOOLEAN -> buffer.get() == 0 ? 0 : 1; + case INT32, DATE, FLOAT -> buffer.getInt(); + case INT64, TIMESTAMP, DOUBLE -> buffer.getLong(); + default -> throw unsupported(type); + }; + } + + public long ramBytesUsed() { + return RamUsageEstimator.shallowSizeOfInstance(NumericRowValues.class) + + RamUsageEstimator.sizeOf(values) + + (tagValues == null ? 0 : RamUsageEstimator.sizeOf(tagValues)) + + RamUsageEstimator.shallowSizeOfInstance(BitSet.class) + + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + + (long) nulls.size() / Byte.SIZE; + } + + private static UnSupportedDataTypeException unsupported(TSDataType type) { + return new UnSupportedDataTypeException(DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE + type); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof NumericRowValues row) + || size != row.size + || tagCount != row.tagCount + || !Arrays.equals(values, row.values) + || !nulls.equals(row.nulls)) { + return false; + } + // An allocated array cleared by a partial failure equals an absent all-null TAG array. + for (int i = 0; i < size(); i++) { + if (!Objects.equals(getTag(i), row.getTag(i))) { + return false; + } + } + return true; + } + + @Override + public int hashCode() { + int hash = 31 * (31 * size + tagCount) + Arrays.hashCode(values) + nulls.hashCode(); + for (int i = 0; i < size(); i++) { + hash = 31 * hash + Objects.hashCode(getTag(i)); + } + return hash; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index e677db7933957..f766d295246d5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager; +import org.apache.iotdb.db.utils.NumericRowValues; import org.apache.iotdb.db.utils.TypeServices; import org.apache.iotdb.rpc.TSStatusCode; @@ -499,6 +500,46 @@ public synchronized void putAlignedValue(long timestamp, Object[] value) { } } + /** Writes row bits into native column arrays without a boxed intermediate row. */ + public synchronized void putAlignedNumericValue( + long timestamp, NumericRowValues row, int[] columnMapping) { + checkExpansion(); + int arrayIndex = rowCount / ARRAY_SIZE; + int elementIndex = rowCount % ARRAY_SIZE; + maxTime = Math.max(maxTime, timestamp); + minTime = Math.min(minTime, timestamp); + timestamps.get(arrayIndex)[elementIndex] = timestamp; + for (int i = 0; i < values.size(); i++) { + int source = columnMapping[i]; + if (source < 0 || row.isNull(source)) { + markNullValue(i, arrayIndex, elementIndex); + continue; + } + Object valueArray = getOrCreateValueArray(i, arrayIndex, elementIndex); + long value = row.get(source); + switch (dataTypes.get(i)) { + case BOOLEAN -> ((boolean[]) valueArray)[elementIndex] = value != 0; + case INT32, DATE -> ((int[]) valueArray)[elementIndex] = (int) value; + case INT64, TIMESTAMP -> ((long[]) valueArray)[elementIndex] = value; + case FLOAT -> ((float[]) valueArray)[elementIndex] = Float.intBitsToFloat((int) value); + case DOUBLE -> ((double[]) valueArray)[elementIndex] = Double.longBitsToDouble(value); + default -> throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); + } + } + if (indices != null) { + indices.get(arrayIndex)[elementIndex] = rowCount; + } + rowCount++; + if (sorted) { + if (rowCount > 1 && timestamp < getTime(rowCount - 2)) { + sorted = false; + segmentMovedMap = null; + } else { + seqRowCount++; + } + } + } + @Override public Object getAlignedValue(int index) { return getAlignedValueForQuery(index, null, null); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatementTest.java index 8c563a84b1db4..a58a05af70ad5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatementTest.java @@ -19,15 +19,26 @@ package org.apache.iotdb.db.pipe.receiver.transform.statement; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.db.pipe.receiver.visitor.PipeTableStatementDataTypeConvertExecutionVisitor; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; +import org.apache.iotdb.db.utils.NumericRowValues; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; import org.junit.Test; import java.time.ZoneId; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; public class PipeConvertedInsertRowStatementTest { @@ -52,4 +63,139 @@ public void testTransferTypeKeepsNullValue() throws Exception { Assert.assertNull(convertedStatement.getValues()[0]); Assert.assertFalse(convertedStatement.isNeedInferType()); } + + @Test + public void testNumericRowDispatchConvertsNumericAndStringFields() throws Exception { + final InsertNumericRowStatement original = numericRow(); + final long[] originalValues = original.getValues().clone(); + final AtomicInteger executions = new AtomicInteger(); + final TSStatus success = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + final PipeTableStatementDataTypeConvertExecutionVisitor visitor = + new PipeTableStatementDataTypeConvertExecutionVisitor( + (statement, database) -> { + executions.incrementAndGet(); + Assert.assertEquals("db", database); + Assert.assertTrue(statement instanceof PipeConvertedInsertRowStatement); + final PipeConvertedInsertRowStatement converted = + (PipeConvertedInsertRowStatement) statement; + Assert.assertTrue(converted.isForceTypeConversion()); + Assert.assertFalse(converted.isNeedInferType()); + Assert.assertTrue(converted.isAligned()); + Assert.assertTrue(converted.isWriteToTable()); + Assert.assertEquals(original.getTime(), converted.getTime()); + Assert.assertEquals(original.getDatabaseName(), converted.getDatabaseName()); + Assert.assertEquals(original.getTableDeviceID(), converted.getTableDeviceID()); + Assert.assertEquals(original.getFirstValueOfIndex(0), converted.getValues()[0]); + Assert.assertTrue(converted.checkAndCastDataType(1, TSDataType.INT32)); + Assert.assertTrue(converted.checkAndCastDataType(2, TSDataType.STRING)); + Assert.assertTrue(converted.checkAndCastDataType(3, TSDataType.STRING)); + Assert.assertEquals(12, converted.getValues()[1]); + Assert.assertEquals( + new Binary("8.5", TSFileConfig.STRING_CHARSET), converted.getValues()[2]); + Assert.assertNull(converted.getValues()[3]); + Assert.assertArrayEquals( + new TSDataType[] { + TSDataType.STRING, TSDataType.INT32, TSDataType.STRING, TSDataType.STRING + }, + converted.getDataTypes()); + return success; + }); + + Assert.assertEquals(success, visitor.process(original, new Pair<>(success, "db")).orElse(null)); + Assert.assertEquals(1, executions.get()); + Assert.assertArrayEquals(originalValues, original.getValues()); + Assert.assertArrayEquals( + new TSDataType[] { + TSDataType.STRING, TSDataType.DOUBLE, TSDataType.DOUBLE, TSDataType.INT64 + }, + original.getDataTypes()); + Assert.assertTrue(original.getNumericValues().isNull(3)); + } + + @Test + public void testNumericRowRetryRestoresFailuresOnlyInCopy() throws Exception { + final InsertNumericRowStatement original = numericRow(); + original.markFailedMeasurement(0, new IllegalArgumentException("tag failure")); + original.markFailedMeasurement(1, new IllegalArgumentException("field failure")); + final long[] failedValues = original.getValues().clone(); + final PipeConvertedInsertRowStatement converted = new PipeConvertedInsertRowStatement(original); + + Assert.assertFalse(converted.hasFailedMeasurements()); + Assert.assertEquals(List.of(0), converted.getTagColumnIndices()); + Assert.assertEquals("tag", converted.getMeasurements()[0]); + Assert.assertEquals("count", converted.getMeasurements()[1]); + Assert.assertEquals(new Binary("d1", TSFileConfig.STRING_CHARSET), converted.getValues()[0]); + Assert.assertEquals(12.5, converted.getValues()[1]); + Assert.assertTrue(converted.checkAndCastDataType(1, TSDataType.INT32)); + Assert.assertEquals(12, converted.getValues()[1]); + + Assert.assertEquals(2, original.getFailedMeasurementNumber()); + Assert.assertNotSame( + original.getFailedMeasurementInfoMap(), converted.getFailedMeasurementInfoMap()); + Assert.assertNull(original.getMeasurements()[0]); + Assert.assertNull(original.getMeasurements()[1]); + Assert.assertNull(original.getDataTypes()[1]); + Assert.assertTrue(original.getNumericValues().isNull(0)); + Assert.assertTrue(original.getNumericValues().isNull(1)); + Assert.assertArrayEquals(failedValues, original.getValues()); + Assert.assertEquals(12.5, original.getFailedMeasurementInfoMap().get(1).getValue()); + } + + @Test + public void testNumericRowConversionOwnsMutableMetadataAndPreservesNulls() throws Exception { + final InsertNumericRowStatement original = numericRow(); + original.getNumericValues().setTag(0, null); + final PipeConvertedInsertRowStatement converted = new PipeConvertedInsertRowStatement(original); + Assert.assertNull(converted.getValues()[0]); + Assert.assertNull(converted.getValues()[3]); + Assert.assertEquals(original.getTableDeviceID(), converted.getTableDeviceID()); + Assert.assertNotSame(original.getMeasurements(), converted.getMeasurements()); + Assert.assertNotSame(original.getDataTypes(), converted.getDataTypes()); + Assert.assertNotSame(original.getMeasurementSchemas(), converted.getMeasurementSchemas()); + Assert.assertNotSame(original.getColumnCategories(), converted.getColumnCategories()); + converted.swapColumn(1, 2); + converted.setColumnCategory(TsTableColumnCategory.ATTRIBUTE, 1); + converted.markFailedMeasurement(2, new IllegalArgumentException("conversion failure")); + + Assert.assertArrayEquals( + new String[] {"tag", "count", "text", "nullable"}, original.getMeasurements()); + Assert.assertEquals(TsTableColumnCategory.FIELD, original.getColumnCategory(1)); + Assert.assertEquals("count", original.getMeasurementSchemas()[1].getMeasurementName()); + Assert.assertEquals(12.5, original.getFirstValueOfIndex(1)); + Assert.assertEquals(8.5, original.getFirstValueOfIndex(2)); + Assert.assertFalse(original.hasFailedMeasurements()); + } + + private static InsertNumericRowStatement numericRow() throws Exception { + final InsertNumericRowStatement statement = new InsertNumericRowStatement(); + statement.setDevicePath(new PartialPath("t")); + statement.setDatabaseName("db"); + statement.setTime(123L); + statement.setMeasurements(new String[] {"tag", "count", "text", "nullable"}); + statement.setDataTypes( + new TSDataType[] { + TSDataType.STRING, TSDataType.DOUBLE, TSDataType.DOUBLE, TSDataType.INT64 + }); + statement.setColumnCategories( + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD + }); + statement.setMeasurementSchemas( + new MeasurementSchema[] { + new MeasurementSchema("tag", TSDataType.STRING), + new MeasurementSchema("count", TSDataType.DOUBLE), + new MeasurementSchema("text", TSDataType.DOUBLE), + new MeasurementSchema("nullable", TSDataType.INT64) + }); + final NumericRowValues values = new NumericRowValues(1, 3); + values.setTag(0, "d1"); + values.setObject(1, TSDataType.DOUBLE, 12.5); + values.setObject(2, TSDataType.DOUBLE, 8.5); + values.setNull(3); + statement.setNumericValues(values); + return statement; + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java new file mode 100644 index 0000000000000..9fc462abfff87 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java @@ -0,0 +1,365 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.protocol.session; + +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.queryengine.common.SessionInfo; +import org.apache.iotdb.commons.schema.table.TreeViewSchema; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.plan.analyze.lock.DataNodeSchemaLockManager; +import org.apache.iotdb.db.queryengine.plan.analyze.lock.SchemaLockType; +import org.apache.iotdb.db.queryengine.plan.analyze.schema.SchemaValidator; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator; +import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertNumericRow; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +public class PreparedNumericRowInfoTest { + private static final String DATABASE = "prepared_numeric_fastpath_test"; + private TsTable table; + private MPPQueryContext context; + private Metadata metadata; + + @Before + public void setUp() { + DataNodeTableCache.getInstance().invalid(DATABASE); + table = new TsTable("t"); + table.addColumnSchema(new TagColumnSchema("tag", TSDataType.STRING)); + table.addColumnSchema(new FieldColumnSchema("v", TSDataType.DOUBLE)); + install(table); + context = new MPPQueryContext(new QueryId("prepared_test")); + context.setSession(new SessionInfo(1, "root", ZoneId.of("UTC"))); + metadata = mock(Metadata.class); + doAnswer( + invocation -> { + TableHeaderSchemaValidator.getInstance() + .validateInsertNodeMeasurements( + invocation.getArgument(0), + invocation.getArgument(1), + invocation.getArgument(2), + invocation.getArgument(3), + invocation.getArgument(4), + invocation.getArgument(5)); + return null; + }) + .when(metadata) + .validateInsertNodeMeasurements(anyString(), any(), any(), anyBoolean(), any(), any()); + } + + @After + public void tearDown() { + DataNodeSchemaLockManager.getInstance().releaseReadLock(context); + DataNodeTableCache.getInstance().invalid(DATABASE); + } + + // An unchanged table skips column validation but still checks access and device metadata. + @Test + public void testUnchangedVersionRetainsOtherChecks() { + PreparedNumericRowInfo info = new PreparedNumericRowInfo(DATABASE, request("tag", "v")); + InsertNumericRowStatement statement = bind(info, 1); + InsertNumericRowStatement next = bind(info, 1); + assertNotNull(statement.getPreparedTableVersion()); + assertNotSame(statement.getMeasurementSchemas(), next.getMeasurementSchemas()); + assertSame(statement.getMeasurementSchemas()[1], next.getMeasurementSchemas()[1]); + AccessControl accessControl = mock(AccessControl.class); + InsertNumericRow insert = new InsertNumericRow(statement, context); + SchemaValidator.validate(metadata, insert, context, accessControl); + verify(metadata, never()) + .validateInsertNodeMeasurements(anyString(), any(), any(), anyBoolean(), any(), any()); + verify(metadata).validateDeviceSchema(insert, context); + verify(accessControl).checkCanInsertIntoTable(eq("root"), any(), eq(context)); + assertTrue(context.getAcquiredLocks().contains(SchemaLockType.VALIDATE_VS_DELETION_TABLE)); + assertEquals(TSDataType.DOUBLE, statement.getMeasurementSchemas()[1].getType()); + statement.getMeasurementSchemas()[1] = null; + assertNotNull(bind(info, 1).getMeasurementSchemas()[1]); + } + + // Adding a TAG after bind forces normal validation to pad the new device-id segment. + @Test + public void testVersionChangeBetweenBindAndAnalysis() { + PreparedNumericRowInfo info = new PreparedNumericRowInfo(DATABASE, request("tag", "v")); + InsertNumericRowStatement statement = bind(info, 1); + table.addColumnSchema(new TagColumnSchema("tag2", TSDataType.STRING)); + validateFallback(statement); + assertArrayEquals(new String[] {"tag", "tag2", "v"}, statement.getMeasurements()); + assertNull(statement.getFirstValueOfIndex(1)); + assertEquals(2.5D, statement.getFirstValueOfIndex(2)); + assertArrayEquals(new String[] {"tag", "v"}, bind(info, 1).getMeasurements()); + } + + // A replacement instance must pass compatibility checks even when its revision is unchanged. + @Test + public void testCompatibleRecreatedTableRefreshesVersion() { + PreparedNumericRowInfo info = new PreparedNumericRowInfo(DATABASE, request("tag", "v")); + InsertNumericRowStatement statement = bind(info, 1); + TsTable replacement = new TsTable("t"); + replacement.addColumnSchema(new TagColumnSchema("tag", TSDataType.STRING)); + replacement.addColumnSchema(new FieldColumnSchema("v", TSDataType.DOUBLE)); + assertEquals(table.getInstanceVersion().right, replacement.getInstanceVersion().right); + install(replacement); + new InsertNumericRow(statement, context).validateTableSchema(metadata, context); + assertEquals(replacement.getInstanceVersion(), statement.getPreparedTableVersion()); + assertEquals(replacement.getInstanceVersion(), bind(info, 1).getPreparedTableVersion()); + verify(metadata, never()) + .validateInsertNodeMeasurements(anyString(), any(), any(), anyBoolean(), any(), any()); + assertNotNull(statement.getMeasurementSchemas()[1]); + } + + // An unrelated FIELD addition refreshes only the version, preserving schemas and memory usage. + @Test + public void testCompatibleVersionChangeRetainsFastPath() { + PreparedNumericRowInfo info = new PreparedNumericRowInfo(DATABASE, request("tag", "v")); + long reservedBytes = info.ramBytesUsed(); + InsertNumericRowStatement statement = bind(info, 1); + Object schemas = statement.getMeasurementSchemas(); + table.addColumnSchema(new FieldColumnSchema("extra", TSDataType.INT64)); + new InsertNumericRow(statement, context).validateTableSchema(metadata, context); + assertSame(schemas, statement.getMeasurementSchemas()); + assertFalse(statement.isTableSchemaValidationFailed()); + assertEquals(table.getInstanceVersion(), statement.getPreparedTableVersion()); + InsertNumericRowStatement next = bind(info, 1); + assertEquals(table.getInstanceVersion(), next.getPreparedTableVersion()); + assertSame(statement.getMeasurementSchemas()[1], next.getMeasurementSchemas()[1]); + new InsertNumericRow(next, context).validateTableSchema(metadata, context); + verify(metadata, never()) + .validateInsertNodeMeasurements(anyString(), any(), any(), anyBoolean(), any(), any()); + assertEquals(reservedBytes, info.ramBytesUsed()); + } + + // Changed FIELD types must use normal validation and preserve its concrete error. + @Test + public void testIncompatibleVersionUsesValidationError() { + PreparedNumericRowInfo info = new PreparedNumericRowInfo(DATABASE, request("tag", "v")); + InsertNumericRowStatement statement = bind(info, 1); + TsTable replacement = new TsTable("t"); + replacement.addColumnSchema(new TagColumnSchema("tag", TSDataType.STRING)); + replacement.addColumnSchema(new FieldColumnSchema("v", TSDataType.BOOLEAN)); + install(replacement); + SemanticException failure = new SemanticException("column v type mismatch"); + doThrow(failure) + .when(metadata) + .validateInsertNodeMeasurements(anyString(), any(), any(), anyBoolean(), any(), any()); + assertSame( + failure, + assertThrows( + SemanticException.class, + () -> new InsertNumericRow(statement, context).validateTableSchema(metadata, context))); + verify(metadata) + .validateInsertNodeMeasurements(eq(DATABASE), any(), eq(context), eq(true), any(), any()); + assertNull(statement.getPreparedTableVersion()); + assertNull(statement.getMeasurementSchemas()); + assertTrue(statement.isTableSchemaValidationFailed()); + assertFalse(replacement.matchesInstanceVersion(bind(info, 1).getPreparedTableVersion())); + } + + // Partial insert errors also invalidate the handle even when validation returns normally. + @Test + public void testPartialSchemaFailureMarksPreparedRowInvalid() { + PreparedNumericRowInfo info = new PreparedNumericRowInfo(DATABASE, request("tag", "v")); + InsertNumericRowStatement statement = bind(info, 1); + table.addColumnSchema(new TagColumnSchema("tag2", TSDataType.STRING)); + doAnswer( + invocation -> { + statement.markFailedMeasurement(1, new SemanticException("column v rejected")); + return null; + }) + .when(metadata) + .validateInsertNodeMeasurements(anyString(), any(), any(), anyBoolean(), any(), any()); + validateFallback(statement); + assertTrue(statement.isTableSchemaValidationFailed()); + } + + // Missing columns/table, views and type conversions remain on normal creation/validation. + @Test + public void testIneligibleMetadataHasNoCachedSchema() { + assertNull( + bind(new PreparedNumericRowInfo(DATABASE, request("tag", "missing")), 1) + .getPreparedTableVersion()); + TSPrepareNumericRowReq req = request("tag", "v"); + req.setDataTypes(Arrays.asList(TSDataType.STRING.ordinal(), TSDataType.INT64.ordinal())); + assertNull( + new PreparedNumericRowInfo(DATABASE, req) + .bind(1, tagBuffer(1), ByteBuffer.allocate(8), null) + .getPreparedTableVersion()); + table.addProp(TreeViewSchema.TREE_PATH_PATTERN, "root.sg.**"); + assertNull( + bind(new PreparedNumericRowInfo(DATABASE, request("tag", "v")), 1) + .getPreparedTableVersion()); + DataNodeTableCache.getInstance().invalid(DATABASE); + assertNull( + bind(new PreparedNumericRowInfo(DATABASE, request("tag", "v")), 1).getMeasurementSchemas()); + } + + // Reversed TAG input must take the normal path so payload values follow schema order. + @Test + public void testTagReorderingFallsBack() { + table.addColumnSchema(new TagColumnSchema("tag2", TSDataType.STRING)); + InsertNumericRowStatement statement = + bind(new PreparedNumericRowInfo(DATABASE, request("tag2", "tag", "v")), 2); + assertNull(statement.getPreparedTableVersion()); + validateFallback(statement); + assertArrayEquals(new String[] {"tag", "tag2", "v"}, statement.getMeasurements()); + assertEquals("b", statement.getNumericValues().getTag(0)); + assertEquals("a", statement.getNumericValues().getTag(1)); + } + + private void validateFallback(InsertNumericRowStatement statement) { + new InsertNumericRow(statement, context).validateTableSchema(metadata, context); + verify(metadata) + .validateInsertNodeMeasurements(eq(DATABASE), any(), eq(context), eq(true), any(), any()); + assertNull(statement.getPreparedTableVersion()); + } + + // The optional bitmap uses FIELD indices, not measurement indices, and accepts sliced buffers. + @Test + public void testNullBitmapDecodingAndValidation() throws Exception { + TSPrepareNumericRowReq req = request("tag", "v"); + req.setMeasurements(new ArrayList<>(req.getMeasurements())); + req.setDataTypes(new ArrayList<>(req.getDataTypes())); + req.setColumnCategories(new ArrayList<>(req.getColumnCategories())); + for (int i = 1; i < 9; i++) { + req.addToMeasurements("v" + i); + req.addToDataTypes(TSDataType.DOUBLE.ordinal()); + req.addToColumnCategories((byte) ColumnCategory.FIELD.ordinal()); + } + PreparedNumericRowInfo info = new PreparedNumericRowInfo(DATABASE, req); + ByteBuffer bitmap = ByteBuffer.wrap(new byte[] {99, (byte) 0x81, 1, 99}); + bitmap.position(1).limit(3); + ByteBuffer values = ByteBuffer.allocate(6 * 8); + for (int i = 1; i <= 6; i++) { + values.putLong(i == 1 ? 0x7ff8000000001234L : Double.doubleToRawLongBits(i)); + } + values.flip(); + InsertNumericRowStatement prepared = info.bind(1, tagBuffer(1), values, bitmap); + assertEquals(1, bitmap.position()); + assertEquals(0, values.position()); + assertTrue(prepared.getNumericValues().isNull(1)); + assertTrue(prepared.getNumericValues().isNull(8)); + assertTrue(prepared.getNumericValues().isNull(9)); + assertFalse(prepared.getNumericValues().isNull(0)); + assertEquals(0x7ff8000000001234L, prepared.getNumericValues().get(2)); + for (byte[] bad : new byte[][] {new byte[0], new byte[1], new byte[3], {0, 2}}) { + assertThrows( + IllegalArgumentException.class, + () -> info.bind(1, tagBuffer(1), values, ByteBuffer.wrap(bad))); + } + assertThrows( + SemanticException.class, + () -> info.bind(1, tagBuffer(1), ByteBuffer.allocate(9 * 8), bitmap)); + ByteBuffer allNulls = ByteBuffer.wrap(new byte[] {(byte) 255, 1}); + InsertNumericRowStatement allNull = + info.bind(1, tagBuffer(1), ByteBuffer.allocate(0), allNulls); + for (int i = 1; i <= 9; i++) { + assertTrue(allNull.getNumericValues().isNull(i)); + } + } + + @Test + public void testUnsupportedColumnTypesRejectedDuringPrepare() { + for (TSDataType type : + new TSDataType[] { + TSDataType.TEXT, + TSDataType.STRING, + TSDataType.BLOB, + TSDataType.OBJECT, + TSDataType.VECTOR, + TSDataType.UNKNOWN + }) { + TSPrepareNumericRowReq req = request("v"); + req.getDataTypes().set(0, type.ordinal()); + assertThrows(SemanticException.class, () -> new PreparedNumericRowInfo(DATABASE, req)); + } + TSPrepareNumericRowReq req = request("tag", "v"); + req.getDataTypes().set(0, TSDataType.INT64.ordinal()); + assertThrows(SemanticException.class, () -> new PreparedNumericRowInfo(DATABASE, req)); + req.getDataTypes().set(0, TSDataType.STRING.ordinal()); + req.getColumnCategories().set(0, (byte) ColumnCategory.ATTRIBUTE.ordinal()); + assertThrows(SemanticException.class, () -> new PreparedNumericRowInfo(DATABASE, req)); + } + + private static void install(TsTable table) { + DataNodeTableCache.getInstance().preUpdateTable(DATABASE, table, null); + DataNodeTableCache.getInstance().commitUpdateTable(DATABASE, "t", null); + } + + private static TSPrepareNumericRowReq request(String... names) { + TSPrepareNumericRowReq req = new TSPrepareNumericRowReq(); + req.setTableName("t"); + req.setMeasurements(Arrays.asList(names)); + for (int i = 0; i < names.length; i++) { + boolean tag = i < names.length - 1; + req.addToDataTypes((tag ? TSDataType.STRING : TSDataType.DOUBLE).ordinal()); + req.addToColumnCategories((byte) (tag ? ColumnCategory.TAG : ColumnCategory.FIELD).ordinal()); + } + return req; + } + + private static ByteBuffer tagBuffer(int count) { + ByteBuffer tags = ByteBuffer.allocate(count * 5); + for (int i = 0; i < count; i++) { + tags.putInt(1).put((byte) ('a' + i)); + } + tags.flip(); + return tags; + } + + private static InsertNumericRowStatement bind(PreparedNumericRowInfo info, int tags) { + ByteBuffer values = ByteBuffer.allocate(8); + values.putDouble(2.5D).flip(); + return info.bind(1, tagBuffer(tags), values, null); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImplPreparedNumericRowTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImplPreparedNumericRowTest.java new file mode 100644 index 0000000000000..917bcfa4d7642 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImplPreparedNumericRowTest.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.protocol.thrift.impl; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.RpcThrottlingException; +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.protocol.session.PreparedNumericRowInfo; +import org.apache.iotdb.db.protocol.session.SessionManager; +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.plan.Coordinator; +import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; +import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeThrottleQuotaManager; +import org.apache.iotdb.db.storageengine.rescon.quotas.OperationQuota; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TSExecutePreparedNumericRowReq; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InOrder; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*", "javax.management.*"}) +@PrepareForTest({ + ClientRPCServiceImpl.class, + DataNodeThrottleQuotaManager.class, + PreparedNumericRowInfo.class +}) +public class ClientRPCServiceImplPreparedNumericRowTest { + private ClientRPCServiceImpl service; + private int originalDataNodeId; + private Coordinator coordinator; + private Coordinator originalCoordinator; + private SessionManager originalSessionManager; + private IClientSession session; + private PreparedNumericRowInfo prepared; + private InsertNumericRowStatement statement; + private DataNodeThrottleQuotaManager quotaManager; + private OperationQuota quota; + private TSExecutePreparedNumericRowReq request; + private TSStatus success; + + @Before + public void setUp() throws Exception { + // Coordinator initialization requires a DataNode ID even when query execution is mocked. + originalDataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0); + // Exercise the actual RPC method while isolating query execution and per-user quota state. + service = mock(ClientRPCServiceImpl.class, CALLS_REAL_METHODS); + originalCoordinator = Whitebox.getInternalState(ClientRPCServiceImpl.class, "COORDINATOR"); + originalSessionManager = + Whitebox.getInternalState(ClientRPCServiceImpl.class, "SESSION_MANAGER"); + coordinator = mock(Coordinator.class); + SessionManager sessions = mock(SessionManager.class); + Whitebox.setInternalState(ClientRPCServiceImpl.class, "COORDINATOR", coordinator); + Whitebox.setInternalState(ClientRPCServiceImpl.class, "SESSION_MANAGER", sessions); + session = mock(IClientSession.class); + prepared = PowerMockito.mock(PreparedNumericRowInfo.class); + statement = new InsertNumericRowStatement(); + quotaManager = mock(DataNodeThrottleQuotaManager.class); + quota = mock(OperationQuota.class); + PowerMockito.mockStatic(DataNodeThrottleQuotaManager.class); + when(DataNodeThrottleQuotaManager.getInstance()).thenReturn(quotaManager); + when(sessions.getCurrSessionAndUpdateIdleTime()).thenReturn(session); + when(sessions.checkLogin(session)).thenReturn(true); + when(session.getId()).thenReturn(11L); + when(session.getUsername()).thenReturn("quota-user"); + when(session.getPreparedNumericRow(101L)).thenReturn(prepared); + when(prepared.bind(anyLong(), any(), any(), any())).thenReturn(statement); + when(quotaManager.checkQuota("quota-user", statement)).thenReturn(quota); + success = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + when(coordinator.executeForTableModel( + same(statement), any(), same(session), anyLong(), any(), eq(""), any(), anyLong())) + .thenReturn(new ExecutionResult(new QueryId("prepared_quota"), success)); + request = + new TSExecutePreparedNumericRowReq( + 11L, 101L, 1L, ByteBuffer.allocate(0), ByteBuffer.allocate(0)); + } + + @After + public void tearDown() { + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(originalDataNodeId); + Whitebox.setInternalState(ClientRPCServiceImpl.class, "COORDINATOR", originalCoordinator); + Whitebox.setInternalState( + ClientRPCServiceImpl.class, "SESSION_MANAGER", originalSessionManager); + } + + @Test + public void testChecksQuotaBeforeExecutionAndClosesOnSuccess() throws Exception { + assertSame(success, service.executePreparedNumericRow(request)); + InOrder order = inOrder(quotaManager, coordinator, quota); + order.verify(quotaManager).checkQuota("quota-user", statement); + order + .verify(coordinator) + .executeForTableModel( + same(statement), any(), same(session), anyLong(), any(), eq(""), any(), anyLong()); + order.verify(quota).close(); + verify(session, never()).removePreparedNumericRow(anyLong()); + } + + @Test + public void testQuotaRejectionDoesNotExecuteOrInvalidatePreparedRow() throws Exception { + int code = TSStatusCode.NUM_WRITE_REQUESTS_EXCEEDED.getStatusCode(); + when(quotaManager.checkQuota("quota-user", statement)) + .thenThrow(new RpcThrottlingException("write request quota exceeded", code)); + TSStatus status = service.executePreparedNumericRow(request); + assertEquals(code, status.getCode()); + // Standard RPC handling adds an operation prefix while retaining the quota failure detail. + assertTrue(status.getMessage().endsWith("write request quota exceeded")); + verifyZeroInteractions(coordinator, quota); + verify(session, never()).removePreparedNumericRow(anyLong()); + } + + @Test + public void testExecutionExceptionClosesQuota() throws Exception { + when(coordinator.executeForTableModel( + same(statement), any(), same(session), anyLong(), any(), eq(""), any(), anyLong())) + .thenThrow(new IllegalStateException("executor failure")); + TSStatus status = service.executePreparedNumericRow(request); + assertNotEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + verify(quota).close(); + verify(session, never()).removePreparedNumericRow(anyLong()); + } + + @Test + public void testSchemaFailureClosesQuotaAndRemovesPreparedRow() throws Exception { + TSStatus schemaFailure = new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()); + when(coordinator.executeForTableModel( + same(statement), any(), same(session), anyLong(), any(), eq(""), any(), anyLong())) + .thenAnswer( + invocation -> { + statement.markTableSchemaValidationFailed(); + return new ExecutionResult(new QueryId("prepared_quota"), schemaFailure); + }); + assertSame(schemaFailure, service.executePreparedNumericRow(request)); + verify(quotaManager).checkQuota("quota-user", statement); + verify(quota).close(); + verify(session).removePreparedNumericRow(101L); + } + + @Test + public void testInvalidPayloadDoesNotConsumeQuota() throws Exception { + when(prepared.bind(anyLong(), any(), any(), any())) + .thenThrow(new SemanticException("invalid numeric payload")); + TSStatus status = service.executePreparedNumericRow(request); + assertNotEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + verifyZeroInteractions(quotaManager, coordinator, quota); + verify(session, never()).removePreparedNumericRow(anyLong()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java new file mode 100644 index 0000000000000..cc85949aaad7b --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java @@ -0,0 +1,537 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.queryengine.plan.parser; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.consensus.index.impl.IoTProgressIndex; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.commons.request.IConsensusRequest; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.consensus.common.request.BatchIndexedConsensusRequest; +import org.apache.iotdb.consensus.common.request.ByteBufferConsensusRequest; +import org.apache.iotdb.consensus.common.request.DeserializedBatchIndexedConsensusRequest; +import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest; +import org.apache.iotdb.consensus.common.request.IoTConsensusRequest; +import org.apache.iotdb.db.consensus.statemachine.dataregion.IoTConsensusDataRegionStateMachine; +import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTablePatternParser; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletBatchReqV2; +import org.apache.iotdb.db.protocol.session.PreparedNumericRowInfo; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; +import org.apache.iotdb.db.queryengine.plan.relational.planner.TableModelPlanner; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertNumericRow; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; +import org.apache.iotdb.db.storageengine.dataregion.memtable.AlignedWritableMemChunkGroup; +import org.apache.iotdb.db.storageengine.dataregion.memtable.PrimitiveMemTable; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALInfoEntry; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; +import org.apache.iotdb.db.subscription.broker.consensus.ConsensusLogToTabletConverter; +import org.apache.iotdb.db.utils.NumericRowValues; +import org.apache.iotdb.db.utils.datastructure.AlignedTVList; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.TypeFactory; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.lang.reflect.Field; +import java.nio.ByteBuffer; +import java.time.LocalDate; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** Exercises one numeric row from prepared binding through schema edits, WAL and memtable. */ +public class NumericTableRowTest { + private static final List NAMES = Arrays.asList("b", "i", "l", "f", "d", "date", "ts"); + private static final List TYPES = + Arrays.asList( + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.DATE, + TSDataType.TIMESTAMP); + private static final long[] BITS = { + 1, Integer.MIN_VALUE, Long.MIN_VALUE, 0x7fc01234, 0x7ff8000000001234L, 20260910, Long.MAX_VALUE + }; + + private InsertNumericRowStatement preparedRow() { + TSPrepareNumericRowReq req = + new TSPrepareNumericRowReq() + .setTableName("numeric_table") + .setMeasurements(NAMES) + .setDataTypes(TYPES.stream().map(TSDataType::ordinal).toList()) + .setColumnCategories( + Collections.nCopies(NAMES.size(), (byte) ColumnCategory.FIELD.ordinal())); + ByteBuffer values = ByteBuffer.allocate(37); + values + .put((byte) BITS[0]) + .putInt((int) BITS[1]) + .putLong(BITS[2]) + .putInt((int) BITS[3]) + .putLong(BITS[4]) + .putInt((int) BITS[5]) + .putLong(BITS[6]) + .flip(); + return new PreparedNumericRowInfo("numeric_test", req) + .bind(42, ByteBuffer.allocate(0), values, null); + } + + @Test + public void testPreparedBindAndSchemaChangesRetainPrimitiveStorage() throws Exception { + // Missing TAG insertion, column swaps and partial failure recovery must retain long[] values. + InsertNumericRowStatement statement = preparedRow(); + assertArrayEquals(BITS, statement.getNumericValues().getValues()); + assertObjectStorageAbsent(InsertNumericRowStatement.class, statement); + assertTrue(statement.isWriteToTable()); + assertTrue(statement.isAligned()); + assertEquals(StatementType.INSERT_NUMERIC_ROW, statement.getType()); + statement.semanticCheck(); + + statement.insertColumn( + 0, + new ColumnSchema( + "tag", TypeFactory.getType(TSDataType.STRING), false, TsTableColumnCategory.TAG)); + assertTrue(statement.getNumericValues().isNull(0)); + assertNull(statement.getFirstValueOfIndex(0)); + assertEquals(20260910, statement.getFirstValueOfIndex(6)); + statement.swapColumn(1, 2); + assertEquals(Integer.MIN_VALUE, statement.getNumericValues().get(1)); + assertEquals(1, statement.getNumericValues().get(2)); + statement.markFailedMeasurement(1, new IllegalArgumentException()); + assertTrue(statement.getNumericValues().isNull(1)); + statement.removeAllFailedMeasurementMarks(); + assertEquals(Integer.MIN_VALUE, statement.getNumericValues().get(1)); + assertObjectStorageAbsent(InsertNumericRowStatement.class, statement); + assertEquals("numeric_table", statement.getTableDeviceID().getTableName()); + } + + @Test + public void testStringTagsSurviveSchemaChanges() throws Exception { + // The client places TAG metadata before FIELD metadata while retaining dense value arrays. + List names = Arrays.asList("tag2", "tag1", "b", "l"); + List types = + Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.INT64); + List categories = + Arrays.asList( + ColumnCategory.TAG, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD); + TSPrepareNumericRowReq req = + new TSPrepareNumericRowReq() + .setTableName("numeric_table") + .setMeasurements(names) + .setDataTypes(types.stream().map(TSDataType::ordinal).toList()) + .setColumnCategories(categories.stream().map(c -> (byte) c.ordinal()).toList()); + ByteBuffer tags = ByteBuffer.allocate(64); + ReadWriteIOUtils.write("设备😀", tags); + ReadWriteIOUtils.write("", tags); + tags.flip(); + ByteBuffer values = ByteBuffer.allocate(9); + values.put((byte) 1).putLong(Long.MIN_VALUE).flip(); + InsertNumericRowStatement statement = + new PreparedNumericRowInfo("numeric_test", req).bind(42, tags, values, null); + assertEquals("设备😀", statement.getNumericValues().getTag(0)); + assertEquals("", statement.getNumericValues().getTag(1)); + assertArrayEquals(new long[] {1, Long.MIN_VALUE}, statement.getValues()); + statement.rebuildArraysAfterExpansion( + new int[] {-1, 1, 0, 2, 3}, new String[] {"missing", "tag1", "tag2", "b", "l"}); + assertArrayEquals( + new String[] {"numeric_table", null, "", "设备😀"}, + statement.getTableDeviceID().getSegments()); + statement.swapColumn(1, 2); + assertEquals("设备😀", statement.getNumericValues().getTag(1)); + statement.markFailedMeasurement(1, new IllegalArgumentException()); + statement.removeAllFailedMeasurementMarks(); + assertEquals("设备😀", statement.getNumericValues().getTag(1)); + assertEquals(1, statement.getValues()[0]); + assertEquals(Long.MIN_VALUE, statement.getValues()[1]); + assertObjectStorageAbsent(InsertNumericRowStatement.class, statement); + } + + @Test + public void testStringTagsSurviveWalConsensusAndTabletAdapters() throws Exception { + // TAGs identify the device through WAL recovery and remain present in Pipe/subscription + // Tablets. + for (String tag : Arrays.asList("设备😀", "", null)) { + RelationalInsertNumericRowNode node = nodeWithNullTag(); + node.getNumericValues().setTag(0, tag); + assertRoundTrips(node); + ByteBuffer bytes = node.serializeToByteBuffer(); + RelationalInsertNumericRowNode decoded = + (RelationalInsertNumericRowNode) PlanNodeType.deserialize(bytes); + assertEquals(tag, decoded.getNumericValues().getTag(0)); + assertEquals(node.getDeviceID(), decoded.getDeviceID()); + assertEquals( + new Binary(tag == null ? "" : tag, TSFileConfig.STRING_CHARSET), + tag == null ? Binary.EMPTY_VALUE : decoded.getValue(0)); + Tablet pipe = + new TabletInsertionEventTablePatternParser(null, null, decoded, null).convertToTablet(); + Tablet subscription = + new ConsensusLogToTabletConverter(null, null, null, "db").convert(decoded).get(0); + for (Tablet tablet : Arrays.asList(pipe, subscription)) { + if (tag == null) { + assertTrue(tablet.getBitMaps()[0].isMarked(0)); + } else { + assertEquals( + tag, + ((Binary[]) tablet.getValues()[0])[0].getStringValue(TSFileConfig.STRING_CHARSET)); + } + } + } + } + + @Test + public void testBulkTagExpansionKeepsValuesAndNulls() throws Exception { + // A reversed schema mapping with two missing TAGs must preserve numeric bits and null slots. + InsertNumericRowStatement statement = preparedRow(); + statement.rebuildArraysAfterExpansion( + new int[] {-1, -1, 6, 5, 4, 3, 2, 1, 0}, + new String[] {"tag1", "tag2", "ts", "date", "d", "f", "l", "i", "b"}); + assertTrue(statement.getNumericValues().isNull(0)); + assertTrue(statement.getNumericValues().isNull(1)); + for (int i = 0; i < BITS.length; i++) { + assertEquals(BITS[BITS.length - 1 - i], statement.getNumericValues().get(i + 2)); + } + assertObjectStorageAbsent(InsertNumericRowStatement.class, statement); + } + + @Test + public void testNumericCastsKeepPrimitiveStorage() { + // Widen INT32 to DOUBLE in place; reject STRING coercion while accepting an inserted null TAG. + NumericRowValues values = new NumericRowValues(0, 2); + values.getValues()[0] = Integer.MIN_VALUE; + values.setNull(1); + assertTrue(values.cast(0, TSDataType.INT32, TSDataType.DOUBLE)); + assertEquals(Double.doubleToRawLongBits((double) Integer.MIN_VALUE), values.get(0)); + assertFalse(values.cast(0, TSDataType.DOUBLE, TSDataType.STRING)); + assertTrue(values.cast(1, null, TSDataType.STRING)); + assertTrue(values.isNull(1)); + } + + @Test + public void testConsensusAndWalRetainPrimitiveStorage() throws Exception { + // Both serializers and WAL readers must retain numeric storage, including partial failures. + RelationalInsertNumericRowNode node = nodeWithNullTag(); + assertRoundTrips(node); + node.markFailedMeasurement(2); + assertRoundTrips(node); + node.setMeasurementSchemas(null); + assertRoundTrips(node); + } + + @Test + public void testWalEnvelopeRetainsTypeAndFrozenIndex() throws Exception { + RelationalInsertNumericRowNode node = nodeWithNullTag(); + node.setSearchIndex(123); + node.setLastFragment(true); + WALInfoEntry entry = new WALInfoEntry(7, node); + // WAL queuing freezes the consensus index even if the caller later reuses the node. + node.setSearchIndex(456); + node.setLastFragment(false); + ByteBuffer buffer = ByteBuffer.allocate(entry.serializedSize()); + entry.serialize(new WALByteBufferForTest(buffer)); + assertFalse(buffer.hasRemaining()); + buffer.flip(); + RelationalInsertNumericRowNode consensus = + (RelationalInsertNumericRowNode) WALEntry.deserializeForConsensus(buffer); + assertEquals(123, consensus.getSearchIndex()); + assertTrue(consensus.isLastFragment()); + assertEquals(node.getNumericValues(), consensus.getNumericValues()); + try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(buffer.array()))) { + WALEntry recovered = WALEntry.deserialize(input); + assertEquals(7, recovered.getMemTableId()); + RelationalInsertNumericRowNode row = (RelationalInsertNumericRowNode) recovered.getValue(); + assertEquals(123, row.getSearchIndex()); + assertTrue(row.isLastFragment()); + assertEquals(node.getNumericValues(), row.getNumericValues()); + assertEquals(0, input.available()); + assertObjectStorageAbsent(RelationalInsertNumericRowNode.class, row); + } + } + + @Test + public void testIoTConsensusBatchFromMemoryAndWal() throws Exception { + // The real state machine must accept both live PlanNode bytes and WAL catch-up envelopes, + // attach the sender progress index, and dispatch each numeric row to DataRegion.insert. + RelationalInsertNumericRowNode original = nodeWithNullTag(); + original.markFailedMeasurement(2); + original.setSearchIndex(42); + original.setLastFragment(true); + WALInfoEntry entry = new WALInfoEntry(7, original); + ByteBuffer wal = ByteBuffer.allocate(entry.serializedSize()); + entry.serialize(new WALByteBufferForTest(wal)); + wal.flip(); + BatchIndexedConsensusRequest batch = new BatchIndexedConsensusRequest(3); + batch.add( + new IndexedConsensusRequest( + -1, + 41, + Collections.singletonList( + new ByteBufferConsensusRequest(original.serializeToByteBuffer())))); + batch.add( + new IndexedConsensusRequest( + -1, 42, Collections.singletonList(new IoTConsensusRequest(wal)))); + DataRegion region = Mockito.mock(DataRegion.class); + IoTConsensusDataRegionStateMachine machine = new IoTConsensusDataRegionStateMachine(region); + DeserializedBatchIndexedConsensusRequest decoded = + (DeserializedBatchIndexedConsensusRequest) machine.deserializeRequest(batch); + assertEquals(2, decoded.getInsertNodes().size()); + for (int i = 0; i < 2; i++) { + IConsensusRequest request = decoded.getInsertNodes().get(i); + assertTrue(request instanceof RelationalInsertNumericRowNode); + RelationalInsertNumericRowNode row = (RelationalInsertNumericRowNode) request; + assertEquals(original.getDeviceID(), row.getDeviceID()); + assertEquals(original.getTime(), row.getTime()); + assertEquals(original.getValidMeasurementNumber(false), row.getMeasurements().length); + assertEquals( + ((RelationalInsertNumericRowNode) decoded.getInsertNodes().get(0)).getNumericValues(), + row.getNumericValues()); + assertEquals(new IoTProgressIndex(3, 41L + i), row.getProgressIndex()); + assertObjectStorageAbsent(RelationalInsertNumericRowNode.class, row); + } + TSStatus status = machine.write(decoded); + assertEquals(2, status.getSubStatusSize()); + for (int i = 0; i < 2; i++) { + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getSubStatus().get(i).getCode()); + RelationalInsertNumericRowNode row = + (RelationalInsertNumericRowNode) decoded.getInsertNodes().get(i); + Mockito.verify(region).insert(Mockito.same(row)); + Mockito.verify(region).insertSeparatorToWAL(Mockito.same(row)); + } + } + + @Test + public void testPipeBatchRetainsIndependentNumericStatements() throws Exception { + RelationalInsertNumericRowNode node = nodeWithNullTag(); + PipeTransferTabletBatchReqV2 request = + PipeTransferTabletBatchReqV2.toTPipeTransferReq( + Arrays.asList(node.serializeToByteBuffer(), node.serializeToByteBuffer()), + Collections.emptyList(), + Arrays.asList("db", "db"), + Collections.emptyList()); + List statements = + PipeTransferTabletBatchReqV2.fromTPipeTransferReq(request).constructStatements(); + // Batching must not replace numeric statements with the object-valued InsertRowsStatement. + assertEquals(2, statements.size()); + for (InsertBaseStatement statement : statements) { + assertEquals(StatementType.INSERT_NUMERIC_ROW, statement.getType()); + assertEquals("db", statement.getDatabaseName().get()); + assertEquals( + node.getNumericValues(), ((InsertNumericRowStatement) statement).getNumericValues()); + assertObjectStorageAbsent(InsertNumericRowStatement.class, statement); + } + } + + @Test + public void testNumericRowRedirectUsesSingleEndpoint() throws Exception { + InsertNumericRowStatement statement = preparedRow(); + Analysis analysis = new Analysis(new InsertNumericRow(statement, null), Collections.emptyMap()); + TEndPoint local = new TEndPoint("127.0.0.1", 6667); + TEndPoint leader = new TEndPoint("127.0.0.1", 6668); + analysis.setRedirectNodeList(Collections.singletonList(leader)); + TableModelPlanner planner = Mockito.mock(TableModelPlanner.class, Mockito.CALLS_REAL_METHODS); + TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + planner.setRedirectInfo(analysis, local, status); + assertEquals(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode(), status.getCode()); + assertEquals(leader, status.getRedirectNode()); + assertFalse(status.isSetSubStatus()); + TSStatus localStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + planner.setRedirectInfo(analysis, leader, localStatus); + assertFalse(localStatus.isSetRedirectNode()); + } + + private void assertRoundTrips(RelationalInsertNumericRowNode node) throws Exception { + ByteBuffer streamed = node.serializeToByteBuffer(); + ByteBuffer direct = ByteBuffer.allocate(streamed.remaining()); + node.serialize(direct); + direct.flip(); + assertEquals(streamed, direct); + RelationalInsertNumericRowNode decoded = + (RelationalInsertNumericRowNode) PlanNodeType.deserialize(direct); + assertObjectStorageAbsent(RelationalInsertNumericRowNode.class, decoded); + assertEquals(node.getNumericValues().isNull(0), decoded.getNumericValues().isNull(0)); + assertEquals(node.getNumericValues().getTag(0), decoded.getNumericValues().getTag(0)); + assertEquals(node.getDeviceID(), decoded.getDeviceID()); + + ByteBuffer wal = ByteBuffer.allocate(node.serializedSize()); + node.setSearchIndex(123); + node.setLastFragment(true); + node.serializeToWAL(new WALByteBufferForTest(wal)); + assertFalse(wal.hasRemaining()); + wal.flip(); + assertEquals(PlanNodeType.RELATIONAL_INSERT_NUMERIC_ROW.getNodeType(), wal.getShort()); + RelationalInsertNumericRowNode recovered = + RelationalInsertNumericRowNode.deserializeFromWAL(wal); + assertFalse(wal.hasRemaining()); + assertEquals(decoded.getNumericValues(), recovered.getNumericValues()); + assertArrayEquals(decoded.getMeasurements(), recovered.getMeasurements()); + assertArrayEquals(decoded.getColumnCategories(), recovered.getColumnCategories()); + assertObjectStorageAbsent(RelationalInsertNumericRowNode.class, recovered); + assertEquals(123, recovered.getSearchIndex()); + assertTrue(recovered.isLastFragment()); + + try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(wal.array()))) { + input.readShort(); + RelationalInsertNumericRowNode fromStream = + RelationalInsertNumericRowNode.deserializeFromWAL(input); + assertEquals(recovered.getNumericValues(), fromStream.getNumericValues()); + assertObjectStorageAbsent(RelationalInsertNumericRowNode.class, fromStream); + assertEquals(0, input.available()); + } + } + + @Test + public void testMemtableAcceptsRecoveredRowWithoutBoxing() throws Exception { + // A WAL-recovered row must reach primitive column arrays without boxing the row or its NaNs. + RelationalInsertNumericRowNode original = nodeWithNullTag(); + ByteBuffer wal = ByteBuffer.allocate(original.serializedSize()); + original.serializeToWAL(new WALByteBufferForTest(wal)); + wal.flip(); + wal.getShort(); + RelationalInsertNumericRowNode recovered = + RelationalInsertNumericRowNode.deserializeFromWAL(wal); + // Fail immediately if the primitive memtable write path boxes even a single cell. + RelationalInsertNumericRowNode row = + new RelationalInsertNumericRowNode(new PlanNodeId("test")) { + @Override + public Object getValue(int index) { + throw new AssertionError("Numeric memtable writes must not box the row"); + } + }; + row.setTargetPath(recovered.getTargetPath()); + row.setTime(42); + row.setAligned(true); + row.setMeasurements(recovered.getMeasurements()); + row.setDataTypes(recovered.getDataTypes()); + row.setMeasurementSchemas(recovered.getMeasurementSchemas()); + row.setColumnCategories(recovered.getColumnCategories()); + row.setNumericValues(recovered.getNumericValues()); + PrimitiveMemTable memtable = new PrimitiveMemTable(); + try { + assertEquals(7, memtable.insertAlignedRow(row)); + AlignedTVList list = + ((AlignedWritableMemChunkGroup) memtable.getMemTableMap().get(row.getDeviceID())) + .getAlignedMemChunk() + .getWorkingTVList(); + assertEquals(42, list.getTime(0)); + assertTrue(list.getBooleanByValueIndex(0, 0)); + assertEquals(Integer.MIN_VALUE, list.getIntByValueIndex(0, 1)); + assertEquals(Long.MIN_VALUE, list.getLongByValueIndex(0, 2)); + assertEquals(0x7fc01234, Float.floatToRawIntBits(list.getFloatByValueIndex(0, 3))); + assertEquals( + 0x7ff8000000001234L, Double.doubleToRawLongBits(list.getDoubleByValueIndex(0, 4))); + assertEquals(20260910, list.getIntByValueIndex(0, 5)); + assertEquals(Long.MAX_VALUE, list.getLongByValueIndex(0, 6)); + assertObjectStorageAbsent(RelationalInsertNumericRowNode.class, row); + } finally { + memtable.release(); + } + } + + @Test + public void testTabletAdaptersDoNotMaterializeObjectRows() throws Exception { + // Pipe and subscription must preserve values and null TAGs without boxing the entire row. + RelationalInsertNumericRowNode node = Mockito.spy(nodeWithNullTag()); + Mockito.doThrow(new AssertionError("Numeric adapters must not box the row")) + .when(node) + .getValues(); + Tablet pipe = + new TabletInsertionEventTablePatternParser(null, null, node, null).convertToTablet(); + List subscription = + new ConsensusLogToTabletConverter(null, null, null, "db").convert(node); + assertEquals(1, subscription.size()); + for (Tablet tablet : Arrays.asList(pipe, subscription.get(0))) { + assertEquals(1, tablet.getRowSize()); + assertEquals("numeric_table", tablet.getTableName()); + assertEquals(42, tablet.getTimestamps()[0]); + assertTrue(tablet.getBitMaps()[0].isMarked(0)); + assertTrue(((boolean[]) tablet.getValues()[1])[0]); + assertEquals(Integer.MIN_VALUE, ((int[]) tablet.getValues()[2])[0]); + assertEquals(Long.MIN_VALUE, ((long[]) tablet.getValues()[3])[0]); + assertEquals(0x7fc01234, Float.floatToRawIntBits(((float[]) tablet.getValues()[4])[0])); + assertEquals( + 0x7ff8000000001234L, Double.doubleToRawLongBits(((double[]) tablet.getValues()[5])[0])); + assertEquals(LocalDate.of(2026, 9, 10), ((LocalDate[]) tablet.getValues()[6])[0]); + assertEquals(Long.MAX_VALUE, ((long[]) tablet.getValues()[7])[0]); + } + assertObjectStorageAbsent(RelationalInsertNumericRowNode.class, node); + } + + private RelationalInsertNumericRowNode nodeWithNullTag() throws Exception { + InsertNumericRowStatement statement = preparedRow(); + statement.insertColumn( + 0, + new ColumnSchema( + "tag", TypeFactory.getType(TSDataType.STRING), false, TsTableColumnCategory.TAG)); + MeasurementSchema[] schemas = new MeasurementSchema[statement.getValueCount()]; + for (int i = 0; i < schemas.length; i++) { + schemas[i] = new MeasurementSchema(statement.getMeasurements()[i], statement.getDataType(i)); + } + RelationalInsertNumericRowNode node = + new RelationalInsertNumericRowNode( + new PlanNodeId("numeric"), + statement.getDevicePath(), + statement.getMeasurements(), + statement.getDataTypes(), + schemas, + statement.getTime(), + statement.getNumericValues(), + statement.getColumnCategories()); + node.setNumericValues(statement.getNumericValues()); + return node; + } + + private void assertObjectStorageAbsent(Class owner, Object row) throws Exception { + // A dedicated numeric type must not inherit the object row or retain an Object[] payload. + assertFalse(InsertRowStatement.class.isInstance(row)); + assertFalse(InsertRowNode.class.isInstance(row)); + for (Class type = owner; type != null; type = type.getSuperclass()) { + for (Field field : type.getDeclaredFields()) { + assertFalse(field.getName(), field.getType() == Object[].class); + } + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java index 0acb6816bfb8a..e664d9cccf996 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java @@ -299,7 +299,7 @@ public void testInsertRecord() throws QueryProcessException, IllegalPathExceptio Arrays.asList("s1", "s2"), ByteBuffer.wrap(new byte[] {0, 0, 0, 0}), 1000L); - InsertRowStatement statement = StatementGenerator.createStatement(req); + InsertRowStatement statement = (InsertRowStatement) StatementGenerator.createStatement(req); assertEquals(1000L, statement.getTime()); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTableNumericRowTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTableNumericRowTest.java new file mode 100644 index 0000000000000..098f6ab72e55b --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTableNumericRowTest.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.memtable; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; +import org.apache.iotdb.db.utils.NumericRowValues; +import org.apache.iotdb.db.utils.datastructure.AlignedTVList; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class AbstractMemTableNumericRowTest { + private PrimitiveMemTable memTable; + private boolean previousIncludeNull; + + @Before + public void setUp() { + memTable = new PrimitiveMemTable("numeric_test", "0"); + previousIncludeNull = + IoTDBDescriptor.getInstance().getConfig().isIncludeNullValueInWriteThroughputMetric(); + IoTDBDescriptor.getInstance().getConfig().setIncludeNullValueInWriteThroughputMetric(false); + } + + @After + public void tearDown() { + memTable.release(); + IoTDBDescriptor.getInstance() + .getConfig() + .setIncludeNullValueInWriteThroughputMetric(previousIncludeNull); + } + + @Test + public void testNullAndFailedFieldAccountingWithBothMetricSettings() throws Exception { + // TAGs and failed FIELDs never count; valid NULL FIELDs count only in the enabled metric. + NumericRowValues values = new NumericRowValues(1, 3); + values.setTag(0, "device"); + values.setRawValue(1, Double.doubleToRawLongBits(10)); + values.setNull(2); + RelationalInsertNumericRowNode row = + numericRow(new String[] {"tag", "a", "b", "failed"}, 1, values, 1); + row.markFailedMeasurement(3); + + assertEquals(1, memTable.insertAlignedRow(row)); + assertEquals(2, memTable.getTotalValueCount()); + assertEquals(1, memTable.getNullValueCount()); + assertEquals(1, memTable.getSeriesNumber()); + assertEquals(20, memTable.memSize()); + + IoTDBDescriptor.getInstance().getConfig().setIncludeNullValueInWriteThroughputMetric(true); + row.setTime(2); + assertEquals(2, memTable.insertAlignedRow(row)); + values.setNull(1); + row.setTime(3); + assertEquals(2, memTable.insertAlignedRow(row)); + assertEquals(5, memTable.getTotalPointsNum()); + assertEquals(6, memTable.getTotalValueCount()); + assertEquals(4, memTable.getNullValueCount()); + assertEquals(52, memTable.memSize()); + assertEquals(1, memTable.getSeriesNumber()); + AlignedTVList list = chunk(row).getWorkingTVList(); + assertEquals(10, list.getDoubleByValueIndex(0, 0), 0); + assertTrue(list.isNullValue(2, 0)); + } + + @Test + public void testMissingSlotsAndInvalidMetadataAreNotCounted() throws Exception { + // A valid schema beyond the value buffer is absent, not a NULL point or a new series. + NumericRowValues values = new NumericRowValues(1, 1); + values.setTag(0, "device"); + values.setRawValue(1, Double.doubleToRawLongBits(12)); + RelationalInsertNumericRowNode row = + numericRow(new String[] {"tag", "a", "missing", "invalid"}, 1, values, 1); + row.getMeasurementSchemas()[3] = null; + assertEquals(1, memTable.insertAlignedRow(row)); + assertEquals(1, memTable.getTotalValueCount()); + assertEquals(0, memTable.getNullValueCount()); + assertEquals(1, memTable.getSeriesNumber()); + assertEquals(20, memTable.memSize()); + assertEquals(12, chunk(row).getWorkingTVList().getDoubleByValueIndex(0, 0), 0); + } + + @Test + public void testExistingGroupCanGrowAfterNumericAndTabletWrites() throws Exception { + // Creation filters NULL schemas; a later Tablet materializes that field in the same group. + NumericRowValues values = new NumericRowValues(0, 2); + values.setRawValue(0, Double.doubleToRawLongBits(1)); + values.setNull(1); + RelationalInsertNumericRowNode row = numericRow(new String[] {"a", "b"}, 0, values, 1); + assertEquals(1, memTable.insertAlignedRow(row)); + AlignedWritableMemChunk original = chunk(row); + row.setTime(2); + assertEquals(1, memTable.insertAlignedRow(row)); + RelationalInsertTabletNode tablet = + new RelationalInsertTabletNode( + new PlanNodeId("tablet"), + new PartialPath("t"), + true, + new String[] {"b"}, + new TSDataType[] {TSDataType.DOUBLE}, + new MeasurementSchema[] {new MeasurementSchema("b", TSDataType.DOUBLE)}, + new long[] {3}, + null, + new Object[] {new double[] {30}}, + 1, + new TsTableColumnCategory[] {TsTableColumnCategory.FIELD}); + assertEquals(1, memTable.insertAlignedTablet(tablet, 0, 1, null)); + values.setRawValue(0, Double.doubleToRawLongBits(4)); + values.setRawValue(1, Double.doubleToRawLongBits(40)); + row.setTime(4); + assertEquals(2, memTable.insertAlignedRow(row)); + assertSame(original, chunk(row)); + assertEquals(2, memTable.getSeriesNumber()); + assertEquals(5, memTable.getTotalPointsNum()); + assertEquals(7, memTable.getTotalValueCount()); + assertEquals(2, memTable.getNullValueCount()); + AlignedTVList list = original.getWorkingTVList(); + assertTrue(list.isNullValue(0, 1)); + assertTrue(list.isNullValue(2, 0)); + assertEquals(30, list.getDoubleByValueIndex(2, 1), 0); + assertEquals(4, list.getDoubleByValueIndex(3, 0), 0); + assertEquals(40, list.getDoubleByValueIndex(3, 1), 0); + } + + @Test + public void testEmptyRowDoesNotCreateGroup() throws Exception { + // Empty input leaves counters and the device map untouched. + assertEquals( + 0, memTable.insertAlignedRow(numericRow(new String[0], 0, new NumericRowValues(0, 0), 1))); + assertTrue(memTable.getMemTableMap().isEmpty()); + assertEquals(0, memTable.memSize()); + assertEquals(0, memTable.getTotalValueCount()); + } + + private AlignedWritableMemChunk chunk(RelationalInsertNumericRowNode row) { + return ((AlignedWritableMemChunkGroup) memTable.getMemTableMap().get(row.getDeviceID())) + .getAlignedMemChunk(); + } + + private static RelationalInsertNumericRowNode numericRow( + String[] names, int tags, NumericRowValues values, long time) throws IllegalPathException { + TSDataType[] types = new TSDataType[names.length]; + TsTableColumnCategory[] categories = new TsTableColumnCategory[names.length]; + Arrays.fill(types, TSDataType.DOUBLE); + Arrays.fill(categories, TsTableColumnCategory.FIELD); + MeasurementSchema[] schemas = new MeasurementSchema[names.length]; + for (int i = 0; i < names.length; i++) { + if (i < tags) { + types[i] = TSDataType.STRING; + categories[i] = TsTableColumnCategory.TAG; + } + schemas[i] = new MeasurementSchema(names[i], types[i]); + } + return new RelationalInsertNumericRowNode( + new PlanNodeId("numeric"), + new PartialPath("t"), + names, + types, + schemas, + time, + values, + categories); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkNumericTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkNumericTest.java new file mode 100644 index 0000000000000..cb8d2a15207f1 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkNumericTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.memtable; + +import org.apache.iotdb.db.utils.NumericRowValues; +import org.apache.iotdb.db.utils.datastructure.AlignedTVList; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class AlignedWritableMemChunkNumericTest { + + @Test + public void testMappingAllocatedOnlyForNumericRowsAndReused() throws Exception { + // Ordinary row/Tablet writes must not allocate scratch space; numeric rows share it. + List schemas = + Collections.singletonList(new MeasurementSchema("a", TSDataType.INT64)); + AlignedWritableMemChunk chunk = new AlignedWritableMemChunk(new ArrayList<>(schemas), true); + try { + assertNull(mapping(chunk)); + chunk.writeAlignedPoints(1, new Object[] {10L}, schemas); + chunk.writeAlignedTablet( + new long[] {2}, new Object[] {new long[] {20}}, null, schemas, 0, 1, null); + assertNull(mapping(chunk)); + + chunk.writeAlignedNumericPoints(3, row(30), schemas); + int[] mapping = mapping(chunk); + assertNotNull(mapping); + chunk.writeAlignedNumericPoints(4, row(40), schemas); + assertSame(mapping, mapping(chunk)); + assertEquals(30, chunk.getWorkingTVList().getLongByValueIndex(2, 0)); + assertEquals(40, chunk.getWorkingTVList().getLongByValueIndex(3, 0)); + } finally { + chunk.release(); + } + } + + @Test + public void testMappingResetAndGrowthWithMixedWrites() throws Exception { + // Reordered/absent columns must stay correct after reuse and schema growth by either API. + IMeasurementSchema a = new MeasurementSchema("a", TSDataType.INT64); + IMeasurementSchema b = new MeasurementSchema("b", TSDataType.INT64); + IMeasurementSchema c = new MeasurementSchema("c", TSDataType.INT64); + IMeasurementSchema d = new MeasurementSchema("d", TSDataType.INT64); + AlignedWritableMemChunk chunk = + new AlignedWritableMemChunk(new ArrayList<>(Arrays.asList(a, b)), true); + try { + chunk.writeAlignedNumericPoints(1, row(20, 10), Arrays.asList(b, a)); + int[] mapping = mapping(chunk); + chunk.writeAlignedNumericPoints(2, row(30, 999), Arrays.asList(a, null)); + assertSame(mapping, mapping(chunk)); + AlignedTVList list = chunk.getWorkingTVList(); + assertEquals(10, list.getLongByValueIndex(0, 0)); + assertEquals(20, list.getLongByValueIndex(0, 1)); + assertEquals(30, list.getLongByValueIndex(1, 0)); + assertTrue(list.isNullValue(1, 1)); + + chunk.writeAlignedNumericPoints(3, row(40, 50), Arrays.asList(c, b)); + int[] extendedMapping = mapping(chunk); + assertEquals(3, extendedMapping.length); + assertTrue(list.isNullValue(2, 0)); + assertEquals(50, list.getLongByValueIndex(2, 1)); + assertEquals(40, list.getLongByValueIndex(2, 2)); + + chunk.writeAlignedTablet( + new long[] {4}, + new Object[] {new long[] {60}}, + null, + Collections.singletonList(d), + 0, + 1, + null); + assertSame(extendedMapping, mapping(chunk)); + chunk.writeAlignedNumericPoints(5, row(70), Collections.singletonList(a)); + assertEquals(4, mapping(chunk).length); + assertEquals(60, list.getLongByValueIndex(3, 3)); + assertEquals(70, list.getLongByValueIndex(4, 0)); + assertTrue(list.isNullValue(4, 1)); + assertTrue(list.isNullValue(4, 2)); + assertTrue(list.isNullValue(4, 3)); + + mapping = mapping(chunk); + chunk.writeAlignedNumericPoints(6, row(80), Collections.singletonList(d)); + assertSame(mapping, mapping(chunk)); + assertTrue(list.isNullValue(5, 0)); + assertEquals(80, list.getLongByValueIndex(5, 3)); + } finally { + chunk.release(); + } + } + + private static NumericRowValues row(long... values) { + NumericRowValues row = new NumericRowValues(0, values.length); + for (int i = 0; i < values.length; i++) { + row.setRawValue(i, values[i]); + } + return row; + } + + private static int[] mapping(AlignedWritableMemChunk chunk) throws Exception { + // Array identity verifies the allocation contract without exposing scratch state as public API. + Field field = AlignedWritableMemChunk.class.getDeclaredField("numericColumnMapping"); + field.setAccessible(true); + return (int[]) field.get(chunk); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java index 8be064d01925b..e324b00555f8b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java @@ -20,6 +20,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; +import org.apache.iotdb.db.utils.NumericRowValues; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.common.conf.TSFileConfig; @@ -44,9 +45,53 @@ import static org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager.ARRAY_SIZE; import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF; +import static org.junit.Assert.assertEquals; public class AlignedTVListTest { + @Test + public void testNumericRowsMatchObjectRowsAcrossBlocksAndSorting() { + // Reordered columns, nulls and descending timestamps must match the existing write path + // across a primitive-array block boundary and after sorting. + List types = + Arrays.asList( + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.INT64, + TSDataType.STRING); + AlignedTVList numeric = AlignedTVList.newAlignedList(types); + AlignedTVList boxed = AlignedTVList.newAlignedList(types); + try { + for (int i = ARRAY_SIZE + 3; i >= 0; i--) { + NumericRowValues row = new NumericRowValues(0, 5); + row.getValues()[0] = i; + row.getValues()[1] = i % 2; + row.getValues()[2] = -i; + row.getValues()[3] = Float.floatToRawIntBits(-0.0f); + row.getValues()[4] = Double.doubleToRawLongBits(-i * 0.25); + if (i % 3 == 0) { + row.setNull(2); + } + // Also exercise a missing STRING column in an existing memtable. + numeric.putAlignedNumericValue(i, row, new int[] {1, 2, 3, 4, 0, -1}); + boxed.putAlignedValue( + i, new Object[] {i % 2 != 0, i % 3 == 0 ? null : -i, -0.0f, -i * 0.25, (long) i, null}); + } + numeric.sort(); + boxed.sort(); + assertEquals(boxed.rowCount(), numeric.rowCount()); + for (int i = 0; i < numeric.rowCount(); i++) { + assertEquals(boxed.getTime(i), numeric.getTime(i)); + assertEquals(boxed.getAlignedValue(i), numeric.getAlignedValue(i)); + } + } finally { + numeric.clear(); + boxed.clear(); + } + } + @Test public void testExtendColumnRetainsImplicitNullsOnFirstWrite() { AlignedTVList list = diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeType.java index 20c3c1bf879a2..9069c6e3a0e83 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -210,6 +210,7 @@ public enum PlanNodeType { RELATIONAL_INSERT_ROWS((short) 2002), RELATIONAL_DELETE_DATA((short) 2003), OBJECT_FILE_NODE((short) 2004), + RELATIONAL_INSERT_NUMERIC_ROW((short) 2005), ; private static final IPlanNodeDeserializer DESERIALIZER; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 681c638bc563e..1d2600ff5e122 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -377,6 +377,16 @@ public Pair getInstanceVersion() { return new Pair<>(creationId, instanceVersion.get()); } + /** Checks both table identity and schema revision, waiting for any ongoing schema write. */ + public boolean matchesInstanceVersion(final Pair version) { + readWriteLock.readLock().lock(); + try { + return creationId.equals(version.left) && instanceVersion.get() == version.right; + } finally { + readWriteLock.readLock().unlock(); + } + } + public boolean containsPropWithoutLock(final String propKey) { return props != null && props.containsKey(propKey); } diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift index 7ff12ceb02154..b01ef1a294652 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift @@ -631,6 +631,12 @@ service IClientRPCService { common.TSStatus insertRecord(1:TSInsertRecordReq req); + TSPrepareNumericRowResp prepareNumericRow(1:TSPrepareNumericRowReq req); + + common.TSStatus executePreparedNumericRow(1:TSExecutePreparedNumericRowReq req); + + common.TSStatus closePreparedNumericRow(1:i64 sessionId, 2:i64 preparedId); + common.TSStatus insertStringRecord(1:TSInsertStringRecordReq req); common.TSStatus insertTablet(1:TSInsertTabletReq req); @@ -705,4 +711,29 @@ service IClientRPCService { /** For other node's call */ common.TSStatus testConnectionEmptyRPC(); -} \ No newline at end of file +} + +struct TSPrepareNumericRowReq { + 1: required i64 sessionId + 2: required string tableName + 3: required list measurements + 4: required list dataTypes + 5: required list columnCategories +} + +struct TSPrepareNumericRowResp { + 1: required common.TSStatus status + 2: optional i64 preparedId +} + +struct TSExecutePreparedNumericRowReq { + 1: required i64 sessionId + 2: required i64 preparedId + 3: required i64 timestamp + 4: required binary values + // Length-prefixed UTF-8 TAGs; length -1 represents a null TAG. + 5: required binary tagValues + // FIELD-only bitmap: bit i (LSB first) marks FIELD i as null; null values omit payload bytes. + // Exactly ceil(FIELD count / 8) bytes, with unused high bits zero; absent means no nulls. + 6: optional binary nullBitmap +}