From 56c20e3840f4057ea370e4311e6f667b10790804 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 10 Sep 2026 19:03:03 +0800 Subject: [PATCH 01/20] tmp --- .../plan/analyze/AnalyzeUtils.java | 1 + .../dataregion/memtable/AbstractMemTable.java | 46 ++++++++++++++++++- .../memtable/AlignedWritableMemChunk.java | 1 + .../db/utils/datastructure/AlignedTVList.java | 41 +++++++++++++++++ .../datastructure/AlignedTVListTest.java | 45 ++++++++++++++++++ 5 files changed, 132 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java index 5da7306656778..c4c16d5af7e26 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java @@ -60,6 +60,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TimeRange; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index e24308cf505da..bd2b7d5770249 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -32,9 +32,11 @@ import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.schemaengine.schemaregion.utils.ResourceByPathUtils; import org.apache.iotdb.db.storageengine.dataregion.flush.FlushStatus; import org.apache.iotdb.db.storageengine.dataregion.flush.NotifyFlushMemTable; @@ -241,10 +243,13 @@ public int insert(InsertRowNode insertRowNode) { } @Override - public int insertAlignedRow(InsertRowNode insertRowNode) { + public int insertAlignedRow(AbstractInsertRowNode insertRowNode) { + if (insertRowNode instanceof RelationalInsertNumericRowNode numeric) { + return insertAlignedNumericRow(numeric); + } String[] measurements = insertRowNode.getMeasurements(); - Object[] values = insertRowNode.getValues(); + Object[] values = ((InsertRowNode) insertRowNode).getValues(); List schemaList = new ArrayList<>(); List dataTypes = new ArrayList<>(); Object[] writableValues = new Object[measurements == null ? 0 : measurements.length]; @@ -286,6 +291,43 @@ public int insertAlignedRow(InsertRowNode insertRowNode) { return pointsInserted; } + private int insertAlignedNumericRow(RelationalInsertNumericRowNode row) { + List schemas = new ArrayList<>(); + int nullPoints = 0; + long rowSize = Long.BYTES + Integer.BYTES; + for (int i = 0; i < row.getMeasurements().length; i++) { + if (!row.isValidMeasurement(i, true) || row.isNull(i)) { + if (row.isValidMeasurement(i, true) && i < row.getValueCount() && row.isNull(i)) { + nullPoints++; + } + schemas.add(null); + } else { + schemas.add(row.getMeasurementSchemas()[i]); + rowSize += row.getDataType(i).getDataTypeSize(); + } + } + if (schemas.isEmpty()) { + return 0; + } + AlignedWritableMemChunkGroup group = + (AlignedWritableMemChunkGroup) + createAlignedMemChunkGroupIfNotExistAndGet(row.getDeviceID(), schemas); + group + .getAlignedMemChunk() + .writeAlignedNumericPoints(row.getTime(), row.getNumericValues(), schemas); + memSize += rowSize; + int validValues = row.getValidMeasurementNumber(true); + int inserted = + validValues + - (IoTDBDescriptor.getInstance().getConfig().isIncludeNullValueInWriteThroughputMetric() + ? 0 + : nullPoints); + totalPointsNum += inserted; + totalValueCount += validValues; + nullValueCount += nullPoints; + return inserted; + } + @Override public int insertTablet(InsertTabletNode insertTabletNode, int start, int end) throws WriteProcessException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index d1c153841733d..37bd7d3c870cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; +import org.apache.iotdb.db.utils.NumericRowValues; import org.apache.iotdb.db.utils.datastructure.AlignedTVList; import org.apache.iotdb.db.utils.datastructure.BatchEncodeInfo; import org.apache.iotdb.db.utils.datastructure.MemPointIterator; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index d2176c6891e49..88c8409cf7a30 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager; import org.apache.iotdb.db.utils.MathUtils; +import org.apache.iotdb.db.utils.NumericRowValues; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.block.column.Column; @@ -523,6 +524,46 @@ public synchronized void putAlignedValue(long timestamp, Object[] value) { } } + /** Writes row bits into native column arrays without a boxed intermediate row. */ + public synchronized void putAlignedNumericValue( + long timestamp, NumericRowValues row, int[] columnMapping) { + checkExpansion(); + int arrayIndex = rowCount / ARRAY_SIZE; + int elementIndex = rowCount % ARRAY_SIZE; + maxTime = Math.max(maxTime, timestamp); + minTime = Math.min(minTime, timestamp); + timestamps.get(arrayIndex)[elementIndex] = timestamp; + for (int i = 0; i < values.size(); i++) { + int source = columnMapping[i]; + if (source < 0 || row.isNull(source)) { + markNullValue(i, arrayIndex, elementIndex); + continue; + } + Object valueArray = getOrCreateValueArray(i, arrayIndex, elementIndex); + long value = row.get(source); + switch (dataTypes.get(i)) { + case BOOLEAN -> ((boolean[]) valueArray)[elementIndex] = value != 0; + case INT32, DATE -> ((int[]) valueArray)[elementIndex] = (int) value; + case INT64, TIMESTAMP -> ((long[]) valueArray)[elementIndex] = value; + case FLOAT -> ((float[]) valueArray)[elementIndex] = Float.intBitsToFloat((int) value); + case DOUBLE -> ((double[]) valueArray)[elementIndex] = Double.longBitsToDouble(value); + default -> throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); + } + } + if (indices != null) { + indices.get(arrayIndex)[elementIndex] = rowCount; + } + rowCount++; + if (sorted) { + if (rowCount > 1 && timestamp < getTime(rowCount - 2)) { + sorted = false; + segmentMovedMap = null; + } else { + seqRowCount++; + } + } + } + @Override public Object getAlignedValue(int index) { return getAlignedValueForQuery(index, null, null); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java index 708f2113cdf23..a39fc45def75d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java @@ -20,6 +20,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; +import org.apache.iotdb.db.utils.NumericRowValues; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.common.conf.TSFileConfig; @@ -44,9 +45,53 @@ import static org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager.ARRAY_SIZE; import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF; +import static org.junit.Assert.assertEquals; public class AlignedTVListTest { + @Test + public void testNumericRowsMatchObjectRowsAcrossBlocksAndSorting() { + // Reordered columns, nulls and descending timestamps must match the existing write path + // across a primitive-array block boundary and after sorting. + List types = + Arrays.asList( + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.INT64, + TSDataType.STRING); + AlignedTVList numeric = AlignedTVList.newAlignedList(types); + AlignedTVList boxed = AlignedTVList.newAlignedList(types); + try { + for (int i = ARRAY_SIZE + 3; i >= 0; i--) { + NumericRowValues row = new NumericRowValues(5); + row.getValues()[0] = i; + row.getValues()[1] = i % 2; + row.getValues()[2] = -i; + row.getValues()[3] = Float.floatToRawIntBits(-0.0f); + row.getValues()[4] = Double.doubleToRawLongBits(-i * 0.25); + if (i % 3 == 0) { + row.setNull(2); + } + // Also exercise a missing STRING column in an existing memtable. + numeric.putAlignedNumericValue(i, row, new int[] {1, 2, 3, 4, 0, -1}); + boxed.putAlignedValue( + i, new Object[] {i % 2 != 0, i % 3 == 0 ? null : -i, -0.0f, -i * 0.25, (long) i, null}); + } + numeric.sort(); + boxed.sort(); + assertEquals(boxed.rowCount(), numeric.rowCount()); + for (int i = 0; i < numeric.rowCount(); i++) { + assertEquals(boxed.getTime(i), numeric.getTime(i)); + assertEquals(boxed.getAlignedValue(i), numeric.getAlignedValue(i)); + } + } finally { + numeric.clear(); + boxed.clear(); + } + } + @Test public void testValueListArrayMemCostExcludesLazyBitmapAndNullPlaceholder() { long expected = (long) ARRAY_SIZE * Long.BYTES + NUM_BYTES_ARRAY_HEADER + NUM_BYTES_OBJECT_REF; From 278d4e01d0a53d40fd3fb80a4beeee7622d5c30c Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 11 Sep 2026 11:18:13 +0800 Subject: [PATCH 02/20] tmp --- .../it/session/IoTDBTableModelSessionIT.java | 152 ++++++++++++++++++ .../apache/iotdb/isession/ITableSession.java | 4 + .../dataregion/DataExecutionVisitor.java | 12 ++ .../memory/InsertNodeMemoryEstimator.java | 14 ++ .../thrift/impl/ClientRPCServiceImpl.java | 28 ++-- .../impl/DataNodeInternalRPCServiceImpl.java | 4 +- .../plan/analyze/AnalyzeUtils.java | 5 + .../node/DataNodePlanNodeDeserializer.java | 7 + .../plan/node/write/InsertRowNode.java | 24 ++- .../plan/relational/sql/ast/AstVisitor.java | 4 + .../metrics/IoTDBInternalLocalReporter.java | 4 +- .../storageengine/dataregion/DataRegion.java | 7 +- .../memtable/AlignedWritableMemChunk.java | 34 ++++ .../dataregion/memtable/IMemTable.java | 3 +- .../quotas/DataNodeThrottleQuotaManager.java | 1 + .../rescon/quotas/DefaultOperationQuota.java | 17 ++ .../ConsensusLogToTabletConverter.java | 41 +++-- .../consensus/ConsensusPrefetchingQueue.java | 5 +- .../src/main/thrift/client.thrift | 2 + 19 files changed, 320 insertions(+), 48 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java index b3014ff56ee62..48a5a994dd5cb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java @@ -21,6 +21,7 @@ import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.ITableSessionPool; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.TableClusterIT; @@ -28,6 +29,8 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.RowRecord; import org.junit.After; import org.junit.Before; @@ -35,8 +38,16 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.time.LocalDate; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showTablesColumnHeaders; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -53,6 +64,147 @@ public void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + @Test + public void testNumericRowInsert() throws Exception { + // Exercise both table-session entry points, schema reordering and missing STRING columns. + // The same values must remain readable from MemTable and after flushing to TsFile. + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE numeric_rows"); + session.executeNonQueryStatement("USE numeric_rows"); + session.executeNonQueryStatement( + "CREATE TABLE readings(tag STRING TAG, attr STRING ATTRIBUTE, " + + "event_time TIMESTAMP FIELD, event_date DATE FIELD, d DOUBLE FIELD, " + + "f FLOAT FIELD, l INT64 FIELD, i INT32 FIELD, b BOOLEAN FIELD, missing STRING FIELD)"); + List names = Arrays.asList("b", "i", "l", "f", "d", "event_date", "event_time"); + List types = + Arrays.asList( + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.DATE, + TSDataType.TIMESTAMP); + long[] values = { + -1, + Integer.MIN_VALUE, + Long.MIN_VALUE, + Float.floatToRawIntBits(-1.25f), + Double.doubleToRawLongBits(2.5), + 20260910, + 1234 + }; + session.insert("readings", 1, names, types, values); + try (ITableSessionPool pool = EnvFactory.getEnv().getTableSessionPool(1, "numeric_rows"); + ITableSession pooledSession = pool.getSession()) { + pooledSession.insert( + "readings", + 2, + names, + types, + Collections.nCopies(types.size(), ColumnCategory.FIELD), + values); + } + assertNumericRows(session); + session.executeNonQueryStatement("FLUSH"); + assertNumericRows(session); + } + } + + @Test + public void testNumericRowInsertWithTags() throws Exception { + // Different TAGs at the same timestamp must stay separate, with identical behavior after flush. + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE numeric_tag_rows"); + session.executeNonQueryStatement("USE numeric_tag_rows"); + session.executeNonQueryStatement( + "CREATE TABLE readings(region STRING TAG, device STRING TAG, " + + "missing STRING TAG, v INT64 FIELD, enabled BOOLEAN FIELD)"); + List names = Arrays.asList("enabled", "device", "v", "region"); + List types = + Arrays.asList(TSDataType.BOOLEAN, TSDataType.STRING, TSDataType.INT64, TSDataType.STRING); + List categories = + Arrays.asList( + ColumnCategory.FIELD, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.TAG); + session.insert( + "readings", + 1, + names, + types, + categories, + new long[] {1, 10}, + new String[] {"设备甲😀", "华北"}); + try (ITableSessionPool pool = EnvFactory.getEnv().getTableSessionPool(1, "numeric_tag_rows"); + ITableSession pooled = pool.getSession()) { + pooled.insert( + "readings", + 1, + names, + types, + categories, + new long[] {0, 20}, + new String[] {"设备乙", "华北"}); + } + session.insert( + "readings", 2, names, types, categories, new long[] {1, 30}, new String[] {"", null}); + session.insert( + "readings", 2, names, types, categories, new long[] {0, 40}, new String[] {null, null}); + assertNumericTagRows(session); + session.executeNonQueryStatement("FLUSH"); + assertNumericTagRows(session); + } + } + + private void assertNumericTagRows(ITableSession session) throws Exception { + String[] devices = {"设备甲😀", "设备乙", "", null}; + try (SessionDataSet dataSet = + session.executeQueryStatement( + "SELECT region,device,missing,v,enabled FROM readings ORDER BY v")) { + for (int i = 0; i < devices.length; i++) { + assertTrue(dataSet.hasNext()); + RowRecord row = dataSet.next(); + if (i < 2) { + assertEquals("华北", row.getFields().get(0).getStringValue()); + } else { + assertNull(row.getFields().get(0).getDataType()); + } + if (devices[i] == null) { + assertNull(row.getFields().get(1).getDataType()); + } else { + assertEquals(devices[i], row.getFields().get(1).getStringValue()); + } + assertNull(row.getFields().get(2).getDataType()); + assertEquals((i + 1) * 10, row.getFields().get(3).getLongV()); + assertEquals(i % 2 == 0, row.getFields().get(4).getBoolV()); + } + assertFalse(dataSet.hasNext()); + } + } + + private void assertNumericRows(ITableSession session) throws Exception { + try (SessionDataSet dataSet = + session.executeQueryStatement( + "SELECT time,b,i,l,f,d,event_date,event_time,tag,attr,missing FROM readings ORDER BY time")) { + for (long time = 1; time <= 2; time++) { + assertTrue(dataSet.hasNext()); + RowRecord row = dataSet.next(); + assertEquals(time, row.getFields().get(0).getLongV()); + assertTrue(row.getFields().get(1).getBoolV()); + assertEquals(Integer.MIN_VALUE, row.getFields().get(2).getIntV()); + assertEquals(Long.MIN_VALUE, row.getFields().get(3).getLongV()); + assertEquals(-1.25f, row.getFields().get(4).getFloatV(), 0); + assertEquals(2.5, row.getFields().get(5).getDoubleV(), 0); + assertEquals(TSDataType.DATE, row.getFields().get(6).getDataType()); + assertEquals(LocalDate.of(2026, 9, 10), row.getFields().get(6).getDateV()); + assertEquals(1234, row.getFields().get(7).getLongV()); + for (int column = 8; column <= 10; column++) { + assertNull(row.getFields().get(column).getDataType()); + } + } + assertFalse(dataSet.hasNext()); + } + } + @Test public void testUseDatabase() { final String[] table1Names = new String[] {"table1"}; diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java index 75a37f8384719..b5868e60ecd3d 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java @@ -22,8 +22,12 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.write.record.Tablet; +import java.util.List; + /** * This interface defines a session for interacting with IoTDB tables. It supports operations such * as data insertion, executing queries, and closing the session. Implementations of this interface diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java index 96bbb6ccc16ba..d5d48dcf38ab4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedInsertNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiTabletsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; @@ -44,6 +45,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; @@ -76,6 +78,16 @@ public TSStatus visitRelationalInsertRows(RelationalInsertRowsNode node, DataReg @Override public TSStatus visitInsertRow(InsertRowNode node, DataRegion dataRegion) { + return executeInsertRow(node, dataRegion); + } + + @Override + public TSStatus visitRelationalInsertNumericRow( + RelationalInsertNumericRowNode node, DataRegion dataRegion) { + return executeInsertRow(node, dataRegion); + } + + private TSStatus executeInsertRow(AbstractInsertRowNode node, DataRegion dataRegion) { try { dataRegion.insert(node); dataRegion.insertSeparatorToWAL(node); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java index 394d67fd6be1b..1ea256b2e2ba3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; @@ -71,6 +72,7 @@ public class InsertNodeMemoryEstimator { private static final String INSERT_MULTI_TABLETS_NODE = "InsertMultiTabletsNode"; private static final String RELATIONAL_INSERT_ROWS_NODE = "RelationalInsertRowsNode"; private static final String RELATIONAL_INSERT_ROW_NODE = "RelationalInsertRowNode"; + private static final String RELATIONAL_INSERT_NUMERIC_ROW_NODE = "RelationalInsertNumericRowNode"; private static final String RELATIONAL_INSERT_TABLET_NODE = "RelationalInsertTabletNode"; private static final long NUM_BYTES_OBJECT_REF = RamUsageEstimator.NUM_BYTES_OBJECT_REF; @@ -103,6 +105,9 @@ public class InsertNodeMemoryEstimator { private static final long RELATIONAL_INSERT_ROW_NODE_SIZE = RamUsageEstimator.shallowSizeOfInstance(RelationalInsertRowNode.class); + private static final long RELATIONAL_INSERT_NUMERIC_ROW_NODE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(RelationalInsertNumericRowNode.class); + private static final long RELATIONAL_INSERT_TABLET_NODE_SIZE = RamUsageEstimator.shallowSizeOfInstance(RelationalInsertTabletNode.class); @@ -181,6 +186,8 @@ public static long sizeOf(final InsertNode insertNode) { return sizeOfRelationalInsertRowsNode((RelationalInsertRowsNode) insertNode); case RELATIONAL_INSERT_ROW_NODE: return sizeOfRelationalInsertRowNode((RelationalInsertRowNode) insertNode); + case RELATIONAL_INSERT_NUMERIC_ROW_NODE: + return sizeOfRelationalInsertNumericRowNode((RelationalInsertNumericRowNode) insertNode); case RELATIONAL_INSERT_TABLET_NODE: return sizeOfRelationalInsertTabletNode((RelationalInsertTabletNode) insertNode); default: @@ -325,6 +332,13 @@ private static long sizeOfRelationalInsertRowNode( return size; } + private static long sizeOfRelationalInsertNumericRowNode( + final RelationalInsertNumericRowNode node) { + return RELATIONAL_INSERT_NUMERIC_ROW_NODE_SIZE + + calculateFullInsertNodeSize(node) + + node.getNumericValues().ramBytesUsed(); + } + private static long sizeOfRelationalInsertTabletNode(final RelationalInsertTabletNode node) { return sizeOfRelationalInsertTabletNode(node, null); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index be0a00bddf829..99d40b3572b0a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -112,8 +112,8 @@ import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; import org.apache.iotdb.db.queryengine.plan.statement.crud.DeleteDataStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; -import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; @@ -2598,22 +2598,24 @@ public TSStatus insertRecord(TSInsertRecordReq req) { PathUtils.checkIsLegalSingleMeasurementsAndUpdate(req.getMeasurements())); } - InsertRowStatement statement = StatementGenerator.createStatement(req); + InsertBaseStatement statement = StatementGenerator.createStatement(req); // return success when this statement is empty because server doesn't need to execute it if (statement.isEmpty()) { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } - // permission check - TSStatus status = - AuthorityChecker.checkAuthority( - statement, - new TreeAccessCheckContext( - clientSession.getUserId(), - clientSession.getUsername(), - clientSession.getClientAddress())); - if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return status; + // Table-model privileges are checked during relational analysis. + if (!statement.isWriteToTable()) { + TSStatus status = + AuthorityChecker.checkAuthority( + statement, + new TreeAccessCheckContext( + clientSession.getUserId(), + clientSession.getUsername(), + clientSession.getClientAddress())); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return status; + } } quota = @@ -3544,7 +3546,7 @@ public TSStatus insertStringRecord(final TSInsertStringRecordReq req) { // check whether measurement is legal according to syntax convention req.setMeasurements(PathUtils.checkIsLegalSingleMeasurementsAndUpdate(req.getMeasurements())); - final InsertRowStatement statement = StatementGenerator.createStatement(req); + final InsertBaseStatement statement = StatementGenerator.createStatement(req); // Permission check final TSStatus status = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 34ef951296ce8..aa9ee3ab06ce1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -191,7 +191,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DeleteDevice; import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; -import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; import org.apache.iotdb.db.schemaengine.SchemaEngine; import org.apache.iotdb.db.schemaengine.lease.MetadataLeaseManager; @@ -3404,7 +3404,7 @@ public TSStatus stopAndClearDataNode() { @Override public TSStatus insertRecord(TSInsertRecordReq req) throws TException { try { - InsertRowStatement statement = StatementGenerator.createStatement(req); + InsertBaseStatement statement = StatementGenerator.createStatement(req); SessionInfo sessionInfo = new SessionInfo( 0, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java index c4c16d5af7e26..9089a9d0242c2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java @@ -218,6 +218,11 @@ public static List computeTableDataPartitionParams( } } return computeDataPartitionParams(timePartitionSlotMap, getDatabaseName(statement, context)); + } else if (statement instanceof InsertNumericRowStatement row) { + return computeDataPartitionParams( + Collections.singletonMap( + row.getTableDeviceID(), Collections.singleton(row.getTimePartitionSlot())), + getDatabaseName(statement, context)); } else if (statement instanceof InsertRowStatement) { final InsertRowStatement insertRowStatement = (InsertRowStatement) statement; return computeDataPartitionParams( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/DataNodePlanNodeDeserializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/DataNodePlanNodeDeserializer.java index 3e5e46a3c0c87..bbf66774d395d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/DataNodePlanNodeDeserializer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/DataNodePlanNodeDeserializer.java @@ -119,6 +119,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; @@ -164,6 +165,8 @@ public PlanNode deserializeFromWAL(DataInputStream stream) throws IOException { return ContinuousSameSearchIndexSeparatorNode.deserializeFromWAL(stream); case 2000: return RelationalInsertTabletNode.deserializeFromWAL(stream); + case 2005: + return RelationalInsertNumericRowNode.deserializeFromWAL(stream); case 2001: return RelationalInsertRowNode.deserializeFromWAL(stream); case 2002: @@ -193,6 +196,8 @@ public PlanNode deserializeFromWAL(ByteBuffer buffer) { return ContinuousSameSearchIndexSeparatorNode.deserializeFromWAL(buffer); case 2000: return RelationalInsertTabletNode.deserializeFromWAL(buffer); + case 2005: + return RelationalInsertNumericRowNode.deserializeFromWAL(buffer); case 2001: return RelationalInsertRowNode.deserializeFromWAL(buffer); case 2002: @@ -473,6 +478,8 @@ public PlanNode deserialize(ByteBuffer buffer, short nodeType) { return NonAlignedAggregationTreeDeviceViewScanNode.deserialize(buffer); case 2000: return RelationalInsertTabletNode.deserialize(buffer); + case 2005: + return RelationalInsertNumericRowNode.deserialize(buffer); case 2001: return RelationalInsertRowNode.deserialize(buffer); case 2002: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index d7a09a6b49d78..bb46eaf6ac156 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -33,12 +33,10 @@ import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; -import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.LastCacheUpdateSource; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunkGroup; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; -import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; import org.apache.iotdb.db.utils.TypeInferenceUtils; @@ -61,7 +59,7 @@ import java.util.List; import java.util.Objects; -public class InsertRowNode extends InsertNode implements WALEntryValue, LastCacheUpdateSource { +public class InsertRowNode extends AbstractInsertRowNode { private static final byte TYPE_RAW_STRING = -1; @@ -213,6 +211,26 @@ public Object[] getValues() { return values; } + @Override + public int getValueCount() { + return values == null ? 0 : values.length; + } + + @Override + public boolean isNull(int index) { + return index < 0 || index >= getValueCount() || values[index] == null; + } + + @Override + public Object getValue(int index) { + return isNull(index) ? null : values[index]; + } + + @Override + public Object getValuesForMemoryAccounting() { + return values; + } + public void setValues(Object[] values) { this.values = values; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index 86084ff2c6a1d..bc730e37b8963 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -231,6 +231,10 @@ default R visitSetSystemStatus(SetSystemStatus node, C context) { return visitStatement(node, context); } + default R visitInsertNumericRow(InsertNumericRow node, C context) { + return visitNode(node, context); + } + default R visitInsertRow(InsertRow node, C context) { return visitStatement(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/IoTDBInternalLocalReporter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/IoTDBInternalLocalReporter.java index f081a7b60fc6a..3d23a65a91f6f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/IoTDBInternalLocalReporter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/IoTDBInternalLocalReporter.java @@ -46,7 +46,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator; -import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateMultiTimeSeriesStatement; import org.apache.iotdb.metrics.config.MetricConfigDescriptor; import org.apache.iotdb.metrics.reporter.iotdb.IoTDBInternalReporter; @@ -211,7 +211,7 @@ private TSStatus insertRecord(Map valueMap, String prefix, long request.setValues(buffer); request.setIsAligned(false); - InsertRowStatement s = StatementGenerator.createStatement(request); + InsertBaseStatement s = StatementGenerator.createStatement(request); final long queryId = SESSION_MANAGER.requestQueryId(); ExecutionResult result = COORDINATOR.executeForTreeModel( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 6b14b43edf5ff..014ca384184d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -84,6 +84,7 @@ import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ContinuousSameSearchIndexSeparatorNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiTabletsNode; @@ -1174,7 +1175,7 @@ private int compareFileName(File o1, File o2) { * * @param insertRowNode one row of data */ - public void insert(InsertRowNode insertRowNode) throws WriteProcessException { + public void insert(AbstractInsertRowNode insertRowNode) throws WriteProcessException { // reject insertions that are out of ttl long ttl = getTTL(insertRowNode); if (!CommonUtils.isAlive(insertRowNode.getTime(), ttl)) { @@ -1800,7 +1801,7 @@ private void tryToUpdateInsertTabletLastCache( } private TsFileProcessor insertToTsFileProcessor( - InsertRowNode insertRowNode, boolean sequence, long timePartitionId) + AbstractInsertRowNode insertRowNode, boolean sequence, long timePartitionId) throws WriteProcessException { if (insertRowNode.allMeasurementFailed()) { return null; @@ -1819,7 +1820,7 @@ private TsFileProcessor insertToTsFileProcessor( return tsFileProcessor; } - private void tryToUpdateInsertRowLastCache(final InsertRowNode node) { + private void tryToUpdateInsertRowLastCache(final AbstractInsertRowNode node) { node.updateLastCache(databaseName); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index 37bd7d3c870cc..6800feb7a45cf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -75,6 +75,8 @@ public class AlignedWritableMemChunk extends AbstractWritableMemChunk { private List sortedList; private long sortedRowCount = 0; private final boolean ignoreAllNullRows; + // Allocate reusable scratch space only when the numeric row path is first used. + private int[] numericColumnMapping; private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); private final int TVLIST_SORT_THRESHOLD = CONFIG.getTvListSortThreshold(); @@ -241,6 +243,38 @@ public void writeAlignedPoints( } } + /** Reorders primitive slots directly, including null placeholders for absent columns. */ + public void writeAlignedNumericPoints( + long time, NumericRowValues row, List incomingSchemas) { + int[] mapping = numericColumnMapping; + // Other write paths may have extended the schema since the last numeric row. + if (mapping == null || mapping.length < schemaList.size()) { + mapping = new int[schemaList.size()]; + } + // Missing columns must not reuse source indices from the previous row. + Arrays.fill(mapping, -1); + for (int i = 0; i < incomingSchemas.size(); i++) { + IMeasurementSchema schema = incomingSchemas.get(i); + if (schema == null) { + continue; + } + Integer index = measurementIndexMap.get(schema.getMeasurementName()); + if (index == null) { + index = list.getTsDataTypes().size(); + measurementIndexMap.put(schema.getMeasurementName(), index); + schemaList.add(schema); + list.extendColumn(schema.getType()); + mapping = Arrays.copyOf(mapping, mapping.length + 1); + } + mapping[index] = i; + } + numericColumnMapping = mapping; + list.putAlignedNumericValue(time, row, mapping); + if (TVLIST_SORT_THRESHOLD > 0 && list.rowCount() >= TVLIST_SORT_THRESHOLD) { + handoverAlignedTvList(); + } + } + @Override public void writeAlignedTablet( long[] times, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java index 4d38fea2c1124..90952af6c7395 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; @@ -106,7 +107,7 @@ void writeAlignedRow( */ int insert(InsertRowNode insertRowNode); - int insertAlignedRow(InsertRowNode insertRowNode); + int insertAlignedRow(AbstractInsertRowNode insertRowNode); /** * insert tablet into this memtable. The rows to be inserted are in the range [start, end). Null diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DataNodeThrottleQuotaManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DataNodeThrottleQuotaManager.java index db7ad470930ea..817a46f4259f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DataNodeThrottleQuotaManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DataNodeThrottleQuotaManager.java @@ -81,6 +81,7 @@ public OperationQuota checkQuota(String userName, Statement s) throws RpcThrottl return NoopOperationQuota.get(); } switch (s.getType()) { + case INSERT_NUMERIC_ROW: case INSERT: case BATCH_INSERT: case BATCH_INSERT_ONE_DEVICE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DefaultOperationQuota.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DefaultOperationQuota.java index 764d6763184ac..b10668d3357d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DefaultOperationQuota.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DefaultOperationQuota.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; @@ -33,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement; import org.apache.iotdb.db.utils.TypeInferenceUtils; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.BitMap; @@ -101,6 +103,21 @@ protected void updateEstimateConsumeQuota(int numWrites, int numReads, Statement } final StatementType statementType = s.getType(); switch (statementType) { + case INSERT_NUMERIC_ROW: + InsertNumericRowStatement numeric = (InsertNumericRowStatement) s; + for (int i = 0; i < numeric.getValueCount(); i++) { + if (!numeric.getNumericValues().isNull(i)) { + avgSize += + numeric.getDataType(i) == TSDataType.STRING + ? numeric + .getNumericValues() + .getTag(i) + .getBytes(TSFileConfig.STRING_CHARSET) + .length + : numeric.getDataType(i).getDataTypeSize(); + } + } + break; case INSERT: // InsertStatement InsertRowStatement if (s instanceof InsertStatement) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java index 0b27de77b123a..df4a12881be1e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusLogToTabletConverter.java @@ -28,6 +28,7 @@ import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest; import org.apache.iotdb.consensus.common.request.IoTConsensusRequest; import org.apache.iotdb.db.i18n.DataNodePipeMessages; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiTabletsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; @@ -222,8 +223,9 @@ public List convert(final InsertNode insertNode) { return convertInsertRowsOfOneDeviceNode((InsertRowsOfOneDeviceNode) insertNode); case INSERT_MULTI_TABLET: return convertInsertMultiTabletsNode((InsertMultiTabletsNode) insertNode); + case RELATIONAL_INSERT_NUMERIC_ROW: case RELATIONAL_INSERT_ROW: - return convertRelationalInsertRowNode((RelationalInsertRowNode) insertNode); + return convertRelationalInsertRowNode((AbstractInsertRowNode) insertNode); case RELATIONAL_INSERT_TABLET: return convertRelationalInsertTabletNode((RelationalInsertTabletNode) insertNode); case RELATIONAL_INSERT_ROWS: @@ -467,8 +469,8 @@ private List convertInsertMultiTabletsNode(final InsertMultiTabletsNode // ======================== Table Model Conversion ======================== - private List convertRelationalInsertRowNode(final RelationalInsertRowNode node) { - final String tableName = node.getTableName(); + private List convertRelationalInsertRowNode(final AbstractInsertRowNode node) { + final String tableName = node.getTargetPath().getFullPath(); // Table-level pattern filtering if (tablePattern != null) { @@ -480,13 +482,7 @@ private List convertRelationalInsertRowNode(final RelationalInsertRowNod } } - final long time = node.getTime(); - final String[] measurements = node.getMeasurements(); - final TSDataType[] dataTypes = node.getDataTypes(); - final Object[] values = node.getValues(); - final Tablet tablet = - buildTableModelTabletFromRow( - tableName, time, measurements, dataTypes, values, node.getColumnCategories()); + final Tablet tablet = buildTableModelTabletFromRow(node); if (Objects.isNull(tablet)) { return Collections.emptyList(); } @@ -499,7 +495,7 @@ private List convertRelationalInsertRowNode(final RelationalInsertRowNod } private List convertRelationalInsertTabletNode(final RelationalInsertTabletNode node) { - final String tableName = node.getTableName(); + final String tableName = node.getTargetPath().getFullPath(); // Table-level pattern filtering if (tablePattern != null) { @@ -578,14 +574,12 @@ private List convertRelationalInsertRowsNode(final RelationalInsertRowsN return tablets; } - private Tablet buildTableModelTabletFromRow( - final String tableName, - final long time, - final String[] measurements, - final TSDataType[] dataTypes, - final Object[] values, - final TsTableColumnCategory[] columnCategories) { - if (Objects.isNull(measurements) || Objects.isNull(dataTypes) || Objects.isNull(values)) { + private Tablet buildTableModelTabletFromRow(final AbstractInsertRowNode node) { + final String tableName = node.getTargetPath().getFullPath(); + final String[] measurements = node.getMeasurements(); + final TSDataType[] dataTypes = node.getDataTypes(); + final TsTableColumnCategory[] columnCategories = node.getColumnCategories(); + if (Objects.isNull(measurements) || Objects.isNull(dataTypes)) { return null; } @@ -593,7 +587,9 @@ private Tablet buildTableModelTabletFromRow( final List columnDataTypes = new ArrayList<>(measurements.length); final List columnTypes = new ArrayList<>(measurements.length); final List originalColumnIndexes = new ArrayList<>(measurements.length); - for (int i = 0; i < measurements.length && i < dataTypes.length && i < values.length; i++) { + for (int i = 0; + i < measurements.length && i < dataTypes.length && i < node.getValueCount(); + i++) { if (Objects.isNull(measurements[i]) || Objects.isNull(dataTypes[i])) { continue; } @@ -613,10 +609,11 @@ private Tablet buildTableModelTabletFromRow( columnDataTypes, columnTypes, 1); - tablet.addTimestamp(0, time); + tablet.addTimestamp(0, node.getTime()); for (int i = 0; i < originalColumnIndexes.size(); i++) { final int originalColumnIndex = originalColumnIndexes.get(i); - final Object value = values[originalColumnIndex]; + // Convert one column at a time, keeping numeric rows in their primitive representation. + final Object value = node.getValue(originalColumnIndex); if (Objects.isNull(value)) { if (Objects.isNull(tablet.getBitMaps())) { tablet.initBitMaps(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java index c6fd52301a428..db1a5364e0f7c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiTabletsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; @@ -3575,8 +3576,8 @@ private void mergeTailProgress( */ private long extractMaxTime(final InsertNode insertNode) { try { - if (insertNode instanceof InsertRowNode) { - return ((InsertRowNode) insertNode).getTime(); + if (insertNode instanceof AbstractInsertRowNode row) { + return row.getTime(); } if (insertNode instanceof InsertTabletNode) { final InsertTabletNode tabletNode = (InsertTabletNode) insertNode; diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift index 7ff12ceb02154..792a42dde5150 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift @@ -248,6 +248,8 @@ struct TSInsertRecordReq { 6: optional bool isAligned 7: optional bool isWriteToTable 8: optional list columnCategoryies + // Numeric table rows retain primitive storage throughout the server write path. + 9: optional bool isNumericRow } struct TSInsertStringRecordReq { From eed7ef0dba99bf2ec390be12b1ff53ca193db2fb Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Fri, 11 Sep 2026 12:30:16 +0800 Subject: [PATCH 03/20] tmp --- .../IoTDBNumericRowInsertPerformanceIT.java | 456 ++++++++++++++++++ .../it/session/IoTDBTableModelSessionIT.java | 75 ++- .../apache/iotdb/isession/ITableSession.java | 16 +- .../org/apache/iotdb/isession/NumericRow.java | 97 ++++ .../iotdb/session/i18n/SessionMessages.java | 4 + .../iotdb/session/i18n/SessionMessages.java | 4 + .../org/apache/iotdb/session/Session.java | 51 ++ .../apache/iotdb/session/TableSession.java | 6 + .../session/pool/TableSessionWrapper.java | 13 + .../iotdb/session/util/SessionUtils.java | 74 +++ .../org/apache/iotdb/session/SessionTest.java | 80 +++ .../iotdb/session/util/SessionUtilsTest.java | 151 ++++++ .../PipeInsertNodeTabletInsertionEvent.java | 12 +- .../parser/TabletInsertionEventParser.java | 19 +- ...abletInsertionEventTablePatternParser.java | 5 +- .../visitor/PipePlanToStatementVisitor.java | 15 + .../request/PipeTransferTabletBatchReqV2.java | 10 +- .../request/PipeTransferTabletBinaryReq.java | 4 +- .../PipeTransferTabletBinaryReqV2.java | 4 +- .../PipeTransferTabletInsertNodeReq.java | 4 +- .../PipeTransferTabletInsertNodeReqV2.java | 4 +- .../executor/RegionWriteExecutor.java | 7 + .../plan/parser/StatementGenerator.java | 46 +- .../plan/planner/plan/node/PlanVisitor.java | 5 + .../node/write/AbstractInsertRowNode.java | 64 +++ .../analyzer/StatementAnalyzer.java | 6 + .../relational/planner/RelationPlanner.java | 23 + .../relational/planner/TableModelPlanner.java | 14 + .../relational/sql/ast/InsertNumericRow.java | 71 +++ .../plan/statement/StatementType.java | 1 + .../plan/statement/StatementVisitor.java | 5 + .../crud/InsertNumericRowStatement.java | 292 +++++++++++ .../iotdb/db/storageengine/StorageEngine.java | 4 +- .../dataregion/memtable/TsFileProcessor.java | 55 ++- .../dataregion/wal/buffer/WALEntry.java | 6 +- .../dataregion/wal/buffer/WALInfoEntry.java | 4 +- .../dataregion/wal/node/IWALNode.java | 4 +- .../dataregion/wal/node/WALFakeNode.java | 4 +- .../dataregion/wal/node/WALNode.java | 4 +- .../wal/recover/file/TsFilePlanRedoer.java | 9 +- .../plan/parser/StatementGeneratorTest.java | 2 +- .../AlignedWritableMemChunkNumericTest.java | 134 +++++ .../plan/planner/plan/node/PlanNodeType.java | 1 + 43 files changed, 1767 insertions(+), 98 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/performance/IoTDBNumericRowInsertPerformanceIT.java create mode 100644 iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRow.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/AbstractInsertRowNode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertNumericRow.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkNumericTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/performance/IoTDBNumericRowInsertPerformanceIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/performance/IoTDBNumericRowInsertPerformanceIT.java new file mode 100644 index 0000000000000..84a52da145189 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/performance/IoTDBNumericRowInsertPerformanceIT.java @@ -0,0 +1,456 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.performance; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.NumericRow; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Properties; + +/** + * Manual end-to-end benchmark for the numeric table-row API and relational {@link Tablet} writes. + * + *

Enable this test with {@code -Diotdb.numeric.row.insert.perf.enabled=true}. The timed region + * includes the Session call and server write confirmation. Schema creation and client-side input + * preparation are outside the timed region. By default both APIs write one row per request; set + * {@code iotdb.numeric.row.insert.perf.tablet.row.count} to compare against batched Tablet writes. + * + *

Results are written to {@code target/numeric-row-insert-performance.properties}. Percentiles + * describe elapsed time per complete round, not individual RPC latency. See the adjacent README for + * reproducible commands and interpretation. + */ +@RunWith(IoTDBTestRunner.class) +@Category(TableLocalStandaloneIT.class) +public class IoTDBNumericRowInsertPerformanceIT { + + private static final String PROPERTY_PREFIX = "iotdb.numeric.row.insert.perf."; + private static final String ENABLED_PROPERTY = PROPERTY_PREFIX + "enabled"; + private static final String ROW_COUNT_PROPERTY = PROPERTY_PREFIX + "row.count"; + private static final String COLUMN_COUNT_PROPERTY = PROPERTY_PREFIX + "column.count"; + private static final String TABLET_ROW_COUNT_PROPERTY = PROPERTY_PREFIX + "tablet.row.count"; + private static final String WARMUP_ROUND_COUNT_PROPERTY = PROPERTY_PREFIX + "warmup.round.count"; + private static final String ROUND_COUNT_PROPERTY = PROPERTY_PREFIX + "round.count"; + private static final String RESULT_PATH_PROPERTY = PROPERTY_PREFIX + "result.path"; + + private static final int ROW_COUNT = Integer.getInteger(ROW_COUNT_PROPERTY, 10_000); + private static final int COLUMN_COUNT = Integer.getInteger(COLUMN_COUNT_PROPERTY, 8); + private static final int TABLET_ROW_COUNT = Integer.getInteger(TABLET_ROW_COUNT_PROPERTY, 1); + private static final int WARMUP_ROUND_COUNT = Integer.getInteger(WARMUP_ROUND_COUNT_PROPERTY, 1); + private static final int ROUND_COUNT = Integer.getInteger(ROUND_COUNT_PROPERTY, 5); + + private static final String DATABASE = "numeric_row_insert_performance"; + private static final List TYPE_SEQUENCE = + Arrays.asList( + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE); + private static List columnNames; + private static List dataTypes; + private static List columnCategories; + + private static boolean environmentInitialized; + + @BeforeClass + public static void setUp() throws Exception { + Assume.assumeTrue(Boolean.getBoolean(ENABLED_PROPERTY)); + // Validate before allocating inputs or starting nodes, including when a count is negative. + assertValidConfiguration(); + columnNames = createColumnNames(); + dataTypes = createDataTypes(); + columnCategories = Collections.nCopies(COLUMN_COUNT, ColumnCategory.FIELD); + environmentInitialized = true; + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + if (environmentInitialized) { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + } + + @Test + // Compare identical, prebuilt numeric data; verify every field count outside the timed calls. + public void testNumericRowInsertPerformanceAgainstTablet() throws Exception { + final long[] numericRowElapsedNanos = new long[ROUND_COUNT]; + final long[] tabletElapsedNanos = new long[ROUND_COUNT]; + + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE " + DATABASE); + session.executeNonQueryStatement("USE " + DATABASE); + + final int totalRoundCount = WARMUP_ROUND_COUNT + ROUND_COUNT; + for (int round = 0; round < totalRoundCount; round++) { + final NumericRowWorkload numericRowWorkload = prepareNumericRowWorkload(session, round); + final TabletWorkload tabletWorkload = prepareTabletWorkload(session, round); + + final long numericRowNanos; + final long tabletNanos; + // Alternate execution order so one path does not consistently receive a warmer server. + if ((round & 1) == 0) { + numericRowNanos = executeNumericRowWorkload(session, numericRowWorkload); + tabletNanos = executeTabletWorkload(session, tabletWorkload); + } else { + tabletNanos = executeTabletWorkload(session, tabletWorkload); + numericRowNanos = executeNumericRowWorkload(session, numericRowWorkload); + } + assertRowCount(session, numericRowWorkload.tableName); + assertRowCount(session, tabletWorkload.tableName); + + if (round >= WARMUP_ROUND_COUNT) { + final int measuredRound = round - WARMUP_ROUND_COUNT; + numericRowElapsedNanos[measuredRound] = numericRowNanos; + tabletElapsedNanos[measuredRound] = tabletNanos; + } + } + } + + final BenchmarkResult result = new BenchmarkResult(numericRowElapsedNanos, tabletElapsedNanos); + writeResult(result); + } + + private static NumericRowWorkload prepareNumericRowWorkload(ITableSession session, int round) + throws Exception { + final String tableName = "numeric_row_" + round; + createTable(session, tableName); + final long startTimestamp = (long) round * ROW_COUNT; + final NumericRow[] rows = new NumericRow[ROW_COUNT]; + final String[] tagValues = new String[0]; + for (int row = 0; row < ROW_COUNT; row++) { + rows[row] = + new NumericRow( + tableName, + startTimestamp + row, + columnNames, + dataTypes, + columnCategories, + createRawValues(startTimestamp + row), + tagValues); + } + return new NumericRowWorkload(tableName, rows); + } + + private static TabletWorkload prepareTabletWorkload(ITableSession session, int round) + throws Exception { + final String tableName = "tablet_" + round; + createTable(session, tableName); + final long startTimestamp = (long) round * ROW_COUNT; + final List tablets = new ArrayList<>((ROW_COUNT - 1) / TABLET_ROW_COUNT + 1); + for (int rowStart = 0; rowStart < ROW_COUNT; ) { + final int rowCount = Math.min(TABLET_ROW_COUNT, ROW_COUNT - rowStart); + final Tablet tablet = + new Tablet(tableName, columnNames, dataTypes, columnCategories, rowCount); + for (int row = 0; row < rowCount; row++) { + final int rowIndex = tablet.getRowSize(); + final long timestamp = startTimestamp + rowStart + row; + final long[] values = createRawValues(timestamp); + tablet.addTimestamp(rowIndex, timestamp); + for (int column = 0; column < COLUMN_COUNT; column++) { + tablet.addValue( + columnNames.get(column), + rowIndex, + toTabletValue(dataTypes.get(column), values[column])); + } + } + tablets.add(tablet); + rowStart += rowCount; + } + return new TabletWorkload(tableName, tablets); + } + + private static long executeNumericRowWorkload( + ITableSession session, NumericRowWorkload numericRowWorkload) throws Exception { + final long startNanos = System.nanoTime(); + for (final NumericRow row : numericRowWorkload.rows) { + session.insert(row); + } + return System.nanoTime() - startNanos; + } + + private static long executeTabletWorkload(ITableSession session, TabletWorkload tabletWorkload) + throws Exception { + final long startNanos = System.nanoTime(); + for (final Tablet tablet : tabletWorkload.tablets) { + session.insert(tablet); + } + return System.nanoTime() - startNanos; + } + + private static void createTable(ITableSession session, String tableName) throws Exception { + final StringBuilder statement = new StringBuilder("CREATE TABLE "); + statement.append(tableName).append('('); + for (int column = 0; column < COLUMN_COUNT; column++) { + if (column > 0) { + statement.append(','); + } + statement + .append(columnNames.get(column)) + .append(' ') + .append(dataTypes.get(column)) + .append(" FIELD"); + } + statement.append(')'); + session.executeNonQueryStatement(statement.toString()); + } + + private static void assertRowCount(ITableSession session, String tableName) throws Exception { + final List counts = new ArrayList<>(COLUMN_COUNT); + for (String columnName : columnNames) { + counts.add("COUNT(" + columnName + ")"); + } + try (SessionDataSet dataSet = + session.executeQueryStatement( + "SELECT " + String.join(",", counts) + " FROM " + tableName)) { + Assert.assertTrue(dataSet.hasNext()); + for (var field : dataSet.next().getFields()) { + Assert.assertEquals(ROW_COUNT, field.getLongV()); + } + Assert.assertFalse(dataSet.hasNext()); + } + } + + private static long[] createRawValues(long timestamp) { + final long[] values = new long[COLUMN_COUNT]; + for (int column = 0; column < COLUMN_COUNT; column++) { + final TSDataType type = dataTypes.get(column); + // Floating-point values occupy raw IEEE 754 bits in the numeric-row API. + values[column] = + switch (type) { + case BOOLEAN -> (timestamp + column) & 1; + case INT32 -> (int) (timestamp + column); + case INT64 -> timestamp + column; + case FLOAT -> Float.floatToRawIntBits((float) timestamp + column / 10.0F); + case DOUBLE -> Double.doubleToRawLongBits(timestamp + column / 10.0D); + default -> throw new AssertionError(type); + }; + } + return values; + } + + private static Object toTabletValue(TSDataType type, long value) { + return switch (type) { + case BOOLEAN -> value != 0; + case INT32 -> (int) value; + case INT64 -> value; + case FLOAT -> Float.intBitsToFloat((int) value); + case DOUBLE -> Double.longBitsToDouble(value); + default -> throw new AssertionError(type); + }; + } + + private static List createColumnNames() { + final List columnNames = new ArrayList<>(COLUMN_COUNT); + for (int column = 0; column < COLUMN_COUNT; column++) { + columnNames.add("v" + column); + } + return Collections.unmodifiableList(columnNames); + } + + private static List createDataTypes() { + final List dataTypes = new ArrayList<>(COLUMN_COUNT); + for (int column = 0; column < COLUMN_COUNT; column++) { + dataTypes.add(TYPE_SEQUENCE.get(column % TYPE_SEQUENCE.size())); + } + return Collections.unmodifiableList(dataTypes); + } + + private static void assertValidConfiguration() { + Assert.assertTrue(ROW_COUNT > 0); + Assert.assertTrue(COLUMN_COUNT > 0); + Assert.assertTrue(TABLET_ROW_COUNT > 0); + Assert.assertTrue(WARMUP_ROUND_COUNT >= 0); + Assert.assertTrue(ROUND_COUNT > 0); + Assert.assertTrue((long) WARMUP_ROUND_COUNT + ROUND_COUNT <= Integer.MAX_VALUE); + } + + private static void writeResult(BenchmarkResult result) throws IOException { + final Properties properties = new Properties(); + properties.setProperty("format.version", "2"); + properties.setProperty("java.version", System.getProperty("java.version")); + properties.setProperty("os.name", System.getProperty("os.name")); + properties.setProperty("os.arch", System.getProperty("os.arch")); + properties.setProperty( + "client.available.processors", + Integer.toString(Runtime.getRuntime().availableProcessors())); + properties.setProperty( + "client.max.heap.bytes", Long.toString(Runtime.getRuntime().maxMemory())); + properties.setProperty("client.count", "1"); + properties.setProperty("row.count", Integer.toString(ROW_COUNT)); + properties.setProperty("column.count", Integer.toString(COLUMN_COUNT)); + properties.setProperty("tablet.row.count", Integer.toString(TABLET_ROW_COUNT)); + properties.setProperty("warmup.round.count", Integer.toString(WARMUP_ROUND_COUNT)); + properties.setProperty("round.count", Integer.toString(ROUND_COUNT)); + properties.setProperty("column.types", join(dataTypes)); + properties.setProperty("numeric.row.requests.per.round", Integer.toString(ROW_COUNT)); + properties.setProperty( + "insert.tablet.requests.per.round", + Integer.toString((ROW_COUNT - 1) / TABLET_ROW_COUNT + 1)); + properties.setProperty("numeric.row.round.elapsed.nanos", join(result.numericRowElapsedNanos)); + properties.setProperty("insert.tablet.round.elapsed.nanos", join(result.tabletElapsedNanos)); + properties.setProperty( + "numeric.row.round.p50.nanos", Long.toString(result.numericRowP50Nanos())); + properties.setProperty( + "numeric.row.round.p95.nanos", Long.toString(result.numericRowP95Nanos())); + properties.setProperty("numeric.row.rows.per.second", format(result.numericRowRowsPerSecond())); + properties.setProperty( + "numeric.row.values.per.second", format(result.numericRowValuesPerSecond())); + properties.setProperty("insert.tablet.round.p50.nanos", Long.toString(result.tabletP50Nanos())); + properties.setProperty("insert.tablet.round.p95.nanos", Long.toString(result.tabletP95Nanos())); + properties.setProperty("insert.tablet.rows.per.second", format(result.tabletRowsPerSecond())); + properties.setProperty( + "insert.tablet.values.per.second", format(result.tabletValuesPerSecond())); + + final Path resultPath = + Paths.get( + System.getProperty( + RESULT_PATH_PROPERTY, "target/numeric-row-insert-performance.properties")); + final Path parent = resultPath.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + try (OutputStream outputStream = Files.newOutputStream(resultPath)) { + properties.store(outputStream, null); + } + } + + private static String join(List values) { + final StringBuilder builder = new StringBuilder(); + for (int index = 0; index < values.size(); index++) { + if (index > 0) { + builder.append(','); + } + builder.append(values.get(index)); + } + return builder.toString(); + } + + private static String join(long[] values) { + final StringBuilder builder = new StringBuilder(); + for (int index = 0; index < values.length; index++) { + if (index > 0) { + builder.append(','); + } + builder.append(values[index]); + } + return builder.toString(); + } + + private static String format(double value) { + return String.format(Locale.ROOT, "%.2f", value); + } + + private static long percentile(long[] latencies, double percentile) { + final long[] sorted = latencies.clone(); + Arrays.sort(sorted); + return sorted[(int) Math.ceil(percentile * sorted.length) - 1]; + } + + private static final class NumericRowWorkload { + + private final String tableName; + private final NumericRow[] rows; + + private NumericRowWorkload(String tableName, NumericRow[] rows) { + this.tableName = tableName; + this.rows = rows; + } + } + + private static final class TabletWorkload { + + private final String tableName; + private final List tablets; + + private TabletWorkload(String tableName, List tablets) { + this.tableName = tableName; + this.tablets = tablets; + } + } + + private static final class BenchmarkResult { + + private final long[] numericRowElapsedNanos; + private final long[] tabletElapsedNanos; + + private BenchmarkResult(long[] numericRowElapsedNanos, long[] tabletElapsedNanos) { + this.numericRowElapsedNanos = numericRowElapsedNanos; + this.tabletElapsedNanos = tabletElapsedNanos; + } + + private long numericRowP50Nanos() { + return percentile(numericRowElapsedNanos, 0.5D); + } + + private long numericRowP95Nanos() { + return percentile(numericRowElapsedNanos, 0.95D); + } + + private double numericRowRowsPerSecond() { + return ROW_COUNT * 1_000_000_000.0D / numericRowP50Nanos(); + } + + private double numericRowValuesPerSecond() { + return numericRowRowsPerSecond() * COLUMN_COUNT; + } + + private long tabletP50Nanos() { + return percentile(tabletElapsedNanos, 0.5D); + } + + private long tabletP95Nanos() { + return percentile(tabletElapsedNanos, 0.95D); + } + + private double tabletRowsPerSecond() { + return ROW_COUNT * 1_000_000_000.0D / tabletP50Nanos(); + } + + private double tabletValuesPerSecond() { + return tabletRowsPerSecond() * COLUMN_COUNT; + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java index 48a5a994dd5cb..f9c7ca00444be 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java @@ -20,6 +20,7 @@ package org.apache.iotdb.relational.it.session; import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.NumericRow; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.isession.pool.ITableSessionPool; import org.apache.iotdb.it.env.EnvFactory; @@ -66,7 +67,7 @@ public void tearDown() throws Exception { @Test public void testNumericRowInsert() throws Exception { - // Exercise both table-session entry points, schema reordering and missing STRING columns. + // Exercise direct and pooled table sessions, schema reordering and missing STRING columns. // The same values must remain readable from MemTable and after flushing to TsFile. try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { session.executeNonQueryStatement("CREATE DATABASE numeric_rows"); @@ -94,16 +95,26 @@ public void testNumericRowInsert() throws Exception { 20260910, 1234 }; - session.insert("readings", 1, names, types, values); + session.insert( + new NumericRow( + "readings", + 1, + names, + types, + Collections.nCopies(types.size(), ColumnCategory.FIELD), + values, + new String[0])); try (ITableSessionPool pool = EnvFactory.getEnv().getTableSessionPool(1, "numeric_rows"); ITableSession pooledSession = pool.getSession()) { pooledSession.insert( - "readings", - 2, - names, - types, - Collections.nCopies(types.size(), ColumnCategory.FIELD), - values); + new NumericRow( + "readings", + 2, + names, + types, + Collections.nCopies(types.size(), ColumnCategory.FIELD), + values, + new String[0])); } assertNumericRows(session); session.executeNonQueryStatement("FLUSH"); @@ -127,28 +138,44 @@ public void testNumericRowInsertWithTags() throws Exception { Arrays.asList( ColumnCategory.FIELD, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.TAG); session.insert( - "readings", - 1, - names, - types, - categories, - new long[] {1, 10}, - new String[] {"设备甲😀", "华北"}); + new NumericRow( + "readings", + 1, + names, + types, + categories, + new long[] {1, 10}, + new String[] {"设备甲😀", "华北"})); try (ITableSessionPool pool = EnvFactory.getEnv().getTableSessionPool(1, "numeric_tag_rows"); ITableSession pooled = pool.getSession()) { pooled.insert( - "readings", - 1, - names, - types, - categories, - new long[] {0, 20}, - new String[] {"设备乙", "华北"}); + new NumericRow( + "readings", + 1, + names, + types, + categories, + new long[] {0, 20}, + new String[] {"设备乙", "华北"})); } session.insert( - "readings", 2, names, types, categories, new long[] {1, 30}, new String[] {"", null}); + new NumericRow( + "readings", + 2, + names, + types, + categories, + new long[] {1, 30}, + new String[] {"", null})); session.insert( - "readings", 2, names, types, categories, new long[] {0, 40}, new String[] {null, null}); + new NumericRow( + "readings", + 2, + names, + types, + categories, + new long[] {0, 40}, + new String[] {null, null})); assertNumericTagRows(session); session.executeNonQueryStatement("FLUSH"); assertNumericTagRows(session); diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java index b5868e60ecd3d..af524f9a853fa 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java @@ -22,12 +22,8 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.tsfile.enums.ColumnCategory; -import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.write.record.Tablet; -import java.util.List; - /** * This interface defines a session for interacting with IoTDB tables. It supports operations such * as data insertion, executing queries, and closing the session. Implementations of this interface @@ -50,6 +46,18 @@ public interface ITableSession extends AutoCloseable { */ void insert(Tablet tablet) throws StatementExecutionException, IoTDBConnectionException; + /** + * Inserts one table-model {@link NumericRow} with numeric FIELDs and optional STRING TAGs. + * + *

This API requires a server supporting numeric row requests. The row's metadata and value + * arrays must not be modified until this call returns. + * + * @param row the numeric row to insert + * @throws StatementExecutionException if an error occurs while executing the statement + * @throws IoTDBConnectionException if there is an issue with the IoTDB connection + */ + void insert(NumericRow row) throws StatementExecutionException, IoTDBConnectionException; + /** * Executes a non-query SQL statement, such as a DDL or DML command. * diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRow.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRow.java new file mode 100644 index 0000000000000..502c75e05e4c9 --- /dev/null +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRow.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.isession; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; + +import java.util.List; + +/** + * A table-model row with numeric FIELDs in {@code long[]} and STRING TAGs in {@code String[]}. + * + *

Column names, types and categories describe all supplied columns in the same order. The value + * arrays contain only their respective FIELD or TAG values, in occurrence order in the column list. + * TAGs may be null or empty; missing TAG columns are null. ATTRIBUTE columns, nonnumeric FIELDs and + * null FIELD values are not supported. + * + *

BOOLEAN uses zero for false and nonzero for true. INT32 and DATE use the low 32 bits (DATE is + * encoded as yyyyMMdd); INT64 and TIMESTAMP use the long value. FLOAT uses {@link + * Float#floatToRawIntBits(float)} in the low 32 bits and DOUBLE uses {@link + * Double#doubleToRawLongBits(double)}. + * + *

Metadata and arrays are retained without copying or boxing. Do not modify them while {@link + * ITableSession#insert(NumericRow)} is in progress. Arguments are validated when inserting the row. + */ +public final class NumericRow { + private final String tableName; + private final long timestamp; + private final List columnNames; + private final List dataTypes; + private final List columnCategories; + private final long[] values; + private final String[] tagValues; + + /** Creates a row with numeric FIELDs and optional STRING TAGs in separate arrays. */ + public NumericRow( + String tableName, + long timestamp, + List columnNames, + List dataTypes, + List columnCategories, + long[] values, + String[] tagValues) { + this.tableName = tableName; + this.timestamp = timestamp; + this.columnNames = columnNames; + this.dataTypes = dataTypes; + this.columnCategories = columnCategories; + this.values = values; + this.tagValues = tagValues; + } + + public String getTableName() { + return tableName; + } + + public long getTimestamp() { + return timestamp; + } + + public List getColumnNames() { + return columnNames; + } + + public List getDataTypes() { + return dataTypes; + } + + public List getColumnCategories() { + return columnCategories; + } + + public long[] getValues() { + return values; + } + + public String[] getTagValues() { + return tagValues; + } +} diff --git a/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java b/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java index 69b87b2db1653..3eb19971a9245 100644 --- a/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java +++ b/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java @@ -20,6 +20,8 @@ package org.apache.iotdb.session.i18n; public final class SessionMessages { + public static final String EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233 = "Numeric row arguments must be non-null; column metadata lengths and per-category value counts must match."; + public static final String EXCEPTION_NUMERIC_ROWS_SUPPORT_NUMERIC_FIELD_COLUMNS_AND_STRING_TAG_COLUMNS_ONLY_87159FFA = "Numeric rows support numeric FIELD columns and STRING TAG columns only."; // Session public static final String NODE_URLS_EMPTY = "nodeUrls shouldn't be empty."; @@ -286,4 +288,6 @@ private SessionMessages() {} public static final String EXCEPTION_RETRY_EXECUTE_STATEMENT_ARG_FAILED_ARG_TIMES_ARG_216C6873 = "retry to execute statement on %s failed %d times: %s"; public static final String EXCEPTION_SESSIONPOOL_DOESN_T_SUPPORT_EXECUTING_ARG_DIRECTLY_B778F701 = "SessionPool doesn't support executing %s directly"; + public static final String EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_AND_COLUMN_NAMES_TYPES_CATEGORIES_AND_VALUES_MUST_HAVE_EQUAL_LENGTHS_AB102B01 = "Numeric row arguments must be non-null and column names, types, categories and values must have equal lengths."; + public static final String EXCEPTION_NUMERIC_ROWS_SUPPORT_ONLY_FIELD_COLUMNS_DA7A8EE7 = "Numeric rows support only FIELD columns."; } diff --git a/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java b/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java index 2f28590ac8840..22f3bea7821bd 100644 --- a/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java +++ b/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java @@ -20,6 +20,8 @@ package org.apache.iotdb.session.i18n; public final class SessionMessages { + public static final String EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233 = "数值行参数不能为 null；列元数据长度必须一致，FIELD 和 TAG 的数量必须分别与对应的值数组长度一致。"; + public static final String EXCEPTION_NUMERIC_ROWS_SUPPORT_NUMERIC_FIELD_COLUMNS_AND_STRING_TAG_COLUMNS_ONLY_87159FFA = "数值行仅支持数值类型的 FIELD 列和 STRING 类型的 TAG 列。"; // Session public static final String NODE_URLS_EMPTY = "nodeUrls 不能为空。"; @@ -284,4 +286,6 @@ private SessionMessages() {} public static final String EXCEPTION_RETRY_EXECUTE_STATEMENT_ARG_FAILED_ARG_TIMES_ARG_216C6873 = "在 %s 上重试执行语句失败 %d 次：%s"; public static final String EXCEPTION_SESSIONPOOL_DOESN_T_SUPPORT_EXECUTING_ARG_DIRECTLY_B778F701 = "SessionPool 不支持直接执行 %s"; + public static final String EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_AND_COLUMN_NAMES_TYPES_CATEGORIES_AND_VALUES_MUST_HAVE_EQUAL_LENGTHS_AB102B01 = "数值行参数不能为 null，列名、类型、类别和值的数量必须相同。"; + public static final String EXCEPTION_NUMERIC_ROWS_SUPPORT_ONLY_FIELD_COLUMNS_DA7A8EE7 = "数值行仅支持 FIELD 列。"; } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index 39bb9c06cba5f..716d14fa80296 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.isession.INodeSupplier; import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.NumericRow; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.isession.template.Template; @@ -66,6 +67,7 @@ import org.apache.thrift.TException; import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; @@ -2873,6 +2875,55 @@ public void insertRelationalTablet(Tablet tablet) } } + /** Inserts numeric FIELDs and STRING TAGs without constructing a boxed row. */ + public void insertRelationalRecord(NumericRow row) + throws IoTDBConnectionException, StatementExecutionException { + if (row == null || row.getTableName() == null) { + throw new IllegalArgumentException( + SessionMessages + .EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233); + } + String tableName = row.getTableName(); + long timestamp = row.getTimestamp(); + List measurements = row.getColumnNames(); + List types = row.getDataTypes(); + List columnCategories = row.getColumnCategories(); + long[] values = row.getValues(); + String[] tagValues = row.getTagValues(); + ByteBuffer valueBuffer = + SessionUtils.getValueBuffer(types, values, measurements, columnCategories, tagValues); + TSInsertRecordReq request = new TSInsertRecordReq(); + request.setPrefixPath(tableName); + request.setTimestamp(timestamp); + request.setMeasurements(measurements); + request.setValues(valueBuffer); + request.setIsWriteToTable(true); + request.setIsNumericRow(true); + request.setIsAligned(true); + request.setColumnCategoryies( + columnCategories.stream() + .map(category -> (byte) category.ordinal()) + .collect(Collectors.toList())); + String[] deviceSegments = new String[tagValues.length + 1]; + deviceSegments[0] = tableName; + System.arraycopy(tagValues, 0, deviceSegments, 1, tagValues.length); + IDeviceID deviceId = IDeviceID.Factory.DEFAULT_FACTORY.create(deviceSegments); + try { + getSessionConnection(deviceId).insertRecord(request); + } catch (RedirectException e) { + handleRedirection(deviceId, e.getEndPoint()); + } catch (IoTDBConnectionException e) { + if (!enableRedirection || tableModelDeviceIdToEndpoint.remove(deviceId) == null) { + throw e; + } + try { + getDefaultSessionConnection().insertRecord(request); + } catch (RedirectException redirect) { + handleRedirection(deviceId, redirect.getEndPoint()); + } + } + } + private void insertRelationalTabletWithLeaderCache(Tablet tablet) throws IoTDBConnectionException, StatementExecutionException { Map relationalTabletGroup = new HashMap<>(); diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java index 059fdea346948..38cc6d72cea3f 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java @@ -20,6 +20,7 @@ package org.apache.iotdb.session; import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.NumericRow; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; @@ -39,6 +40,11 @@ public void insert(Tablet tablet) throws StatementExecutionException, IoTDBConne session.insertRelationalTablet(tablet); } + @Override + public void insert(NumericRow row) throws StatementExecutionException, IoTDBConnectionException { + session.insertRelationalRecord(row); + } + @Override public void executeNonQueryStatement(String sql) throws IoTDBConnectionException, StatementExecutionException { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java index 4cf7ca980f347..8f158f4ab4ff6 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java @@ -20,6 +20,7 @@ package org.apache.iotdb.session.pool; import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.NumericRow; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; @@ -68,6 +69,18 @@ public void insert(Tablet tablet) throws StatementExecutionException, IoTDBConne } } + @Override + public void insert(NumericRow row) throws StatementExecutionException, IoTDBConnectionException { + try { + session.insertRelationalRecord(row); + } catch (IoTDBConnectionException e) { + sessionPool.cleanSessionAndMayThrowConnectionException(session); + closed.set(true); + session = null; + throw e; + } + } + @Override public SessionDataSet executeQueryStatement(String sql) throws StatementExecutionException, IoTDBConnectionException { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java index b3b29f0be932e..0d838b6925449 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java @@ -163,6 +163,80 @@ public static ByteBuffer getValueBuffer( return buffer; } + /** Encodes separate, densely packed FIELD and TAG arrays in the metadata column order. */ + public static ByteBuffer getValueBuffer( + List types, + long[] values, + List measurements, + List categories, + String[] tagValues) + throws IoTDBConnectionException { + if (types == null + || values == null + || measurements == null + || categories == null + || tagValues == null + || types.size() != measurements.size() + || types.size() != categories.size()) { + throw invalidNumericRowArguments(); + } + int fieldCount = 0; + int tagCount = 0; + int length = types.size(); + for (int i = 0; i < types.size(); i++) { + TSDataType type = types.get(i); + if (categories.get(i) == ColumnCategory.TAG && type == TSDataType.STRING) { + if (tagCount >= tagValues.length) { + throw invalidNumericRowArguments(); + } + length += ReadWriteIOUtils.sizeToWrite(tagValues[tagCount++]); + } else if (categories.get(i) == ColumnCategory.FIELD && type != null) { + fieldCount++; + length += + switch (type) { + case BOOLEAN -> Byte.BYTES; + case INT32, DATE, FLOAT -> Integer.BYTES; + case INT64, TIMESTAMP, DOUBLE -> Long.BYTES; + default -> throw new IoTDBConnectionException(MSG_UNSUPPORTED_DATA_TYPE + type); + }; + } else { + throw new IllegalArgumentException( + SessionMessages + .EXCEPTION_NUMERIC_ROWS_SUPPORT_NUMERIC_FIELD_COLUMNS_AND_STRING_TAG_COLUMNS_ONLY_87159FFA); + } + } + if (fieldCount != values.length || tagCount != tagValues.length) { + throw invalidNumericRowArguments(); + } + ByteBuffer buffer = ByteBuffer.allocate(length); + int fieldIndex = 0; + int tagIndex = 0; + for (int i = 0; i < types.size(); i++) { + TSDataType type = types.get(i); + ReadWriteIOUtils.write(type, buffer); + if (categories.get(i) == ColumnCategory.TAG) { + // A length of -1 represents a null TAG; an empty TAG has length zero. + ReadWriteIOUtils.write(tagValues[tagIndex++], buffer); + } else { + long value = values[fieldIndex++]; + switch (type) { + case BOOLEAN -> buffer.put((byte) (value == 0 ? 0 : 1)); + case INT32, DATE, FLOAT -> buffer.putInt((int) value); + case INT64, TIMESTAMP, DOUBLE -> buffer.putLong(value); + default -> throw new IoTDBConnectionException(MSG_UNSUPPORTED_DATA_TYPE + type); + } + } + } + buffer.flip(); + return buffer; + } + + private static IllegalArgumentException invalidNumericRowArguments() { + return new IllegalArgumentException( + SessionMessages + .EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233); + } + public static int calculateLength(List types, List values) throws IoTDBConnectionException { int res = 0; diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java index e4d2dbbb41dc8..88280ac8b4b9d 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java @@ -22,13 +22,18 @@ import org.apache.iotdb.common.rpc.thrift.TAggregationType; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.NumericRow; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.isession.util.Version; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.service.rpc.thrift.TSInsertRecordReq; import org.apache.iotdb.service.rpc.thrift.TSQueryTemplateResp; +import org.apache.iotdb.session.util.SessionUtils; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -41,6 +46,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; @@ -134,6 +140,80 @@ public void testBuildSession() { .build(); } + @Test + public void testNumericTableRowRequest() throws Exception { + // A BOOLEAN/DOUBLE row must carry the table RPC flags and explicit FIELD categories. + ITableSession tableSession = new TableSession((Session) session); + List names = Arrays.asList("enabled", "temperature"); + List types = Arrays.asList(TSDataType.BOOLEAN, TSDataType.DOUBLE); + tableSession.insert( + new NumericRow( + "readings", + 123L, + names, + types, + Collections.nCopies(types.size(), ColumnCategory.FIELD), + new long[] {1L, Double.doubleToRawLongBits(-0.0)}, + new String[0])); + ArgumentCaptor captor = ArgumentCaptor.forClass(TSInsertRecordReq.class); + Mockito.verify(sessionConnection).insertRecord(captor.capture()); + TSInsertRecordReq request = captor.getValue(); + Assert.assertTrue(request.isIsNumericRow()); + Assert.assertTrue(request.isIsWriteToTable()); + Assert.assertTrue(request.isIsAligned()); + assertEquals("readings", request.getPrefixPath()); + assertEquals(123L, request.getTimestamp()); + assertEquals(names, request.getMeasurements()); + assertEquals( + Collections.nCopies(2, (byte) ColumnCategory.FIELD.ordinal()), + request.getColumnCategoryies()); + } + + @Test + public void testNumericTableRowRejectsNonFieldColumnsBeforeRpc() throws Exception { + // Numeric TAG values are invalid and must fail locally without sending a request. + ITableSession tableSession = new TableSession((Session) session); + Assert.assertThrows( + IllegalArgumentException.class, () -> tableSession.insert((NumericRow) null)); + Assert.assertThrows( + IllegalArgumentException.class, + () -> + tableSession.insert( + new NumericRow( + "readings", + 1L, + Collections.singletonList("tag"), + Collections.singletonList(TSDataType.INT64), + Collections.singletonList(ColumnCategory.TAG), + new long[] {1}, + new String[0]))); + Mockito.verify(sessionConnection, Mockito.never()).insertRecord(any(TSInsertRecordReq.class)); + } + + @Test + public void testNumericTableRowWithTagsRequest() throws Exception { + // The NumericRow entry point must forward the TAG payload and metadata with numeric flags. + ITableSession tableSession = new TableSession((Session) session); + List names = Arrays.asList("tag", "enabled"); + List types = Arrays.asList(TSDataType.STRING, TSDataType.BOOLEAN); + List categories = Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD); + tableSession.insert( + new NumericRow( + "readings", 123L, names, types, categories, new long[] {1}, new String[] {"device"})); + ArgumentCaptor captor = ArgumentCaptor.forClass(TSInsertRecordReq.class); + Mockito.verify(sessionConnection).insertRecord(captor.capture()); + TSInsertRecordReq request = captor.getValue(); + Assert.assertTrue(request.isIsNumericRow()); + Assert.assertTrue(request.isIsWriteToTable()); + assertEquals( + Arrays.asList((byte) ColumnCategory.TAG.ordinal(), (byte) ColumnCategory.FIELD.ordinal()), + request.getColumnCategoryies()); + assertEquals( + SessionUtils.getValueBuffer( + types, new long[] {1}, names, categories, new String[] {"device"}), + request.bufferForValues()); + } + @Test public void testTimeZone() throws IoTDBConnectionException, StatementExecutionException { String timeZone = session.getTimeZone(); diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java index b8524d4b31122..d2c24a4da8e1b 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java @@ -28,6 +28,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -45,6 +46,156 @@ public class SessionUtilsTest { + @Test + public void testNumericRowEncodingPreservesRawBits() throws IoTDBConnectionException { + List types = + Arrays.asList( + TSDataType.BOOLEAN, + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.DATE, + TSDataType.TIMESTAMP); + // Noncanonical NaN payloads must survive encoding without a float/double round trip. + long[] values = { + 0, + -7, + Integer.MIN_VALUE, + Long.MIN_VALUE, + 0x7fc01234, + 0x7ff8000000001234L, + 20260910, + Long.MAX_VALUE + }; + ByteBuffer actual = + SessionUtils.getValueBuffer( + types, + values, + Collections.nCopies(8, "s"), + Collections.nCopies(8, ColumnCategory.FIELD), + new String[0]); + ByteBuffer expected = ByteBuffer.allocate(46); + expected.put(TSDataType.BOOLEAN.serialize()).put((byte) 0); + expected.put(TSDataType.BOOLEAN.serialize()).put((byte) 1); + expected.put(TSDataType.INT32.serialize()).putInt(Integer.MIN_VALUE); + expected.put(TSDataType.INT64.serialize()).putLong(Long.MIN_VALUE); + expected.put(TSDataType.FLOAT.serialize()).putInt(0x7fc01234); + expected.put(TSDataType.DOUBLE.serialize()).putLong(0x7ff8000000001234L); + expected.put(TSDataType.DATE.serialize()).putInt(20260910); + expected.put(TSDataType.TIMESTAMP.serialize()).putLong(Long.MAX_VALUE); + expected.flip(); + Assert.assertEquals(expected, actual); + } + + @Test + public void testNumericRowRejectsInvalidValues() { + // Reject mismatched lengths, null types and every unsupported type before encoding. + assertThrows( + IllegalArgumentException.class, + () -> + SessionUtils.getValueBuffer( + Collections.singletonList(TSDataType.INT64), + new long[0], + Collections.singletonList("s"), + Collections.singletonList(ColumnCategory.FIELD), + new String[0])); + assertThrows( + IllegalArgumentException.class, + () -> + SessionUtils.getValueBuffer( + Collections.singletonList(null), + new long[] {1}, + Collections.singletonList("s"), + Collections.singletonList(ColumnCategory.FIELD), + new String[0])); + for (TSDataType type : + Arrays.asList( + TSDataType.TEXT, + TSDataType.STRING, + TSDataType.BLOB, + TSDataType.OBJECT, + TSDataType.VECTOR, + TSDataType.UNKNOWN)) { + assertThrows( + IoTDBConnectionException.class, + () -> + SessionUtils.getValueBuffer( + Collections.singletonList(type), + new long[] {1}, + Collections.singletonList("s"), + Collections.singletonList(ColumnCategory.FIELD), + new String[0])); + } + } + + @Test + public void testNumericRowWithSeparateTags() throws Exception { + // TAG strings, including null and empty, must not consume slots from the compact long[]. + List types = + Arrays.asList( + TSDataType.STRING, + TSDataType.DOUBLE, + TSDataType.STRING, + TSDataType.BOOLEAN, + TSDataType.STRING); + List categories = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.TAG); + ByteBuffer buffer = + SessionUtils.getValueBuffer( + types, + new long[] {0x7ff8000000001234L, -1}, + Arrays.asList("a", "d", "b", "enabled", "c"), + categories, + new String[] {"设备😀", null, ""}); + Assert.assertEquals(TSDataType.STRING.serialize(), buffer.get()); + Assert.assertEquals("设备😀", ReadWriteIOUtils.readString(buffer)); + Assert.assertEquals(TSDataType.DOUBLE.serialize(), buffer.get()); + Assert.assertEquals(0x7ff8000000001234L, buffer.getLong()); + Assert.assertEquals(TSDataType.STRING.serialize(), buffer.get()); + Assert.assertEquals(-1, buffer.getInt()); + Assert.assertEquals(TSDataType.BOOLEAN.serialize(), buffer.get()); + Assert.assertEquals(1, buffer.get()); + Assert.assertEquals(TSDataType.STRING.serialize(), buffer.get()); + Assert.assertEquals(0, buffer.getInt()); + Assert.assertFalse(buffer.hasRemaining()); + } + + @Test + public void testNumericRowRejectsInvalidTagArrays() { + // Mismatched per-category counts and ATTRIBUTE values must fail before sending the RPC. + List types = Arrays.asList(TSDataType.STRING, TSDataType.INT64); + List names = Arrays.asList("tag", "field"); + List categories = Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD); + for (String[] tags : Arrays.asList(new String[0], new String[] {"a", "b"})) { + assertThrows( + IllegalArgumentException.class, + () -> SessionUtils.getValueBuffer(types, new long[] {1}, names, categories, tags)); + } + assertThrows( + IllegalArgumentException.class, + () -> + SessionUtils.getValueBuffer(types, new long[0], names, categories, new String[] {"a"})); + assertThrows( + IllegalArgumentException.class, + () -> SessionUtils.getValueBuffer(types, new long[] {1}, names, categories, null)); + assertThrows( + IllegalArgumentException.class, + () -> + SessionUtils.getValueBuffer( + types, + new long[] {1}, + names, + Arrays.asList(ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD), + new String[] {"a"})); + } + @Test public void testGetTimeBuffer() { List schemas = new ArrayList<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java index 3bebbc9115305..dc8a8877b6d7c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java @@ -46,10 +46,12 @@ import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; import org.apache.iotdb.db.pipe.resource.memory.PipeTabletMemoryBlock; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; @@ -366,8 +368,8 @@ public boolean mayEventTimeOverlappedWithTimeRange() { return true; } - if (insertNode instanceof InsertRowNode) { - final long timestamp = ((InsertRowNode) insertNode).getTime(); + if (insertNode instanceof AbstractInsertRowNode row) { + final long timestamp = row.getTime(); return startTime <= timestamp && timestamp <= endTime; } @@ -409,13 +411,14 @@ public boolean mayEventPathsOverlappedWithPattern() { try { final InsertNode insertNode = getInsertNode(); - if (insertNode instanceof RelationalInsertRowNode + if (insertNode instanceof RelationalInsertNumericRowNode + || insertNode instanceof RelationalInsertRowNode || insertNode instanceof RelationalInsertTabletNode || insertNode instanceof RelationalInsertRowsNode) { return true; } - if (insertNode instanceof InsertRowNode || insertNode instanceof InsertTabletNode) { + if (insertNode instanceof AbstractInsertRowNode || insertNode instanceof InsertTabletNode) { final PartialPath devicePartialPath = insertNode.getTargetPath(); return Objects.isNull(devicePartialPath) || treePattern.mayOverlapWithDevice(devicePartialPath.getIDeviceIDAsFullDevice()); @@ -539,6 +542,7 @@ private List initEventParsers() { pipeTaskMeta, this, insertRowNode, treePattern, userEntity)); } break; + case RELATIONAL_INSERT_NUMERIC_ROW: case RELATIONAL_INSERT_ROW: case RELATIONAL_INSERT_TABLET: eventParsers.add( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java index 545a587aec496..886b070a84d2a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java @@ -25,7 +25,7 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.pipe.api.access.Row; import org.apache.iotdb.pipe.api.collector.RowCollector; @@ -115,7 +115,7 @@ public void markAsNeedToReport() { //////////////////////////// parse //////////////////////////// - protected void parse(final InsertRowNode insertRowNode) { + protected void parse(final AbstractInsertRowNode insertRowNode) { final String[] originColumnNameStringList = insertRowNode.getMeasurements(); final int originColumnSize = originColumnNameStringList == null ? 0 : originColumnNameStringList.length; @@ -133,7 +133,8 @@ protected void parse(final InsertRowNode insertRowNode) { final MeasurementSchema[] originMeasurementSchemaList = insertRowNode.getMeasurementSchemas(); final TsTableColumnCategory[] originColumnCategories = insertRowNode.getColumnCategories(); final TSDataType[] originValueDataTypes = insertRowNode.getDataTypes(); - final Object[] originValues = insertRowNode.getValues(); + // Read individual columns so numeric rows never need an Object[] row snapshot. + final int originValueCount = insertRowNode.getValueCount(); generateColumnIndexMapper( originColumnNameStringList, originColumnIndex2FilteredColumnIndexMapperList); @@ -147,8 +148,7 @@ protected void parse(final InsertRowNode insertRowNode) { originMeasurementSchemaList, originValueDataTypes, i) - && originValues != null - && i < originValues.length); + && i < originValueCount); this.measurementSchemaList = new MeasurementSchema[filteredColumnSize]; this.columnNameStringList = new String[filteredColumnSize]; @@ -170,7 +170,7 @@ protected void parse(final InsertRowNode insertRowNode) { : ColumnCategory.FIELD; this.valueColumnDataTypes[filteredColumnIndex] = originValueDataTypes[i]; final BitMap bitMap = new BitMap(this.timestampColumn.length); - if (Objects.isNull(originValues[i]) || Objects.isNull(originValueDataTypes[i])) { + if (insertRowNode.isNull(i) || Objects.isNull(originValueDataTypes[i])) { fillNullValue( originValueDataTypes[i], this.valueColumns, @@ -180,7 +180,12 @@ protected void parse(final InsertRowNode insertRowNode) { } else { this.valueColumns[filteredColumnIndex] = filterValueColumnsByRowIndexList( - originValueDataTypes[i], originValues[i], rowIndexList, true, null, bitMap); + originValueDataTypes[i], + insertRowNode.getValue(i), + rowIndexList, + true, + null, + bitMap); } this.nullValueColumnBitmaps[filteredColumnIndex] = bitMap; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTablePatternParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTablePatternParser.java index 14708aa4be95e..84dfb70baccbb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTablePatternParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTablePatternParser.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.pipe.api.access.Row; @@ -60,7 +61,9 @@ public TabletInsertionEventTablePatternParser( super(pipeTaskMeta, sourceEvent); this.pattern = pattern; - if (insertNode instanceof RelationalInsertRowNode) { + if (insertNode instanceof RelationalInsertNumericRowNode numeric) { + parse(numeric); + } else if (insertNode instanceof RelationalInsertRowNode) { parse((RelationalInsertRowNode) insertNode); } else if (insertNode instanceof RelationalInsertTabletNode) { parse((RelationalInsertTabletNode) insertNode); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java index 6158c19bb9950..210defbeb7013 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipePlanToStatementVisitor.java @@ -42,6 +42,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; @@ -50,6 +51,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Update; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.crud.DeleteDataStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; @@ -84,6 +86,19 @@ public Statement visitPlan(final PlanNode node, final Void context) { node)); } + @Override + public InsertNumericRowStatement visitRelationalInsertNumericRow( + final RelationalInsertNumericRowNode node, final Void context) { + InsertNumericRowStatement statement = new InsertNumericRowStatement(); + statement.setDevicePath(node.getTargetPath()); + statement.setTime(node.getTime()); + statement.setMeasurements(node.getMeasurements()); + statement.setDataTypes(node.getDataTypes()); + statement.setNumericValues(node.getNumericValues()); + statement.setColumnCategories(node.getColumnCategories()); + return statement; + } + @Override public InsertRowStatement visitInsertRow(final InsertRowNode node, final Void context) { // MeasurementSchema is only allowed to be set at the analysis type stage. However, InsertNode diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBatchReqV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBatchReqV2.java index 2cec219fa7524..e8b765366c182 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBatchReqV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBatchReqV2.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; @@ -78,9 +79,12 @@ public List constructStatements() { continue; } if (statement.isWriteToTable()) { - if (statement instanceof InsertRowStatement) { - addTableModelInsertRowStatement( - tableModelDatabaseInsertRowStatementMap, (InsertRowStatement) statement); + if (statement instanceof InsertNumericRowStatement) { + // Keep numeric rows independent: grouping into InsertRowsStatement would box their + // values. + statements.add(statement); + } else if (statement instanceof InsertRowStatement row) { + addTableModelInsertRowStatement(tableModelDatabaseInsertRowStatementMap, row); } else if (statement instanceof InsertTabletStatement) { statements.add(statement); } else if (statement instanceof InsertRowsStatement) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReq.java index cf4a746cb0eaf..cf89d64461ab7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReq.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReq.java @@ -24,8 +24,8 @@ import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.receiver.protocol.thrift.IoTDBDataNodeReceiver; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; @@ -55,7 +55,7 @@ public ByteBuffer getByteBuffer() { public InsertBaseStatement constructStatement() { final InsertNode insertNode = parseByteBuffer(); - if (!(insertNode instanceof InsertRowNode + if (!(insertNode instanceof AbstractInsertRowNode || insertNode instanceof InsertTabletNode || insertNode instanceof InsertRowsNode)) { throw new UnsupportedOperationException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReqV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReqV2.java index 196af0b16d2d1..d764dc7ce9847 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReqV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBinaryReqV2.java @@ -24,8 +24,8 @@ import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.receiver.protocol.thrift.IoTDBDataNodeReceiver; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; @@ -56,7 +56,7 @@ public String getDataBaseName() { public InsertBaseStatement constructStatement() { final InsertNode insertNode = parseByteBuffer(); - if (!(insertNode instanceof InsertRowNode + if (!(insertNode instanceof AbstractInsertRowNode || insertNode instanceof InsertTabletNode || insertNode instanceof InsertRowsNode)) { throw new UnsupportedOperationException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java index 42f353c250667..40fd29ad98fb4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java @@ -24,8 +24,8 @@ import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.receiver.protocol.thrift.IoTDBDataNodeReceiver; import org.apache.iotdb.db.queryengine.plan.planner.plan.PlanFragment; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; @@ -51,7 +51,7 @@ public InsertNode getInsertNode() { } public InsertBaseStatement constructStatement() { - if (!(insertNode instanceof InsertRowNode + if (!(insertNode instanceof AbstractInsertRowNode || insertNode instanceof InsertTabletNode || insertNode instanceof InsertRowsNode)) { throw new UnsupportedOperationException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReqV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReqV2.java index 4fc18398dda75..45dd2607ef625 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReqV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletInsertNodeReqV2.java @@ -25,8 +25,8 @@ import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.receiver.protocol.thrift.IoTDBDataNodeReceiver; import org.apache.iotdb.db.queryengine.plan.planner.plan.PlanFragment; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; @@ -56,7 +56,7 @@ public String getDataBaseName() { @Override public InsertBaseStatement constructStatement() { - if (!(insertNode instanceof InsertRowNode + if (!(insertNode instanceof AbstractInsertRowNode || insertNode instanceof InsertTabletNode || insertNode instanceof InsertRowsNode)) { throw new UnsupportedOperationException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java index 6fc521a7f724c..a49fd33161c78 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java @@ -71,6 +71,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; @@ -231,6 +232,12 @@ private TSStatus executePlanNodeInConsensusLayer( } } + @Override + public RegionExecutionResult visitRelationalInsertNumericRow( + final RelationalInsertNumericRowNode node, final WritePlanNodeExecutionContext context) { + return executeDataInsert(node, context); + } + @Override public RegionExecutionResult visitInsertRow( final InsertRowNode node, final WritePlanNodeExecutionContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java index 7464e6d99fbfc..0303eed2777f0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java @@ -47,7 +47,9 @@ import org.apache.iotdb.db.queryengine.plan.statement.component.SelectComponent; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; import org.apache.iotdb.db.queryengine.plan.statement.crud.DeleteDataStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; @@ -70,6 +72,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.UnsetSchemaTemplateStatement; import org.apache.iotdb.db.schemaengine.schemaregion.utils.MetaFormatUtils; import org.apache.iotdb.db.schemaengine.template.TemplateQueryType; +import org.apache.iotdb.db.utils.NumericRowValues; import org.apache.iotdb.db.utils.QueryDataSetUtils; import org.apache.iotdb.db.utils.TabletDecoder; import org.apache.iotdb.mpp.rpc.thrift.TFetchTimeseriesReq; @@ -278,19 +281,38 @@ public static Statement createStatement(TSAggregationQueryReq req) throws Illega return queryStatement; } - public static InsertRowStatement createStatement(TSInsertRecordReq insertRecordReq) + public static InsertBaseStatement createStatement(TSInsertRecordReq insertRecordReq) throws IllegalPathException, QueryProcessException { final long startTime = System.nanoTime(); // construct insert statement - InsertRowStatement insertStatement = new InsertRowStatement(); + InsertBaseStatement insertStatement = + insertRecordReq.isIsNumericRow() + ? new InsertNumericRowStatement() + : new InsertRowStatement(); insertStatement.setDevicePath( - DEVICE_PATH_CACHE.getPartialPath(insertRecordReq.getPrefixPath())); + insertRecordReq.isIsWriteToTable() + ? new PartialPath(insertRecordReq.getPrefixPath(), false) + : DEVICE_PATH_CACHE.getPartialPath(insertRecordReq.getPrefixPath())); TimestampPrecisionUtils.checkTimestampPrecision(insertRecordReq.getTimestamp()); - insertStatement.setTime(insertRecordReq.getTimestamp()); + insertStatement.setMeasurements(insertRecordReq.getMeasurements().toArray(new String[0])); insertStatement.setAligned(insertRecordReq.isAligned); - insertStatement.fillValues(insertRecordReq.values); - if (insertRecordReq.isSetIsWriteToTable()) { + if (insertRecordReq.isIsNumericRow()) { + if (!insertRecordReq.isIsWriteToTable()) { + throw new IllegalArgumentException( + DataNodeQueryMessages + .QUERY_EXCEPTION_MISSING_OR_INVALID_COLUMN_CATEGORIES_FOR_TABLE_INSERTION_5DF990B9); + } + InsertNumericRowStatement numeric = (InsertNumericRowStatement) insertStatement; + numeric.setTime(insertRecordReq.getTimestamp()); + numeric.fillValues(insertRecordReq.values); + insertStatement.setAligned(true); + } else { + InsertRowStatement row = (InsertRowStatement) insertStatement; + row.setTime(insertRecordReq.getTimestamp()); + row.fillValues(insertRecordReq.values); + } + if (insertRecordReq.isIsWriteToTable()) { insertStatement.setWriteToTable(insertRecordReq.isIsWriteToTable()); if (!insertRecordReq.isSetColumnCategoryies() || insertRecordReq.getColumnCategoryiesSize() != insertRecordReq.getMeasurementsSize()) { @@ -305,6 +327,18 @@ public static InsertRowStatement createStatement(TSInsertRecordReq insertRecordR TsTableColumnCategory.fromTsFileColumnCategory( ColumnCategory.values()[insertRecordReq.getColumnCategoryies().get(i).intValue()]); } + if (insertRecordReq.isIsNumericRow()) { + for (int i = 0; i < columnCategories.length; i++) { + TSDataType type = insertStatement.getDataType(i); + if (!(columnCategories[i] == TsTableColumnCategory.TAG && type == TSDataType.STRING) + && !(columnCategories[i] == TsTableColumnCategory.FIELD + && NumericRowValues.supports(type))) { + throw new IllegalArgumentException( + DataNodeQueryMessages + .QUERY_EXCEPTION_MISSING_OR_INVALID_COLUMN_CATEGORIES_FOR_TABLE_INSERTION_5DF990B9); + } + } + } insertStatement.setColumnCategories(columnCategories); } PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index a5b886ff4b058..864c7401c9c8c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -117,6 +117,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; @@ -581,6 +582,10 @@ default R visitInsertRow(InsertRowNode node, C context) { return visitPlan(node, context); } + default R visitRelationalInsertNumericRow(RelationalInsertNumericRowNode node, C context) { + return visitPlan(node, context); + } + default R visitRelationalInsertRow(RelationalInsertRowNode node, C context) { return visitInsertRow(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/AbstractInsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/AbstractInsertRowNode.java new file mode 100644 index 0000000000000..f1c0785afa3aa --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/AbstractInsertRowNode.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.LastCacheUpdateSource; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; + +import org.apache.tsfile.enums.TSDataType; + +/** + * Storage-facing contract for single-row writes. Concrete nodes own their value representation and + * wire format; sharing this contract never converts a primitive row to an object row. + */ +public abstract class AbstractInsertRowNode extends InsertNode + implements WALEntryValue, LastCacheUpdateSource { + protected AbstractInsertRowNode(PlanNodeId id) { + super(id); + } + + protected AbstractInsertRowNode( + PlanNodeId id, + PartialPath path, + boolean aligned, + String[] measurements, + TSDataType[] dataTypes) { + super(id, path, aligned, measurements, dataTypes); + } + + public abstract long getTime(); + + public abstract int getValueCount(); + + public abstract boolean isNull(int index); + + /** Boxes only the requested cell for cache and Tablet consumers. */ + public abstract Object getValue(int index); + + /** Returns the original backing storage for allocation accounting, without copying it. */ + public abstract Object getValuesForMemoryAccounting(); + + public abstract void updateLastCache(String databaseName); + + public abstract void serializeToWAL(IWALByteBufferView buffer, long encodedSearchIndex); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 777e0de391a41..20bef3dc8eedb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -185,6 +185,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Insert; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertNumericRow; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRow; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRows; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertTablet; @@ -1223,6 +1224,11 @@ public Scope visitInsert(Insert insert, Optional scope) { insert, scope, Field.newUnqualified(Insert.ROWS, Insert.ROWS_TYPE, Insert.ROWS_CATEGORY)); } + @Override + public Scope visitInsertNumericRow(InsertNumericRow node, Optional context) { + return visitInsert(node, context); + } + @Override public Scope visitInsertRow(InsertRow node, Optional context) { return visitInsert(node, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 6f5f3c278ed3d..c047be7c2b466 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -100,6 +100,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; @@ -130,11 +131,13 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AsofJoinOn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Delete; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertNumericRow; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRow; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRows; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertTablet; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeEnriched; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; @@ -1370,6 +1373,26 @@ public RelationPlan visitInsertTablet(InsertTablet node, Void context) { insertNode, analysis.getRootScope(), Collections.emptyList(), outerContext); } + @Override + public RelationPlan visitInsertNumericRow(InsertNumericRow node, Void context) { + InsertNumericRowStatement statement = node.getInnerTreeStatement(); + String[] measurements = statement.getMeasurements(); + MeasurementSchema[] schemas = statement.getMeasurementSchemas(); + stayConsistent(measurements, schemas); + RelationalInsertNumericRowNode insert = + new RelationalInsertNumericRowNode( + idAllocator.genPlanNodeId(), + statement.getDevicePath(), + measurements, + statement.getDataTypes(), + schemas, + statement.getTime(), + statement.getNumericValues(), + statement.getColumnCategories()); + insert.setFailedMeasurementNumber(statement.getFailedMeasurementNumber()); + return new RelationPlan(insert, analysis.getRootScope(), Collections.emptyList(), outerContext); + } + @Override public RelationPlan visitInsertRow(InsertRow node, Void context) { InsertRowStatement insertRowStatement = node.getInnerTreeStatement(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java index cca616e0a3b17..c91eb3e63fcd1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java @@ -55,6 +55,7 @@ import org.apache.iotdb.db.queryengine.plan.scheduler.IScheduler; import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.rpc.RpcUtils; @@ -239,6 +240,19 @@ public void setRedirectInfo(IAnalysis iAnalysis, TEndPoint localEndPoint, TSStat ((WrappedInsertStatement) statementToRedirect).getInnerTreeStatement(); if (!analysis.isFinishQueryAfterAnalyze()) { + if (insertStatement instanceof InsertNumericRowStatement) { + // A numeric row uses insertRecord, whose redirect response carries one endpoint. + if (tsstatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + List endpoints = analysis.getRedirectNodeList(); + if (endpoints != null + && !endpoints.isEmpty() + && !localEndPoint.equals(endpoints.get(0))) { + tsstatus.setCode(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()); + tsstatus.setRedirectNode(endpoints.get(0)); + } + } + return; + } // Table Model Session supports insertTablet and pipe-generated insertRows statements. if (insertStatement instanceof InsertTabletStatement || insertStatement instanceof InsertRowsStatement) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertNumericRow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertNumericRow.java new file mode 100644 index 0000000000000..38f29f01d2c96 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertNumericRow.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IAstVisitor; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** Relational analysis entry for a primitive numeric row. */ +public class InsertNumericRow extends WrappedInsertStatement { + public InsertNumericRow(InsertNumericRowStatement statement, MPPQueryContext context) { + super(statement, context); + } + + @Override + public R accept(IAstVisitor visitor, C context) { + return ((AstVisitor) visitor).visitInsertNumericRow(this, context); + } + + @Override + public InsertNumericRowStatement getInnerTreeStatement() { + return (InsertNumericRowStatement) super.getInnerTreeStatement(); + } + + @Override + public void updateAfterSchemaValidation(MPPQueryContext context) { + // Numeric rows are already typed; schema validation casts values in their primitive storage. + } + + @Override + public String getTableName() { + return getInnerTreeStatement().getTableName(); + } + + @Override + public List getDeviceIdList() { + Object[] segments = getInnerTreeStatement().getTableDeviceID().getSegments(); + return Collections.singletonList(Arrays.copyOfRange(segments, 1, segments.length)); + } + + @Override + public List getAttributeColumnNameList() { + return Collections.emptyList(); + } + + @Override + public List getAttributeValueList() { + return Collections.singletonList(new Object[0]); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java index 00d36a03e236f..6488f76f56d10 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java @@ -203,4 +203,5 @@ public enum StatementType { SHOW_EXTERNAL_SERVICE, SHOW_DISK_USAGE, + INSERT_NUMERIC_ROW, } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index 5f2f72f431397..d107a5fcde883 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.DeleteDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; @@ -405,6 +406,10 @@ public R visitLoadFile(LoadTsFileStatement loadTsFileStatement, C context) { return visitStatement(loadTsFileStatement, context); } + public R visitInsertNumericRow(InsertNumericRowStatement statement, C context) { + return visitInsertBase(statement, context); + } + public R visitInsertRow(InsertRowStatement insertRowStatement, C context) { return visitInsertBase(insertRowStatement, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java new file mode 100644 index 0000000000000..db90503d4646a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.crud; + +import org.apache.iotdb.calc.exception.QueryProcessException; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Statement; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaValidation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertNumericRow; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; +import org.apache.iotdb.db.utils.NumericRowValues; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +/** A table-only numeric row. Schema edits and partial failures keep the payload in long[]. */ +public class InsertNumericRowStatement extends InsertBaseStatement { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(InsertNumericRowStatement.class); + private long time; + private NumericRowValues numericValues; + private IDeviceID deviceID; + + public InsertNumericRowStatement() { + statementType = StatementType.INSERT_NUMERIC_ROW; + writeToTable = true; + isAligned = true; + } + + public long getTime() { + return time; + } + + public void setTime(long time) { + this.time = time; + } + + public NumericRowValues getNumericValues() { + return numericValues; + } + + public long[] getValues() { + return numericValues.getValues(); + } + + public void setNumericValues(NumericRowValues values) { + numericValues = values; + deviceID = null; + } + + public int getValueCount() { + return numericValues == null ? 0 : numericValues.size(); + } + + /** RPC cells carry typed primitive FIELD bits or a length-prefixed STRING TAG. */ + public void fillValues(ByteBuffer buffer) throws QueryProcessException { + numericValues = new NumericRowValues(measurements.length); + dataTypes = new TSDataType[measurements.length]; + for (int i = 0; i < dataTypes.length; i++) { + byte type = buffer.get(); + if (type < 0 || type >= TSDataType.values().length) { + throw new QueryProcessException(DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE + type); + } + dataTypes[i] = TSDataType.values()[type]; + numericValues.read(i, dataTypes[i], buffer); + } + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitInsertNumericRow(this, context); + } + + @Override + public Statement toRelationalStatement(MPPQueryContext context) { + return new InsertNumericRow(this, context); + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } + + @Override + public long getMinTime() { + return time; + } + + public TTimePartitionSlot getTimePartitionSlot() { + return TimePartitionUtils.getTimePartitionSlot(time); + } + + @Override + public boolean isEmpty() { + return getValueCount() == 0; + } + + @Override + public Object getFirstValueOfIndex(int index) { + return index < 0 || index >= getValueCount() + ? null + : numericValues.getObject(index, getDataType(index)); + } + + @Override + public boolean isColumnPresent(int index) { + return super.isColumnPresent(index) && index < getValueCount(); + } + + @Override + protected boolean checkAndCastDataType(int index, TSDataType type) { + if (index < 0 + || index >= getValueCount() + || !numericValues.cast(index, getDataType(index), type)) { + return false; + } + dataTypes[index] = type; + return true; + } + + @Override + public void semanticCheck() { + super.semanticCheck(); + if (measurements.length != getValueCount()) { + throw new SemanticException( + String.format( + DataNodeQueryMessages + .THE_MEASUREMENTLIST_S_SIZE_D_IS_NOT_CONSISTENT_WITH_THE_VALUELIST_S_SIZE_D, + measurements.length, + getValueCount())); + } + } + + @Override + public void markFailedMeasurement(int index, Exception cause) { + if (!isColumnPresent(index)) { + return; + } + if (failedMeasurementIndex2Info == null) { + failedMeasurementIndex2Info = new HashMap<>(); + } + failedMeasurementIndex2Info.putIfAbsent( + index, + new FailedMeasurementInfo( + measurements[index], getDataType(index), getFirstValueOfIndex(index), cause)); + measurements[index] = null; + if (dataTypes != null && index < dataTypes.length) { + dataTypes[index] = null; + } + numericValues.setNull(index); + tagColumnIndices = null; + deviceID = null; + } + + @Override + public void removeAllFailedMeasurementMarks() { + if (failedMeasurementIndex2Info == null) { + return; + } + failedMeasurementIndex2Info.forEach( + (index, info) -> { + measurements[index] = info.getMeasurement(); + dataTypes[index] = info.getDataType(); + numericValues.setObject(index, info.getDataType(), info.getValue()); + }); + failedMeasurementIndex2Info.clear(); + tagColumnIndices = null; + deviceID = null; + } + + @Override + public void insertColumn(int pos, ColumnSchema columnSchema) { + super.insertColumn(pos, columnSchema); + int[] mapping = new int[measurements.length]; + for (int i = 0; i < mapping.length; i++) { + mapping[i] = i < pos ? i : i == pos ? -1 : i - 1; + } + numericValues = numericValues.remap(mapping); + deviceID = null; + } + + @Override + public void swapColumn(int src, int target) { + super.swapColumn(src, target); + numericValues.swap(src, target); + deviceID = null; + } + + @Override + public void rebuildArraysAfterExpansion(int[] mapping, String[] newMeasurements) { + super.rebuildArraysAfterExpansion(mapping, newMeasurements); + numericValues = numericValues.remap(mapping); + deviceID = null; + } + + @Override + protected void subRemoveAttributeColumns(List columnsToKeep) { + numericValues = + numericValues.remap(columnsToKeep.stream().mapToInt(Integer::intValue).toArray()); + deviceID = null; + } + + @Override + public void setColumnCategories(TsTableColumnCategory[] categories) { + super.setColumnCategories(categories); + deviceID = null; + } + + public IDeviceID getTableDeviceID() { + if (deviceID == null) { + String[] segments = new String[getTagColumnIndices().size() + 1]; + segments[0] = getTableName(); + for (int i = 0; i < getTagColumnIndices().size(); i++) { + segments[i + 1] = numericValues.getTag(getTagColumnIndices().get(i)); + } + deviceID = IDeviceID.Factory.DEFAULT_FACTORY.create(segments); + } + return deviceID; + } + + @Override + protected long calculateBytesUsed() { + return INSTANCE_SIZE + + (numericValues == null ? 0 : numericValues.ramBytesUsed()) + + InsertNodeMemoryEstimator.sizeOfIDeviceID(deviceID); + } + + // Tree schema validation and logical view splitting do not apply to numeric table rows. + @Override + public ISchemaValidation getSchemaValidation() { + throw new UnsupportedOperationException(); + } + + @Override + public List getSchemaValidationList() { + throw new UnsupportedOperationException(); + } + + @Override + public InsertBaseStatement removeLogicalView() { + return this; + } + + @Override + public String getPipeLoggingString() { + return "InsertNumericRowStatement{table=" + + getTableName() + + ", time=" + + time + + ", valueLength=" + + getValueCount() + + '}'; + } + + @Override + public String toString() { + return getPipeLoggingString() + Arrays.toString(getValues()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index 1ac6c15488b11..ba5349edc0e51 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -1199,8 +1199,8 @@ public Runnable executeCompactFileTimeIndexCache() { return () -> { if (!isReadyForNonReadWriteFunctions()) { return; - } - for (DataRegion dataRegion : dataRegionMap.values()) { + } + for (DataRegion dataRegion : dataRegionMap.values()) { if (dataRegion != null) { dataRegion.compactFileTimeIndexCache(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index de1a5a09b7ddc..df24cefdf30ac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -42,6 +42,7 @@ import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; @@ -80,6 +81,7 @@ import org.apache.iotdb.db.utils.EncryptDBUtils; import org.apache.iotdb.db.utils.MemUtils; import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.iotdb.db.utils.NumericRowValues; import org.apache.iotdb.db.utils.datastructure.AlignedTVList; import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.iotdb.rpc.RpcUtils; @@ -290,7 +292,7 @@ private void ensureMemTable(long[] infoForMetrics) { } } - private static void clearDataRegionReplicaSet(final InsertRowNode insertRowNode) { + private static void clearDataRegionReplicaSet(final AbstractInsertRowNode insertRowNode) { insertRowNode.setDataRegionReplicaSet(null); } @@ -310,7 +312,7 @@ private static void clearDataRegionReplicaSet(final InsertTabletNode insertTable * * @param insertRowNode physical plan of insertion */ - public void insert(InsertRowNode insertRowNode, long[] infoForMetrics) + public void insert(AbstractInsertRowNode insertRowNode, long[] infoForMetrics) throws WriteProcessException { ensureMemTable(infoForMetrics); @@ -329,7 +331,7 @@ public void insert(InsertRowNode insertRowNode, long[] infoForMetrics) insertRowNode.getDeviceID(), insertRowNode.getMeasurements(), insertRowNode.getDataTypes(), - insertRowNode.getValues(), + insertRowNode.getValuesForMemoryAccounting(), insertRowNode.getColumnCategories()); } else { memIncrements = @@ -337,7 +339,7 @@ public void insert(InsertRowNode insertRowNode, long[] infoForMetrics) insertRowNode.getDeviceID(), insertRowNode.getMeasurements(), insertRowNode.getDataTypes(), - insertRowNode.getValues(), + insertRowNode.getValuesForMemoryAccounting(), insertRowNode.getColumnCategories()); } // recordScheduleMemoryBlockCost @@ -390,7 +392,7 @@ public void insert(InsertRowNode insertRowNode, long[] infoForMetrics) if (insertRowNode.isAligned()) { pointInserted = workMemTable.insertAlignedRow(insertRowNode); } else { - pointInserted = workMemTable.insert(insertRowNode); + pointInserted = workMemTable.insert((InsertRowNode) insertRowNode); } } finally { reconcileAlignedTVListRamCost(alignedRamCostSnapshot, memIncrements[0]); @@ -514,7 +516,7 @@ public void insertRows(InsertRowsNode insertRowsNode, long[] infoForMetrics) if (insertRowNode.isAligned()) { pointInserted += workMemTable.insertAlignedRow(insertRowNode); } else { - pointInserted += workMemTable.insert(insertRowNode); + pointInserted += workMemTable.insert((InsertRowNode) insertRowNode); } // update start time of this memtable tsFileResource.updateStartTime(insertRowNode.getDeviceID(), insertRowNode.getTime()); @@ -770,7 +772,7 @@ private long[] checkMemCostAndAddToTspInfoForRow( IDeviceID deviceId, String[] measurements, TSDataType[] dataTypes, - Object[] values, + Object values, TsTableColumnCategory[] columnCategories) throws WriteProcessException { // Fixed-size TVList structures and materialized value primitive arrays. @@ -798,8 +800,8 @@ private long[] checkMemCostAndAddToTspInfoForRow( } } // TEXT data mem size - if (dataTypes[i].isBinary() && values[i] != null) { - textDataIncrement += MemUtils.getBinarySize((Binary) values[i]); + if (dataTypes[i].isBinary() && values instanceof Object[] rowValues && rowValues[i] != null) { + textDataIncrement += MemUtils.getBinarySize((Binary) rowValues[i]); } } updateMemoryInfo(memTableIncrement, chunkMetadataIncrement, textDataIncrement); @@ -818,7 +820,7 @@ private long[] checkMemCostAndAddToTspInfoForRows(List insertRowN for (InsertRowNode insertRowNode : insertRowNodeList) { IDeviceID deviceId = insertRowNode.getDeviceID(); TSDataType[] dataTypes = insertRowNode.getDataTypes(); - Object[] values = insertRowNode.getValues(); + Object values = insertRowNode.getValuesForMemoryAccounting(); String[] measurements = insertRowNode.getMeasurements(); for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { // Skip failed Measurements @@ -852,8 +854,10 @@ private long[] checkMemCostAndAddToTspInfoForRows(List insertRowN increasingMemTableInfo.get(deviceId).computeIfPresent(measurements[i], (k, v) -> v + 1); } // TEXT data mem size - if (dataTypes[i].isBinary() && values[i] != null) { - textDataIncrement += MemUtils.getBinarySize((Binary) values[i]); + if (dataTypes[i].isBinary() + && values instanceof Object[] rowValues + && rowValues[i] != null) { + textDataIncrement += MemUtils.getBinarySize((Binary) rowValues[i]); } } } @@ -866,7 +870,7 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRow( IDeviceID deviceId, String[] measurements, TSDataType[] dataTypes, - Object[] values, + Object values, TsTableColumnCategory[] columnCategories) throws WriteProcessException { // Fixed-size TVList structures and materialized value primitive arrays. @@ -915,9 +919,10 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRow( } for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { if (isWritableFieldMeasurement(measurements, dataTypes, values, columnCategories, i) - && dataTypes[i].isBinary()) { + && dataTypes[i].isBinary() + && values instanceof Object[] rowValues) { // Binary values keep their variable-length content outside the fixed Binary[] array. - textDataIncrement += MemUtils.getBinarySize((Binary) values[i]); + textDataIncrement += MemUtils.getBinarySize((Binary) rowValues[i]); } } updateMemoryInfo(memTableIncrement, chunkMetadataIncrement, textDataIncrement); @@ -939,7 +944,7 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins for (InsertRowNode insertRowNode : insertRowNodeList) { IDeviceID deviceId = insertRowNode.getDeviceID(); TSDataType[] dataTypes = insertRowNode.getDataTypes(); - Object[] values = insertRowNode.getValues(); + Object values = insertRowNode.getValuesForMemoryAccounting(); String[] measurements = insertRowNode.getMeasurements(); IWritableMemChunk memChunk = @@ -1031,9 +1036,11 @@ private long[] checkAlignedMemCostAndAddToTspInfoForRows(List ins measurements, dataTypes, values, insertRowNode.getColumnCategories(), i)) { continue; } - if (dataTypes[i].isBinary() && values[i] != null) { + if (dataTypes[i].isBinary() + && values instanceof Object[] rowValues + && rowValues[i] != null) { // Binary values keep their variable-length content outside the fixed Binary[] array. - textDataIncrement += MemUtils.getBinarySize((Binary) values[i]); + textDataIncrement += MemUtils.getBinarySize((Binary) rowValues[i]); } } } @@ -1326,7 +1333,7 @@ private static boolean containsNonNullValue( private static TSDataType[] getWritableFieldDataTypes( String[] measurementIds, TSDataType[] dataTypes, - Object[] valuesOrColumns, + Object valuesOrColumns, TsTableColumnCategory[] columnCategories) { List writableFieldDataTypes = new ArrayList<>(); for (int i = 0; dataTypes != null && i < dataTypes.length; i++) { @@ -1341,13 +1348,15 @@ private static TSDataType[] getWritableFieldDataTypes( private static boolean isWritableFieldMeasurement( String[] measurementIds, TSDataType[] dataTypes, - Object[] valuesOrColumns, + Object valuesOrColumns, TsTableColumnCategory[] columnCategories, int index) { return isFieldMeasurement(measurementIds, dataTypes, columnCategories, index) - && valuesOrColumns != null - && index < valuesOrColumns.length - && valuesOrColumns[index] != null; + && (valuesOrColumns instanceof NumericRowValues row + ? index < row.size() && !row.isNull(index) + : valuesOrColumns instanceof Object[] columns + && index < columns.length + && columns[index] != null); } private static boolean isFieldMeasurement( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java index 8c0a5e353291e..346e3007c7e5c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java @@ -22,9 +22,9 @@ import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.i18n.StorageEngineMessages; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ContinuousSameSearchIndexSeparatorNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; @@ -65,7 +65,7 @@ protected WALEntry(long memTableId, WALEntryValue value, boolean wait) { this.value = value; if (value instanceof IMemTable) { this.type = WALEntryType.MEMORY_TABLE_SNAPSHOT; - } else if (value instanceof InsertRowNode) { + } else if (value instanceof AbstractInsertRowNode) { this.type = WALEntryType.INSERT_ROW_NODE; } else if (value instanceof InsertTabletNode) { this.type = WALEntryType.INSERT_TABLET_NODE; @@ -121,7 +121,7 @@ public static WALEntry deserialize(DataInputStream stream) throws IOException { value = AbstractMemTable.Factory.createFromOldMemTableSnapshot(stream); break; case INSERT_ROW_NODE: - value = (InsertRowNode) PlanNodeType.deserializeFromWAL(stream); + value = (AbstractInsertRowNode) PlanNodeType.deserializeFromWAL(stream); break; case INSERT_TABLET_NODE: value = (InsertTabletNode) PlanNodeType.deserializeFromWAL(stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java index 957cf03d89237..37cc8aff863b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java @@ -22,9 +22,9 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; @@ -102,7 +102,7 @@ public void serialize(IWALByteBufferView buffer) { .serializeToWAL(buffer, tabletInfo.tabletRangeList, encodedSearchIndex); break; case INSERT_ROW_NODE: - ((InsertRowNode) value).serializeToWAL(buffer, encodedSearchIndex); + ((AbstractInsertRowNode) value).serializeToWAL(buffer, encodedSearchIndex); break; case INSERT_ROWS_NODE: ((InsertRowsNode) value).serializeToWAL(buffer, encodedSearchIndex); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java index a8fbbee0dc4f5..b2689a5f99ae7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java @@ -21,9 +21,9 @@ import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.consensus.iot.log.ConsensusReqReader; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ContinuousSameSearchIndexSeparatorNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; @@ -38,7 +38,7 @@ public interface IWALNode extends FlushListener, AutoCloseable, ConsensusReqReader, DataSet { /** Log InsertRowNode. */ - WALFlushListener log(long memTableId, InsertRowNode insertRowNode); + WALFlushListener log(long memTableId, AbstractInsertRowNode insertRowNode); /** Log InsertRowsNode. */ WALFlushListener log(long memTableId, InsertRowsNode insertRowsNode); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java index d24d1a1d82348..93ac79c05a982 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java @@ -20,9 +20,9 @@ package org.apache.iotdb.db.storageengine.dataregion.wal.node; import org.apache.iotdb.db.i18n.StorageEngineMessages; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ContinuousSameSearchIndexSeparatorNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; @@ -53,7 +53,7 @@ public WALFakeNode(Status status, Exception cause) { } @Override - public WALFlushListener log(long memTableId, InsertRowNode insertRowNode) { + public WALFlushListener log(long memTableId, AbstractInsertRowNode insertRowNode) { return getResult(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java index 0e66509d35099..825a83ffe2c8f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java @@ -29,9 +29,9 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ContinuousSameSearchIndexSeparatorNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; @@ -222,7 +222,7 @@ private void removeDeletedWalFilesFromCache(List deletedVersionIds) { } @Override - public WALFlushListener log(long memTableId, InsertRowNode insertRowNode) { + public WALFlushListener log(long memTableId, AbstractInsertRowNode insertRowNode) { logger.debug( StorageEngineMessages.STORAGE_LOG_WAL_NODE_LOGS_INSERTROWNODE_THE_SEARCH_INDEX_IS_027450AC, identifier, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java index 221d39cdfccb3..dbd26029418f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; import org.apache.iotdb.db.exception.WriteProcessException; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; @@ -96,8 +97,8 @@ void redoInsert(InsertNode node) throws WriteProcessException { // the last chunk group may contain the same data with the logs, ignore such logs in seq file Optional lastEndTime = tsFileResource.getEndTime(node.getDeviceID()); long minTimeInNode; - if (node instanceof InsertRowNode) { - minTimeInNode = ((InsertRowNode) node).getTime(); + if (node instanceof AbstractInsertRowNode row) { + minTimeInNode = row.getTime(); } else { minTimeInNode = ((InsertTabletNode) node).getTimes()[0]; } @@ -107,9 +108,9 @@ void redoInsert(InsertNode node) throws WriteProcessException { } int pointsInserted; - if (node instanceof InsertRowNode) { + if (node instanceof AbstractInsertRowNode row) { if (node.isAligned()) { - pointsInserted = recoveryMemTable.insertAlignedRow((InsertRowNode) node); + pointsInserted = recoveryMemTable.insertAlignedRow(row); } else { pointsInserted = recoveryMemTable.insert((InsertRowNode) node); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java index 5512e312cf025..7a56ac2ef037a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java @@ -271,7 +271,7 @@ public void testInsertRecord() throws QueryProcessException, IllegalPathExceptio Arrays.asList("s1", "s2"), ByteBuffer.wrap(new byte[] {0, 0, 0, 0}), 1000L); - InsertRowStatement statement = StatementGenerator.createStatement(req); + InsertRowStatement statement = (InsertRowStatement) StatementGenerator.createStatement(req); assertEquals(1000L, statement.getTime()); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkNumericTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkNumericTest.java new file mode 100644 index 0000000000000..8620f17f71bb8 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkNumericTest.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.memtable; + +import org.apache.iotdb.db.utils.NumericRowValues; +import org.apache.iotdb.db.utils.datastructure.AlignedTVList; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class AlignedWritableMemChunkNumericTest { + + @Test + public void testMappingAllocatedOnlyForNumericRowsAndReused() throws Exception { + // Ordinary row/Tablet writes must not allocate scratch space; numeric rows share it. + List schemas = + Collections.singletonList(new MeasurementSchema("a", TSDataType.INT64)); + AlignedWritableMemChunk chunk = new AlignedWritableMemChunk(new ArrayList<>(schemas), true); + try { + assertNull(mapping(chunk)); + chunk.writeAlignedPoints(1, new Object[] {10L}, schemas); + chunk.writeAlignedTablet( + new long[] {2}, new Object[] {new long[] {20}}, null, schemas, 0, 1, null); + assertNull(mapping(chunk)); + + chunk.writeAlignedNumericPoints(3, row(30), schemas); + int[] mapping = mapping(chunk); + assertNotNull(mapping); + chunk.writeAlignedNumericPoints(4, row(40), schemas); + assertSame(mapping, mapping(chunk)); + assertEquals(30, chunk.getWorkingTVList().getLongByValueIndex(2, 0)); + assertEquals(40, chunk.getWorkingTVList().getLongByValueIndex(3, 0)); + } finally { + chunk.release(); + } + } + + @Test + public void testMappingResetAndGrowthWithMixedWrites() throws Exception { + // Reordered/absent columns must stay correct after reuse and schema growth by either API. + IMeasurementSchema a = new MeasurementSchema("a", TSDataType.INT64); + IMeasurementSchema b = new MeasurementSchema("b", TSDataType.INT64); + IMeasurementSchema c = new MeasurementSchema("c", TSDataType.INT64); + IMeasurementSchema d = new MeasurementSchema("d", TSDataType.INT64); + AlignedWritableMemChunk chunk = + new AlignedWritableMemChunk(new ArrayList<>(Arrays.asList(a, b)), true); + try { + chunk.writeAlignedNumericPoints(1, row(20, 10), Arrays.asList(b, a)); + int[] mapping = mapping(chunk); + chunk.writeAlignedNumericPoints(2, row(30, 999), Arrays.asList(a, null)); + assertSame(mapping, mapping(chunk)); + AlignedTVList list = chunk.getWorkingTVList(); + assertEquals(10, list.getLongByValueIndex(0, 0)); + assertEquals(20, list.getLongByValueIndex(0, 1)); + assertEquals(30, list.getLongByValueIndex(1, 0)); + assertTrue(list.isNullValue(1, 1)); + + chunk.writeAlignedNumericPoints(3, row(40, 50), Arrays.asList(c, b)); + int[] extendedMapping = mapping(chunk); + assertEquals(3, extendedMapping.length); + assertTrue(list.isNullValue(2, 0)); + assertEquals(50, list.getLongByValueIndex(2, 1)); + assertEquals(40, list.getLongByValueIndex(2, 2)); + + chunk.writeAlignedTablet( + new long[] {4}, + new Object[] {new long[] {60}}, + null, + Collections.singletonList(d), + 0, + 1, + null); + assertSame(extendedMapping, mapping(chunk)); + chunk.writeAlignedNumericPoints(5, row(70), Collections.singletonList(a)); + assertEquals(4, mapping(chunk).length); + assertEquals(60, list.getLongByValueIndex(3, 3)); + assertEquals(70, list.getLongByValueIndex(4, 0)); + assertTrue(list.isNullValue(4, 1)); + assertTrue(list.isNullValue(4, 2)); + assertTrue(list.isNullValue(4, 3)); + + mapping = mapping(chunk); + chunk.writeAlignedNumericPoints(6, row(80), Collections.singletonList(d)); + assertSame(mapping, mapping(chunk)); + assertTrue(list.isNullValue(5, 0)); + assertEquals(80, list.getLongByValueIndex(5, 3)); + } finally { + chunk.release(); + } + } + + private static NumericRowValues row(long... values) { + return new NumericRowValues(values, new BitSet()); + } + + private static int[] mapping(AlignedWritableMemChunk chunk) throws Exception { + // Array identity verifies the allocation contract without exposing scratch state as public API. + Field field = AlignedWritableMemChunk.class.getDeclaredField("numericColumnMapping"); + field.setAccessible(true); + return (int[]) field.get(chunk); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeType.java index e36796fca43c3..64da340870eca 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -209,6 +209,7 @@ public enum PlanNodeType { RELATIONAL_INSERT_ROWS((short) 2002), RELATIONAL_DELETE_DATA((short) 2003), OBJECT_FILE_NODE((short) 2004), + RELATIONAL_INSERT_NUMERIC_ROW((short) 2005), ; private static final IPlanNodeDeserializer DESERIALIZER; From 0d296b082d493d3e5a4a7d6361ffe163333640fe Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 15 Sep 2026 14:54:57 +0800 Subject: [PATCH 04/20] Add insert numeric row interface --- .../IoTDBNumericRowInsertPerformanceIT.java | 220 +++++-- .../org/apache/iotdb/session/Session.java | 37 +- .../iotdb/session/util/SessionUtils.java | 15 +- .../org/apache/iotdb/session/SessionTest.java | 36 +- .../iotdb/session/util/SessionUtilsTest.java | 31 +- .../plan/parser/StatementGenerator.java | 4 +- .../write/RelationalInsertNumericRowNode.java | 590 ++++++++++++++++++ .../crud/InsertNumericRowStatement.java | 71 ++- .../iotdb/db/storageengine/StorageEngine.java | 4 +- .../iotdb/db/utils/NumericRowValues.java | 307 +++++++++ .../plan/parser/NumericTableRowTest.java | 524 ++++++++++++++++ 11 files changed, 1732 insertions(+), 107 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertNumericRowNode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/NumericRowValues.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/performance/IoTDBNumericRowInsertPerformanceIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/performance/IoTDBNumericRowInsertPerformanceIT.java index 84a52da145189..e2d6a481743eb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/performance/IoTDBNumericRowInsertPerformanceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/performance/IoTDBNumericRowInsertPerformanceIT.java @@ -48,14 +48,19 @@ import java.util.List; import java.util.Locale; import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; /** * Manual end-to-end benchmark for the numeric table-row API and relational {@link Tablet} writes. * *

Enable this test with {@code -Diotdb.numeric.row.insert.perf.enabled=true}. The timed region - * includes the Session call and server write confirmation. Schema creation and client-side input - * preparation are outside the timed region. By default both APIs write one row per request; set - * {@code iotdb.numeric.row.insert.perf.tablet.row.count} to compare against batched Tablet writes. + * includes the concurrent Session calls and server write confirmations. Schema creation and + * client-side input preparation are outside the timed region. By default 16 client sessions write + * one row per request; set {@code iotdb.numeric.row.insert.perf.client.count} or {@code + * iotdb.numeric.row.insert.perf.tablet.row.count} to vary concurrency or Tablet batching. * *

Results are written to {@code target/numeric-row-insert-performance.properties}. Percentiles * describe elapsed time per complete round, not individual RPC latency. See the adjacent README for @@ -69,13 +74,15 @@ public class IoTDBNumericRowInsertPerformanceIT { private static final String ENABLED_PROPERTY = PROPERTY_PREFIX + "enabled"; private static final String ROW_COUNT_PROPERTY = PROPERTY_PREFIX + "row.count"; private static final String COLUMN_COUNT_PROPERTY = PROPERTY_PREFIX + "column.count"; + private static final String CLIENT_COUNT_PROPERTY = PROPERTY_PREFIX + "client.count"; private static final String TABLET_ROW_COUNT_PROPERTY = PROPERTY_PREFIX + "tablet.row.count"; private static final String WARMUP_ROUND_COUNT_PROPERTY = PROPERTY_PREFIX + "warmup.round.count"; private static final String ROUND_COUNT_PROPERTY = PROPERTY_PREFIX + "round.count"; private static final String RESULT_PATH_PROPERTY = PROPERTY_PREFIX + "result.path"; private static final int ROW_COUNT = Integer.getInteger(ROW_COUNT_PROPERTY, 10_000); - private static final int COLUMN_COUNT = Integer.getInteger(COLUMN_COUNT_PROPERTY, 8); + private static final int COLUMN_COUNT = Integer.getInteger(COLUMN_COUNT_PROPERTY, 100); + private static final int CLIENT_COUNT = Integer.getInteger(CLIENT_COUNT_PROPERTY, 16); private static final int TABLET_ROW_COUNT = Integer.getInteger(TABLET_ROW_COUNT_PROPERTY, 1); private static final int WARMUP_ROUND_COUNT = Integer.getInteger(WARMUP_ROUND_COUNT_PROPERTY, 1); private static final int ROUND_COUNT = Integer.getInteger(ROUND_COUNT_PROPERTY, 5); @@ -119,27 +126,41 @@ public void testNumericRowInsertPerformanceAgainstTablet() throws Exception { final long[] numericRowElapsedNanos = new long[ROUND_COUNT]; final long[] tabletElapsedNanos = new long[ROUND_COUNT]; - try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { - session.executeNonQueryStatement("CREATE DATABASE " + DATABASE); - session.executeNonQueryStatement("USE " + DATABASE); + final List sessions = new ArrayList<>(CLIENT_COUNT); + final ExecutorService executor = Executors.newFixedThreadPool(CLIENT_COUNT); + try { + final ITableSession setupSession = EnvFactory.getEnv().getTableSessionConnection(); + setupSession.executeNonQueryStatement("CREATE DATABASE " + DATABASE); + setupSession.executeNonQueryStatement("USE " + DATABASE); + sessions.add(setupSession); + for (int client = 1; client < CLIENT_COUNT; client++) { + final ITableSession session = EnvFactory.getEnv().getTableSessionConnection(); + session.executeNonQueryStatement("USE " + DATABASE); + sessions.add(session); + } final int totalRoundCount = WARMUP_ROUND_COUNT + ROUND_COUNT; for (int round = 0; round < totalRoundCount; round++) { - final NumericRowWorkload numericRowWorkload = prepareNumericRowWorkload(session, round); - final TabletWorkload tabletWorkload = prepareTabletWorkload(session, round); + final List numericRowWorkloads = + prepareNumericRowWorkloads(setupSession, round); + final List tabletWorkloads = prepareTabletWorkloads(setupSession, round); final long numericRowNanos; final long tabletNanos; // Alternate execution order so one path does not consistently receive a warmer server. if ((round & 1) == 0) { - numericRowNanos = executeNumericRowWorkload(session, numericRowWorkload); - tabletNanos = executeTabletWorkload(session, tabletWorkload); + numericRowNanos = executeNumericRowWorkload(sessions, numericRowWorkloads, executor); + tabletNanos = executeTabletWorkload(sessions, tabletWorkloads, executor); } else { - tabletNanos = executeTabletWorkload(session, tabletWorkload); - numericRowNanos = executeNumericRowWorkload(session, numericRowWorkload); + tabletNanos = executeTabletWorkload(sessions, tabletWorkloads, executor); + numericRowNanos = executeNumericRowWorkload(sessions, numericRowWorkloads, executor); + } + for (NumericRowWorkload workload : numericRowWorkloads) { + assertRowCount(setupSession, workload.tableName); + } + for (TabletWorkload workload : tabletWorkloads) { + assertRowCount(setupSession, workload.tableName); } - assertRowCount(session, numericRowWorkload.tableName); - assertRowCount(session, tabletWorkload.tableName); if (round >= WARMUP_ROUND_COUNT) { final int measuredRound = round - WARMUP_ROUND_COUNT; @@ -147,79 +168,138 @@ public void testNumericRowInsertPerformanceAgainstTablet() throws Exception { tabletElapsedNanos[measuredRound] = tabletNanos; } } + } finally { + executor.shutdownNow(); + for (ITableSession session : sessions) { + session.close(); + } } final BenchmarkResult result = new BenchmarkResult(numericRowElapsedNanos, tabletElapsedNanos); writeResult(result); } - private static NumericRowWorkload prepareNumericRowWorkload(ITableSession session, int round) - throws Exception { - final String tableName = "numeric_row_" + round; - createTable(session, tableName); - final long startTimestamp = (long) round * ROW_COUNT; - final NumericRow[] rows = new NumericRow[ROW_COUNT]; - final String[] tagValues = new String[0]; - for (int row = 0; row < ROW_COUNT; row++) { - rows[row] = - new NumericRow( - tableName, - startTimestamp + row, - columnNames, - dataTypes, - columnCategories, - createRawValues(startTimestamp + row), - tagValues); + private static List prepareNumericRowWorkloads( + ITableSession session, int round) throws Exception { + final List workloads = new ArrayList<>(CLIENT_COUNT); + for (int client = 0; client < CLIENT_COUNT; client++) { + final String tableName = "numeric_row_" + round + "_" + client; + createTable(session, tableName); + final long startTimestamp = ((long) round * CLIENT_COUNT + client) * ROW_COUNT; + final NumericRow[] rows = new NumericRow[ROW_COUNT]; + final String[] tagValues = new String[0]; + for (int row = 0; row < ROW_COUNT; row++) { + rows[row] = + new NumericRow( + tableName, + startTimestamp + row, + columnNames, + dataTypes, + columnCategories, + createRawValues(startTimestamp + row), + tagValues); + } + workloads.add(new NumericRowWorkload(tableName, rows)); } - return new NumericRowWorkload(tableName, rows); + return workloads; } - private static TabletWorkload prepareTabletWorkload(ITableSession session, int round) + private static List prepareTabletWorkloads(ITableSession session, int round) throws Exception { - final String tableName = "tablet_" + round; - createTable(session, tableName); - final long startTimestamp = (long) round * ROW_COUNT; - final List tablets = new ArrayList<>((ROW_COUNT - 1) / TABLET_ROW_COUNT + 1); - for (int rowStart = 0; rowStart < ROW_COUNT; ) { - final int rowCount = Math.min(TABLET_ROW_COUNT, ROW_COUNT - rowStart); - final Tablet tablet = - new Tablet(tableName, columnNames, dataTypes, columnCategories, rowCount); - for (int row = 0; row < rowCount; row++) { - final int rowIndex = tablet.getRowSize(); - final long timestamp = startTimestamp + rowStart + row; - final long[] values = createRawValues(timestamp); - tablet.addTimestamp(rowIndex, timestamp); - for (int column = 0; column < COLUMN_COUNT; column++) { - tablet.addValue( - columnNames.get(column), - rowIndex, - toTabletValue(dataTypes.get(column), values[column])); + final List workloads = new ArrayList<>(CLIENT_COUNT); + for (int client = 0; client < CLIENT_COUNT; client++) { + final String tableName = "tablet_" + round + "_" + client; + createTable(session, tableName); + final long startTimestamp = ((long) round * CLIENT_COUNT + client) * ROW_COUNT; + final List tablets = new ArrayList<>((ROW_COUNT - 1) / TABLET_ROW_COUNT + 1); + for (int rowStart = 0; rowStart < ROW_COUNT; ) { + final int rowCount = Math.min(TABLET_ROW_COUNT, ROW_COUNT - rowStart); + final Tablet tablet = + new Tablet(tableName, columnNames, dataTypes, columnCategories, rowCount); + for (int row = 0; row < rowCount; row++) { + final int rowIndex = tablet.getRowSize(); + final long timestamp = startTimestamp + rowStart + row; + final long[] values = createRawValues(timestamp); + tablet.addTimestamp(rowIndex, timestamp); + for (int column = 0; column < COLUMN_COUNT; column++) { + tablet.addValue( + columnNames.get(column), + rowIndex, + toTabletValue(dataTypes.get(column), values[column])); + } } + tablets.add(tablet); + rowStart += rowCount; } - tablets.add(tablet); - rowStart += rowCount; + workloads.add(new TabletWorkload(tableName, tablets)); } - return new TabletWorkload(tableName, tablets); + return workloads; } private static long executeNumericRowWorkload( - ITableSession session, NumericRowWorkload numericRowWorkload) throws Exception { - final long startNanos = System.nanoTime(); - for (final NumericRow row : numericRowWorkload.rows) { - session.insert(row); - } - return System.nanoTime() - startNanos; + List sessions, List workloads, ExecutorService executor) + throws Exception { + return executeConcurrently( + sessions, + workloads, + executor, + (session, workload) -> { + for (NumericRow row : workload.rows) { + session.insert(row); + } + }); } - private static long executeTabletWorkload(ITableSession session, TabletWorkload tabletWorkload) + private static long executeTabletWorkload( + List sessions, List workloads, ExecutorService executor) throws Exception { + return executeConcurrently( + sessions, + workloads, + executor, + (session, workload) -> { + for (Tablet tablet : workload.tablets) { + session.insert(tablet); + } + }); + } + + private static long executeConcurrently( + List sessions, + List workloads, + ExecutorService executor, + WorkloadWriter writer) + throws Exception { + final CountDownLatch ready = new CountDownLatch(CLIENT_COUNT); + final CountDownLatch start = new CountDownLatch(1); + final List> futures = new ArrayList<>(CLIENT_COUNT); + for (int client = 0; client < CLIENT_COUNT; client++) { + final ITableSession session = sessions.get(client); + final T workload = workloads.get(client); + futures.add( + executor.submit( + () -> { + ready.countDown(); + start.await(); + writer.write(session, workload); + return null; + })); + } + ready.await(); final long startNanos = System.nanoTime(); - for (final Tablet tablet : tabletWorkload.tablets) { - session.insert(tablet); + start.countDown(); + for (Future future : futures) { + future.get(); } return System.nanoTime() - startNanos; } + @FunctionalInterface + private interface WorkloadWriter { + + void write(ITableSession session, T workload) throws Exception; + } + private static void createTable(ITableSession session, String tableName) throws Exception { final StringBuilder statement = new StringBuilder("CREATE TABLE "); statement.append(tableName).append('('); @@ -301,6 +381,7 @@ private static List createDataTypes() { private static void assertValidConfiguration() { Assert.assertTrue(ROW_COUNT > 0); Assert.assertTrue(COLUMN_COUNT > 0); + Assert.assertTrue(CLIENT_COUNT > 0); Assert.assertTrue(TABLET_ROW_COUNT > 0); Assert.assertTrue(WARMUP_ROUND_COUNT >= 0); Assert.assertTrue(ROUND_COUNT > 0); @@ -318,17 +399,18 @@ private static void writeResult(BenchmarkResult result) throws IOException { Integer.toString(Runtime.getRuntime().availableProcessors())); properties.setProperty( "client.max.heap.bytes", Long.toString(Runtime.getRuntime().maxMemory())); - properties.setProperty("client.count", "1"); + properties.setProperty("client.count", Integer.toString(CLIENT_COUNT)); properties.setProperty("row.count", Integer.toString(ROW_COUNT)); properties.setProperty("column.count", Integer.toString(COLUMN_COUNT)); properties.setProperty("tablet.row.count", Integer.toString(TABLET_ROW_COUNT)); properties.setProperty("warmup.round.count", Integer.toString(WARMUP_ROUND_COUNT)); properties.setProperty("round.count", Integer.toString(ROUND_COUNT)); properties.setProperty("column.types", join(dataTypes)); - properties.setProperty("numeric.row.requests.per.round", Integer.toString(ROW_COUNT)); + properties.setProperty( + "numeric.row.requests.per.round", Integer.toString(CLIENT_COUNT * ROW_COUNT)); properties.setProperty( "insert.tablet.requests.per.round", - Integer.toString((ROW_COUNT - 1) / TABLET_ROW_COUNT + 1)); + Integer.toString(CLIENT_COUNT * ((ROW_COUNT - 1) / TABLET_ROW_COUNT + 1))); properties.setProperty("numeric.row.round.elapsed.nanos", join(result.numericRowElapsedNanos)); properties.setProperty("insert.tablet.round.elapsed.nanos", join(result.tabletElapsedNanos)); properties.setProperty( @@ -430,7 +512,7 @@ private long numericRowP95Nanos() { } private double numericRowRowsPerSecond() { - return ROW_COUNT * 1_000_000_000.0D / numericRowP50Nanos(); + return CLIENT_COUNT * ROW_COUNT * 1_000_000_000.0D / numericRowP50Nanos(); } private double numericRowValuesPerSecond() { @@ -446,7 +528,7 @@ private long tabletP95Nanos() { } private double tabletRowsPerSecond() { - return ROW_COUNT * 1_000_000_000.0D / tabletP50Nanos(); + return CLIENT_COUNT * ROW_COUNT * 1_000_000_000.0D / tabletP50Nanos(); } private double tabletValuesPerSecond() { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index 716d14fa80296..835cf924777c0 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -2890,18 +2890,49 @@ public void insertRelationalRecord(NumericRow row) List columnCategories = row.getColumnCategories(); long[] values = row.getValues(); String[] tagValues = row.getTagValues(); + if (measurements == null + || types == null + || columnCategories == null + || values == null + || tagValues == null + || measurements.size() != types.size() + || measurements.size() != columnCategories.size()) { + throw new IllegalArgumentException( + SessionMessages + .EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233); + } + // Keep wire metadata aligned with the compact payload: TAG columns precede FIELD columns. + List rpcMeasurements = new ArrayList<>(measurements.size()); + List rpcTypes = new ArrayList<>(types.size()); + List rpcColumnCategories = new ArrayList<>(columnCategories.size()); + for (int pass = 0; pass < 2; pass++) { + ColumnCategory category = pass == 0 ? ColumnCategory.TAG : ColumnCategory.FIELD; + for (int i = 0; i < columnCategories.size(); i++) { + if (columnCategories.get(i) == category) { + rpcMeasurements.add(measurements.get(i)); + rpcTypes.add(types.get(i)); + rpcColumnCategories.add(category); + } + } + } + if (rpcMeasurements.size() != measurements.size()) { + throw new IllegalArgumentException( + SessionMessages + .EXCEPTION_NUMERIC_ROWS_SUPPORT_NUMERIC_FIELD_COLUMNS_AND_STRING_TAG_COLUMNS_ONLY_87159FFA); + } ByteBuffer valueBuffer = - SessionUtils.getValueBuffer(types, values, measurements, columnCategories, tagValues); + SessionUtils.getValueBuffer( + rpcTypes, values, rpcMeasurements, rpcColumnCategories, tagValues); TSInsertRecordReq request = new TSInsertRecordReq(); request.setPrefixPath(tableName); request.setTimestamp(timestamp); - request.setMeasurements(measurements); + request.setMeasurements(rpcMeasurements); request.setValues(valueBuffer); request.setIsWriteToTable(true); request.setIsNumericRow(true); request.setIsAligned(true); request.setColumnCategoryies( - columnCategories.stream() + rpcColumnCategories.stream() .map(category -> (byte) category.ordinal()) .collect(Collectors.toList())); String[] deviceSegments = new String[tagValues.length + 1]; diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java index 0d838b6925449..40fa0fae41ff7 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java @@ -211,13 +211,20 @@ public static ByteBuffer getValueBuffer( ByteBuffer buffer = ByteBuffer.allocate(length); int fieldIndex = 0; int tagIndex = 0; + for (TSDataType type : types) { + ReadWriteIOUtils.write(type, buffer); + } + tagIndex = 0; for (int i = 0; i < types.size(); i++) { TSDataType type = types.get(i); - ReadWriteIOUtils.write(type, buffer); - if (categories.get(i) == ColumnCategory.TAG) { - // A length of -1 represents a null TAG; an empty TAG has length zero. + if (categories.get(i) == ColumnCategory.TAG && type == TSDataType.STRING) { ReadWriteIOUtils.write(tagValues[tagIndex++], buffer); - } else { + } + } + fieldIndex = 0; + for (int i = 0; i < types.size(); i++) { + TSDataType type = types.get(i); + if (categories.get(i) == ColumnCategory.FIELD) { long value = values[fieldIndex++]; switch (type) { case BOOLEAN -> buffer.put((byte) (value == 0 ? 0 : 1)); diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java index 88280ac8b4b9d..9f79ee200d91f 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java @@ -194,23 +194,47 @@ public void testNumericTableRowRejectsNonFieldColumnsBeforeRpc() throws Exceptio public void testNumericTableRowWithTagsRequest() throws Exception { // The NumericRow entry point must forward the TAG payload and metadata with numeric flags. ITableSession tableSession = new TableSession((Session) session); - List names = Arrays.asList("tag", "enabled"); - List types = Arrays.asList(TSDataType.STRING, TSDataType.BOOLEAN); - List categories = Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD); + List names = Arrays.asList("enabled", "tag", "value", "region"); + List types = + Arrays.asList(TSDataType.BOOLEAN, TSDataType.STRING, TSDataType.DOUBLE, TSDataType.STRING); + List categories = + Arrays.asList( + ColumnCategory.FIELD, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.TAG); + List rpcNames = Arrays.asList("tag", "region", "enabled", "value"); + List rpcTypes = + Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.DOUBLE); + List rpcCategories = + Arrays.asList( + ColumnCategory.TAG, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD); tableSession.insert( new NumericRow( - "readings", 123L, names, types, categories, new long[] {1}, new String[] {"device"})); + "readings", + 123L, + names, + types, + categories, + new long[] {1, Double.doubleToRawLongBits(2.5)}, + new String[] {"device", "cn"})); ArgumentCaptor captor = ArgumentCaptor.forClass(TSInsertRecordReq.class); Mockito.verify(sessionConnection).insertRecord(captor.capture()); TSInsertRecordReq request = captor.getValue(); Assert.assertTrue(request.isIsNumericRow()); Assert.assertTrue(request.isIsWriteToTable()); assertEquals( - Arrays.asList((byte) ColumnCategory.TAG.ordinal(), (byte) ColumnCategory.FIELD.ordinal()), + Arrays.asList( + (byte) ColumnCategory.TAG.ordinal(), + (byte) ColumnCategory.TAG.ordinal(), + (byte) ColumnCategory.FIELD.ordinal(), + (byte) ColumnCategory.FIELD.ordinal()), request.getColumnCategoryies()); + assertEquals(rpcNames, request.getMeasurements()); assertEquals( SessionUtils.getValueBuffer( - types, new long[] {1}, names, categories, new String[] {"device"}), + rpcTypes, + new long[] {1, Double.doubleToRawLongBits(2.5)}, + rpcNames, + rpcCategories, + new String[] {"device", "cn"}), request.bufferForValues()); } diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java index d2c24a4da8e1b..dfb1ef4a2bf76 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java @@ -77,14 +77,17 @@ public void testNumericRowEncodingPreservesRawBits() throws IoTDBConnectionExcep Collections.nCopies(8, ColumnCategory.FIELD), new String[0]); ByteBuffer expected = ByteBuffer.allocate(46); - expected.put(TSDataType.BOOLEAN.serialize()).put((byte) 0); - expected.put(TSDataType.BOOLEAN.serialize()).put((byte) 1); - expected.put(TSDataType.INT32.serialize()).putInt(Integer.MIN_VALUE); - expected.put(TSDataType.INT64.serialize()).putLong(Long.MIN_VALUE); - expected.put(TSDataType.FLOAT.serialize()).putInt(0x7fc01234); - expected.put(TSDataType.DOUBLE.serialize()).putLong(0x7ff8000000001234L); - expected.put(TSDataType.DATE.serialize()).putInt(20260910); - expected.put(TSDataType.TIMESTAMP.serialize()).putLong(Long.MAX_VALUE); + for (TSDataType type : types) { + expected.put(type.serialize()); + } + expected.put((byte) 0); + expected.put((byte) 1); + expected.putInt(Integer.MIN_VALUE); + expected.putLong(Long.MIN_VALUE); + expected.putInt(0x7fc01234); + expected.putLong(0x7ff8000000001234L); + expected.putInt(20260910); + expected.putLong(Long.MAX_VALUE); expected.flip(); Assert.assertEquals(expected, actual); } @@ -154,16 +157,14 @@ public void testNumericRowWithSeparateTags() throws Exception { Arrays.asList("a", "d", "b", "enabled", "c"), categories, new String[] {"设备😀", null, ""}); - Assert.assertEquals(TSDataType.STRING.serialize(), buffer.get()); + for (TSDataType type : types) { + Assert.assertEquals(type.serialize(), buffer.get()); + } Assert.assertEquals("设备😀", ReadWriteIOUtils.readString(buffer)); - Assert.assertEquals(TSDataType.DOUBLE.serialize(), buffer.get()); + Assert.assertNull(ReadWriteIOUtils.readString(buffer)); + Assert.assertEquals("", ReadWriteIOUtils.readString(buffer)); Assert.assertEquals(0x7ff8000000001234L, buffer.getLong()); - Assert.assertEquals(TSDataType.STRING.serialize(), buffer.get()); - Assert.assertEquals(-1, buffer.getInt()); - Assert.assertEquals(TSDataType.BOOLEAN.serialize(), buffer.get()); Assert.assertEquals(1, buffer.get()); - Assert.assertEquals(TSDataType.STRING.serialize(), buffer.get()); - Assert.assertEquals(0, buffer.getInt()); Assert.assertFalse(buffer.hasRemaining()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java index 0303eed2777f0..3aac7cfaccb2d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java @@ -305,7 +305,6 @@ public static InsertBaseStatement createStatement(TSInsertRecordReq insertRecord } InsertNumericRowStatement numeric = (InsertNumericRowStatement) insertStatement; numeric.setTime(insertRecordReq.getTimestamp()); - numeric.fillValues(insertRecordReq.values); insertStatement.setAligned(true); } else { InsertRowStatement row = (InsertRowStatement) insertStatement; @@ -327,7 +326,9 @@ public static InsertBaseStatement createStatement(TSInsertRecordReq insertRecord TsTableColumnCategory.fromTsFileColumnCategory( ColumnCategory.values()[insertRecordReq.getColumnCategoryies().get(i).intValue()]); } + insertStatement.setColumnCategories(columnCategories); if (insertRecordReq.isIsNumericRow()) { + ((InsertNumericRowStatement) insertStatement).fillValues(insertRecordReq.values); for (int i = 0; i < columnCategories.length; i++) { TSDataType type = insertStatement.getDataType(i); if (!(columnCategories[i] == TsTableColumnCategory.TAG && type == TSDataType.STRING) @@ -339,7 +340,6 @@ public static InsertBaseStatement createStatement(TSInsertRecordReq insertRecord } } } - insertStatement.setColumnCategories(columnCategories); } PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return insertStatement; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertNumericRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertNumericRowNode.java new file mode 100644 index 0000000000000..0f8a78ed91447 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertNumericRowNode.java @@ -0,0 +1,590 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write; + +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.IPlanVisitor; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.utils.TimePartitionUtils; +import org.apache.iotdb.db.exception.DataTypeInconsistentException; +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; +import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; +import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunkGroup; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; +import org.apache.iotdb.db.utils.NumericRowValues; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * A numeric table row with its own plan type and wire format. FIELDs retain raw long bits and TAGs + * retain strings through consensus and WAL recovery. + */ +public class RelationalInsertNumericRowNode extends AbstractInsertRowNode { + private long time; + private NumericRowValues numericValues; + + public RelationalInsertNumericRowNode(PlanNodeId id) { + super(id); + isAligned = true; + } + + public RelationalInsertNumericRowNode( + PlanNodeId id, + PartialPath table, + String[] measurements, + TSDataType[] dataTypes, + MeasurementSchema[] schemas, + long time, + NumericRowValues values, + TsTableColumnCategory[] categories) { + super(id, table, true, measurements, dataTypes); + this.measurementSchemas = schemas; + this.time = time; + this.numericValues = values; + setColumnCategories(categories); + } + + @Override + public PlanNodeType getType() { + return PlanNodeType.RELATIONAL_INSERT_NUMERIC_ROW; + } + + @Override + public R accept(IPlanVisitor visitor, C context) { + return ((PlanVisitor) visitor).visitRelationalInsertNumericRow(this, context); + } + + @Override + public InsertNode mergeInsertNode(List nodes) { + // One numeric row has exactly one device and timestamp, so it cannot be split into fragments. + // InsertNode.merge returns the sole node before reaching this method. + throw new UnsupportedOperationException(); + } + + @Override + public long getTime() { + return time; + } + + public void setTime(long time) { + this.time = time; + } + + @Override + public long getMinTime() { + return time; + } + + public NumericRowValues getNumericValues() { + return numericValues; + } + + public long[] getValues() { + return numericValues.getValues(); + } + + public void setNumericValues(NumericRowValues values) { + numericValues = values; + deviceID = null; + } + + @Override + public Object getValuesForMemoryAccounting() { + return numericValues; + } + + @Override + public int getValueCount() { + return numericValues == null ? 0 : numericValues.size(); + } + + @Override + public boolean isNull(int index) { + return index < 0 || index >= getValueCount() || numericValues.isNull(index); + } + + @Override + public Object getValue(int index) { + return isNull(index) ? null : numericValues.getObject(index, getDataType(index)); + } + + public String getTableName() { + return targetPath.getFullPath(); + } + + @Override + public IDeviceID getDeviceID() { + if (deviceID == null) { + int tagCount = 0; + for (int i = 0; columnCategories != null && i < columnCategories.length; i++) { + if (columnCategories[i] == TsTableColumnCategory.TAG && isPresent(i)) { + tagCount++; + } + } + String[] segments = new String[tagCount + 1]; + segments[0] = getTableName(); + int tagIndex = 1; + for (int i = 0; columnCategories != null && i < columnCategories.length; i++) { + if (columnCategories[i] == TsTableColumnCategory.TAG && isPresent(i)) { + segments[tagIndex++] = numericValues.getTag(i); + } + } + deviceID = IDeviceID.Factory.DEFAULT_FACTORY.create(segments); + } + return deviceID; + } + + private boolean isPresent(int index) { + return measurements != null + && index >= 0 + && index < measurements.length + && measurements[index] != null + && index < getValueCount() + && columnCategories != null + && index < columnCategories.length + && columnCategories[index] != null + && (measurementSchemas == null + || index < measurementSchemas.length && measurementSchemas[index] != null); + } + + @Override + protected int getValidMeasurementNumber() { + int count = 0; + for (int i = 0; i < getValueCount(); i++) { + if (isPresent(i)) { + count++; + } + } + return count; + } + + @Override + public int getValidMeasurementNumber(boolean fieldOnly) { + int count = 0; + for (int i = 0; i < getValueCount(); i++) { + if (isPresent(i) && isValidMeasurement(i, fieldOnly)) { + count++; + } + } + return count; + } + + @Override + public void markFailedMeasurement(int index) { + if (!isPresent(index)) { + return; + } + measurements[index] = null; + if (dataTypes != null && index < dataTypes.length) { + dataTypes[index] = null; + } + numericValues.setNull(index); + deviceID = null; + measurementColumnCnt = -1; + } + + @Override + public List splitByPartition(IAnalysis analysis) { + TTimePartitionSlot slot = TimePartitionUtils.getTimePartitionSlot(time); + dataRegionReplicaSet = + analysis + .getDataPartitionInfo() + .getDataRegionReplicaSetForWriting(getDeviceID(), slot, analysis.getDatabaseName()); + analysis.setRedirectNodeList( + Collections.singletonList( + dataRegionReplicaSet.getDataNodeLocations().get(0).getClientRpcEndPoint())); + return Collections.singletonList(this); + } + + @Override + public void addChild(PlanNode child) { + /* A single-row write has no children. */ + } + + @Override + public int allowedChildCount() { + return NO_CHILD_ALLOWED; + } + + @Override + public List getOutputColumnNames() { + return null; + } + + @Override + public PlanNode clone() { + throw new UnsupportedOperationException(); + } + + @Override + public long getLastCacheTimestamp() { + return time; + } + + @Override + public boolean hasLastCacheValue(int index) { + return isPresent(index) && !isNull(index) && isWritableFieldMeasurement(index); + } + + @Override + public TimeValuePair getLastCacheValue(int index) { + return hasLastCacheValue(index) + ? new TimeValuePair(time, TsPrimitiveType.getByType(getDataType(index), getValue(index))) + : null; + } + + @Override + public void updateLastCache(String database) { + TableDeviceSchemaCache.getInstance() + .updateLastCacheIfExists(database, getDeviceID(), measurements, measurementSchemas, this); + } + + @Override + public void checkDataType(AbstractMemTable memTable) throws DataTypeInconsistentException { + IWritableMemChunkGroup group = memTable.getWritableMemChunkGroup(getDeviceID()); + if (group != null) { + group.checkDataType(this); + } + } + + private int bodySize() { + // Fixed body fields: timestamp, table name, number of valid columns, and schema-present flag. + int size = + Long.BYTES + ReadWriteIOUtils.sizeToWrite(getTableName()) + Integer.BYTES + Byte.BYTES; + for (int i = 0; i < getValueCount(); i++) { + if (!isPresent(i)) { + continue; + } + // Column identity: either the measurement name or the serialized measurement schema. + size += + measurementSchemas == null + ? ReadWriteIOUtils.sizeToWrite(measurements[i]) + : measurementSchemas[i].serializedSize(); + // Column data type, column category, and null marker. + size += 3 * Byte.BYTES; + if (!isNull(i)) { + // Non-null TAGs carry a length-prefixed String; numeric FIELDs carry one long value. + size += + columnCategories[i] == TsTableColumnCategory.TAG + ? ReadWriteIOUtils.sizeToWrite(numericValues.getTag(i)) + : Long.BYTES; + } + } + return size; + } + + @Override + protected int serializedAttributesSize() { + return PlanNodeType.BYTES + bodySize(); + } + + @Override + public int serializedSize() { + return PlanNodeType.BYTES + Long.BYTES + bodySize(); + } + + @Override + protected void serializeAttributes(ByteBuffer buffer) { + getType().serialize(buffer); + serializeBody(buffer); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + getType().serialize(stream); + serializeBody(stream); + } + + @Override + public void serializeToWAL(IWALByteBufferView buffer) { + serializeToWAL(buffer, getEncodedSearchIndex()); + } + + @Override + public void serializeToWAL(IWALByteBufferView buffer, long encodedSearchIndex) { + buffer.putShort(getType().getNodeType()); + buffer.putLong(encodedSearchIndex); + serializeBody(buffer); + } + + private void serializeBody(ByteBuffer buffer) { + ReadWriteIOUtils.write(time, buffer); + ReadWriteIOUtils.write(getTableName(), buffer); + ReadWriteIOUtils.write(getValidMeasurementNumber(), buffer); + ReadWriteIOUtils.write((byte) (measurementSchemas == null ? 0 : 1), buffer); + for (int i = 0; i < getValueCount(); i++) { + if (!isPresent(i)) { + continue; + } + if (measurementSchemas == null) { + ReadWriteIOUtils.write(measurements[i], buffer); + } else { + measurementSchemas[i].serializeTo(buffer); + } + ReadWriteIOUtils.write( + getDataType(i) == null ? (byte) -1 : getDataType(i).serialize(), buffer); + ReadWriteIOUtils.write(columnCategories[i].getCategory(), buffer); + ReadWriteIOUtils.write((byte) (isNull(i) ? 1 : 0), buffer); + if (!isNull(i)) { + if (columnCategories[i] == TsTableColumnCategory.TAG) { + ReadWriteIOUtils.write(numericValues.getTag(i), buffer); + } else { + ReadWriteIOUtils.write(numericValues.get(i), buffer); + } + } + } + } + + private void serializeBody(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(time, stream); + ReadWriteIOUtils.write(getTableName(), stream); + ReadWriteIOUtils.write(getValidMeasurementNumber(), stream); + ReadWriteIOUtils.write((byte) (measurementSchemas == null ? 0 : 1), stream); + for (int i = 0; i < getValueCount(); i++) { + if (!isPresent(i)) { + continue; + } + if (measurementSchemas == null) { + ReadWriteIOUtils.write(measurements[i], stream); + } else { + measurementSchemas[i].serializeTo(stream); + } + ReadWriteIOUtils.write( + getDataType(i) == null ? (byte) -1 : getDataType(i).serialize(), stream); + ReadWriteIOUtils.write(columnCategories[i].getCategory(), stream); + ReadWriteIOUtils.write((byte) (isNull(i) ? 1 : 0), stream); + if (!isNull(i)) { + if (columnCategories[i] == TsTableColumnCategory.TAG) { + ReadWriteIOUtils.write(numericValues.getTag(i), stream); + } else { + ReadWriteIOUtils.write(numericValues.get(i), stream); + } + } + } + } + + private void serializeBody(IWALByteBufferView buffer) { + WALWriteUtils.write(time, buffer); + WALWriteUtils.write(getTableName(), buffer); + WALWriteUtils.write(getValidMeasurementNumber(), buffer); + WALWriteUtils.write((byte) (measurementSchemas == null ? 0 : 1), buffer); + for (int i = 0; i < getValueCount(); i++) { + if (!isPresent(i)) { + continue; + } + if (measurementSchemas == null) { + WALWriteUtils.write(measurements[i], buffer); + } else { + WALWriteUtils.write(measurementSchemas[i], buffer); + } + WALWriteUtils.write(getDataType(i) == null ? (byte) -1 : getDataType(i).serialize(), buffer); + WALWriteUtils.write(columnCategories[i].getCategory(), buffer); + WALWriteUtils.write((byte) (isNull(i) ? 1 : 0), buffer); + if (!isNull(i)) { + if (columnCategories[i] == TsTableColumnCategory.TAG) { + WALWriteUtils.write(numericValues.getTag(i), buffer); + } else { + WALWriteUtils.write(numericValues.get(i), buffer); + } + } + } + } + + private static RelationalInsertNumericRowNode deserializeBody(ByteBuffer buffer) { + RelationalInsertNumericRowNode node = new RelationalInsertNumericRowNode(new PlanNodeId("")); + node.time = buffer.getLong(); + node.targetPath = new PartialPath(ReadWriteIOUtils.readString(buffer), false); + int count = buffer.getInt(); + boolean hasSchemas = buffer.get() != 0; + node.measurements = new String[count]; + node.dataTypes = new TSDataType[count]; + node.measurementSchemas = hasSchemas ? new MeasurementSchema[count] : null; + TsTableColumnCategory[] categories = new TsTableColumnCategory[count]; + String[] tagPayload = new String[count]; + long[] valuePayload = new long[count]; + boolean[] nonNull = new boolean[count]; + for (int i = 0; i < count; i++) { + if (hasSchemas) { + node.measurementSchemas[i] = MeasurementSchema.deserializeFrom(buffer); + node.measurements[i] = node.measurementSchemas[i].getMeasurementName(); + } else { + node.measurements[i] = ReadWriteIOUtils.readString(buffer); + } + byte type = buffer.get(); + node.dataTypes[i] = type == -1 ? null : TSDataType.deserialize(type); + categories[i] = TsTableColumnCategory.deserialize(buffer); + nonNull[i] = buffer.get() == 0; + if (nonNull[i]) { + if (categories[i] == TsTableColumnCategory.TAG && node.dataTypes[i] == TSDataType.STRING) { + tagPayload[i] = ReadWriteIOUtils.readString(buffer); + } else { + if (!NumericRowValues.supports(node.dataTypes[i]) + || categories[i] != TsTableColumnCategory.FIELD) { + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE + node.dataTypes[i]); + } + valuePayload[i] = buffer.getLong(); + } + } + } + node.numericValues = buildNumericValues(categories, tagPayload, valuePayload, nonNull); + node.setColumnCategories(categories); + return node; + } + + public static RelationalInsertNumericRowNode deserializeFromWAL(ByteBuffer buffer) { + long searchIndex = buffer.getLong(); + RelationalInsertNumericRowNode node = deserializeBody(buffer); + node.setSearchIndexFromWAL(searchIndex); + return node; + } + + private static RelationalInsertNumericRowNode deserializeBody(DataInputStream stream) + throws IOException { + RelationalInsertNumericRowNode node = new RelationalInsertNumericRowNode(new PlanNodeId("")); + node.time = stream.readLong(); + node.targetPath = new PartialPath(ReadWriteIOUtils.readString(stream), false); + int count = stream.readInt(); + boolean hasSchemas = stream.readByte() != 0; + node.measurements = new String[count]; + node.dataTypes = new TSDataType[count]; + node.measurementSchemas = hasSchemas ? new MeasurementSchema[count] : null; + TsTableColumnCategory[] categories = new TsTableColumnCategory[count]; + String[] tagPayload = new String[count]; + long[] valuePayload = new long[count]; + boolean[] nonNull = new boolean[count]; + for (int i = 0; i < count; i++) { + if (hasSchemas) { + node.measurementSchemas[i] = MeasurementSchema.deserializeFrom(stream); + node.measurements[i] = node.measurementSchemas[i].getMeasurementName(); + } else { + node.measurements[i] = ReadWriteIOUtils.readString(stream); + } + byte type = stream.readByte(); + node.dataTypes[i] = type == -1 ? null : TSDataType.deserialize(type); + categories[i] = TsTableColumnCategory.deserialize(stream); + nonNull[i] = stream.readByte() == 0; + if (nonNull[i]) { + if (categories[i] == TsTableColumnCategory.TAG && node.dataTypes[i] == TSDataType.STRING) { + tagPayload[i] = ReadWriteIOUtils.readString(stream); + } else { + if (!NumericRowValues.supports(node.dataTypes[i]) + || categories[i] != TsTableColumnCategory.FIELD) { + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE + node.dataTypes[i]); + } + valuePayload[i] = stream.readLong(); + } + } + } + node.numericValues = buildNumericValues(categories, tagPayload, valuePayload, nonNull); + node.setColumnCategories(categories); + return node; + } + + private static NumericRowValues buildNumericValues( + TsTableColumnCategory[] categories, + String[] tagPayload, + long[] valuePayload, + boolean[] nonNull) { + int tagCount = 0; + while (tagCount < categories.length && categories[tagCount] == TsTableColumnCategory.TAG) { + tagCount++; + } + for (int i = tagCount; i < categories.length; i++) { + if (categories[i] == TsTableColumnCategory.TAG) { + throw new IllegalArgumentException(); + } + } + NumericRowValues values = new NumericRowValues(tagCount, categories.length - tagCount); + for (int i = 0; i < categories.length; i++) { + if (!nonNull[i]) { + values.setNull(i); + } else if (i < tagCount) { + values.setTag(i, tagPayload[i]); + } else { + values.setRawValue(i, valuePayload[i]); + } + } + return values; + } + + public static RelationalInsertNumericRowNode deserializeFromWAL(DataInputStream stream) + throws IOException { + long searchIndex = stream.readLong(); + RelationalInsertNumericRowNode node = deserializeBody(stream); + node.setSearchIndexFromWAL(searchIndex); + return node; + } + + public static RelationalInsertNumericRowNode deserialize(ByteBuffer buffer) { + RelationalInsertNumericRowNode node = deserializeBody(buffer); + node.setPlanNodeId(PlanNodeId.deserialize(buffer)); + return node; + } + + @Override + public boolean equals(Object other) { + return other instanceof RelationalInsertNumericRowNode node + && super.equals(other) + && time == node.time + && Objects.equals(numericValues, node.numericValues); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), time, numericValues); + } + + @Override + public String toString() { + return "RelationalInsertNumericRowNode{table=" + + targetPath + + ", time=" + + time + + ", values=" + + Arrays.toString(getValues()) + + '}'; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java index db90503d4646a..fe77996fabe98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java @@ -36,9 +36,11 @@ import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.utils.NumericRowValues; +import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.write.UnSupportedDataTypeException; import java.nio.ByteBuffer; import java.util.Arrays; @@ -87,7 +89,6 @@ public int getValueCount() { /** RPC cells carry typed primitive FIELD bits or a length-prefixed STRING TAG. */ public void fillValues(ByteBuffer buffer) throws QueryProcessException { - numericValues = new NumericRowValues(measurements.length); dataTypes = new TSDataType[measurements.length]; for (int i = 0; i < dataTypes.length; i++) { byte type = buffer.get(); @@ -95,7 +96,41 @@ public void fillValues(ByteBuffer buffer) throws QueryProcessException { throw new QueryProcessException(DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE + type); } dataTypes[i] = TSDataType.values()[type]; - numericValues.read(i, dataTypes[i], buffer); + if (columnCategories[i] == TsTableColumnCategory.TAG && dataTypes[i] != TSDataType.STRING) { + throw new IllegalArgumentException( + DataNodeQueryMessages + .QUERY_EXCEPTION_MISSING_OR_INVALID_COLUMN_CATEGORIES_FOR_TABLE_INSERTION_5DF990B9); + } + if (columnCategories[i] == TsTableColumnCategory.FIELD + && !NumericRowValues.supports(dataTypes[i])) { + if (dataTypes[i] == TSDataType.STRING) { + throw new IllegalArgumentException( + DataNodeQueryMessages + .QUERY_EXCEPTION_MISSING_OR_INVALID_COLUMN_CATEGORIES_FOR_TABLE_INSERTION_5DF990B9); + } + throw new UnSupportedDataTypeException( + DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE + dataTypes[i]); + } + } + int tagCount = countTagColumns(columnCategories); + numericValues = new NumericRowValues(tagCount, measurements.length - tagCount); + ColumnCategory[] categories = new ColumnCategory[measurements.length]; + for (int i = 0; i < measurements.length; i++) { + categories[i] = + columnCategories[i] == TsTableColumnCategory.TAG + ? ColumnCategory.TAG + : ColumnCategory.FIELD; + } + numericValues.setColumnCategories(categories); + for (int i = 0; i < dataTypes.length; i++) { + if (columnCategories[i] == TsTableColumnCategory.TAG) { + numericValues.read(i, dataTypes[i], buffer); + } + } + for (int i = 0; i < dataTypes.length; i++) { + if (columnCategories[i] == TsTableColumnCategory.FIELD) { + numericValues.read(i, dataTypes[i], buffer); + } } } @@ -208,37 +243,61 @@ public void insertColumn(int pos, ColumnSchema columnSchema) { for (int i = 0; i < mapping.length; i++) { mapping[i] = i < pos ? i : i == pos ? -1 : i - 1; } - numericValues = numericValues.remap(mapping); + numericValues = numericValues.remap(mapping, countTagColumns(columnCategories)); deviceID = null; } @Override public void swapColumn(int src, int target) { super.swapColumn(src, target); - numericValues.swap(src, target); + int[] newToOld = new int[measurements.length]; + for (int i = 0; i < newToOld.length; i++) { + newToOld[i] = i; + } + newToOld[src] = target; + newToOld[target] = src; + numericValues = numericValues.remap(newToOld, countTagColumns(columnCategories)); deviceID = null; } @Override public void rebuildArraysAfterExpansion(int[] mapping, String[] newMeasurements) { super.rebuildArraysAfterExpansion(mapping, newMeasurements); - numericValues = numericValues.remap(mapping); + numericValues = numericValues.remap(mapping, countTagColumns(columnCategories)); deviceID = null; } @Override protected void subRemoveAttributeColumns(List columnsToKeep) { numericValues = - numericValues.remap(columnsToKeep.stream().mapToInt(Integer::intValue).toArray()); + numericValues.remap( + columnsToKeep.stream().mapToInt(Integer::intValue).toArray(), + countTagColumns(columnCategories)); deviceID = null; } @Override public void setColumnCategories(TsTableColumnCategory[] categories) { super.setColumnCategories(categories); + if (numericValues != null) { + ColumnCategory[] compactCategories = new ColumnCategory[categories.length]; + for (int i = 0; i < categories.length; i++) { + compactCategories[i] = + categories[i] == TsTableColumnCategory.TAG ? ColumnCategory.TAG : ColumnCategory.FIELD; + } + numericValues.setColumnCategories(compactCategories); + } deviceID = null; } + private static int countTagColumns(TsTableColumnCategory[] categories) { + int count = 0; + while (count < categories.length && categories[count] == TsTableColumnCategory.TAG) { + count++; + } + return count; + } + public IDeviceID getTableDeviceID() { if (deviceID == null) { String[] segments = new String[getTagColumnIndices().size() + 1]; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index ba5349edc0e51..1ac6c15488b11 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -1199,8 +1199,8 @@ public Runnable executeCompactFileTimeIndexCache() { return () -> { if (!isReadyForNonReadWriteFunctions()) { return; - } - for (DataRegion dataRegion : dataRegionMap.values()) { + } + for (DataRegion dataRegion : dataRegionMap.values()) { if (dataRegion != null) { dataRegion.compactFileTimeIndexCache(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/NumericRowValues.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/NumericRowValues.java new file mode 100644 index 0000000000000..8b9a03a44ccdf --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/NumericRowValues.java @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.i18n.DataNodeQueryMessages; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.write.UnSupportedDataTypeException; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Objects; + +/** + * Primitive storage for a numeric row. FLOAT and DOUBLE use raw IEEE 754 bits; BOOLEAN uses 0/1. + * TAGs live in a separate String[] and occupy the first logical columns; FIELDs follow them in + * {@code long[]}. Nulls need separate bits because every long bit pattern is a valid numeric value. + */ +public final class NumericRowValues { + private long[] values; + private final BitSet nulls; + private String[] tagValues; + private final int size; + private int tagCount; + + @TestOnly + public NumericRowValues(int size) { + this(0, size); + } + + public NumericRowValues(int tagLength, int valueLength) { + this.values = new long[valueLength]; + this.nulls = new BitSet(tagLength + valueLength); + this.tagValues = tagLength == 0 ? null : new String[tagLength]; + this.size = tagLength + valueLength; + this.tagCount = tagLength; + } + + @TestOnly + public NumericRowValues(long[] values, BitSet nulls) { + this.values = values; + this.nulls = nulls; + this.size = values.length; + this.tagCount = 0; + } + + public int size() { + return size; + } + + public long[] getValues() { + return values; + } + + public String getTag(int index) { + return index < 0 || index >= size || isNull(index) || tagValues == null || index >= tagCount + ? null + : tagValues[index]; + } + + public void setTag(int index, String value) { + if (index < 0 || index >= tagCount) { + throw new IndexOutOfBoundsException(index); + } + if (value == null) { + tagValues[index] = null; + nulls.set(index); + return; + } + tagValues[index] = value; + nulls.clear(index); + } + + public boolean isNull(int index) { + return nulls.get(index); + } + + public void setNull(int index) { + if (index < 0 || index >= size) { + throw new IndexOutOfBoundsException(index); + } + // Null slots have no payload on the wire; keep their in-memory representation canonical too. + if (index >= tagCount) { + values[index - tagCount] = 0; + } + if (tagValues != null && index < tagCount) { + tagValues[index] = null; + } + nulls.set(index); + } + + public long get(int index) { + if (index < 0 || index >= size) { + throw new IndexOutOfBoundsException(index); + } + return index < tagCount ? 0 : values[index - tagCount]; + } + + public void setRawValue(int index, long value) { + if (index < tagCount) { + throw new IllegalArgumentException(); + } + if (index < 0 || index >= size) { + throw new IndexOutOfBoundsException(index); + } + values[index - tagCount] = value; + nulls.clear(index); + } + + public String[] getTagValues() { + return tagValues; + } + + public void setColumnCategories(ColumnCategory[] categories) { + if (categories == null || categories.length != size) { + throw new IllegalArgumentException(); + } + int newTagCount = 0; + while (newTagCount < size && categories[newTagCount] == ColumnCategory.TAG) { + newTagCount++; + } + for (int i = newTagCount; i < size; i++) { + if (categories[i] == ColumnCategory.TAG) { + throw new IllegalArgumentException(); + } + } + if (newTagCount != tagCount) { + throw new IllegalArgumentException(); + } + } + + public static boolean supports(TSDataType type) { + return type != null + && switch (type) { + case BOOLEAN, INT32, INT64, FLOAT, DOUBLE, DATE, TIMESTAMP -> true; + case TEXT, STRING, BLOB, OBJECT, VECTOR, UNKNOWN -> false; + }; + } + + public Object getObject(int index, TSDataType type) { + if (isNull(index)) { + return null; + } + return switch (type) { + case BOOLEAN -> get(index) != 0; + case INT32, DATE -> (int) get(index); + case INT64, TIMESTAMP -> get(index); + case FLOAT -> Float.intBitsToFloat((int) get(index)); + case DOUBLE -> Double.longBitsToDouble(get(index)); + // Binary is required only at existing per-cell APIs (schema, Pipe and subscription). + case STRING -> { + if (index >= tagCount) { + throw new IllegalArgumentException(); + } + yield new Binary(getTag(index), TSFileConfig.STRING_CHARSET); + } + default -> throw unsupported(type); + }; + } + + // Used only at APIs that consume individual objects (last cache, diagnostics and type coercion). + public void setObject(int index, TSDataType type, Object value) { + if (value == null) { + setNull(index); + return; + } + if (type == TSDataType.STRING) { + setTag( + index, + value instanceof Binary binary + ? binary.getStringValue(TSFileConfig.STRING_CHARSET) + : (String) value); + return; + } + if (index < tagCount || index < 0 || index >= size) { + throw new IllegalArgumentException(); + } + values[index - tagCount] = + switch (type) { + case BOOLEAN -> (boolean) value ? 1 : 0; + case INT32, DATE -> (int) value; + case INT64, TIMESTAMP -> (long) value; + case FLOAT -> Float.floatToRawIntBits((float) value); + case DOUBLE -> Double.doubleToRawLongBits((double) value); + default -> throw unsupported(type); + }; + nulls.clear(index); + } + + public boolean cast(int index, TSDataType source, TSDataType target) { + if (isNull(index)) { + return true; + } + if (source == TSDataType.STRING && target == TSDataType.STRING) { + return true; + } + if (!supports(target) || source == null || !target.isCompatible(source)) { + return false; + } + if (source == target) { + return true; + } + setObject(index, target, target.castFromSingleValue(source, getObject(index, source))); + return true; + } + + /** Returns a new primitive row; -1 in the mapping represents an inserted null column. */ + public NumericRowValues remap(int[] newToOld, int newTagCount) { + NumericRowValues result = new NumericRowValues(newTagCount, newToOld.length - newTagCount); + for (int i = 0; i < newToOld.length; i++) { + int old = newToOld[i]; + if (old < 0 || old >= size() || isNull(old)) { + result.setNull(i); + } else if (i < newTagCount) { + if (tagValues != null && old < this.tagCount && tagValues[old] != null) { + result.setTag(i, tagValues[old]); + } + } else if (old >= this.tagCount) { + result.setRawValue(i, get(old)); + } else { + result.setNull(i); + } + } + return result; + } + + public void read(int index, TSDataType type, ByteBuffer buffer) { + if (type == TSDataType.STRING) { + setTag(index, ReadWriteIOUtils.readString(buffer)); + return; + } + if (index < tagCount || index < 0 || index >= size) { + throw new IllegalArgumentException(); + } + values[index - tagCount] = + switch (type) { + case BOOLEAN -> buffer.get() == 0 ? 0 : 1; + case INT32, DATE, FLOAT -> buffer.getInt(); + case INT64, TIMESTAMP, DOUBLE -> buffer.getLong(); + default -> throw unsupported(type); + }; + } + + public long ramBytesUsed() { + return RamUsageEstimator.shallowSizeOfInstance(NumericRowValues.class) + + RamUsageEstimator.sizeOf(values) + + (tagValues == null ? 0 : RamUsageEstimator.sizeOf(tagValues)) + + RamUsageEstimator.shallowSizeOfInstance(BitSet.class) + + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + + (long) nulls.size() / Byte.SIZE; + } + + private static UnSupportedDataTypeException unsupported(TSDataType type) { + return new UnSupportedDataTypeException(DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE + type); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof NumericRowValues row) + || size != row.size + || tagCount != row.tagCount + || !Arrays.equals(values, row.values) + || !nulls.equals(row.nulls)) { + return false; + } + // An allocated array cleared by a partial failure equals an absent all-null TAG array. + for (int i = 0; i < size(); i++) { + if (!Objects.equals(getTag(i), row.getTag(i))) { + return false; + } + } + return true; + } + + @Override + public int hashCode() { + int hash = 31 * (31 * size + tagCount) + Arrays.hashCode(values) + nulls.hashCode(); + for (int i = 0; i < size(); i++) { + hash = 31 * hash + Objects.hashCode(getTag(i)); + } + return hash; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java new file mode 100644 index 0000000000000..0541ecd217319 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java @@ -0,0 +1,524 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.queryengine.plan.parser; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTablePatternParser; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletBatchReqV2; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; +import org.apache.iotdb.db.queryengine.plan.relational.planner.TableModelPlanner; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertNumericRow; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; +import org.apache.iotdb.db.storageengine.dataregion.memtable.AlignedWritableMemChunkGroup; +import org.apache.iotdb.db.storageengine.dataregion.memtable.PrimitiveMemTable; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALInfoEntry; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; +import org.apache.iotdb.db.subscription.broker.consensus.ConsensusLogToTabletConverter; +import org.apache.iotdb.db.utils.NumericRowValues; +import org.apache.iotdb.db.utils.datastructure.AlignedTVList; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TSInsertRecordReq; +import org.apache.iotdb.session.util.SessionUtils; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.TypeFactory; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.lang.reflect.Field; +import java.nio.ByteBuffer; +import java.time.LocalDate; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +/** Exercises one numeric row from the client codec through schema edits, WAL and memtable. */ +public class NumericTableRowTest { + private static final List NAMES = Arrays.asList("b", "i", "l", "f", "d", "date", "ts"); + private static final List TYPES = + Arrays.asList( + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.DATE, + TSDataType.TIMESTAMP); + private static final long[] BITS = { + 1, Integer.MIN_VALUE, Long.MIN_VALUE, 0x7fc01234, 0x7ff8000000001234L, 20260910, Long.MAX_VALUE + }; + + private TSInsertRecordReq request() throws Exception { + return new TSInsertRecordReq() + .setPrefixPath("numeric_table") + .setTimestamp(42) + .setMeasurements(NAMES) + .setValues( + SessionUtils.getValueBuffer( + TYPES, + BITS, + NAMES, + Collections.nCopies(NAMES.size(), ColumnCategory.FIELD), + new String[0])) + .setIsAligned(true) + .setIsWriteToTable(true) + .setIsNumericRow(true) + .setColumnCategoryies( + Collections.nCopies(NAMES.size(), (byte) ColumnCategory.FIELD.ordinal())); + } + + @Test + public void testRpcDecodeAndSchemaChangesRetainPrimitiveStorage() throws Exception { + // Missing TAG insertion, column swaps and partial failure recovery must retain long[] values. + InsertNumericRowStatement statement = + (InsertNumericRowStatement) StatementGenerator.createStatement(request()); + assertArrayEquals(BITS, statement.getNumericValues().getValues()); + assertObjectStorageAbsent(InsertNumericRowStatement.class, statement); + assertTrue(statement.isWriteToTable()); + assertTrue(statement.isAligned()); + assertEquals(StatementType.INSERT_NUMERIC_ROW, statement.getType()); + statement.semanticCheck(); + + statement.insertColumn( + 0, + new ColumnSchema( + "tag", TypeFactory.getType(TSDataType.STRING), false, TsTableColumnCategory.TAG)); + assertTrue(statement.getNumericValues().isNull(0)); + assertNull(statement.getFirstValueOfIndex(0)); + assertEquals(20260910, statement.getFirstValueOfIndex(6)); + statement.swapColumn(1, 2); + assertEquals(Integer.MIN_VALUE, statement.getNumericValues().get(1)); + assertEquals(1, statement.getNumericValues().get(2)); + statement.markFailedMeasurement(1, new IllegalArgumentException()); + assertTrue(statement.getNumericValues().isNull(1)); + statement.removeAllFailedMeasurementMarks(); + assertEquals(Integer.MIN_VALUE, statement.getNumericValues().get(1)); + assertObjectStorageAbsent(InsertNumericRowStatement.class, statement); + assertEquals("numeric_table", statement.getTableDeviceID().getTableName()); + } + + @Test + public void testRpcRejectsNumericTreeAndNonStringTagRequests() throws Exception { + // Raw RPC callers must use STRING TAGs and numeric FIELDs in the table model. + TSInsertRecordReq tree = request().setIsWriteToTable(false); + assertThrows(IllegalArgumentException.class, () -> StatementGenerator.createStatement(tree)); + TSInsertRecordReq tag = + request() + .setColumnCategoryies( + Collections.nCopies(NAMES.size(), (byte) ColumnCategory.TAG.ordinal())); + assertThrows(IllegalArgumentException.class, () -> StatementGenerator.createStatement(tag)); + } + + @Test + public void testStringTagsSurviveSchemaChanges() throws Exception { + // The client places TAG metadata before FIELD metadata while retaining dense value arrays. + List names = Arrays.asList("tag2", "tag1", "b", "l"); + List types = + Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.INT64); + List categories = + Arrays.asList( + ColumnCategory.TAG, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD); + TSInsertRecordReq req = + request() + .setMeasurements(names) + .setValues( + SessionUtils.getValueBuffer( + types, + new long[] {1, Long.MIN_VALUE}, + names, + categories, + new String[] {"设备😀", ""})) + .setColumnCategoryies( + Arrays.asList( + (byte) ColumnCategory.TAG.ordinal(), + (byte) ColumnCategory.TAG.ordinal(), + (byte) ColumnCategory.FIELD.ordinal(), + (byte) ColumnCategory.FIELD.ordinal())); + InsertNumericRowStatement statement = + (InsertNumericRowStatement) StatementGenerator.createStatement(req); + assertEquals("设备😀", statement.getNumericValues().getTag(0)); + assertEquals("", statement.getNumericValues().getTag(1)); + assertArrayEquals(new long[] {1, Long.MIN_VALUE}, statement.getValues()); + statement.rebuildArraysAfterExpansion( + new int[] {-1, 1, 0, 2, 3}, new String[] {"missing", "tag1", "tag2", "b", "l"}); + assertArrayEquals( + new String[] {"numeric_table", null, "", "设备😀"}, + statement.getTableDeviceID().getSegments()); + statement.swapColumn(1, 2); + assertEquals("设备😀", statement.getNumericValues().getTag(1)); + statement.markFailedMeasurement(1, new IllegalArgumentException()); + statement.removeAllFailedMeasurementMarks(); + assertEquals("设备😀", statement.getNumericValues().getTag(1)); + assertEquals(1, statement.getValues()[0]); + assertEquals(Long.MIN_VALUE, statement.getValues()[1]); + assertObjectStorageAbsent(InsertNumericRowStatement.class, statement); + } + + @Test + public void testStringTagsSurviveWalConsensusAndTabletAdapters() throws Exception { + // TAGs identify the device through WAL recovery and remain present in Pipe/subscription + // Tablets. + for (String tag : Arrays.asList("设备😀", "", null)) { + RelationalInsertNumericRowNode node = nodeWithNullTag(); + node.getNumericValues().setTag(0, tag); + assertRoundTrips(node); + ByteBuffer bytes = node.serializeToByteBuffer(); + RelationalInsertNumericRowNode decoded = + (RelationalInsertNumericRowNode) PlanNodeType.deserialize(bytes); + assertEquals(tag, decoded.getNumericValues().getTag(0)); + assertEquals(node.getDeviceID(), decoded.getDeviceID()); + assertEquals( + new Binary(tag == null ? "" : tag, TSFileConfig.STRING_CHARSET), + tag == null ? Binary.EMPTY_VALUE : decoded.getValue(0)); + Tablet pipe = + new TabletInsertionEventTablePatternParser(null, null, decoded, null).convertToTablet(); + Tablet subscription = + new ConsensusLogToTabletConverter(null, null, null, "db").convert(decoded).get(0); + for (Tablet tablet : Arrays.asList(pipe, subscription)) { + if (tag == null) { + assertTrue(tablet.getBitMaps()[0].isMarked(0)); + } else { + assertEquals( + tag, + ((Binary[]) tablet.getValues()[0])[0].getStringValue(TSFileConfig.STRING_CHARSET)); + } + } + } + } + + @Test + public void testBulkTagExpansionKeepsValuesAndNulls() throws Exception { + // A reversed schema mapping with two missing TAGs must preserve numeric bits and null slots. + InsertNumericRowStatement statement = + (InsertNumericRowStatement) StatementGenerator.createStatement(request()); + statement.rebuildArraysAfterExpansion( + new int[] {-1, -1, 6, 5, 4, 3, 2, 1, 0}, + new String[] {"tag1", "tag2", "ts", "date", "d", "f", "l", "i", "b"}); + assertTrue(statement.getNumericValues().isNull(0)); + assertTrue(statement.getNumericValues().isNull(1)); + for (int i = 0; i < BITS.length; i++) { + assertEquals(BITS[BITS.length - 1 - i], statement.getNumericValues().get(i + 2)); + } + assertObjectStorageAbsent(InsertNumericRowStatement.class, statement); + } + + @Test + public void testNumericCastsKeepPrimitiveStorage() { + // Widen INT32 to DOUBLE in place; reject STRING coercion while accepting an inserted null TAG. + NumericRowValues values = new NumericRowValues(2); + values.getValues()[0] = Integer.MIN_VALUE; + values.setNull(1); + assertTrue(values.cast(0, TSDataType.INT32, TSDataType.DOUBLE)); + assertEquals(Double.doubleToRawLongBits((double) Integer.MIN_VALUE), values.get(0)); + assertFalse(values.cast(0, TSDataType.DOUBLE, TSDataType.STRING)); + assertTrue(values.cast(1, null, TSDataType.STRING)); + assertTrue(values.isNull(1)); + } + + @Test + public void testConsensusAndWalRetainPrimitiveStorage() throws Exception { + // Both serializers and WAL readers must retain numeric storage, including partial failures. + RelationalInsertNumericRowNode node = nodeWithNullTag(); + assertRoundTrips(node); + node.markFailedMeasurement(2); + assertRoundTrips(node); + node.setMeasurementSchemas(null); + assertRoundTrips(node); + } + + @Test + public void testWalEnvelopeRetainsTypeAndFrozenIndex() throws Exception { + RelationalInsertNumericRowNode node = nodeWithNullTag(); + node.setSearchIndex(123); + node.setLastFragment(true); + WALInfoEntry entry = new WALInfoEntry(7, node); + // WAL queuing freezes the consensus index even if the caller later reuses the node. + node.setSearchIndex(456); + node.setLastFragment(false); + ByteBuffer buffer = ByteBuffer.allocate(entry.serializedSize()); + entry.serialize(new WALByteBufferForTest(buffer)); + assertFalse(buffer.hasRemaining()); + buffer.flip(); + RelationalInsertNumericRowNode consensus = + (RelationalInsertNumericRowNode) WALEntry.deserializeForConsensus(buffer); + assertEquals(123, consensus.getSearchIndex()); + assertTrue(consensus.isLastFragment()); + assertEquals(node.getNumericValues(), consensus.getNumericValues()); + try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(buffer.array()))) { + WALEntry recovered = WALEntry.deserialize(input); + assertEquals(7, recovered.getMemTableId()); + RelationalInsertNumericRowNode row = (RelationalInsertNumericRowNode) recovered.getValue(); + assertEquals(123, row.getSearchIndex()); + assertTrue(row.isLastFragment()); + assertEquals(node.getNumericValues(), row.getNumericValues()); + assertEquals(0, input.available()); + assertObjectStorageAbsent(RelationalInsertNumericRowNode.class, row); + } + } + + @Test + public void testPipeBatchRetainsIndependentNumericStatements() throws Exception { + RelationalInsertNumericRowNode node = nodeWithNullTag(); + PipeTransferTabletBatchReqV2 request = + PipeTransferTabletBatchReqV2.toTPipeTransferReq( + Arrays.asList(node.serializeToByteBuffer(), node.serializeToByteBuffer()), + Collections.emptyList(), + Arrays.asList("db", "db"), + Collections.emptyList()); + List statements = + PipeTransferTabletBatchReqV2.fromTPipeTransferReq(request).constructStatements(); + // Batching must not replace numeric statements with the object-valued InsertRowsStatement. + assertEquals(2, statements.size()); + for (InsertBaseStatement statement : statements) { + assertEquals(StatementType.INSERT_NUMERIC_ROW, statement.getType()); + assertEquals("db", statement.getDatabaseName().get()); + assertEquals( + node.getNumericValues(), ((InsertNumericRowStatement) statement).getNumericValues()); + assertObjectStorageAbsent(InsertNumericRowStatement.class, statement); + } + } + + @Test + public void testNumericRowRedirectUsesSingleEndpoint() throws Exception { + InsertNumericRowStatement statement = + (InsertNumericRowStatement) StatementGenerator.createStatement(request()); + Analysis analysis = new Analysis(new InsertNumericRow(statement, null), Collections.emptyMap()); + TEndPoint local = new TEndPoint("127.0.0.1", 6667); + TEndPoint leader = new TEndPoint("127.0.0.1", 6668); + analysis.setRedirectNodeList(Collections.singletonList(leader)); + TableModelPlanner planner = Mockito.mock(TableModelPlanner.class, Mockito.CALLS_REAL_METHODS); + TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + planner.setRedirectInfo(analysis, local, status); + assertEquals(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode(), status.getCode()); + assertEquals(leader, status.getRedirectNode()); + assertFalse(status.isSetSubStatus()); + TSStatus localStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + planner.setRedirectInfo(analysis, leader, localStatus); + assertFalse(localStatus.isSetRedirectNode()); + } + + private void assertRoundTrips(RelationalInsertNumericRowNode node) throws Exception { + ByteBuffer streamed = node.serializeToByteBuffer(); + ByteBuffer direct = ByteBuffer.allocate(streamed.remaining()); + node.serialize(direct); + direct.flip(); + assertEquals(streamed, direct); + RelationalInsertNumericRowNode decoded = + (RelationalInsertNumericRowNode) PlanNodeType.deserialize(direct); + assertObjectStorageAbsent(RelationalInsertNumericRowNode.class, decoded); + assertEquals(node.getNumericValues().isNull(0), decoded.getNumericValues().isNull(0)); + assertEquals(node.getNumericValues().getTag(0), decoded.getNumericValues().getTag(0)); + assertEquals(node.getDeviceID(), decoded.getDeviceID()); + + ByteBuffer wal = ByteBuffer.allocate(node.serializedSize()); + node.setSearchIndex(123); + node.setLastFragment(true); + node.serializeToWAL(new WALByteBufferForTest(wal)); + assertFalse(wal.hasRemaining()); + wal.flip(); + assertEquals(PlanNodeType.RELATIONAL_INSERT_NUMERIC_ROW.getNodeType(), wal.getShort()); + RelationalInsertNumericRowNode recovered = + RelationalInsertNumericRowNode.deserializeFromWAL(wal); + assertFalse(wal.hasRemaining()); + assertEquals(decoded.getNumericValues(), recovered.getNumericValues()); + assertArrayEquals(decoded.getMeasurements(), recovered.getMeasurements()); + assertArrayEquals(decoded.getColumnCategories(), recovered.getColumnCategories()); + assertObjectStorageAbsent(RelationalInsertNumericRowNode.class, recovered); + assertEquals(123, recovered.getSearchIndex()); + assertTrue(recovered.isLastFragment()); + + try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(wal.array()))) { + input.readShort(); + RelationalInsertNumericRowNode fromStream = + RelationalInsertNumericRowNode.deserializeFromWAL(input); + assertEquals(recovered.getNumericValues(), fromStream.getNumericValues()); + assertObjectStorageAbsent(RelationalInsertNumericRowNode.class, fromStream); + assertEquals(0, input.available()); + } + } + + @Test + public void testMemtableAcceptsRecoveredRowWithoutBoxing() throws Exception { + // A WAL-recovered row must reach primitive column arrays without boxing the row or its NaNs. + RelationalInsertNumericRowNode original = nodeWithNullTag(); + ByteBuffer wal = ByteBuffer.allocate(original.serializedSize()); + original.serializeToWAL(new WALByteBufferForTest(wal)); + wal.flip(); + wal.getShort(); + RelationalInsertNumericRowNode recovered = + RelationalInsertNumericRowNode.deserializeFromWAL(wal); + // Fail immediately if the primitive memtable write path boxes even a single cell. + RelationalInsertNumericRowNode row = + new RelationalInsertNumericRowNode(new PlanNodeId("test")) { + @Override + public Object getValue(int index) { + throw new AssertionError("Numeric memtable writes must not box the row"); + } + }; + row.setTargetPath(recovered.getTargetPath()); + row.setTime(42); + row.setAligned(true); + row.setMeasurements(recovered.getMeasurements()); + row.setDataTypes(recovered.getDataTypes()); + row.setMeasurementSchemas(recovered.getMeasurementSchemas()); + row.setColumnCategories(recovered.getColumnCategories()); + row.setNumericValues(recovered.getNumericValues()); + PrimitiveMemTable memtable = new PrimitiveMemTable(); + try { + assertEquals(7, memtable.insertAlignedRow(row)); + AlignedTVList list = + ((AlignedWritableMemChunkGroup) memtable.getMemTableMap().get(row.getDeviceID())) + .getAlignedMemChunk() + .getWorkingTVList(); + assertEquals(42, list.getTime(0)); + assertTrue(list.getBooleanByValueIndex(0, 0)); + assertEquals(Integer.MIN_VALUE, list.getIntByValueIndex(0, 1)); + assertEquals(Long.MIN_VALUE, list.getLongByValueIndex(0, 2)); + assertEquals(0x7fc01234, Float.floatToRawIntBits(list.getFloatByValueIndex(0, 3))); + assertEquals( + 0x7ff8000000001234L, Double.doubleToRawLongBits(list.getDoubleByValueIndex(0, 4))); + assertEquals(20260910, list.getIntByValueIndex(0, 5)); + assertEquals(Long.MAX_VALUE, list.getLongByValueIndex(0, 6)); + assertObjectStorageAbsent(RelationalInsertNumericRowNode.class, row); + } finally { + memtable.release(); + } + } + + @Test + public void testRpcRejectsNonNumericTypes() throws Exception { + // Raw RPC callers cannot smuggle unsupported values into numeric FIELD storage. + for (TSDataType type : + Arrays.asList( + TSDataType.TEXT, + TSDataType.BLOB, + TSDataType.OBJECT, + TSDataType.VECTOR, + TSDataType.UNKNOWN)) { + TSInsertRecordReq invalid = + request().setValues(ByteBuffer.wrap(new byte[] {type.serialize()})); + assertThrows( + UnSupportedDataTypeException.class, () -> StatementGenerator.createStatement(invalid)); + } + // STRING has a valid wire representation for TAGs, but must still be rejected as a FIELD. + ByteBuffer stringValue = ByteBuffer.allocate(32); + ReadWriteIOUtils.write(TSDataType.STRING, stringValue); + ReadWriteIOUtils.write("invalid", stringValue); + stringValue.flip(); + TSInsertRecordReq stringField = + request() + .setMeasurements(Collections.singletonList("s")) + .setColumnCategoryies(Collections.singletonList((byte) ColumnCategory.FIELD.ordinal())) + .setValues(stringValue); + assertThrows( + IllegalArgumentException.class, () -> StatementGenerator.createStatement(stringField)); + } + + @Test + public void testTabletAdaptersDoNotMaterializeObjectRows() throws Exception { + // Pipe and subscription must preserve values and null TAGs without boxing the entire row. + RelationalInsertNumericRowNode node = Mockito.spy(nodeWithNullTag()); + Mockito.doThrow(new AssertionError("Numeric adapters must not box the row")) + .when(node) + .getValues(); + Tablet pipe = + new TabletInsertionEventTablePatternParser(null, null, node, null).convertToTablet(); + List subscription = + new ConsensusLogToTabletConverter(null, null, null, "db").convert(node); + assertEquals(1, subscription.size()); + for (Tablet tablet : Arrays.asList(pipe, subscription.get(0))) { + assertEquals(1, tablet.getRowSize()); + assertEquals("numeric_table", tablet.getTableName()); + assertEquals(42, tablet.getTimestamps()[0]); + assertTrue(tablet.getBitMaps()[0].isMarked(0)); + assertTrue(((boolean[]) tablet.getValues()[1])[0]); + assertEquals(Integer.MIN_VALUE, ((int[]) tablet.getValues()[2])[0]); + assertEquals(Long.MIN_VALUE, ((long[]) tablet.getValues()[3])[0]); + assertEquals(0x7fc01234, Float.floatToRawIntBits(((float[]) tablet.getValues()[4])[0])); + assertEquals( + 0x7ff8000000001234L, Double.doubleToRawLongBits(((double[]) tablet.getValues()[5])[0])); + assertEquals(LocalDate.of(2026, 9, 10), ((LocalDate[]) tablet.getValues()[6])[0]); + assertEquals(Long.MAX_VALUE, ((long[]) tablet.getValues()[7])[0]); + } + assertObjectStorageAbsent(RelationalInsertNumericRowNode.class, node); + } + + private RelationalInsertNumericRowNode nodeWithNullTag() throws Exception { + InsertNumericRowStatement statement = + (InsertNumericRowStatement) StatementGenerator.createStatement(request()); + statement.insertColumn( + 0, + new ColumnSchema( + "tag", TypeFactory.getType(TSDataType.STRING), false, TsTableColumnCategory.TAG)); + MeasurementSchema[] schemas = new MeasurementSchema[statement.getValueCount()]; + for (int i = 0; i < schemas.length; i++) { + schemas[i] = new MeasurementSchema(statement.getMeasurements()[i], statement.getDataType(i)); + } + RelationalInsertNumericRowNode node = + new RelationalInsertNumericRowNode( + new PlanNodeId("numeric"), + statement.getDevicePath(), + statement.getMeasurements(), + statement.getDataTypes(), + schemas, + statement.getTime(), + statement.getNumericValues(), + statement.getColumnCategories()); + node.setNumericValues(statement.getNumericValues()); + return node; + } + + private void assertObjectStorageAbsent(Class owner, Object row) throws Exception { + // A dedicated numeric type must not inherit the object row or retain an Object[] payload. + assertFalse(InsertRowStatement.class.isInstance(row)); + assertFalse(InsertRowNode.class.isInstance(row)); + for (Class type = owner; type != null; type = type.getSuperclass()) { + for (Field field : type.getDeclaredFields()) { + assertFalse(field.getName(), field.getType() == Object[].class); + } + } + } +} From fb255decbb54b5f703ae0a2c11ddc88ef38ab53e Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Tue, 15 Sep 2026 16:29:59 +0800 Subject: [PATCH 05/20] add client.thrift --- .../src/main/thrift/client.thrift | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift index 792a42dde5150..c08959e5fe487 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift @@ -633,6 +633,12 @@ service IClientRPCService { common.TSStatus insertRecord(1:TSInsertRecordReq req); + TSPrepareNumericRowResp prepareNumericRow(1:TSPrepareNumericRowReq req); + + common.TSStatus executePreparedNumericRow(1:TSExecutePreparedNumericRowReq req); + + common.TSStatus closePreparedNumericRow(1:i64 sessionId, 2:i64 preparedId); + common.TSStatus insertStringRecord(1:TSInsertStringRecordReq req); common.TSStatus insertTablet(1:TSInsertTabletReq req); @@ -707,4 +713,26 @@ service IClientRPCService { /** For other node's call */ common.TSStatus testConnectionEmptyRPC(); -} \ No newline at end of file +} + +struct TSPrepareNumericRowReq { + 1: required i64 sessionId + 2: required string tableName + 3: required list measurements + 4: required list dataTypes + 5: required list columnCategories +} + +struct TSPrepareNumericRowResp { + 1: required common.TSStatus status + 2: optional i64 preparedId +} + +struct TSExecutePreparedNumericRowReq { + 1: required i64 sessionId + 2: required i64 preparedId + 3: required i64 timestamp + 4: required binary values + // Length-prefixed UTF-8 TAGs; length -1 represents a null TAG. + 5: required binary tagValues +} From 1d6186fface82890ef72cfcaa8279f87f96ca0d8 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 16 Sep 2026 15:11:10 +0800 Subject: [PATCH 06/20] tmp --- .../it/session/IoTDBTableModelSessionIT.java | 214 ++++++++++++++++++ .../apache/iotdb/isession/ITableSession.java | 15 ++ .../iotdb/session/i18n/SessionMessages.java | 4 + .../iotdb/session/i18n/SessionMessages.java | 5 + .../iotdb/session/SessionConnection.java | 42 ++++ .../apache/iotdb/session/TableSession.java | 12 +- .../iotdb/db/i18n/DataNodeMiscMessages.java | 2 + .../iotdb/db/i18n/DataNodeMiscMessages.java | 4 + .../db/protocol/session/IClientSession.java | 53 +++++ .../db/protocol/session/SessionManager.java | 2 + .../thrift/impl/ClientRPCServiceImpl.java | 65 ++++++ 11 files changed, 417 insertions(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java index f9c7ca00444be..828ddba1fbd74 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java @@ -21,6 +21,7 @@ import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.NumericRow; +import org.apache.iotdb.isession.NumericRowPreparedStatement; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.isession.pool.ITableSessionPool; import org.apache.iotdb.it.env.EnvFactory; @@ -33,16 +34,26 @@ import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.record.Tablet; import org.junit.After; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import java.time.LocalDate; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.IntConsumer; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showTablesColumnHeaders; import static org.junit.Assert.assertEquals; @@ -182,6 +193,209 @@ public void testNumericRowInsertWithTags() throws Exception { } } + @Test + public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { + Assume.assumeTrue( + "Performance benchmark disabled; set iotdb.numeric.row.insert.perf.enabled=true to run", + Boolean.getBoolean("iotdb.numeric.row.insert.perf.enabled")); + final int concurrency = 16; + // Compare sustained single-row writes after warming both paths; verify all acknowledged rows. + final int warmupRounds = 2; + final int measuredRounds = 6; + final int fieldCount = 99; + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE numeric_perf"); + session.executeNonQueryStatement("USE numeric_perf"); + StringBuilder ddl = new StringBuilder("CREATE TABLE prepared_perf (tag STRING TAG"); + for (int i = 0; i < fieldCount; i++) { + ddl.append(", f").append(i).append(" DOUBLE FIELD"); + } + ddl.append(")"); + session.executeNonQueryStatement(ddl.toString()); + session.executeNonQueryStatement(ddl.toString().replace("prepared_perf", "tablet_perf")); + } + + List names = new ArrayList<>(); + List types = new ArrayList<>(); + List categories = new ArrayList<>(); + names.add("tag"); + types.add(TSDataType.STRING); + categories.add(ColumnCategory.TAG); + for (int i = 0; i < fieldCount; i++) { + names.add("f" + i); + types.add(TSDataType.DOUBLE); + categories.add(ColumnCategory.FIELD); + } + ITableSession[] sessions = new ITableSession[concurrency]; + NumericRowPreparedStatement[] prepared = new NumericRowPreparedStatement[concurrency]; + Tablet[] tablets = new Tablet[concurrency]; + String[][] tags = new String[concurrency][]; + long[][] numericValues = new long[concurrency][fieldCount]; + long[] preparedRows = new long[concurrency]; + long[] tabletRows = new long[concurrency]; + ExecutorService executor = Executors.newFixedThreadPool(concurrency); + try { + // Connections, USE, prepare and reusable input allocation are outside every timed round. + for (int i = 0; i < concurrency; i++) { + sessions[i] = EnvFactory.getEnv().getTableSessionConnection(); + sessions[i].executeNonQueryStatement("USE numeric_perf"); + prepared[i] = + sessions[i].prepareInsertNumericRow("prepared_perf", names, types, categories); + tablets[i] = new Tablet("tablet_perf", names, types, categories, 1); + tags[i] = new String[] {"device-" + i}; + Arrays.fill(numericValues[i], Double.doubleToRawLongBits(1.0)); + } + IntConsumer preparedWrite = + threadId -> { + try { + sessions[threadId].executePreparedInsertNumericRow( + prepared[threadId], + preparedRows[threadId], + numericValues[threadId], + tags[threadId]); + preparedRows[threadId]++; + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + IntConsumer tabletWrite = + threadId -> { + try { + Tablet tablet = tablets[threadId]; + tablet.addTimestamp(0, tabletRows[threadId]); + tablet.addValue("tag", 0, tags[threadId][0]); + for (int i = 0; i < fieldCount; i++) { + tablet.addValue(names.get(i + 1), 0, 1.0D); + } + sessions[threadId].insert(tablet); + tablet.reset(); + tabletRows[threadId]++; + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + double[][] rates = new double[2][measuredRounds]; + long[] totalRows = new long[2]; + long[] totalNanos = new long[2]; + for (int round = 0; round < warmupRounds + measuredRounds; round++) { + boolean warmup = round < warmupRounds; + int seconds = warmup ? 3 : 5; + // Alternate AB/BA to distribute shared JVM warmup and background storage work. + for (int position = 0; position < 2; position++) { + int path = (round + position) % 2; + WritePerformance result = + runConcurrentWrites( + executor, concurrency, seconds, path == 0 ? preparedWrite : tabletWrite); + System.out.printf( + "%s round %d position %d %s: %d rows, %.3f s, %.1f rows/s%n", + warmup ? "warmup" : "measured", + warmup ? round + 1 : round - warmupRounds + 1, + position + 1, + path == 0 ? "prepared numeric row" : "insertTablet batch=1", + result.rows(), + result.nanos() / 1_000_000_000.0, + result.rowsPerSecond()); + if (!warmup) { + rates[path][round - warmupRounds] = result.rowsPerSecond(); + totalRows[path] += result.rows(); + totalNanos[path] += result.nanos(); + } + } + } + for (int path = 0; path < 2; path++) { + Arrays.sort(rates[path]); + System.out.printf( + "summary %s: median %.1f rows/s, aggregate %.1f rows/s, min %.1f, max %.1f%n", + path == 0 ? "prepared numeric row" : "insertTablet batch=1", + (rates[path][measuredRounds / 2 - 1] + rates[path][measuredRounds / 2]) / 2, + totalRows[path] * 1_000_000_000.0 / totalNanos[path], + rates[path][0], + rates[path][measuredRounds - 1]); + } + // Per-device monotonic timestamps span warmup and measurement, so no rows overwrite others. + for (int path = 0; path < 2; path++) { + String table = path == 0 ? "prepared_perf" : "tablet_perf"; + long expectedRows = Arrays.stream(path == 0 ? preparedRows : tabletRows).sum(); + try (SessionDataSet dataSet = + sessions[0].executeQueryStatement("SELECT count(*) FROM " + table)) { + assertTrue(dataSet.hasNext()); + assertEquals(expectedRows, dataSet.next().getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + System.out.printf("verified %s: %d rows%n", table, expectedRows); + } + } finally { + executor.shutdownNow(); + try { + assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES)); + } finally { + // Closing each session also releases its server-side prepared metadata. + Exception closeFailure = null; + for (ITableSession session : sessions) { + if (session != null) { + try { + session.close(); + } catch (Exception e) { + if (closeFailure == null) closeFailure = e; + else closeFailure.addSuppressed(e); + } + } + } + if (closeFailure != null) throw closeFailure; + } + } + } + + private record WritePerformance(long rows, long nanos) { + double rowsPerSecond() { + return rows * 1_000_000_000.0 / nanos; + } + } + + private static WritePerformance runConcurrentWrites( + ExecutorService executor, int concurrency, int seconds, IntConsumer write) throws Exception { + CountDownLatch ready = new CountDownLatch(concurrency); + CountDownLatch start = new CountDownLatch(1); + AtomicLong deadline = new AtomicLong(); + List> futures = new ArrayList<>(); + try { + for (int i = 0; i < concurrency; i++) { + final int threadId = i; + futures.add( + executor.submit( + () -> { + ready.countDown(); + start.await(); + long rows = 0; + while (!Thread.currentThread().isInterrupted() + && System.nanoTime() < deadline.get()) { + write.accept(threadId); + rows++; + } + return new WritePerformance(rows, System.nanoTime()); + })); + } + assertTrue(ready.await(30, TimeUnit.SECONDS)); + long startNanos = System.nanoTime(); + deadline.set(startNanos + TimeUnit.SECONDS.toNanos(seconds)); + start.countDown(); + long rows = 0; + long endNanos = startNanos; + for (Future future : futures) { + WritePerformance result = future.get(seconds + 60, TimeUnit.SECONDS); + rows += result.rows(); + endNanos = Math.max(endNanos, result.nanos()); + } + // Include completion of in-flight RPCs, but exclude main-thread collection/printing time. + return new WritePerformance(rows, endNanos - startNanos); + } finally { + start.countDown(); + for (Future future : futures) { + if (!future.isDone()) future.cancel(true); + } + } + } + private void assertNumericTagRows(ITableSession session) throws Exception { String[] devices = {"设备甲😀", "设备乙", "", null}; try (SessionDataSet dataSet = diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java index af524f9a853fa..d14abab1238f7 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java @@ -22,8 +22,12 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.write.record.Tablet; +import java.util.List; + /** * This interface defines a session for interacting with IoTDB tables. It supports operations such * as data insertion, executing queries, and closing the session. Implementations of this interface @@ -58,6 +62,17 @@ public interface ITableSession extends AutoCloseable { */ void insert(NumericRow row) throws StatementExecutionException, IoTDBConnectionException; + NumericRowPreparedStatement prepareInsertNumericRow( + String tableName, + List columnNames, + List dataTypes, + List columnCategories) + throws StatementExecutionException, IoTDBConnectionException; + + void executePreparedInsertNumericRow( + NumericRowPreparedStatement prepared, long timestamp, long[] values, String[] tagValues) + throws StatementExecutionException, IoTDBConnectionException; + /** * Executes a non-query SQL statement, such as a DDL or DML command. * diff --git a/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java b/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java index 3eb19971a9245..173260e4a07cf 100644 --- a/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java +++ b/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java @@ -290,4 +290,8 @@ private SessionMessages() {} public static final String EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_AND_COLUMN_NAMES_TYPES_CATEGORIES_AND_VALUES_MUST_HAVE_EQUAL_LENGTHS_AB102B01 = "Numeric row arguments must be non-null and column names, types, categories and values must have equal lengths."; public static final String EXCEPTION_NUMERIC_ROWS_SUPPORT_ONLY_FIELD_COLUMNS_DA7A8EE7 = "Numeric rows support only FIELD columns."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_BELONGS_TO_ANOTHER_CONNECTION_PREPARE_IT_AGAIN_055CC7BC = "Prepared numeric row is closed or belongs to another connection; prepare it again."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_PREPARE_RETURNED_NO_STATUS_6FF7BC24 = "Prepared numeric row prepare returned no status"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_COUNT_MISMATCH_9ACD3A0C = "Prepared numeric row value count mismatch"; + public static final String EXCEPTION_UNSUPPORTED_NUMERIC_TYPE_06DD3044 = "Unsupported numeric type"; } diff --git a/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java b/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java index 22f3bea7821bd..b45704d8738ac 100644 --- a/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java +++ b/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java @@ -288,4 +288,9 @@ private SessionMessages() {} public static final String EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_AND_COLUMN_NAMES_TYPES_CATEGORIES_AND_VALUES_MUST_HAVE_EQUAL_LENGTHS_AB102B01 = "数值行参数不能为 null，列名、类型、类别和值的数量必须相同。"; public static final String EXCEPTION_NUMERIC_ROWS_SUPPORT_ONLY_FIELD_COLUMNS_DA7A8EE7 = "数值行仅支持 FIELD 列。"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_BELONGS_TO_ANOTHER_CONNECTION_PREPARE_IT_AGAIN_055CC7BC = + "预备数值行已关闭或属于其他连接，请重新准备"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_PREPARE_RETURNED_NO_STATUS_6FF7BC24 = "预备数值行准备请求未返回状态"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_COUNT_MISMATCH_9ACD3A0C = "预备数值行的值数量不匹配"; + public static final String EXCEPTION_UNSUPPORTED_NUMERIC_TYPE_06DD3044 = "不支持的数值类型"; } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java index 2e144448b58a7..e01c5dd8a2073 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java @@ -44,6 +44,7 @@ import org.apache.iotdb.service.rpc.thrift.TSCreateTimeseriesReq; import org.apache.iotdb.service.rpc.thrift.TSDeleteDataReq; import org.apache.iotdb.service.rpc.thrift.TSDropSchemaTemplateReq; +import org.apache.iotdb.service.rpc.thrift.TSExecutePreparedNumericRowReq; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; import org.apache.iotdb.service.rpc.thrift.TSFastLastDataQueryForOneDeviceReq; @@ -59,6 +60,8 @@ import org.apache.iotdb.service.rpc.thrift.TSLastDataQueryReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowResp; import org.apache.iotdb.service.rpc.thrift.TSPruneSchemaTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSQueryTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSQueryTemplateResp; @@ -79,6 +82,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.ByteBuffer; import java.security.SecureRandom; import java.time.ZoneId; import java.util.ArrayList; @@ -759,6 +763,44 @@ protected void insertRecord(TSInsertRecordReq request) callWithRetryAndVerifyWithRedirection(() -> insertRecordInternal(request)); } + protected long prepareNumericRow(TSPrepareNumericRowReq request) + throws IoTDBConnectionException, StatementExecutionException { + request.setSessionId(sessionId); + try { + TSPrepareNumericRowResp resp = client.prepareNumericRow(request); + if (resp.getStatus() == null) { + throw new IoTDBConnectionException( + SessionMessages.EXCEPTION_PREPARED_NUMERIC_ROW_PREPARE_RETURNED_NO_STATUS_6FF7BC24); + } + RpcUtils.verifySuccess(resp.getStatus()); + return resp.getPreparedId(); + } catch (TException e) { + throw new IoTDBConnectionException(e); + } + } + + protected void executePreparedNumericRow( + long id, long timestamp, ByteBuffer values, ByteBuffer tags) + throws IoTDBConnectionException, StatementExecutionException { + TSExecutePreparedNumericRowReq request = + new TSExecutePreparedNumericRowReq(sessionId, id, timestamp, values, tags); + try { + RpcUtils.verifySuccess(client.executePreparedNumericRow(request)); + } catch (TException e) { + throw new IoTDBConnectionException(e); + } + } + + protected void closePreparedNumericRow(long id) throws IoTDBConnectionException { + try { + RpcUtils.verifySuccess(client.closePreparedNumericRow(sessionId, id)); + } catch (StatementExecutionException e) { + throw new IoTDBConnectionException(e); + } catch (TException e) { + throw new IoTDBConnectionException(e); + } + } + private TSStatus insertRecordInternal(TSInsertRecordReq request) throws TException { request.setSessionId(sessionId); return client.insertRecord(request); diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java index 38cc6d72cea3f..f7154b3c5f73f 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java @@ -21,17 +21,27 @@ import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.NumericRow; +import org.apache.iotdb.isession.NumericRowPreparedStatement; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; +import org.apache.iotdb.session.i18n.SessionMessages; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.record.Tablet; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + public class TableSession implements ITableSession { private final Session session; - TableSession(Session session) { + public TableSession(Session session) { this.session = session; } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index 12f4e374ca7a7..d4ce7f97ba848 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1489,4 +1489,6 @@ private DataNodeMiscMessages() {} LOG_TABLE_QUERY_DEVICE_ENTRY_BATCH_SIZE_IN_BYTES_ARG_EXCEEDS_DN_THRIFT_MAX_FRAME_SIZE_ARG_USING_ARG_AS_THE_EFFECTIVE_VALUE_2AE1BEDA = "table_query_device_entry_batch_size_in_bytes (%d) exceeds the maximum RPC payload (dn_thrift_max_frame_size %d minus 1024 bytes); using %d as the effective value"; + public static final String EXCEPTION_INVALID_PREPARED_NUMERIC_ROW_METADATA_OR_PAYLOAD_AECCCEAA = "Invalid prepared numeric row metadata or payload."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_DOES_NOT_BELONG_TO_THIS_SESSION_5CC5C50A = "Prepared numeric row is closed or does not belong to this session."; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index d809fea199259..2d19c74be8b7f 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1469,4 +1469,8 @@ private DataNodeMiscMessages() {} LOG_TABLE_QUERY_DEVICE_ENTRY_BATCH_SIZE_IN_BYTES_ARG_EXCEEDS_DN_THRIFT_MAX_FRAME_SIZE_ARG_USING_ARG_AS_THE_EFFECTIVE_VALUE_2AE1BEDA = "table_query_device_entry_batch_size_in_bytes（%d）超过最大 RPC payload（dn_thrift_max_frame_size %d 减去 1024 字节），将使用 %d 作为生效值"; + public static final String EXCEPTION_INVALID_PREPARED_NUMERIC_ROW_METADATA_OR_PAYLOAD_AECCCEAA = + "预备数值行元数据或数据载荷无效"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_DOES_NOT_BELONG_TO_THIS_SESSION_5CC5C50A = + "预备数值行已关闭或不属于当前会话"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/IClientSession.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/IClientSession.java index bac4b15e342dd..0283b226bdf85 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/IClientSession.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/IClientSession.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; +import org.apache.iotdb.db.queryengine.plan.execution.config.session.PreparedStatementMemoryManager; import org.apache.iotdb.rpc.subscription.annotation.TableModel; import org.apache.iotdb.service.rpc.thrift.TSConnectionInfo; import org.apache.iotdb.service.rpc.thrift.TSConnectionType; @@ -33,12 +34,64 @@ import javax.annotation.Nullable; import java.time.ZoneId; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicLong; public abstract class IClientSession { + private static final AtomicLong NUMERIC_ROW_IDS = new AtomicLong(); + private Map preparedNumericRows; + private boolean numericRowsClosed; + + public synchronized long addPreparedNumericRow(PreparedNumericRowInfo info) { + if (numericRowsClosed) { + throw new SemanticException( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_DOES_NOT_BELONG_TO_THIS_SESSION_5CC5C50A); + } + long preparedId = NUMERIC_ROW_IDS.incrementAndGet(); + if (preparedNumericRows == null) { + preparedNumericRows = new HashMap<>(); + } + PreparedStatementMemoryManager.getInstance() + .allocate(Long.toString(preparedId), info.ramBytesUsed()); + preparedNumericRows.put(preparedId, info); + return preparedId; + } + + public synchronized PreparedNumericRowInfo getPreparedNumericRow(long preparedId) { + PreparedNumericRowInfo info = + preparedNumericRows == null ? null : preparedNumericRows.get(preparedId); + if (info == null) { + throw new SemanticException( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_DOES_NOT_BELONG_TO_THIS_SESSION_5CC5C50A); + } + return info; + } + + public synchronized void removePreparedNumericRow(long preparedId) { + PreparedNumericRowInfo info = + preparedNumericRows == null ? null : preparedNumericRows.remove(preparedId); + if (info != null) { + PreparedStatementMemoryManager.getInstance().release(info.ramBytesUsed()); + } + } + + public synchronized void releasePreparedNumericRows() { + numericRowsClosed = true; + if (preparedNumericRows != null) { + for (PreparedNumericRowInfo info : preparedNumericRows.values()) { + PreparedStatementMemoryManager.getInstance().release(info.ramBytesUsed()); + } + preparedNumericRows = null; + } + } + private long id; private ClientVersion clientVersion; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java index 21dbffb606ebd..fa1da5d8ea56e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java @@ -260,6 +260,8 @@ private void releaseSessionResource(IClientSession session, LongConsumer release } } + session.releasePreparedNumericRows(); + // Release PreparedStatement memory resources try { PreparedStatementMemoryManager.getInstance().releaseAllForSession(session); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 99d40b3572b0a..d48aef4cbf671 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -69,6 +69,7 @@ import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.protocol.session.PreparedNumericRowInfo; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.protocol.thrift.OperationType; import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; @@ -170,6 +171,7 @@ import org.apache.iotdb.service.rpc.thrift.TSDeleteDataReq; import org.apache.iotdb.service.rpc.thrift.TSDropSchemaTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementReq; +import org.apache.iotdb.service.rpc.thrift.TSExecutePreparedNumericRowReq; import org.apache.iotdb.service.rpc.thrift.TSExecutePreparedReq; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; @@ -191,6 +193,8 @@ import org.apache.iotdb.service.rpc.thrift.TSLastDataQueryReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowResp; import org.apache.iotdb.service.rpc.thrift.TSPrepareReq; import org.apache.iotdb.service.rpc.thrift.TSPrepareResp; import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion; @@ -2582,6 +2586,67 @@ public TSStatus insertStringRecordsOfOneDevice(TSInsertStringRecordsOfOneDeviceR } } + @Override + public TSPrepareNumericRowResp prepareNumericRow(TSPrepareNumericRowReq req) { + IClientSession session = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); + if (!SESSION_MANAGER.checkLogin(session)) { + return new TSPrepareNumericRowResp(getNotLoggedInStatus()); + } + try { + if (req.getSessionId() != session.getId()) { + return new TSPrepareNumericRowResp(RpcUtils.getStatus(TSStatusCode.NOT_LOGIN)); + } + PreparedNumericRowInfo info = new PreparedNumericRowInfo(session.getDatabaseName(), req); + TSPrepareNumericRowResp response = new TSPrepareNumericRowResp(); + response.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + response.setPreparedId(session.addPreparedNumericRow(info)); + return response; + } catch (Exception e) { + return new TSPrepareNumericRowResp( + onQueryException( + e, OperationType.INSERT_RECORD.getName(), TSStatusCode.EXECUTE_STATEMENT_ERROR)); + } + } + + @Override + public TSStatus executePreparedNumericRow(TSExecutePreparedNumericRowReq req) { + IClientSession session = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); + if (!SESSION_MANAGER.checkLogin(session)) return getNotLoggedInStatus(); + try { + if (req.getSessionId() != session.getId()) return RpcUtils.getStatus(TSStatusCode.NOT_LOGIN); + InsertBaseStatement statement = + session + .getPreparedNumericRow(req.getPreparedId()) + .bind( + req.getTimestamp(), + ByteBuffer.wrap(req.getTagValues()), + ByteBuffer.wrap(req.getValues())); + long queryId = SESSION_MANAGER.requestQueryId(); + return COORDINATOR.executeForTableModel( + statement, + relationSqlParser, + session, + queryId, + SESSION_MANAGER.getSessionInfo(session), + "", + metadata, + config.getConnectionTimeoutInMS()) + .status; + } catch (Exception e) { + return onQueryException( + e, OperationType.INSERT_RECORD.getName(), TSStatusCode.EXECUTE_STATEMENT_ERROR); + } + } + + @Override + public TSStatus closePreparedNumericRow(long sessionId, long preparedId) { + IClientSession session = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); + if (!SESSION_MANAGER.checkLogin(session) || session.getId() != sessionId) + return getNotLoggedInStatus(); + session.removePreparedNumericRow(preparedId); + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } + @Override public TSStatus insertRecord(TSInsertRecordReq req) { long t1 = System.nanoTime(); From 07d73d28b1b737928c4492fbacd345d95d18dea8 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 16 Sep 2026 15:56:00 +0800 Subject: [PATCH 07/20] tmp --- .../iotdb/isession/i18n/ISessionMessages.java | 2 + .../iotdb/isession/i18n/ISessionMessages.java | 2 + .../apache/iotdb/session/TableSession.java | 87 +++++++++++++++++++ .../session/pool/TableSessionWrapper.java | 2 + 4 files changed, 93 insertions(+) diff --git a/iotdb-client/isession/src/main/i18n/en/org/apache/iotdb/isession/i18n/ISessionMessages.java b/iotdb-client/isession/src/main/i18n/en/org/apache/iotdb/isession/i18n/ISessionMessages.java index bc593534c60d2..3a15370611a4c 100644 --- a/iotdb-client/isession/src/main/i18n/en/org/apache/iotdb/isession/i18n/ISessionMessages.java +++ b/iotdb-client/isession/src/main/i18n/en/org/apache/iotdb/isession/i18n/ISessionMessages.java @@ -31,6 +31,8 @@ public final class ISessionMessages { public static final String NOT_DIRECT_CHILD_OF_TEMPLATE = "It is not a direct child of the template: "; + public static final String EXCEPTION_UNSUPPORTED_NUMERIC_TYPE_06DD3044 = "Unsupported numeric type"; + private ISessionMessages() {} // --------------------------------------------------------------------------- // Additional auto-collected messages diff --git a/iotdb-client/isession/src/main/i18n/zh/org/apache/iotdb/isession/i18n/ISessionMessages.java b/iotdb-client/isession/src/main/i18n/zh/org/apache/iotdb/isession/i18n/ISessionMessages.java index 5514a76c0f71e..a6316202ebc84 100644 --- a/iotdb-client/isession/src/main/i18n/zh/org/apache/iotdb/isession/i18n/ISessionMessages.java +++ b/iotdb-client/isession/src/main/i18n/zh/org/apache/iotdb/isession/i18n/ISessionMessages.java @@ -31,6 +31,8 @@ public final class ISessionMessages { public static final String NOT_DIRECT_CHILD_OF_TEMPLATE = "这不是该模板的直接子节点："; + public static final String EXCEPTION_UNSUPPORTED_NUMERIC_TYPE_06DD3044 = "不支持的数值类型"; + private ISessionMessages() {} // --------------------------------------------------------------------------- // Additional auto-collected messages diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java index f7154b3c5f73f..7833557e10b1a 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java @@ -55,6 +55,93 @@ public void insert(NumericRow row) throws StatementExecutionException, IoTDBConn session.insertRelationalRecord(row); } + @Override + public NumericRowPreparedStatement prepareInsertNumericRow( + String tableName, + List columnNames, + List dataTypes, + List columnCategories) + throws StatementExecutionException, IoTDBConnectionException { + List names = new ArrayList<>(); + List types = new ArrayList<>(); + List categories = new ArrayList<>(); + for (ColumnCategory category : + new ColumnCategory[] {ColumnCategory.TAG, ColumnCategory.FIELD}) { + for (int i = 0; i < columnNames.size(); i++) { + if (columnCategories.get(i) == category) { + names.add(columnNames.get(i)); + types.add(dataTypes.get(i)); + categories.add(category); + } + } + } + TSPrepareNumericRowReq request = new TSPrepareNumericRowReq(); + request + .setTableName(tableName) + .setMeasurements(names) + .setDataTypes(types.stream().map(TSDataType::ordinal).toList()) + .setColumnCategories(categories.stream().map(c -> (byte) c.ordinal()).toList()); + NumericRowPreparedStatement prepared = + new NumericRowPreparedStatement(tableName, names, types, categories); + SessionConnection connection = session.getDefaultSessionConnection(); + prepared.bind( + connection.prepareNumericRow(request), + connection, + () -> { + try { + connection.closePreparedNumericRow(prepared.getPreparedId()); + } catch (IoTDBConnectionException ignored) { + // Closing the session releases all server-side prepared rows. + } + }); + return prepared; + } + + @Override + public void executePreparedInsertNumericRow( + NumericRowPreparedStatement prepared, long timestamp, long[] values, String[] tagValues) + throws StatementExecutionException, IoTDBConnectionException { + SessionConnection connection = session.getDefaultSessionConnection(); + if (prepared == null || prepared.isClosed() || prepared.getOwner() != connection) { + throw new IoTDBConnectionException( + SessionMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_BELONGS_TO_ANOTHER_CONNECTION_PREPARE_IT_AGAIN_055CC7BC); + } + int tagCount = prepared.getTagCount(); + int fieldCount = prepared.getDataTypes().size() - tagCount; + if (values.length != fieldCount || tagValues.length != tagCount) { + throw new IllegalArgumentException( + SessionMessages.EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_COUNT_MISMATCH_9ACD3A0C); + } + int tagBytes = 0; + for (String tag : tagValues) { + tagBytes += ReadWriteIOUtils.sizeToWrite(tag); + } + ByteBuffer tagBuffer = prepared.prepareTagBuffer(tagBytes); + for (String tag : tagValues) { + ReadWriteIOUtils.write(tag, tagBuffer); + } + tagBuffer.flip(); + ByteBuffer valueBuffer = prepared.prepareValueBuffer(); + int valueIndex = 0; + for (int i = 0; i < prepared.getDataTypes().size(); i++) { + if (prepared.getColumnCategories().get(i) == ColumnCategory.FIELD) { + long value = values[valueIndex++]; + switch (prepared.getDataTypes().get(i)) { + case BOOLEAN -> valueBuffer.put((byte) (value == 0 ? 0 : 1)); + case INT32, DATE, FLOAT -> valueBuffer.putInt((int) value); + case INT64, TIMESTAMP, DOUBLE -> valueBuffer.putLong(value); + default -> + throw new IllegalArgumentException( + SessionMessages.EXCEPTION_UNSUPPORTED_NUMERIC_TYPE_06DD3044); + } + } + } + valueBuffer.flip(); + connection.executePreparedNumericRow( + prepared.getPreparedId(), timestamp, valueBuffer, tagBuffer); + } + @Override public void executeNonQueryStatement(String sql) throws IoTDBConnectionException, StatementExecutionException { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java index 8f158f4ab4ff6..9478c35d3cb22 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java @@ -21,10 +21,12 @@ import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.NumericRow; +import org.apache.iotdb.isession.NumericRowPreparedStatement; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.TableSession; import org.apache.iotdb.session.i18n.SessionMessages; import org.apache.tsfile.write.record.Tablet; From 8386cdfce3249a6e1e7219e3a15b17869761f786 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 16 Sep 2026 16:48:32 +0800 Subject: [PATCH 08/20] tmp --- .../session/pool/TableSessionWrapper.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java index 9478c35d3cb22..627501a320084 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java @@ -29,10 +29,13 @@ import org.apache.iotdb.session.TableSession; import org.apache.iotdb.session.i18n.SessionMessages; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.write.record.Tablet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; @@ -49,12 +52,15 @@ public class TableSessionWrapper implements ITableSession { private Session session; + private final TableSession tableSession; + private final SessionPool sessionPool; private final AtomicBoolean closed; protected TableSessionWrapper(Session session, SessionPool sessionPool) { this.session = session; + this.tableSession = new TableSession(session); this.sessionPool = sessionPool; this.closed = new AtomicBoolean(false); } @@ -83,6 +89,31 @@ public void insert(NumericRow row) throws StatementExecutionException, IoTDBConn } } + @Override + public NumericRowPreparedStatement prepareInsertNumericRow( + String tableName, + List columnNames, + List dataTypes, + List columnCategories) + throws StatementExecutionException, IoTDBConnectionException { + return tableSession.prepareInsertNumericRow( + tableName, columnNames, dataTypes, columnCategories); + } + + @Override + public void executePreparedInsertNumericRow( + NumericRowPreparedStatement prepared, long timestamp, long[] values, String[] tagValues) + throws StatementExecutionException, IoTDBConnectionException { + try { + tableSession.executePreparedInsertNumericRow(prepared, timestamp, values, tagValues); + } catch (IoTDBConnectionException e) { + sessionPool.cleanSessionAndMayThrowConnectionException(session); + closed.set(true); + session = null; + throw e; + } + } + @Override public SessionDataSet executeQueryStatement(String sql) throws StatementExecutionException, IoTDBConnectionException { From f0bed1c20aeaf6fdc84087a24a044ccf3e7a5cbc Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 16 Sep 2026 18:13:04 +0800 Subject: [PATCH 09/20] tmp --- .../isession/NumericRowPreparedStatement.java | 132 ++++++++++ .../session/PreparedNumericRowBufferTest.java | 141 ++++++++++ .../iotdb/db/i18n/DataNodeMiscMessages.java | 22 ++ .../iotdb/db/i18n/DataNodeMiscMessages.java | 22 ++ .../session/PreparedNumericRowInfo.java | 242 ++++++++++++++++++ 5 files changed, 559 insertions(+) create mode 100644 iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java create mode 100644 iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java new file mode 100644 index 0000000000000..8852d05a8a8ea --- /dev/null +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. + */ +package org.apache.iotdb.isession; + +import org.apache.iotdb.isession.i18n.ISessionMessages; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; + +import java.nio.ByteBuffer; +import java.util.List; + +/** + * Immutable metadata and reusable serialization buffers for repeated numeric table row inserts. + * + *

This statement is not thread-safe. An execution must return before the statement is reused or + * closed, because its buffers are retained until the synchronous RPC completes. + */ +public final class NumericRowPreparedStatement implements AutoCloseable { + private final String tableName; + private final List columnNames; + private final List dataTypes; + private final List columnCategories; + private final int tagCount; + private final int valueBytes; + private ByteBuffer tagBuffer; + private ByteBuffer valueBuffer; + private long preparedId = -1; + private Object owner; + private Runnable closeAction = () -> {}; + private boolean closed; + + public NumericRowPreparedStatement( + String tableName, + List columnNames, + List dataTypes, + List columnCategories) { + this.tableName = tableName; + this.columnNames = List.copyOf(columnNames); + this.dataTypes = List.copyOf(dataTypes); + this.columnCategories = List.copyOf(columnCategories); + // Column categories are immutable, so repeated inserts can reuse this count. + this.tagCount = + (int) this.columnCategories.stream().filter(c -> c == ColumnCategory.TAG).count(); + // FIELD encodings have fixed widths for the lifetime of the prepared schema. + int bytes = 0; + for (int i = 0; i < this.dataTypes.size(); i++) { + if (this.columnCategories.get(i) == ColumnCategory.FIELD) { + bytes += + switch (this.dataTypes.get(i)) { + case BOOLEAN -> 1; + case INT32, DATE, FLOAT -> Integer.BYTES; + case INT64, TIMESTAMP, DOUBLE -> Long.BYTES; + default -> + throw new IllegalArgumentException( + ISessionMessages.EXCEPTION_UNSUPPORTED_NUMERIC_TYPE_06DD3044); + }; + } + } + this.valueBytes = bytes; + } + + public String getTableName() { + return tableName; + } + + public List getColumnNames() { + return columnNames; + } + + public List getDataTypes() { + return dataTypes; + } + + public List getColumnCategories() { + return columnCategories; + } + + public int getTagCount() { + return tagCount; + } + + /** + * Returns the TAG buffer cleared for writing, growing it when the encoded tags need more space. + */ + public ByteBuffer prepareTagBuffer(int requiredCapacity) { + if (tagBuffer == null || tagBuffer.capacity() < requiredCapacity) { + tagBuffer = ByteBuffer.allocate(requiredCapacity); + } + tagBuffer.clear(); + return tagBuffer; + } + + /** Returns the FIELD buffer cleared for writing; a prepared schema has a fixed encoded size. */ + public ByteBuffer prepareValueBuffer() { + if (valueBuffer == null) { + valueBuffer = ByteBuffer.allocate(valueBytes); + } + valueBuffer.clear(); + return valueBuffer; + } + + public long getPreparedId() { + return preparedId; + } + + public Object getOwner() { + return owner; + } + + public void bind(long id, Object owner, Runnable closeAction) { + this.preparedId = id; + this.owner = owner; + this.closeAction = closeAction; + this.closed = false; + } + + public boolean isClosed() { + return closed; + } + + @Override + public void close() { + closeAction.run(); + closeAction = () -> {}; + closed = true; + } +} diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java new file mode 100644 index 0000000000000..a409d6844f008 --- /dev/null +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session; + +import org.apache.iotdb.isession.NumericRowPreparedStatement; +import org.apache.iotdb.rpc.IoTDBConnectionException; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PreparedNumericRowBufferTest { + @Test + public void testBuffersAcrossTagGrowthShrinkAndRpcFailure() throws Exception { + Session session = mock(Session.class); + SessionConnection connection = mock(SessionConnection.class); + when(session.getDefaultSessionConnection()).thenReturn(connection); + TableSession tableSession = new TableSession(session); + NumericRowPreparedStatement prepared = + new NumericRowPreparedStatement( + "t", + List.of("tag", "flag", "count", "value"), + List.of(TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.DOUBLE), + List.of( + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD)); + prepared.bind(1, connection, () -> {}); + List tagBuffers = new ArrayList<>(); + List valueBuffers = new ArrayList<>(); + String[] tags = {"x", "long-tag-\u4e2d\u6587", "", null, "retry"}; + doAnswer( + invocation -> { + int index = tagBuffers.size(); + ByteBuffer values = invocation.getArgument(2); + ByteBuffer encodedTags = invocation.getArgument(3); + tagBuffers.add(encodedTags); + valueBuffers.add(values); + // Consume both buffers as a transport may do, and verify no previous row bytes + // remain. + assertEquals(tags[index], ReadWriteIOUtils.readString(encodedTags)); + assertFalse(encodedTags.hasRemaining()); + assertEquals(13, values.remaining()); + assertEquals((byte) (index % 2), values.get()); + assertEquals(index, values.getInt()); + assertEquals(Double.doubleToRawLongBits(index + 0.5), values.getLong()); + assertFalse(values.hasRemaining()); + if (index == 3) { + throw new IoTDBConnectionException("simulated transport failure"); + } + return null; + }) + .when(connection) + .executePreparedNumericRow(eq(1L), anyLong(), any(ByteBuffer.class), any(ByteBuffer.class)); + for (int i = 0; i < tags.length; i++) { + try { + tableSession.executePreparedInsertNumericRow( + prepared, + i, + new long[] {i % 2, i, Double.doubleToRawLongBits(i + 0.5)}, + new String[] {tags[i]}); + assertNotEquals(3, i); + } catch (IoTDBConnectionException e) { + assertEquals(3, i); + } + } + assertEquals(tags.length, tagBuffers.size()); + assertNotSame(tagBuffers.get(0), tagBuffers.get(1)); + for (int i = 1; i < tags.length; i++) { + assertSame(valueBuffers.get(0), valueBuffers.get(i)); + assertSame(tagBuffers.get(1), tagBuffers.get(i)); + } + } + + @Test + public void testBuffersAreOwnedByEachStatementWithoutTags() throws Exception { + Session session = mock(Session.class); + SessionConnection connection = mock(SessionConnection.class); + when(session.getDefaultSessionConnection()).thenReturn(connection); + TableSession tableSession = new TableSession(session); + List buffers = new ArrayList<>(); + doAnswer( + invocation -> { + ByteBuffer values = invocation.getArgument(2); + ByteBuffer tags = invocation.getArgument(3); + assertEquals(0, tags.remaining()); + assertEquals(42L, values.getLong()); + assertFalse(values.hasRemaining()); + buffers.add(values); + buffers.add(tags); + return null; + }) + .when(connection) + .executePreparedNumericRow( + anyLong(), anyLong(), any(ByteBuffer.class), any(ByteBuffer.class)); + for (int i = 0; i < 2; i++) { + NumericRowPreparedStatement prepared = + new NumericRowPreparedStatement( + "t", List.of("v"), List.of(TSDataType.INT64), List.of(ColumnCategory.FIELD)); + prepared.bind(i, connection, () -> {}); + tableSession.executePreparedInsertNumericRow(prepared, 0, new long[] {42}, new String[0]); + } + assertNotSame(buffers.get(0), buffers.get(2)); + assertNotSame(buffers.get(1), buffers.get(3)); + } +} diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index d4ce7f97ba848..d7e2081e7602b 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1490,5 +1490,27 @@ private DataNodeMiscMessages() {} "table_query_device_entry_batch_size_in_bytes (%d) exceeds the maximum RPC payload (dn_thrift_max_frame_size %d minus 1024 bytes); using %d as the effective value"; public static final String EXCEPTION_INVALID_PREPARED_NUMERIC_ROW_METADATA_OR_PAYLOAD_AECCCEAA = "Invalid prepared numeric row metadata or payload."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_REQUEST_IS_NULL_5A9F9BA9 = "Prepared numeric row request is null."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATABASE_IS_NULL_F2A2E0AC = "Prepared numeric row database is null."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TABLE_NAME_IS_NULL_OR_EMPTY_F2D38BE5 = "Prepared numeric row table name is null or empty."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_MEASUREMENTS_ARE_NULL_830DE646 = "Prepared numeric row measurements are null."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPES_ARE_NULL_7C15F477 = "Prepared numeric row data types are null."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_CATEGORIES_ARE_NULL_6F2EE1BE = "Prepared numeric row column categories are null."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_HAS_NO_COLUMNS_D4E40365 = "Prepared numeric row has no columns."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_METADATA_SIZE_MISMATCH_MEASUREMENTS_ARG_DATA_TYPES_ARG_CATEGORIES_ARG_23DAB34B = "Prepared numeric row column metadata size mismatch: measurements %d, data types %d, categories %d."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_NAME_IS_NULL_OR_EMPTY_AT_INDEX_ARG_DEBDCFF8 = "Prepared numeric row column name is null or empty at index %d."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_NAME_IS_DUPLICATED_AT_INDEX_ARG_ARG_73AAFECC = "Prepared numeric row column name is duplicated at index %d: %s."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPE_IS_NULL_AT_INDEX_ARG_FB2A1FD6 = "Prepared numeric row data type is null at index %d."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPE_ORDINAL_IS_INVALID_AT_INDEX_ARG_ARG_BB5059BB = "Prepared numeric row data type ordinal is invalid at index %d: %d."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_CATEGORY_IS_NULL_AT_INDEX_ARG_3ED8315E = "Prepared numeric row column category is null at index %d."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_IS_INVALID_AT_INDEX_ARG_CATEGORY_ARG_AND_TYPE_ARG_9DAEEA10 = "Prepared numeric row column is invalid at index %d: category %d and type %s."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_HAS_NO_FIELD_COLUMNS_A76209AE = "Prepared numeric row has no FIELD columns."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_BUFFER_IS_NULL_3FF27BF7 = "Prepared numeric row tag buffer is null."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_BUFFER_IS_NULL_A928F919 = "Prepared numeric row value buffer is null."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_BUFFER_SIZE_MISMATCH_EXPECTED_ARG_BYTES_BUT_GOT_ARG_B75D48C8 = "Prepared numeric row value buffer size mismatch: expected %d bytes but got %d."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_LENGTH_PREFIX_IS_INCOMPLETE_AT_INDEX_ARG_ARG_BYTES_REMAINING_6D342F3A = "Prepared numeric row tag length prefix is incomplete at index %d: %d bytes remaining."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_LENGTH_IS_INVALID_AT_INDEX_ARG_LENGTH_ARG_ARG_BYTES_REMAINING_DEDD1E8F = "Prepared numeric row tag length is invalid at index %d: length %d, %d bytes remaining."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_BUFFER_HAS_ARG_TRAILING_BYTES_042CF4D2 = "Prepared numeric row tag buffer has %d trailing bytes."; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_PAYLOAD_DECODING_FAILED_ARG_F87689DD = "Prepared numeric row payload decoding failed: %s."; public static final String EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_DOES_NOT_BELONG_TO_THIS_SESSION_5CC5C50A = "Prepared numeric row is closed or does not belong to this session."; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java index 2d19c74be8b7f..c8ecb24f9e89c 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java @@ -1471,6 +1471,28 @@ private DataNodeMiscMessages() {} public static final String EXCEPTION_INVALID_PREPARED_NUMERIC_ROW_METADATA_OR_PAYLOAD_AECCCEAA = "预备数值行元数据或数据载荷无效"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_REQUEST_IS_NULL_5A9F9BA9 = "预备数值行请求为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATABASE_IS_NULL_F2A2E0AC = "预备数值行数据库为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TABLE_NAME_IS_NULL_OR_EMPTY_F2D38BE5 = "预备数值行表名为 null 或为空"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_MEASUREMENTS_ARE_NULL_830DE646 = "预备数值行 measurements 为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPES_ARE_NULL_7C15F477 = "预备数值行数据类型为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_CATEGORIES_ARE_NULL_6F2EE1BE = "预备数值行列类别为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_HAS_NO_COLUMNS_D4E40365 = "预备数值行没有列"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_METADATA_SIZE_MISMATCH_MEASUREMENTS_ARG_DATA_TYPES_ARG_CATEGORIES_ARG_23DAB34B = "预备数值行列元数据数量不匹配：measurements %d，数据类型 %d，列类别 %d"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_NAME_IS_NULL_OR_EMPTY_AT_INDEX_ARG_DEBDCFF8 = "预备数值行列名在下标 %d 处为 null 或为空"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_NAME_IS_DUPLICATED_AT_INDEX_ARG_ARG_73AAFECC = "预备数值行列名在下标 %d 处重复：%s"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPE_IS_NULL_AT_INDEX_ARG_FB2A1FD6 = "预备数值行数据类型在下标 %d 处为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPE_ORDINAL_IS_INVALID_AT_INDEX_ARG_ARG_BB5059BB = "预备数值行数据类型序号在下标 %d 处无效：%d"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_CATEGORY_IS_NULL_AT_INDEX_ARG_3ED8315E = "预备数值行列类别在下标 %d 处为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_IS_INVALID_AT_INDEX_ARG_CATEGORY_ARG_AND_TYPE_ARG_9DAEEA10 = "预备数值行列在下标 %d 处无效：类别 %d，类型 %s"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_HAS_NO_FIELD_COLUMNS_A76209AE = "预备数值行没有 FIELD 列"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_BUFFER_IS_NULL_3FF27BF7 = "预备数值行 TAG 缓冲区为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_BUFFER_IS_NULL_A928F919 = "预备数值行 FIELD 缓冲区为 null"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_BUFFER_SIZE_MISMATCH_EXPECTED_ARG_BYTES_BUT_GOT_ARG_B75D48C8 = "预备数值行 FIELD 缓冲区大小不匹配：期望 %d 字节，实际 %d 字节"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_LENGTH_PREFIX_IS_INCOMPLETE_AT_INDEX_ARG_ARG_BYTES_REMAINING_6D342F3A = "预备数值行 TAG 长度前缀在下标 %d 处不完整：剩余 %d 字节"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_LENGTH_IS_INVALID_AT_INDEX_ARG_LENGTH_ARG_ARG_BYTES_REMAINING_DEDD1E8F = "预备数值行 TAG 长度在下标 %d 处无效：长度 %d，剩余 %d 字节"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_TAG_BUFFER_HAS_ARG_TRAILING_BYTES_042CF4D2 = "预备数值行 TAG 缓冲区有 %d 个尾随字节"; + public static final String EXCEPTION_PREPARED_NUMERIC_ROW_PAYLOAD_DECODING_FAILED_ARG_F87689DD = "预备数值行数据载荷解码失败：%s"; public static final String EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_DOES_NOT_BELONG_TO_THIS_SESSION_5CC5C50A = "预备数值行已关闭或不属于当前会话"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java new file mode 100644 index 0000000000000..a1a67e4510027 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.protocol.session; + +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.utils.TimestampPrecisionUtils; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.db.i18n.DataNodeMiscMessages; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; +import org.apache.iotdb.db.utils.NumericRowValues; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; + +/** + * Session-owned metadata. Each execution gets fresh arrays because schema analysis mutates them. + */ +public final class PreparedNumericRowInfo { + private final String database; + private final String table; + private final String[] measurements; + private final TSDataType[] types; + private final TsTableColumnCategory[] categories; + private final int tagCount; + private final int valueBytes; + + public PreparedNumericRowInfo(String database, TSPrepareNumericRowReq req) { + if (req == null) { + throw invalid(DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_REQUEST_IS_NULL_5A9F9BA9); + } + if (database == null) { + throw invalid(DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_DATABASE_IS_NULL_F2A2E0AC); + } + if (req.getTableName() == null || req.getTableName().isEmpty()) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_TABLE_NAME_IS_NULL_OR_EMPTY_F2D38BE5); + } + if (req.getMeasurements() == null) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_MEASUREMENTS_ARE_NULL_830DE646); + } + if (req.getDataTypes() == null) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPES_ARE_NULL_7C15F477); + } + if (req.getColumnCategories() == null) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_CATEGORIES_ARE_NULL_6F2EE1BE); + } + if (req.getMeasurementsSize() == 0) { + throw invalid(DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_HAS_NO_COLUMNS_D4E40365); + } + if (req.getMeasurementsSize() != req.getDataTypesSize() + || req.getMeasurementsSize() != req.getColumnCategoriesSize()) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_METADATA_SIZE_MISMATCH_MEASUREMENTS_ARG_DATA_TYPES_ARG_CATEGORIES_ARG_23DAB34B, + req.getMeasurementsSize(), + req.getDataTypesSize(), + req.getColumnCategoriesSize()); + } + this.database = database; + table = req.getTableName(); + measurements = req.getMeasurements().toArray(new String[0]); + types = new TSDataType[measurements.length]; + categories = new TsTableColumnCategory[measurements.length]; + int tags = 0; + int bytes = 0; + Set names = new HashSet<>(); + for (int i = 0; i < measurements.length; i++) { + Integer type = req.getDataTypes().get(i); + Byte category = req.getColumnCategories().get(i); + if (measurements[i] == null || measurements[i].isEmpty()) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_NAME_IS_NULL_OR_EMPTY_AT_INDEX_ARG_DEBDCFF8, + i); + } + if (!names.add(measurements[i].toLowerCase(Locale.ROOT))) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_NAME_IS_DUPLICATED_AT_INDEX_ARG_ARG_73AAFECC, + i, + measurements[i]); + } + if (type == null) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPE_IS_NULL_AT_INDEX_ARG_FB2A1FD6, + i); + } + if (type < 0 || type >= TSDataType.values().length) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_DATA_TYPE_ORDINAL_IS_INVALID_AT_INDEX_ARG_ARG_BB5059BB, + i, + type); + } + if (category == null) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_CATEGORY_IS_NULL_AT_INDEX_ARG_3ED8315E, + i); + } + types[i] = TSDataType.values()[type]; + if (category == ColumnCategory.TAG.ordinal() && types[i] == TSDataType.STRING && tags == i) { + tags++; + categories[i] = TsTableColumnCategory.TAG; + } else if (category == ColumnCategory.FIELD.ordinal() + && NumericRowValues.supports(types[i])) { + categories[i] = TsTableColumnCategory.FIELD; + bytes = Math.addExact(bytes, types[i].getDataTypeSize()); + } else { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_COLUMN_IS_INVALID_AT_INDEX_ARG_CATEGORY_ARG_AND_TYPE_ARG_9DAEEA10, + i, + category, + types[i]); + } + } + tagCount = tags; + valueBytes = bytes; + if (tagCount == measurements.length) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_HAS_NO_FIELD_COLUMNS_A76209AE); + } + } + + public InsertNumericRowStatement bind(long timestamp, ByteBuffer tags, ByteBuffer values) { + if (tags == null) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_TAG_BUFFER_IS_NULL_3FF27BF7); + } + if (values == null) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_BUFFER_IS_NULL_A928F919); + } + if (values.remaining() != valueBytes) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_BUFFER_SIZE_MISMATCH_EXPECTED_ARG_BYTES_BUT_GOT_ARG_B75D48C8, + valueBytes, + values.remaining()); + } + TimestampPrecisionUtils.checkTimestampPrecision(timestamp); + NumericRowValues row = new NumericRowValues(tagCount, types.length - tagCount); + // Duplicate cursors so validation never changes a caller's buffers, even on failure. + tags = tags.duplicate(); + values = values.duplicate(); + try { + for (int i = 0; i < tagCount; i++) { + if (tags.remaining() < Integer.BYTES) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_TAG_LENGTH_PREFIX_IS_INCOMPLETE_AT_INDEX_ARG_ARG_BYTES_REMAINING_6D342F3A, + i, + tags.remaining()); + } + int length = tags.getInt(tags.position()); + if (length < -1 || length > tags.remaining() - Integer.BYTES) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_TAG_LENGTH_IS_INVALID_AT_INDEX_ARG_LENGTH_ARG_ARG_BYTES_REMAINING_DEDD1E8F, + i, + length, + tags.remaining()); + } + row.setTag(i, ReadWriteIOUtils.readString(tags)); + } + if (tags.hasRemaining()) { + throw invalid( + DataNodeMiscMessages + .EXCEPTION_PREPARED_NUMERIC_ROW_TAG_BUFFER_HAS_ARG_TRAILING_BYTES_042CF4D2, + tags.remaining()); + } + for (int i = tagCount; i < types.length; i++) { + row.read(i, types[i], values); + } + } catch (BufferUnderflowException e) { + throw invalid( + DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_PAYLOAD_DECODING_FAILED_ARG_F87689DD, + e.getMessage()); + } + InsertNumericRowStatement statement = new InsertNumericRowStatement(); + statement.setDevicePath(new PartialPath(table, false)); + statement.setDatabaseName(database); + statement.setMeasurements(measurements.clone()); + statement.setDataTypes(types.clone()); + statement.setColumnCategories(categories.clone()); + statement.setTime(timestamp); + statement.setNumericValues(row); + return statement; + } + + public long ramBytesUsed() { + return RamUsageEstimator.shallowSizeOfInstance(PreparedNumericRowInfo.class) + + RamUsageEstimator.sizeOf(database) + + RamUsageEstimator.sizeOf(table) + + RamUsageEstimator.sizeOf(measurements) + + RamUsageEstimator.shallowSizeOf(types) + + RamUsageEstimator.shallowSizeOf(categories) + // Account for the map entry and boxed ID retained by the session. + + 64; + } + + private static SemanticException invalid() { + return new SemanticException( + DataNodeMiscMessages.EXCEPTION_INVALID_PREPARED_NUMERIC_ROW_METADATA_OR_PAYLOAD_AECCCEAA); + } + + private static SemanticException invalid(String message, Object... args) { + return new SemanticException(String.format(message, args)); + } +} From 028ceb4341f3b2e13e8d94e5eb60ac712cdea9a4 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 17 Sep 2026 10:33:50 +0800 Subject: [PATCH 10/20] Add schema check bypass --- .../it/session/IoTDBTableModelSessionIT.java | 39 +++ .../session/PreparedNumericRowInfo.java | 59 +++++ .../relational/sql/ast/InsertNumericRow.java | 29 +++ .../crud/InsertNumericRowStatement.java | 11 + .../session/PreparedNumericRowInfoTest.java | 225 ++++++++++++++++++ .../iotdb/commons/schema/table/TsTable.java | 10 + 6 files changed, 373 insertions(+) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java index 828ddba1fbd74..29ee26b30c05a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java @@ -193,6 +193,45 @@ public void testNumericRowInsertWithTags() throws Exception { } } + @Test + public void testPreparedNumericRowSchemaVersionChange() throws Exception { + // Reuse a prepared row before and after adding a TAG, including a flush/read round trip. + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE numeric_version"); + session.executeNonQueryStatement("USE numeric_version"); + session.executeNonQueryStatement("CREATE TABLE readings (tag STRING TAG, v DOUBLE FIELD)"); + NumericRowPreparedStatement prepared = + session.prepareInsertNumericRow( + "readings", + Arrays.asList("tag", "v"), + Arrays.asList(TSDataType.STRING, TSDataType.DOUBLE), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); + for (int time = 1; time <= 3; time++) { + if (time == 3) { + session.executeNonQueryStatement("ALTER TABLE readings ADD COLUMN tag2 STRING TAG"); + } + session.executePreparedInsertNumericRow( + prepared, + time, + new long[] {Double.doubleToRawLongBits(time + 0.5)}, + new String[] {"device"}); + } + session.executeNonQueryStatement("FLUSH"); + try (SessionDataSet dataSet = + session.executeQueryStatement("SELECT time,tag,tag2,v FROM readings ORDER BY time")) { + for (int time = 1; time <= 3; time++) { + assertTrue(dataSet.hasNext()); + RowRecord row = dataSet.next(); + assertEquals(time, row.getFields().get(0).getLongV()); + assertEquals("device", row.getFields().get(1).getStringValue()); + assertNull(row.getFields().get(2).getDataType()); + assertEquals(time + 0.5, row.getFields().get(3).getDoubleV(), 0); + } + assertFalse(dataSet.hasNext()); + } + } + } + @Test public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { Assume.assumeTrue( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java index a1a67e4510027..6d5dc1fce4b1a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java @@ -22,16 +22,24 @@ import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.queryengine.utils.TimestampPrecisionUtils; +import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.db.i18n.DataNodeMiscMessages; +import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.db.schemaengine.table.DataNodeTreeViewSchemaUtils; import org.apache.iotdb.db.utils.NumericRowValues; import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.write.schema.MeasurementSchema; import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; @@ -39,6 +47,8 @@ import java.util.Locale; import java.util.Set; +import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; + /** * Session-owned metadata. Each execution gets fresh arrays because schema analysis mutates them. */ @@ -50,6 +60,8 @@ public final class PreparedNumericRowInfo { private final TsTableColumnCategory[] categories; private final int tagCount; private final int valueBytes; + private final Pair tableVersion; + private final MeasurementSchema[] measurementSchemas; public PreparedNumericRowInfo(String database, TSPrepareNumericRowReq req) { if (req == null) { @@ -152,6 +164,44 @@ public PreparedNumericRowInfo(String database, TSPrepareNumericRowReq req) { throw invalid( DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_HAS_NO_FIELD_COLUMNS_A76209AE); } + TsTable currentTable = DataNodeTableCache.getInstance().getTableInWrite(database, table); + Pair version = currentTable == null ? null : currentTable.getInstanceVersion(); + MeasurementSchema[] schemas = prepareMeasurementSchemas(currentTable); + // Never retain a snapshot assembled across concurrent schema revisions. + if (schemas != null && currentTable.matchesInstanceVersion(version)) { + tableVersion = version; + measurementSchemas = schemas; + } else { + tableVersion = null; + measurementSchemas = null; + } + } + + private MeasurementSchema[] prepareMeasurementSchemas(TsTable currentTable) { + // Views and requests needing TAG padding/reordering or type conversion use normal analysis. + if (currentTable == null + || currentTable.getClass() != TsTable.class + || DataNodeTreeViewSchemaUtils.isTreeViewTable(currentTable) + || currentTable.getTagColumnSchemaList().size() != tagCount) { + return null; + } + MeasurementSchema[] schemas = new MeasurementSchema[measurements.length]; + for (int i = 0; i < measurements.length; i++) { + TsTableColumnSchema column = currentTable.getColumnSchema(measurements[i]); + if (column == null + || column.getDataType() != types[i] + || column.getColumnCategory() != categories[i] + || (i < tagCount && currentTable.getTagColumnOrdinal(measurements[i]) != i)) { + return null; + } + schemas[i] = + new MeasurementSchema( + column.getColumnName(), + types[i], + getDefaultEncoding(types[i]), + TSFileDescriptor.getInstance().getConfig().getCompressor(types[i])); + } + return schemas; } public InsertNumericRowStatement bind(long timestamp, ByteBuffer tags, ByteBuffer values) { @@ -217,6 +267,10 @@ public InsertNumericRowStatement bind(long timestamp, ByteBuffer tags, ByteBuffe statement.setColumnCategories(categories.clone()); statement.setTime(timestamp); statement.setNumericValues(row); + if (measurementSchemas != null) { + statement.setPreparedTableVersion(tableVersion); + statement.setMeasurementSchemas(measurementSchemas.clone()); + } return statement; } @@ -227,6 +281,11 @@ public long ramBytesUsed() { + RamUsageEstimator.sizeOf(measurements) + RamUsageEstimator.shallowSizeOf(types) + RamUsageEstimator.shallowSizeOf(categories) + + InsertNodeMemoryEstimator.sizeOfMeasurementSchemas(measurementSchemas) + + (tableVersion == null + ? 0 + : RamUsageEstimator.shallowSizeOf(tableVersion) + + 2 * RamUsageEstimator.shallowSizeOfInstance(Long.class)) // Account for the map entry and boxed ID retained by the session. + 64; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertNumericRow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertNumericRow.java index 38f29f01d2c96..4510973f7535f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertNumericRow.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertNumericRow.java @@ -20,8 +20,13 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IAstVisitor; +import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.analyze.lock.DataNodeSchemaLockManager; +import org.apache.iotdb.db.queryengine.plan.analyze.lock.SchemaLockType; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import java.util.Arrays; import java.util.Collections; @@ -43,6 +48,30 @@ public InsertNumericRowStatement getInnerTreeStatement() { return (InsertNumericRowStatement) super.getInnerTreeStatement(); } + @Override + public void validateTableSchema(Metadata metadata, MPPQueryContext context) { + InsertNumericRowStatement statement = getInnerTreeStatement(); + if (statement.getPreparedTableVersion() != null) { + // Check at analysis time under the same deletion lock and metadata lease as normal inserts. + DataNodeSchemaLockManager.getInstance() + .takeReadLock(context, SchemaLockType.VALIDATE_VS_DELETION_TABLE); + TsTable currentTable = + DataNodeTableCache.getInstance().getTableInWrite(getDatabase(), getTableName()); + if (currentTable != null + && currentTable.matchesInstanceVersion(statement.getPreparedTableVersion())) { + statement.setAttributeColumnsPresent(false); + // Cached schema names must not be mutated by later normalization. + statement.setToLowerCaseApplied(true); + statement.semanticCheck(); + return; + } + // Full validation may cast, reorder, or fail columns; discard all cached schema state first. + statement.setPreparedTableVersion(null); + statement.setMeasurementSchemas(null); + } + super.validateTableSchema(metadata, context); + } + @Override public void updateAfterSchemaValidation(MPPQueryContext context) { // Numeric rows are already typed; schema validation casts values in their primitive storage. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java index fe77996fabe98..7093b1615fb10 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java @@ -39,6 +39,7 @@ import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.UnSupportedDataTypeException; @@ -55,6 +56,8 @@ public class InsertNumericRowStatement extends InsertBaseStatement { private long time; private NumericRowValues numericValues; private IDeviceID deviceID; + // A candidate snapshot, checked again under the schema analysis lock before reuse. + private Pair preparedTableVersion; public InsertNumericRowStatement() { statementType = StatementType.INSERT_NUMERIC_ROW; @@ -62,6 +65,14 @@ public InsertNumericRowStatement() { isAligned = true; } + public Pair getPreparedTableVersion() { + return preparedTableVersion; + } + + public void setPreparedTableVersion(Pair preparedTableVersion) { + this.preparedTableVersion = preparedTableVersion; + } + public long getTime() { return time; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java new file mode 100644 index 0000000000000..e195d69660cf1 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.protocol.session; + +import org.apache.iotdb.commons.queryengine.common.SessionInfo; +import org.apache.iotdb.commons.schema.table.TreeViewSchema; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.plan.analyze.lock.DataNodeSchemaLockManager; +import org.apache.iotdb.db.queryengine.plan.analyze.lock.SchemaLockType; +import org.apache.iotdb.db.queryengine.plan.analyze.schema.SchemaValidator; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator; +import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertNumericRow; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.time.ZoneId; +import java.util.Arrays; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +public class PreparedNumericRowInfoTest { + private static final String DATABASE = "prepared_numeric_fastpath_test"; + private TsTable table; + private MPPQueryContext context; + private Metadata metadata; + + @Before + public void setUp() { + DataNodeTableCache.getInstance().invalid(DATABASE); + table = new TsTable("t"); + table.addColumnSchema(new TagColumnSchema("tag", TSDataType.STRING)); + table.addColumnSchema(new FieldColumnSchema("v", TSDataType.DOUBLE)); + install(table); + context = new MPPQueryContext(new QueryId("prepared_test")); + context.setSession(new SessionInfo(1, "root", ZoneId.of("UTC"))); + metadata = mock(Metadata.class); + doAnswer( + invocation -> { + TableHeaderSchemaValidator.getInstance() + .validateInsertNodeMeasurements( + invocation.getArgument(0), + invocation.getArgument(1), + invocation.getArgument(2), + invocation.getArgument(3), + invocation.getArgument(4), + invocation.getArgument(5)); + return null; + }) + .when(metadata) + .validateInsertNodeMeasurements(anyString(), any(), any(), anyBoolean(), any(), any()); + } + + @After + public void tearDown() { + DataNodeSchemaLockManager.getInstance().releaseReadLock(context); + DataNodeTableCache.getInstance().invalid(DATABASE); + } + + // An unchanged table skips column validation but still checks access and device metadata. + @Test + public void testUnchangedVersionRetainsOtherChecks() { + PreparedNumericRowInfo info = new PreparedNumericRowInfo(DATABASE, request("tag", "v")); + InsertNumericRowStatement statement = bind(info, 1); + InsertNumericRowStatement next = bind(info, 1); + assertNotNull(statement.getPreparedTableVersion()); + assertNotSame(statement.getMeasurementSchemas(), next.getMeasurementSchemas()); + assertSame(statement.getMeasurementSchemas()[1], next.getMeasurementSchemas()[1]); + AccessControl accessControl = mock(AccessControl.class); + InsertNumericRow insert = new InsertNumericRow(statement, context); + SchemaValidator.validate(metadata, insert, context, accessControl); + verify(metadata, never()) + .validateInsertNodeMeasurements(anyString(), any(), any(), anyBoolean(), any(), any()); + verify(metadata).validateDeviceSchema(insert, context); + verify(accessControl).checkCanInsertIntoTable(eq("root"), any(), eq(context)); + assertTrue(context.getAcquiredLocks().contains(SchemaLockType.VALIDATE_VS_DELETION_TABLE)); + assertEquals(TSDataType.DOUBLE, statement.getMeasurementSchemas()[1].getType()); + statement.getMeasurementSchemas()[1] = null; + assertNotNull(bind(info, 1).getMeasurementSchemas()[1]); + } + + // Adding a TAG after bind forces normal validation to pad the new device-id segment. + @Test + public void testVersionChangeBetweenBindAndAnalysis() { + PreparedNumericRowInfo info = new PreparedNumericRowInfo(DATABASE, request("tag", "v")); + InsertNumericRowStatement statement = bind(info, 1); + table.addColumnSchema(new TagColumnSchema("tag2", TSDataType.STRING)); + validateFallback(statement); + assertArrayEquals(new String[] {"tag", "tag2", "v"}, statement.getMeasurements()); + assertNull(statement.getFirstValueOfIndex(1)); + assertEquals(2.5D, statement.getFirstValueOfIndex(2)); + assertArrayEquals(new String[] {"tag", "v"}, bind(info, 1).getMeasurements()); + } + + // A replacement table with the same name/revision must not reuse the old instance's schema. + @Test + public void testRecreatedTableUsesFullValidation() { + InsertNumericRowStatement statement = + bind(new PreparedNumericRowInfo(DATABASE, request("tag", "v")), 1); + TsTable replacement = new TsTable("t"); + replacement.addColumnSchema(new TagColumnSchema("tag", TSDataType.STRING)); + replacement.addColumnSchema(new FieldColumnSchema("v", TSDataType.DOUBLE)); + assertEquals(table.getInstanceVersion().right, replacement.getInstanceVersion().right); + install(replacement); + validateFallback(statement); + assertNotNull(statement.getMeasurementSchemas()[1]); + } + + // Missing columns/table, views and type conversions remain on normal creation/validation. + @Test + public void testIneligibleMetadataHasNoCachedSchema() { + assertNull( + bind(new PreparedNumericRowInfo(DATABASE, request("tag", "missing")), 1) + .getPreparedTableVersion()); + TSPrepareNumericRowReq req = request("tag", "v"); + req.setDataTypes(Arrays.asList(TSDataType.STRING.ordinal(), TSDataType.INT64.ordinal())); + assertNull( + new PreparedNumericRowInfo(DATABASE, req) + .bind(1, tagBuffer(1), ByteBuffer.allocate(8)) + .getPreparedTableVersion()); + table.addProp(TreeViewSchema.TREE_PATH_PATTERN, "root.sg.**"); + assertNull( + bind(new PreparedNumericRowInfo(DATABASE, request("tag", "v")), 1) + .getPreparedTableVersion()); + DataNodeTableCache.getInstance().invalid(DATABASE); + assertNull( + bind(new PreparedNumericRowInfo(DATABASE, request("tag", "v")), 1).getMeasurementSchemas()); + } + + // Reversed TAG input must take the normal path so payload values follow schema order. + @Test + public void testTagReorderingFallsBack() { + table.addColumnSchema(new TagColumnSchema("tag2", TSDataType.STRING)); + InsertNumericRowStatement statement = + bind(new PreparedNumericRowInfo(DATABASE, request("tag2", "tag", "v")), 2); + assertNull(statement.getPreparedTableVersion()); + validateFallback(statement); + assertArrayEquals(new String[] {"tag", "tag2", "v"}, statement.getMeasurements()); + assertEquals("b", statement.getNumericValues().getTag(0)); + assertEquals("a", statement.getNumericValues().getTag(1)); + } + + private void validateFallback(InsertNumericRowStatement statement) { + new InsertNumericRow(statement, context).validateTableSchema(metadata, context); + verify(metadata) + .validateInsertNodeMeasurements(eq(DATABASE), any(), eq(context), eq(true), any(), any()); + assertNull(statement.getPreparedTableVersion()); + } + + private static void install(TsTable table) { + DataNodeTableCache.getInstance().preUpdateTable(DATABASE, table, null); + DataNodeTableCache.getInstance().commitUpdateTable(DATABASE, "t", null); + } + + private static TSPrepareNumericRowReq request(String... names) { + TSPrepareNumericRowReq req = new TSPrepareNumericRowReq(); + req.setTableName("t"); + req.setMeasurements(Arrays.asList(names)); + for (int i = 0; i < names.length; i++) { + boolean tag = i < names.length - 1; + req.addToDataTypes((tag ? TSDataType.STRING : TSDataType.DOUBLE).ordinal()); + req.addToColumnCategories((byte) (tag ? ColumnCategory.TAG : ColumnCategory.FIELD).ordinal()); + } + return req; + } + + private static ByteBuffer tagBuffer(int count) { + ByteBuffer tags = ByteBuffer.allocate(count * 5); + for (int i = 0; i < count; i++) { + tags.putInt(1).put((byte) ('a' + i)); + } + tags.flip(); + return tags; + } + + private static InsertNumericRowStatement bind(PreparedNumericRowInfo info, int tags) { + ByteBuffer values = ByteBuffer.allocate(8); + values.putDouble(2.5D).flip(); + return info.bind(1, tagBuffer(tags), values); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 14481bfa9bcaa..8ddabbd96d1af 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -375,6 +375,16 @@ public Pair getInstanceVersion() { return new Pair<>(creationId, instanceVersion.get()); } + /** Checks both table identity and schema revision, waiting for any ongoing schema write. */ + public boolean matchesInstanceVersion(final Pair version) { + readWriteLock.readLock().lock(); + try { + return creationId.equals(version.left) && instanceVersion.get() == version.right; + } finally { + readWriteLock.readLock().unlock(); + } + } + public boolean containsPropWithoutLock(final String propKey) { return props != null && props.containsKey(propKey); } From 7ad6ab3edd03f81f07ed5eaa9e8e6dbe4e80b5f0 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 17 Sep 2026 11:07:48 +0800 Subject: [PATCH 11/20] add null bitmap --- .../it/session/IoTDBTableModelSessionIT.java | 79 +++++++++++++++++++ .../apache/iotdb/isession/ITableSession.java | 14 ++++ .../org/apache/iotdb/isession/NumericRow.java | 29 ++++++- .../isession/NumericRowPreparedStatement.java | 10 +++ .../iotdb/session/i18n/SessionMessages.java | 2 + .../iotdb/session/i18n/SessionMessages.java | 2 + .../org/apache/iotdb/session/Session.java | 5 +- .../iotdb/session/SessionConnection.java | 9 +++ .../apache/iotdb/session/TableSession.java | 35 +++++++- .../session/pool/TableSessionWrapper.java | 15 +++- .../iotdb/session/util/SessionUtils.java | 53 ++++++++++++- .../session/PreparedNumericRowBufferTest.java | 56 +++++++++++++ .../iotdb/session/util/SessionUtilsTest.java | 63 +++++++++++++++ .../iotdb/db/i18n/DataNodeQueryMessages.java | 6 ++ .../iotdb/db/i18n/DataNodeQueryMessages.java | 6 ++ .../session/PreparedNumericRowInfo.java | 25 +++++- .../thrift/impl/ClientRPCServiceImpl.java | 3 +- .../plan/parser/StatementGenerator.java | 3 +- .../crud/InsertNumericRowStatement.java | 17 +++- .../iotdb/db/utils/NumericRowValues.java | 30 +++++++ .../session/PreparedNumericRowInfoTest.java | 75 ++++++++++++++++++ .../src/main/thrift/client.thrift | 5 ++ 22 files changed, 527 insertions(+), 15 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java index 29ee26b30c05a..9ac67ad9653ec 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java @@ -34,6 +34,7 @@ import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.record.Tablet; import org.junit.After; import org.junit.Assume; @@ -232,6 +233,84 @@ public void testPreparedNumericRowSchemaVersionChange() throws Exception { } } + @Test + public void testNumericRowNullBitmap() throws Exception { + // Exercise bits 0/7/8, interleaved TAG metadata, prepared fast/fallback paths and pooled + // sessions. + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE numeric_nulls"); + session.executeNonQueryStatement("USE numeric_nulls"); + StringBuilder ddl = new StringBuilder("CREATE TABLE readings(tag STRING TAG"); + List names = new ArrayList<>(); + List types = new ArrayList<>(); + List categories = new ArrayList<>(); + long[] values = new long[9]; + for (int i = 0; i < 9; i++) { + ddl.append(", f").append(i).append(" DOUBLE FIELD"); + names.add("f" + i); + types.add(TSDataType.DOUBLE); + categories.add(ColumnCategory.FIELD); + values[i] = Double.doubleToRawLongBits(i + 0.25); + } + session.executeNonQueryStatement(ddl.append(")").toString()); + names.add(1, "tag"); + types.add(1, TSDataType.STRING); + categories.add(1, ColumnCategory.TAG); + BitMap nulls = new BitMap(9); + nulls.mark(0); + nulls.mark(7); + nulls.mark(8); + String[] tags = {"device"}; + session.insert(new NumericRow("readings", 1, names, types, categories, values, tags, nulls)); + try (NumericRowPreparedStatement prepared = + session.prepareInsertNumericRow("readings", names, types, categories)) { + session.executePreparedInsertNumericRow(prepared, 2, values, tags, nulls); + session.executePreparedInsertNumericRow(prepared, 3, values, tags); + // All-null table rows retain their timestamp/TAGs; null slots must never become zeros. + BitMap allNulls = new BitMap(9); + allNulls.markAll(); + session.executePreparedInsertNumericRow(prepared, 3, values, tags, allNulls); + session.insert( + new NumericRow("readings", 10, names, types, categories, values, tags, allNulls)); + session.executePreparedInsertNumericRow(prepared, 11, values, tags, allNulls); + session.executeNonQueryStatement("ALTER TABLE readings ADD COLUMN tag2 STRING TAG"); + session.executePreparedInsertNumericRow(prepared, 4, values, tags, nulls); + } + try (ITableSessionPool pool = EnvFactory.getEnv().getTableSessionPool(1, "numeric_nulls"); + ITableSession pooled = pool.getSession(); + NumericRowPreparedStatement prepared = + pooled.prepareInsertNumericRow("readings", names, types, categories)) { + pooled.insert(new NumericRow("readings", 5, names, types, categories, values, tags, nulls)); + pooled.executePreparedInsertNumericRow(prepared, 6, values, tags, nulls); + } + assertNumericNullRows(session); + session.executeNonQueryStatement("FLUSH"); + assertNumericNullRows(session); + } + } + + private void assertNumericNullRows(ITableSession session) throws Exception { + try (SessionDataSet dataSet = + session.executeQueryStatement( + "SELECT time,tag,tag2,f0,f1,f2,f3,f4,f5,f6,f7,f8 FROM readings ORDER BY time")) { + for (int time : new int[] {1, 2, 3, 4, 5, 6, 10, 11}) { + assertTrue(dataSet.hasNext()); + RowRecord row = dataSet.next(); + assertEquals(time, row.getFields().get(0).getLongV()); + assertEquals("device", row.getFields().get(1).getStringValue()); + assertNull(row.getFields().get(2).getDataType()); + for (int field = 0; field < 9; field++) { + if (time >= 10 || (time != 3 && (field == 0 || field == 7 || field == 8))) { + assertNull(row.getFields().get(field + 3).getDataType()); + } else { + assertEquals(field + 0.25, row.getFields().get(field + 3).getDoubleV(), 0); + } + } + } + assertFalse(dataSet.hasNext()); + } + } + @Test public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { Assume.assumeTrue( diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java index d14abab1238f7..35a2f71b7dcbb 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java @@ -24,6 +24,7 @@ import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.record.Tablet; import java.util.List; @@ -73,6 +74,19 @@ void executePreparedInsertNumericRow( NumericRowPreparedStatement prepared, long timestamp, long[] values, String[] tagValues) throws StatementExecutionException, IoTDBConnectionException; + /** + * Executes a prepared row with an optional FIELD-only null bitmap sized to {@code values.length}. + * Marked slots are null and their values are ignored. Do not mutate the bitmap during execution. + * Requires a server supporting numeric row null bitmaps. + */ + void executePreparedInsertNumericRow( + NumericRowPreparedStatement prepared, + long timestamp, + long[] values, + String[] tagValues, + BitMap nullBitmap) + throws StatementExecutionException, IoTDBConnectionException; + /** * Executes a non-query SQL statement, such as a DDL or DML command. * diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRow.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRow.java index 502c75e05e4c9..040cb7632ba18 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRow.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRow.java @@ -21,6 +21,7 @@ import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; import java.util.List; @@ -29,8 +30,9 @@ * *

Column names, types and categories describe all supplied columns in the same order. The value * arrays contain only their respective FIELD or TAG values, in occurrence order in the column list. - * TAGs may be null or empty; missing TAG columns are null. ATTRIBUTE columns, nonnumeric FIELDs and - * null FIELD values are not supported. + * TAGs may be null or empty; missing TAG columns are null. An optional null bitmap marks null + * FIELDs by their index in {@code values}; marked slots retain their position but are not sent. + * ATTRIBUTE columns and nonnumeric FIELDs are not supported. * *

BOOLEAN uses zero for false and nonzero for true. INT32 and DATE use the low 32 bits (DATE is * encoded as yyyyMMdd); INT64 and TIMESTAMP use the long value. FLOAT uses {@link @@ -48,6 +50,7 @@ public final class NumericRow { private final List columnCategories; private final long[] values; private final String[] tagValues; + private final BitMap nullBitmap; /** Creates a row with numeric FIELDs and optional STRING TAGs in separate arrays. */ public NumericRow( @@ -58,6 +61,23 @@ public NumericRow( List columnCategories, long[] values, String[] tagValues) { + this(tableName, timestamp, columnNames, dataTypes, columnCategories, values, tagValues, null); + } + + /** + * Creates a row with an optional FIELD-only null bitmap, sized exactly to {@code values.length}. + * A marked bit denotes null. The bitmap is retained and must not change during insertion. + * Requires a server supporting numeric row null bitmaps when the bitmap is supplied. + */ + public NumericRow( + String tableName, + long timestamp, + List columnNames, + List dataTypes, + List columnCategories, + long[] values, + String[] tagValues, + BitMap nullBitmap) { this.tableName = tableName; this.timestamp = timestamp; this.columnNames = columnNames; @@ -65,6 +85,7 @@ public NumericRow( this.columnCategories = columnCategories; this.values = values; this.tagValues = tagValues; + this.nullBitmap = nullBitmap; } public String getTableName() { @@ -94,4 +115,8 @@ public long[] getValues() { public String[] getTagValues() { return tagValues; } + + public BitMap getNullBitmap() { + return nullBitmap; + } } diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java index 8852d05a8a8ea..ee0b0f21804fd 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java @@ -29,6 +29,7 @@ public final class NumericRowPreparedStatement implements AutoCloseable { private final int valueBytes; private ByteBuffer tagBuffer; private ByteBuffer valueBuffer; + private ByteBuffer nullBitmapBuffer; private long preparedId = -1; private Object owner; private Runnable closeAction = () -> {}; @@ -108,6 +109,15 @@ public long getPreparedId() { return preparedId; } + /** Returns a reusable FIELD null bitmap buffer, allocated only for nullable executions. */ + public ByteBuffer prepareNullBitmapBuffer() { + if (nullBitmapBuffer == null) { + nullBitmapBuffer = ByteBuffer.allocate((dataTypes.size() - tagCount + 7) / 8); + } + nullBitmapBuffer.clear(); + return nullBitmapBuffer; + } + public Object getOwner() { return owner; } diff --git a/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java b/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java index 173260e4a07cf..a1bdf45262ec0 100644 --- a/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java +++ b/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java @@ -20,6 +20,8 @@ package org.apache.iotdb.session.i18n; public final class SessionMessages { + public static final String EXCEPTION_NUMERIC_ROW_NULL_BITMAP_SIZE_MISMATCH_EXPECTED_ARG_FIELDS_BUT_GOT_ARG_8D30F6A0 = + "Numeric row null bitmap size mismatch: expected %d FIELDs but got %d"; public static final String EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233 = "Numeric row arguments must be non-null; column metadata lengths and per-category value counts must match."; public static final String EXCEPTION_NUMERIC_ROWS_SUPPORT_NUMERIC_FIELD_COLUMNS_AND_STRING_TAG_COLUMNS_ONLY_87159FFA = "Numeric rows support numeric FIELD columns and STRING TAG columns only."; diff --git a/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java b/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java index b45704d8738ac..260b000e61a6b 100644 --- a/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java +++ b/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java @@ -20,6 +20,8 @@ package org.apache.iotdb.session.i18n; public final class SessionMessages { + public static final String EXCEPTION_NUMERIC_ROW_NULL_BITMAP_SIZE_MISMATCH_EXPECTED_ARG_FIELDS_BUT_GOT_ARG_8D30F6A0 = + "Numeric row 的 null bitmap 大小不匹配：预期 %d 个 FIELD，实际为 %d"; public static final String EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233 = "数值行参数不能为 null；列元数据长度必须一致，FIELD 和 TAG 的数量必须分别与对应的值数组长度一致。"; public static final String EXCEPTION_NUMERIC_ROWS_SUPPORT_NUMERIC_FIELD_COLUMNS_AND_STRING_TAG_COLUMNS_ONLY_87159FFA = "数值行仅支持数值类型的 FIELD 列和 STRING 类型的 TAG 列。"; diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index 835cf924777c0..f5c6ebd1b809e 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -2922,7 +2922,7 @@ public void insertRelationalRecord(NumericRow row) } ByteBuffer valueBuffer = SessionUtils.getValueBuffer( - rpcTypes, values, rpcMeasurements, rpcColumnCategories, tagValues); + rpcTypes, values, rpcMeasurements, rpcColumnCategories, tagValues, row.getNullBitmap()); TSInsertRecordReq request = new TSInsertRecordReq(); request.setPrefixPath(tableName); request.setTimestamp(timestamp); @@ -2930,6 +2930,9 @@ public void insertRelationalRecord(NumericRow row) request.setValues(valueBuffer); request.setIsWriteToTable(true); request.setIsNumericRow(true); + if (row.getNullBitmap() != null) { + request.setNullBitmap(SessionUtils.encodeNumericRowNullBitmap(row.getNullBitmap(), null)); + } request.setIsAligned(true); request.setColumnCategoryies( rpcColumnCategories.stream() diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java index e01c5dd8a2073..f98e58965aeca 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java @@ -782,8 +782,17 @@ protected long prepareNumericRow(TSPrepareNumericRowReq request) protected void executePreparedNumericRow( long id, long timestamp, ByteBuffer values, ByteBuffer tags) throws IoTDBConnectionException, StatementExecutionException { + executePreparedNumericRow(id, timestamp, values, tags, null); + } + + protected void executePreparedNumericRow( + long id, long timestamp, ByteBuffer values, ByteBuffer tags, ByteBuffer nullBitmap) + throws IoTDBConnectionException, StatementExecutionException { TSExecutePreparedNumericRowReq request = new TSExecutePreparedNumericRowReq(sessionId, id, timestamp, values, tags); + if (nullBitmap != null) { + request.setNullBitmap(nullBitmap); + } try { RpcUtils.verifySuccess(client.executePreparedNumericRow(request)); } catch (TException e) { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java index 7833557e10b1a..da93c37586fbf 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java @@ -27,9 +27,11 @@ import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; import org.apache.iotdb.session.i18n.SessionMessages; +import org.apache.iotdb.session.util.SessionUtils; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.record.Tablet; @@ -101,6 +103,17 @@ public NumericRowPreparedStatement prepareInsertNumericRow( public void executePreparedInsertNumericRow( NumericRowPreparedStatement prepared, long timestamp, long[] values, String[] tagValues) throws StatementExecutionException, IoTDBConnectionException { + executePreparedInsertNumericRow(prepared, timestamp, values, tagValues, null); + } + + @Override + public void executePreparedInsertNumericRow( + NumericRowPreparedStatement prepared, + long timestamp, + long[] values, + String[] tagValues, + BitMap nullBitmap) + throws StatementExecutionException, IoTDBConnectionException { SessionConnection connection = session.getDefaultSessionConnection(); if (prepared == null || prepared.isClosed() || prepared.getOwner() != connection) { throw new IoTDBConnectionException( @@ -109,10 +122,14 @@ public void executePreparedInsertNumericRow( } int tagCount = prepared.getTagCount(); int fieldCount = prepared.getDataTypes().size() - tagCount; - if (values.length != fieldCount || tagValues.length != tagCount) { + if (values == null + || tagValues == null + || values.length != fieldCount + || tagValues.length != tagCount) { throw new IllegalArgumentException( SessionMessages.EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_COUNT_MISMATCH_9ACD3A0C); } + SessionUtils.validateNumericRowNullBitmap(nullBitmap, fieldCount); int tagBytes = 0; for (String tag : tagValues) { tagBytes += ReadWriteIOUtils.sizeToWrite(tag); @@ -127,6 +144,9 @@ public void executePreparedInsertNumericRow( for (int i = 0; i < prepared.getDataTypes().size(); i++) { if (prepared.getColumnCategories().get(i) == ColumnCategory.FIELD) { long value = values[valueIndex++]; + if (nullBitmap != null && nullBitmap.isMarked(valueIndex - 1)) { + continue; + } switch (prepared.getDataTypes().get(i)) { case BOOLEAN -> valueBuffer.put((byte) (value == 0 ? 0 : 1)); case INT32, DATE, FLOAT -> valueBuffer.putInt((int) value); @@ -138,8 +158,17 @@ public void executePreparedInsertNumericRow( } } valueBuffer.flip(); - connection.executePreparedNumericRow( - prepared.getPreparedId(), timestamp, valueBuffer, tagBuffer); + if (nullBitmap == null) { + connection.executePreparedNumericRow( + prepared.getPreparedId(), timestamp, valueBuffer, tagBuffer); + } else { + connection.executePreparedNumericRow( + prepared.getPreparedId(), + timestamp, + valueBuffer, + tagBuffer, + SessionUtils.encodeNumericRowNullBitmap(nullBitmap, prepared.prepareNullBitmapBuffer())); + } } @Override diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java index 627501a320084..7c950949d0da5 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java @@ -31,6 +31,7 @@ import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.record.Tablet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -104,8 +105,20 @@ public NumericRowPreparedStatement prepareInsertNumericRow( public void executePreparedInsertNumericRow( NumericRowPreparedStatement prepared, long timestamp, long[] values, String[] tagValues) throws StatementExecutionException, IoTDBConnectionException { + executePreparedInsertNumericRow(prepared, timestamp, values, tagValues, null); + } + + @Override + public void executePreparedInsertNumericRow( + NumericRowPreparedStatement prepared, + long timestamp, + long[] values, + String[] tagValues, + BitMap nullBitmap) + throws StatementExecutionException, IoTDBConnectionException { try { - tableSession.executePreparedInsertNumericRow(prepared, timestamp, values, tagValues); + tableSession.executePreparedInsertNumericRow( + prepared, timestamp, values, tagValues, nullBitmap); } catch (IoTDBConnectionException e) { sessionPool.cleanSessionAndMayThrowConnectionException(session); closed.set(true); diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java index 40fa0fae41ff7..fd1dd00cc3862 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java @@ -171,6 +171,18 @@ public static ByteBuffer getValueBuffer( List categories, String[] tagValues) throws IoTDBConnectionException { + return getValueBuffer(types, values, measurements, categories, tagValues, null); + } + + /** Null FIELD slots retain their indices but consume no payload bytes. */ + public static ByteBuffer getValueBuffer( + List types, + long[] values, + List measurements, + List categories, + String[] tagValues, + BitMap nullBitmap) + throws IoTDBConnectionException { if (types == null || values == null || measurements == null @@ -180,6 +192,7 @@ public static ByteBuffer getValueBuffer( || types.size() != categories.size()) { throw invalidNumericRowArguments(); } + validateNumericRowNullBitmap(nullBitmap, values.length); int fieldCount = 0; int tagCount = 0; int length = types.size(); @@ -191,14 +204,20 @@ public static ByteBuffer getValueBuffer( } length += ReadWriteIOUtils.sizeToWrite(tagValues[tagCount++]); } else if (categories.get(i) == ColumnCategory.FIELD && type != null) { - fieldCount++; - length += + if (fieldCount >= values.length) { + throw invalidNumericRowArguments(); + } + int valueSize = switch (type) { case BOOLEAN -> Byte.BYTES; case INT32, DATE, FLOAT -> Integer.BYTES; case INT64, TIMESTAMP, DOUBLE -> Long.BYTES; default -> throw new IoTDBConnectionException(MSG_UNSUPPORTED_DATA_TYPE + type); }; + if (nullBitmap == null || !nullBitmap.isMarked(fieldCount)) { + length += valueSize; + } + fieldCount++; } else { throw new IllegalArgumentException( SessionMessages @@ -226,6 +245,9 @@ public static ByteBuffer getValueBuffer( TSDataType type = types.get(i); if (categories.get(i) == ColumnCategory.FIELD) { long value = values[fieldIndex++]; + if (nullBitmap != null && nullBitmap.isMarked(fieldIndex - 1)) { + continue; + } switch (type) { case BOOLEAN -> buffer.put((byte) (value == 0 ? 0 : 1)); case INT32, DATE, FLOAT -> buffer.putInt((int) value); @@ -244,6 +266,33 @@ private static IllegalArgumentException invalidNumericRowArguments() { .EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233); } + public static void validateNumericRowNullBitmap(BitMap nullBitmap, int fieldCount) { + if (nullBitmap != null && nullBitmap.getSize() != fieldCount) { + throw new IllegalArgumentException( + String.format( + SessionMessages + .EXCEPTION_NUMERIC_ROW_NULL_BITMAP_SIZE_MISMATCH_EXPECTED_ARG_FIELDS_BUT_GOT_ARG_8D30F6A0, + fieldCount, + nullBitmap.getSize())); + } + } + + /** Encodes FIELD bits LSB first with zero padding, including when BitMap.markAll() was used. */ + public static ByteBuffer encodeNumericRowNullBitmap(BitMap nullBitmap, ByteBuffer buffer) { + int fieldCount = nullBitmap.getSize(); + int length = (fieldCount + 7) / 8; + if (buffer == null) { + buffer = ByteBuffer.allocate(length); + } + buffer.clear(); + buffer.put(nullBitmap.getByteArray(), 0, length); + if (fieldCount % 8 != 0) { + buffer.put(length - 1, (byte) (buffer.get(length - 1) & ((1 << (fieldCount % 8)) - 1))); + } + buffer.flip(); + return buffer; + } + public static int calculateLength(List types, List values) throws IoTDBConnectionException { int res = 0; diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java index a409d6844f008..500e4a7b08801 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java @@ -24,6 +24,7 @@ import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.junit.Test; @@ -44,6 +45,61 @@ import static org.mockito.Mockito.when; public class PreparedNumericRowBufferTest { + // Reusing the statement after sparse/all-null rows must reset both payload and bitmap cursors. + @Test + public void testNullBitmapBufferReuse() throws Exception { + Session session = mock(Session.class); + SessionConnection connection = mock(SessionConnection.class); + when(session.getDefaultSessionConnection()).thenReturn(connection); + TableSession tableSession = new TableSession(session); + NumericRowPreparedStatement prepared = + new NumericRowPreparedStatement( + "t", + List.of("a", "b", "c"), + List.of(TSDataType.INT64, TSDataType.INT64, TSDataType.INT64), + List.of(ColumnCategory.FIELD, ColumnCategory.FIELD, ColumnCategory.FIELD)); + prepared.bind(1, connection, () -> {}); + List bitmaps = new ArrayList<>(); + List payloads = new ArrayList<>(); + int[] masks = {2, 7, 0}; + doAnswer( + invocation -> { + int index = bitmaps.size(); + ByteBuffer values = invocation.getArgument(2); + ByteBuffer bitmap = invocation.getArgument(4); + bitmaps.add(bitmap); + payloads.add(values); + assertEquals(masks[index], bitmap.get()); + assertFalse(bitmap.hasRemaining()); + for (int field = 0; field < 3; field++) { + if ((masks[index] & (1 << field)) == 0) { + assertEquals(field + 10, values.getLong()); + } + } + assertFalse(values.hasRemaining()); + return null; + }) + .when(connection) + .executePreparedNumericRow( + eq(1L), anyLong(), any(ByteBuffer.class), any(ByteBuffer.class), any(ByteBuffer.class)); + BitMap nulls = new BitMap(3); + for (int mask : masks) { + nulls.reset(); + for (int i = 0; i < 3; i++) { + if ((mask & (1 << i)) != 0) { + nulls.mark(i); + } + } + tableSession.executePreparedInsertNumericRow( + prepared, 1, new long[] {10, 11, 12}, new String[0], nulls); + } + assertEquals(3, bitmaps.size()); + for (int i = 1; i < 3; i++) { + assertSame(bitmaps.get(0), bitmaps.get(i)); + assertSame(payloads.get(0), payloads.get(i)); + } + } + @Test public void testBuffersAcrossTagGrowthShrinkAndRpcFailure() throws Exception { Session session = mock(Session.class); diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java index dfb1ef4a2bf76..632fbbda8fbf1 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java @@ -46,6 +46,69 @@ public class SessionUtilsTest { + // Null FIELDs are omitted without shifting indices around interleaved TAG metadata or NaN bits. + @Test + public void testNumericRowNullBitmapEncoding() throws Exception { + BitMap nulls = new BitMap(9); + nulls.mark(0); + nulls.mark(7); + nulls.mark(8); + List types = new ArrayList<>(Collections.nCopies(9, TSDataType.DOUBLE)); + List categories = new ArrayList<>(Collections.nCopies(9, ColumnCategory.FIELD)); + types.add(3, TSDataType.STRING); + categories.add(3, ColumnCategory.TAG); + long[] values = new long[9]; + Arrays.fill(values, 0x7ff8000000001234L); + ByteBuffer payload = + SessionUtils.getValueBuffer( + types, values, Collections.nCopies(10, "v"), categories, new String[] {"tag"}, nulls); + for (TSDataType type : types) { + Assert.assertEquals(type.serialize(), payload.get()); + } + Assert.assertEquals("tag", ReadWriteIOUtils.readString(payload)); + for (int i = 0; i < 6; i++) { + Assert.assertEquals(0x7ff8000000001234L, payload.getLong()); + } + Assert.assertFalse(payload.hasRemaining()); + Assert.assertEquals( + ByteBuffer.wrap(new byte[] {(byte) 0x81, 1}), + SessionUtils.encodeNumericRowNullBitmap(nulls, null)); + } + + // markAll may set padding in BitMap; the wire representation uses exactly ceil(FIELDs/8) bytes. + @Test + public void testNumericRowNullBitmapBoundaries() throws Exception { + for (int size : new int[] {1, 7, 8, 9, 16}) { + BitMap nulls = new BitMap(size); + nulls.markAll(); + byte[] original = nulls.getByteArray().clone(); + ByteBuffer bitmap = SessionUtils.encodeNumericRowNullBitmap(nulls, null); + Assert.assertEquals((size + 7) / 8, bitmap.remaining()); + Assert.assertEquals( + size % 8 == 0 ? 255 : (1 << (size % 8)) - 1, bitmap.get(bitmap.limit() - 1) & 255); + Assert.assertArrayEquals(original, nulls.getByteArray()); + ByteBuffer payload = + SessionUtils.getValueBuffer( + Collections.nCopies(size, TSDataType.INT64), + new long[size], + Collections.nCopies(size, "v"), + Collections.nCopies(size, ColumnCategory.FIELD), + new String[0], + nulls); + Assert.assertEquals(size, payload.remaining()); + } + assertThrows( + IllegalArgumentException.class, + () -> + SessionUtils.getValueBuffer( + List.of(TSDataType.INT64), + new long[1], + List.of("v"), + List.of(ColumnCategory.FIELD), + new String[0], + new BitMap(2))); + } + @Test public void testNumericRowEncodingPreservesRawBits() throws IoTDBConnectionException { List types = diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index bf7a36010030a..63de7dbb60394 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -20,6 +20,12 @@ package org.apache.iotdb.db.i18n; public final class DataNodeQueryMessages { + public static final String EXCEPTION_NUMERIC_ROW_NULL_BITMAP_LENGTH_MISMATCH_EXPECTED_ARG_BYTES_FOR_ARG_FIELDS_BUT_GOT_ARG_31F1DB42 = + "Numeric row null bitmap length mismatch: expected %d bytes for %d FIELDs but got %d"; + public static final String EXCEPTION_NUMERIC_ROW_NULL_BITMAP_HAS_SET_BITS_OUTSIDE_ARG_FIELDS_AD6066D1 = + "Numeric row null bitmap has set bits outside %d FIELDs"; + public static final String EXCEPTION_NUMERIC_ROW_PAYLOAD_HAS_ARG_TRAILING_BYTES_89E46A34 = + "Numeric row payload has %d trailing bytes"; // --- Common --- diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index f26f2306be2fa..12e61875802a5 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -20,6 +20,12 @@ package org.apache.iotdb.db.i18n; public final class DataNodeQueryMessages { + public static final String EXCEPTION_NUMERIC_ROW_NULL_BITMAP_LENGTH_MISMATCH_EXPECTED_ARG_BYTES_FOR_ARG_FIELDS_BUT_GOT_ARG_31F1DB42 = + "Numeric row 的 null bitmap 长度不匹配：预期 %d 字节（对应 %d 个 FIELD），实际为 %d"; + public static final String EXCEPTION_NUMERIC_ROW_NULL_BITMAP_HAS_SET_BITS_OUTSIDE_ARG_FIELDS_AD6066D1 = + "Numeric row 的 null bitmap 在 %d 个 FIELD 范围外存在置位"; + public static final String EXCEPTION_NUMERIC_ROW_PAYLOAD_HAS_ARG_TRAILING_BYTES_89E46A34 = + "Numeric row 数据存在 %d 个多余的尾随字节"; // --- Common --- diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java index 6d5dc1fce4b1a..938afbdc85227 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java @@ -205,6 +205,11 @@ private MeasurementSchema[] prepareMeasurementSchemas(TsTable currentTable) { } public InsertNumericRowStatement bind(long timestamp, ByteBuffer tags, ByteBuffer values) { + return bind(timestamp, tags, values, null); + } + + public InsertNumericRowStatement bind( + long timestamp, ByteBuffer tags, ByteBuffer values, ByteBuffer nullBitmap) { if (tags == null) { throw invalid( DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_TAG_BUFFER_IS_NULL_3FF27BF7); @@ -213,11 +218,21 @@ public InsertNumericRowStatement bind(long timestamp, ByteBuffer tags, ByteBuffe throw invalid( DataNodeMiscMessages.EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_BUFFER_IS_NULL_A928F919); } - if (values.remaining() != valueBytes) { + NumericRowValues.validateNullBitmap(nullBitmap, types.length - tagCount); + // Preserve the cached fixed-width length for the common non-null path. + int expectedBytes = valueBytes; + if (nullBitmap != null) { + for (int i = tagCount; i < types.length; i++) { + if (NumericRowValues.isNullField(nullBitmap, i - tagCount)) { + expectedBytes -= types[i].getDataTypeSize(); + } + } + } + if (values.remaining() != expectedBytes) { throw invalid( DataNodeMiscMessages .EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_BUFFER_SIZE_MISMATCH_EXPECTED_ARG_BYTES_BUT_GOT_ARG_B75D48C8, - valueBytes, + expectedBytes, values.remaining()); } TimestampPrecisionUtils.checkTimestampPrecision(timestamp); @@ -252,7 +267,11 @@ public InsertNumericRowStatement bind(long timestamp, ByteBuffer tags, ByteBuffe tags.remaining()); } for (int i = tagCount; i < types.length; i++) { - row.read(i, types[i], values); + if (NumericRowValues.isNullField(nullBitmap, i - tagCount)) { + row.setNull(i); + } else { + row.read(i, types[i], values); + } } } catch (BufferUnderflowException e) { throw invalid( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index d48aef4cbf671..f74ed0b8a5832 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -2620,7 +2620,8 @@ public TSStatus executePreparedNumericRow(TSExecutePreparedNumericRowReq req) { .bind( req.getTimestamp(), ByteBuffer.wrap(req.getTagValues()), - ByteBuffer.wrap(req.getValues())); + ByteBuffer.wrap(req.getValues()), + req.bufferForNullBitmap()); long queryId = SESSION_MANAGER.requestQueryId(); return COORDINATOR.executeForTableModel( statement, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java index 3aac7cfaccb2d..eb8cebad679ad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java @@ -328,7 +328,8 @@ public static InsertBaseStatement createStatement(TSInsertRecordReq insertRecord } insertStatement.setColumnCategories(columnCategories); if (insertRecordReq.isIsNumericRow()) { - ((InsertNumericRowStatement) insertStatement).fillValues(insertRecordReq.values); + ((InsertNumericRowStatement) insertStatement) + .fillValues(insertRecordReq.values, insertRecordReq.bufferForNullBitmap()); for (int i = 0; i < columnCategories.length; i++) { TSDataType type = insertStatement.getDataType(i); if (!(columnCategories[i] == TsTableColumnCategory.TAG && type == TSDataType.STRING) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java index 7093b1615fb10..98e255de911c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java @@ -100,6 +100,10 @@ public int getValueCount() { /** RPC cells carry typed primitive FIELD bits or a length-prefixed STRING TAG. */ public void fillValues(ByteBuffer buffer) throws QueryProcessException { + fillValues(buffer, null); + } + + public void fillValues(ByteBuffer buffer, ByteBuffer nullBitmap) throws QueryProcessException { dataTypes = new TSDataType[measurements.length]; for (int i = 0; i < dataTypes.length; i++) { byte type = buffer.get(); @@ -124,6 +128,7 @@ public void fillValues(ByteBuffer buffer) throws QueryProcessException { } } int tagCount = countTagColumns(columnCategories); + NumericRowValues.validateNullBitmap(nullBitmap, measurements.length - tagCount); numericValues = new NumericRowValues(tagCount, measurements.length - tagCount); ColumnCategory[] categories = new ColumnCategory[measurements.length]; for (int i = 0; i < measurements.length; i++) { @@ -140,9 +145,19 @@ public void fillValues(ByteBuffer buffer) throws QueryProcessException { } for (int i = 0; i < dataTypes.length; i++) { if (columnCategories[i] == TsTableColumnCategory.FIELD) { - numericValues.read(i, dataTypes[i], buffer); + if (NumericRowValues.isNullField(nullBitmap, i - tagCount)) { + numericValues.setNull(i); + } else { + numericValues.read(i, dataTypes[i], buffer); + } } } + if (buffer.hasRemaining()) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages.EXCEPTION_NUMERIC_ROW_PAYLOAD_HAS_ARG_TRAILING_BYTES_89E46A34, + buffer.remaining())); + } } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/NumericRowValues.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/NumericRowValues.java index 8b9a03a44ccdf..1638f0fd00d8b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/NumericRowValues.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/NumericRowValues.java @@ -161,6 +161,36 @@ && switch (type) { }; } + /** Validates the optional FIELD-only RPC bitmap without changing its cursor. */ + public static void validateNullBitmap(ByteBuffer bitmap, int fieldCount) { + if (bitmap == null) { + return; + } + int length = (fieldCount + 7) / 8; + if (bitmap.remaining() != length) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages + .EXCEPTION_NUMERIC_ROW_NULL_BITMAP_LENGTH_MISMATCH_EXPECTED_ARG_BYTES_FOR_ARG_FIELDS_BUT_GOT_ARG_31F1DB42, + length, + fieldCount, + bitmap.remaining())); + } + if (fieldCount % 8 != 0 + && (bitmap.get(bitmap.limit() - 1) & ~((1 << (fieldCount % 8)) - 1)) != 0) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages + .EXCEPTION_NUMERIC_ROW_NULL_BITMAP_HAS_SET_BITS_OUTSIDE_ARG_FIELDS_AD6066D1, + fieldCount)); + } + } + + public static boolean isNullField(ByteBuffer bitmap, int fieldIndex) { + return bitmap != null + && (bitmap.get(bitmap.position() + fieldIndex / 8) & (1 << (fieldIndex % 8))) != 0; + } + public Object getObject(int index, TSDataType type) { if (isNull(index)) { return null; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java index e195d69660cf1..e8e6ef31153e3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.protocol.session; +import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.queryengine.common.SessionInfo; import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.TsTable; @@ -29,12 +30,14 @@ import org.apache.iotdb.db.queryengine.plan.analyze.lock.DataNodeSchemaLockManager; import org.apache.iotdb.db.queryengine.plan.analyze.lock.SchemaLockType; import org.apache.iotdb.db.queryengine.plan.analyze.schema.SchemaValidator; +import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertNumericRow; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.service.rpc.thrift.TSInsertRecordReq; import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; import org.apache.tsfile.enums.ColumnCategory; @@ -45,14 +48,17 @@ import java.nio.ByteBuffer; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Arrays; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -191,6 +197,75 @@ private void validateFallback(InsertNumericRowStatement statement) { assertNull(statement.getPreparedTableVersion()); } + // The optional bitmap uses FIELD indices, not measurement indices, and accepts sliced buffers. + @Test + public void testNullBitmapDecodingAndValidation() throws Exception { + TSPrepareNumericRowReq req = request("tag", "v"); + req.setMeasurements(new ArrayList<>(req.getMeasurements())); + req.setDataTypes(new ArrayList<>(req.getDataTypes())); + req.setColumnCategories(new ArrayList<>(req.getColumnCategories())); + for (int i = 1; i < 9; i++) { + req.addToMeasurements("v" + i); + req.addToDataTypes(TSDataType.DOUBLE.ordinal()); + req.addToColumnCategories((byte) ColumnCategory.FIELD.ordinal()); + } + PreparedNumericRowInfo info = new PreparedNumericRowInfo(DATABASE, req); + ByteBuffer bitmap = ByteBuffer.wrap(new byte[] {99, (byte) 0x81, 1, 99}); + bitmap.position(1).limit(3); + ByteBuffer values = ByteBuffer.allocate(6 * 8); + for (int i = 1; i <= 6; i++) { + values.putLong(i == 1 ? 0x7ff8000000001234L : Double.doubleToRawLongBits(i)); + } + values.flip(); + InsertNumericRowStatement prepared = info.bind(1, tagBuffer(1), values, bitmap); + assertEquals(1, bitmap.position()); + assertEquals(0, values.position()); + ByteBuffer payload = ByteBuffer.allocate(10 + 5 + values.remaining()); + payload.put(TSDataType.STRING.serialize()); + for (int i = 0; i < 9; i++) { + payload.put(TSDataType.DOUBLE.serialize()); + } + payload.put(tagBuffer(1)).put(values.duplicate()).flip(); + TSInsertRecordReq insertReq = new TSInsertRecordReq(); + insertReq + .setPrefixPath("t") + .setTimestamp(1) + .setMeasurements(req.getMeasurements()) + .setColumnCategoryies(req.getColumnCategories()) + .setIsNumericRow(true) + .setIsWriteToTable(true) + .setValues(payload) + .setNullBitmap(bitmap); + InsertNumericRowStatement direct = + (InsertNumericRowStatement) StatementGenerator.createStatement(insertReq); + assertEquals(prepared.getNumericValues(), direct.getNumericValues()); + assertTrue(prepared.getNumericValues().isNull(1)); + assertTrue(prepared.getNumericValues().isNull(8)); + assertTrue(prepared.getNumericValues().isNull(9)); + assertFalse(prepared.getNumericValues().isNull(0)); + assertEquals(0x7ff8000000001234L, prepared.getNumericValues().get(2)); + for (byte[] bad : new byte[][] {new byte[0], new byte[1], new byte[3], {0, 2}}) { + assertThrows( + IllegalArgumentException.class, + () -> info.bind(1, tagBuffer(1), values, ByteBuffer.wrap(bad))); + insertReq.setValues(payload.duplicate().rewind()).setNullBitmap(bad); + assertThrows( + IllegalArgumentException.class, () -> StatementGenerator.createStatement(insertReq)); + } + assertThrows( + SemanticException.class, + () -> info.bind(1, tagBuffer(1), ByteBuffer.allocate(9 * 8), bitmap)); + ByteBuffer allNulls = ByteBuffer.wrap(new byte[] {(byte) 255, 1}); + insertReq.setValues(payload.duplicate().rewind()).setNullBitmap(allNulls); + assertThrows( + IllegalArgumentException.class, () -> StatementGenerator.createStatement(insertReq)); + InsertNumericRowStatement allNull = + info.bind(1, tagBuffer(1), ByteBuffer.allocate(0), allNulls); + for (int i = 1; i <= 9; i++) { + assertTrue(allNull.getNumericValues().isNull(i)); + } + } + private static void install(TsTable table) { DataNodeTableCache.getInstance().preUpdateTable(DATABASE, table, null); DataNodeTableCache.getInstance().commitUpdateTable(DATABASE, "t", null); diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift index c08959e5fe487..edcf8834804fc 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift @@ -250,6 +250,9 @@ struct TSInsertRecordReq { 8: optional list columnCategoryies // Numeric table rows retain primitive storage throughout the server write path. 9: optional bool isNumericRow + // Numeric FIELDs only, in FIELD occurrence order: bit i (LSB first) means null. + // Exactly ceil(FIELD count / 8) bytes, unused high bits zero; null FIELD payloads are omitted. + 10: optional binary nullBitmap } struct TSInsertStringRecordReq { @@ -735,4 +738,6 @@ struct TSExecutePreparedNumericRowReq { 4: required binary values // Length-prefixed UTF-8 TAGs; length -1 represents a null TAG. 5: required binary tagValues + // Same FIELD-only bitmap and omitted-null-payload encoding as TSInsertRecordReq. + 6: optional binary nullBitmap } From ed0e2661d3e3f30c832aa560133a3ae688f17027 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 17 Sep 2026 14:22:22 +0800 Subject: [PATCH 12/20] retain compatible table schema --- .../it/session/IoTDBTableModelSessionIT.java | 60 +++++++++++++- .../session/PreparedNumericRowInfo.java | 49 +++++++++--- .../thrift/impl/ClientRPCServiceImpl.java | 9 ++- .../relational/sql/ast/InsertNumericRow.java | 29 ++++++- .../crud/InsertNumericRowStatement.java | 19 +++++ .../session/PreparedNumericRowInfoTest.java | 79 +++++++++++++++++-- 6 files changed, 223 insertions(+), 22 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java index 9ac67ad9653ec..59169cb8554d9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java @@ -60,6 +60,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -196,7 +197,8 @@ public void testNumericRowInsertWithTags() throws Exception { @Test public void testPreparedNumericRowSchemaVersionChange() throws Exception { - // Reuse a prepared row before and after adding a TAG, including a flush/read round trip. + // Refresh compatible FIELD additions, validate TAG padding, then verify a flush/read round + // trip. try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { session.executeNonQueryStatement("CREATE DATABASE numeric_version"); session.executeNonQueryStatement("USE numeric_version"); @@ -207,8 +209,11 @@ public void testPreparedNumericRowSchemaVersionChange() throws Exception { Arrays.asList("tag", "v"), Arrays.asList(TSDataType.STRING, TSDataType.DOUBLE), Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); - for (int time = 1; time <= 3; time++) { - if (time == 3) { + for (int time = 1; time <= 5; time++) { + if (time == 2) { + session.executeNonQueryStatement("ALTER TABLE readings ADD COLUMN extra INT64 FIELD"); + } + if (time == 4) { session.executeNonQueryStatement("ALTER TABLE readings ADD COLUMN tag2 STRING TAG"); } session.executePreparedInsertNumericRow( @@ -220,7 +225,7 @@ public void testPreparedNumericRowSchemaVersionChange() throws Exception { session.executeNonQueryStatement("FLUSH"); try (SessionDataSet dataSet = session.executeQueryStatement("SELECT time,tag,tag2,v FROM readings ORDER BY time")) { - for (int time = 1; time <= 3; time++) { + for (int time = 1; time <= 5; time++) { assertTrue(dataSet.hasNext()); RowRecord row = dataSet.next(); assertEquals(time, row.getFields().get(0).getLongV()); @@ -289,6 +294,53 @@ public void testNumericRowNullBitmap() throws Exception { } } + // Failed schema validation closes only its handle; fixing the table still requires a new prepare. + @Test + public void testPreparedNumericRowClosesOnSchemaFailure() throws Exception { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE numeric_invalid"); + session.executeNonQueryStatement("USE numeric_invalid"); + session.executeNonQueryStatement("CREATE TABLE readings(tag STRING TAG, v DOUBLE FIELD)"); + List names = Arrays.asList("tag", "v"); + List types = Arrays.asList(TSDataType.STRING, TSDataType.DOUBLE); + List categories = Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD); + long[] values = {Double.doubleToRawLongBits(2.5)}; + String[] tags = {"device"}; + try (NumericRowPreparedStatement prepared = + session.prepareInsertNumericRow("readings", names, types, categories); + NumericRowPreparedStatement other = + session.prepareInsertNumericRow("readings", names, types, categories)) { + session.executeNonQueryStatement("DROP TABLE readings"); + session.executeNonQueryStatement("CREATE TABLE readings(tag STRING TAG, v BOOLEAN FIELD)"); + StatementExecutionException schemaError = + assertThrows( + StatementExecutionException.class, + () -> session.executePreparedInsertNumericRow(prepared, 1, values, tags)); + assertTrue(schemaError.getMessage().contains("v")); + session.executeNonQueryStatement("DROP TABLE readings"); + session.executeNonQueryStatement("CREATE TABLE readings(tag STRING TAG, v DOUBLE FIELD)"); + StatementExecutionException closedError = + assertThrows( + StatementExecutionException.class, + () -> session.executePreparedInsertNumericRow(prepared, 2, values, tags)); + assertTrue(closedError.getMessage().contains("closed")); + session.executePreparedInsertNumericRow(other, 3, values, tags); + try (NumericRowPreparedStatement replacement = + session.prepareInsertNumericRow("readings", names, types, categories)) { + session.executePreparedInsertNumericRow(replacement, 4, values, tags); + } + try (SessionDataSet dataSet = + session.executeQueryStatement("SELECT v FROM readings ORDER BY time")) { + for (int i = 0; i < 2; i++) { + assertTrue(dataSet.hasNext()); + assertEquals(2.5, dataSet.next().getFields().get(0).getDoubleV(), 0); + } + assertFalse(dataSet.hasNext()); + } + } + } + } + private void assertNumericNullRows(ITableSession session) throws Exception { try (SessionDataSet dataSet = session.executeQueryStatement( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java index 938afbdc85227..7d3003276eedb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java @@ -60,7 +60,7 @@ public final class PreparedNumericRowInfo { private final TsTableColumnCategory[] categories; private final int tagCount; private final int valueBytes; - private final Pair tableVersion; + private volatile Pair tableVersion; private final MeasurementSchema[] measurementSchemas; public PreparedNumericRowInfo(String database, TSPrepareNumericRowReq req) { @@ -178,20 +178,13 @@ public PreparedNumericRowInfo(String database, TSPrepareNumericRowReq req) { } private MeasurementSchema[] prepareMeasurementSchemas(TsTable currentTable) { - // Views and requests needing TAG padding/reordering or type conversion use normal analysis. - if (currentTable == null - || currentTable.getClass() != TsTable.class - || DataNodeTreeViewSchemaUtils.isTreeViewTable(currentTable) - || currentTable.getTagColumnSchemaList().size() != tagCount) { + if (!matchesTableSchema(currentTable)) { return null; } MeasurementSchema[] schemas = new MeasurementSchema[measurements.length]; for (int i = 0; i < measurements.length; i++) { TsTableColumnSchema column = currentTable.getColumnSchema(measurements[i]); - if (column == null - || column.getDataType() != types[i] - || column.getColumnCategory() != categories[i] - || (i < tagCount && currentTable.getTagColumnOrdinal(measurements[i]) != i)) { + if (column == null) { return null; } schemas[i] = @@ -204,6 +197,41 @@ private MeasurementSchema[] prepareMeasurementSchemas(TsTable currentTable) { return schemas; } + private boolean matchesTableSchema(TsTable currentTable) { + // Views and requests needing TAG padding/reordering or type conversion use normal analysis. + if (currentTable == null + || currentTable.getClass() != TsTable.class + || DataNodeTreeViewSchemaUtils.isTreeViewTable(currentTable) + || currentTable.getTagColumnSchemaList().size() != tagCount) { + return false; + } + for (int i = 0; i < measurements.length; i++) { + TsTableColumnSchema column = currentTable.getColumnSchema(measurements[i]); + if (column == null + || column.getDataType() != types[i] + || column.getColumnCategory() != categories[i] + || (i < tagCount && currentTable.getTagColumnOrdinal(measurements[i]) != i)) { + return false; + } + } + return true; + } + + /** Returns a stable compatible version, retaining the cached schemas and wire layout. */ + public Pair refreshTableVersionIfCompatible(TsTable currentTable) { + if (measurementSchemas == null || currentTable == null) { + return null; + } + Pair version = currentTable.getInstanceVersion(); + if (!matchesTableSchema(currentTable) || !currentTable.matchesInstanceVersion(version)) { + return null; + } + // Publish only a compatible candidate. Each execution rechecks it under the schema lock; + // retaining the old candidate on failure also keeps the session's memory reservation stable. + tableVersion = version; + return version; + } + public InsertNumericRowStatement bind(long timestamp, ByteBuffer tags, ByteBuffer values) { return bind(timestamp, tags, values, null); } @@ -287,6 +315,7 @@ public InsertNumericRowStatement bind( statement.setTime(timestamp); statement.setNumericValues(row); if (measurementSchemas != null) { + statement.setPreparedNumericRowInfo(this); statement.setPreparedTableVersion(tableVersion); statement.setMeasurementSchemas(measurementSchemas.clone()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index f74ed0b8a5832..7435e901d2fdd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -115,6 +115,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.DeleteDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; @@ -2612,9 +2613,10 @@ public TSPrepareNumericRowResp prepareNumericRow(TSPrepareNumericRowReq req) { public TSStatus executePreparedNumericRow(TSExecutePreparedNumericRowReq req) { IClientSession session = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); if (!SESSION_MANAGER.checkLogin(session)) return getNotLoggedInStatus(); + InsertNumericRowStatement statement = null; try { if (req.getSessionId() != session.getId()) return RpcUtils.getStatus(TSStatusCode.NOT_LOGIN); - InsertBaseStatement statement = + statement = session .getPreparedNumericRow(req.getPreparedId()) .bind( @@ -2636,6 +2638,11 @@ public TSStatus executePreparedNumericRow(TSExecutePreparedNumericRowReq req) { } catch (Exception e) { return onQueryException( e, OperationType.INSERT_RECORD.getName(), TSStatusCode.EXECUTE_STATEMENT_ERROR); + } finally { + if (statement != null && statement.isTableSchemaValidationFailed()) { + // Remove only this session's invalid handle, releasing its reservation exactly once. + session.removePreparedNumericRow(req.getPreparedId()); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertNumericRow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertNumericRow.java index 4510973f7535f..8145485a96e9a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertNumericRow.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertNumericRow.java @@ -28,6 +28,8 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.tsfile.utils.Pair; + import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -50,6 +52,19 @@ public InsertNumericRowStatement getInnerTreeStatement() { @Override public void validateTableSchema(Metadata metadata, MPPQueryContext context) { + try { + validatePreparedTableSchema(metadata, context); + if (getInnerTreeStatement().hasFailedMeasurements()) { + getInnerTreeStatement().markTableSchemaValidationFailed(); + } + } catch (RuntimeException e) { + // The coordinator may convert exceptions to statuses; retain the failure for RPC cleanup. + getInnerTreeStatement().markTableSchemaValidationFailed(); + throw e; + } + } + + private void validatePreparedTableSchema(Metadata metadata, MPPQueryContext context) { InsertNumericRowStatement statement = getInnerTreeStatement(); if (statement.getPreparedTableVersion() != null) { // Check at analysis time under the same deletion lock and metadata lease as normal inserts. @@ -57,8 +72,18 @@ public void validateTableSchema(Metadata metadata, MPPQueryContext context) { .takeReadLock(context, SchemaLockType.VALIDATE_VS_DELETION_TABLE); TsTable currentTable = DataNodeTableCache.getInstance().getTableInWrite(getDatabase(), getTableName()); - if (currentTable != null - && currentTable.matchesInstanceVersion(statement.getPreparedTableVersion())) { + boolean compatible = + currentTable != null + && currentTable.matchesInstanceVersion(statement.getPreparedTableVersion()); + if (!compatible && statement.getPreparedNumericRowInfo() != null) { + Pair version = + statement.getPreparedNumericRowInfo().refreshTableVersionIfCompatible(currentTable); + if (version != null) { + statement.setPreparedTableVersion(version); + compatible = true; + } + } + if (compatible) { statement.setAttributeColumnsPresent(false); // Cached schema names must not be mutated by later normalization. statement.setToLowerCaseApplied(true); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java index 98e255de911c9..62970448eb401 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator; +import org.apache.iotdb.db.protocol.session.PreparedNumericRowInfo; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaValidation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertNumericRow; @@ -58,6 +59,8 @@ public class InsertNumericRowStatement extends InsertBaseStatement { private IDeviceID deviceID; // A candidate snapshot, checked again under the schema analysis lock before reuse. private Pair preparedTableVersion; + private PreparedNumericRowInfo preparedNumericRowInfo; + private volatile boolean tableSchemaValidationFailed; public InsertNumericRowStatement() { statementType = StatementType.INSERT_NUMERIC_ROW; @@ -73,6 +76,22 @@ public void setPreparedTableVersion(Pair preparedTableVersion) { this.preparedTableVersion = preparedTableVersion; } + public PreparedNumericRowInfo getPreparedNumericRowInfo() { + return preparedNumericRowInfo; + } + + public void setPreparedNumericRowInfo(PreparedNumericRowInfo preparedNumericRowInfo) { + this.preparedNumericRowInfo = preparedNumericRowInfo; + } + + public boolean isTableSchemaValidationFailed() { + return tableSchemaValidationFailed; + } + + public void markTableSchemaValidationFailed() { + tableSchemaValidationFailed = true; + } + public long getTime() { return time; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java index e8e6ef31153e3..45cb991615f63 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java @@ -65,6 +65,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -142,20 +143,88 @@ public void testVersionChangeBetweenBindAndAnalysis() { assertArrayEquals(new String[] {"tag", "v"}, bind(info, 1).getMeasurements()); } - // A replacement table with the same name/revision must not reuse the old instance's schema. + // A replacement instance must pass compatibility checks even when its revision is unchanged. @Test - public void testRecreatedTableUsesFullValidation() { - InsertNumericRowStatement statement = - bind(new PreparedNumericRowInfo(DATABASE, request("tag", "v")), 1); + public void testCompatibleRecreatedTableRefreshesVersion() { + PreparedNumericRowInfo info = new PreparedNumericRowInfo(DATABASE, request("tag", "v")); + InsertNumericRowStatement statement = bind(info, 1); TsTable replacement = new TsTable("t"); replacement.addColumnSchema(new TagColumnSchema("tag", TSDataType.STRING)); replacement.addColumnSchema(new FieldColumnSchema("v", TSDataType.DOUBLE)); assertEquals(table.getInstanceVersion().right, replacement.getInstanceVersion().right); install(replacement); - validateFallback(statement); + new InsertNumericRow(statement, context).validateTableSchema(metadata, context); + assertEquals(replacement.getInstanceVersion(), statement.getPreparedTableVersion()); + assertEquals(replacement.getInstanceVersion(), bind(info, 1).getPreparedTableVersion()); + verify(metadata, never()) + .validateInsertNodeMeasurements(anyString(), any(), any(), anyBoolean(), any(), any()); assertNotNull(statement.getMeasurementSchemas()[1]); } + // An unrelated FIELD addition refreshes only the version, preserving schemas and memory usage. + @Test + public void testCompatibleVersionChangeRetainsFastPath() { + PreparedNumericRowInfo info = new PreparedNumericRowInfo(DATABASE, request("tag", "v")); + long reservedBytes = info.ramBytesUsed(); + InsertNumericRowStatement statement = bind(info, 1); + Object schemas = statement.getMeasurementSchemas(); + table.addColumnSchema(new FieldColumnSchema("extra", TSDataType.INT64)); + new InsertNumericRow(statement, context).validateTableSchema(metadata, context); + assertSame(schemas, statement.getMeasurementSchemas()); + assertFalse(statement.isTableSchemaValidationFailed()); + assertEquals(table.getInstanceVersion(), statement.getPreparedTableVersion()); + InsertNumericRowStatement next = bind(info, 1); + assertEquals(table.getInstanceVersion(), next.getPreparedTableVersion()); + assertSame(statement.getMeasurementSchemas()[1], next.getMeasurementSchemas()[1]); + new InsertNumericRow(next, context).validateTableSchema(metadata, context); + verify(metadata, never()) + .validateInsertNodeMeasurements(anyString(), any(), any(), anyBoolean(), any(), any()); + assertEquals(reservedBytes, info.ramBytesUsed()); + } + + // Changed FIELD types must use normal validation and preserve its concrete error. + @Test + public void testIncompatibleVersionUsesValidationError() { + PreparedNumericRowInfo info = new PreparedNumericRowInfo(DATABASE, request("tag", "v")); + InsertNumericRowStatement statement = bind(info, 1); + TsTable replacement = new TsTable("t"); + replacement.addColumnSchema(new TagColumnSchema("tag", TSDataType.STRING)); + replacement.addColumnSchema(new FieldColumnSchema("v", TSDataType.BOOLEAN)); + install(replacement); + SemanticException failure = new SemanticException("column v type mismatch"); + doThrow(failure) + .when(metadata) + .validateInsertNodeMeasurements(anyString(), any(), any(), anyBoolean(), any(), any()); + assertSame( + failure, + assertThrows( + SemanticException.class, + () -> new InsertNumericRow(statement, context).validateTableSchema(metadata, context))); + verify(metadata) + .validateInsertNodeMeasurements(eq(DATABASE), any(), eq(context), eq(true), any(), any()); + assertNull(statement.getPreparedTableVersion()); + assertNull(statement.getMeasurementSchemas()); + assertTrue(statement.isTableSchemaValidationFailed()); + assertFalse(replacement.matchesInstanceVersion(bind(info, 1).getPreparedTableVersion())); + } + + // Partial insert errors also invalidate the handle even when validation returns normally. + @Test + public void testPartialSchemaFailureMarksPreparedRowInvalid() { + PreparedNumericRowInfo info = new PreparedNumericRowInfo(DATABASE, request("tag", "v")); + InsertNumericRowStatement statement = bind(info, 1); + table.addColumnSchema(new TagColumnSchema("tag2", TSDataType.STRING)); + doAnswer( + invocation -> { + statement.markFailedMeasurement(1, new SemanticException("column v rejected")); + return null; + }) + .when(metadata) + .validateInsertNodeMeasurements(anyString(), any(), any(), anyBoolean(), any(), any()); + validateFallback(statement); + assertTrue(statement.isTableSchemaValidationFailed()); + } + // Missing columns/table, views and type conversions remain on normal creation/validation. @Test public void testIneligibleMetadataHasNoCachedSchema() { From 67af28c728a79396cf2871d8f2b689d36ab6fcdb Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 17 Sep 2026 16:04:31 +0800 Subject: [PATCH 13/20] Add redirection --- .../it/session/IoTDBTableModelSessionIT.java | 70 +++++- .../apache/iotdb/isession/ITableSession.java | 6 + .../isession/NumericRowPreparedStatement.java | 47 +++- .../org/apache/iotdb/session/Session.java | 5 +- .../iotdb/session/SessionConnection.java | 12 +- .../apache/iotdb/session/TableSession.java | 95 ++++++-- .../PreparedNumericRowRedirectionTest.java | 219 ++++++++++++++++++ 7 files changed, 414 insertions(+), 40 deletions(-) create mode 100644 iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowRedirectionTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java index 59169cb8554d9..db971c7cc020c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java @@ -43,11 +43,13 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.lang.reflect.Field; import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -319,10 +321,11 @@ public void testPreparedNumericRowClosesOnSchemaFailure() throws Exception { assertTrue(schemaError.getMessage().contains("v")); session.executeNonQueryStatement("DROP TABLE readings"); session.executeNonQueryStatement("CREATE TABLE readings(tag STRING TAG, v DOUBLE FIELD)"); - StatementExecutionException closedError = + IoTDBConnectionException closedError = assertThrows( - StatementExecutionException.class, + IoTDBConnectionException.class, () -> session.executePreparedInsertNumericRow(prepared, 2, values, tags)); + assertTrue(prepared.isClosed()); assertTrue(closedError.getMessage().contains("closed")); session.executePreparedInsertNumericRow(other, 3, values, tags); try (NumericRowPreparedStatement replacement = @@ -363,6 +366,69 @@ private void assertNumericNullRows(ITableSession session) throws Exception { } } + // Pin clients to every DataNode so at least one must learn a remote leader in a cluster. + @Test + public void testPreparedNumericRowRedirection() throws Exception { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE numeric_redirect"); + session.executeNonQueryStatement("USE numeric_redirect"); + session.executeNonQueryStatement( + "CREATE TABLE readings(tag STRING TAG, v INT64 FIELD, n INT64 FIELD)"); + int clientIndex = 0; + boolean redirected = false; + for (var node : EnvFactory.getEnv().getDataNodeWrapperList()) { + try (ITableSession pinned = + EnvFactory.getEnv().getTableSessionConnection(List.of(node.getIpAndPortString()))) { + pinned.executeNonQueryStatement("USE numeric_redirect"); + redirected |= writeRedirectedNumericRows(pinned, clientIndex++ * 1000L); + } + } + try (ITableSessionPool pool = EnvFactory.getEnv().getTableSessionPool(1, "numeric_redirect"); + ITableSession pooled = pool.getSession()) { + redirected |= writeRedirectedNumericRows(pooled, clientIndex++ * 1000L); + } + if (EnvFactory.getEnv().getDataNodeWrapperList().size() > 1) { + assertTrue("Expected a prepared handle on a remote leader", redirected); + } + session.executeNonQueryStatement("FLUSH"); + try (SessionDataSet result = + session.executeQueryStatement("SELECT count(v),count(n) FROM readings")) { + assertTrue(result.hasNext()); + RowRecord counts = result.next(); + assertEquals(clientIndex * 32L, counts.getFields().get(0).getLongV()); + assertEquals(clientIndex * 16L, counts.getFields().get(1).getLongV()); + assertFalse(result.hasNext()); + } + } + } + + private boolean writeRedirectedNumericRows(ITableSession session, long start) throws Exception { + try (NumericRowPreparedStatement prepared = + session.prepareInsertNumericRow( + "readings", + List.of("tag", "v", "n"), + List.of(TSDataType.STRING, TSDataType.INT64, TSDataType.INT64), + List.of(ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD))) { + BitMap nulls = new BitMap(2); + nulls.mark(1); + for (int round = 0; round < 4; round++) { + for (int device = 0; device < 8; device++) { + long time = start + round * 8 + device; + session.executePreparedInsertNumericRow( + prepared, + time, + new long[] {time, time}, + new String[] {"d" + device}, + round % 2 == 0 ? nulls : null); + } + } + // Count actual physical bindings: row counts alone cannot prove the client followed a hint. + Field bindings = NumericRowPreparedStatement.class.getDeclaredField("connectionBindings"); + bindings.setAccessible(true); + return ((Map) bindings.get(prepared)).size() > 1; + } + } + @Test public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { Assume.assumeTrue( diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java index 35a2f71b7dcbb..1cfb0b86eedf0 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java @@ -70,6 +70,12 @@ NumericRowPreparedStatement prepareInsertNumericRow( List columnCategories) throws StatementExecutionException, IoTDBConnectionException; + /** + * Executes a prepared row, following write redirection recommendations for subsequent rows. + * Destination handles are prepared lazily and released when the statement is closed. An execution + * error reported by the server closes the statement; prepare it again before reuse. Transport + * failures are propagated without replaying a write whose outcome is unknown. + */ void executePreparedInsertNumericRow( NumericRowPreparedStatement prepared, long timestamp, long[] values, String[] tagValues) throws StatementExecutionException, IoTDBConnectionException; diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java index ee0b0f21804fd..2fa9ee2125754 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java @@ -12,7 +12,9 @@ import org.apache.tsfile.enums.TSDataType; import java.nio.ByteBuffer; +import java.util.IdentityHashMap; import java.util.List; +import java.util.Map; /** * Immutable metadata and reusable serialization buffers for repeated numeric table row inserts. @@ -22,6 +24,7 @@ */ public final class NumericRowPreparedStatement implements AutoCloseable { private final String tableName; + private final String database; private final List columnNames; private final List dataTypes; private final List columnCategories; @@ -30,17 +33,18 @@ public final class NumericRowPreparedStatement implements AutoCloseable { private ByteBuffer tagBuffer; private ByteBuffer valueBuffer; private ByteBuffer nullBitmapBuffer; - private long preparedId = -1; private Object owner; - private Runnable closeAction = () -> {}; + private final Map connectionBindings = new IdentityHashMap<>(); private boolean closed; public NumericRowPreparedStatement( String tableName, List columnNames, List dataTypes, - List columnCategories) { + List columnCategories, + String database) { this.tableName = tableName; + this.database = database; this.columnNames = List.copyOf(columnNames); this.dataTypes = List.copyOf(dataTypes); this.columnCategories = List.copyOf(columnCategories); @@ -69,6 +73,10 @@ public String getTableName() { return tableName; } + public String getDatabase() { + return database; + } + public List getColumnNames() { return columnNames; } @@ -105,10 +113,6 @@ public ByteBuffer prepareValueBuffer() { return valueBuffer; } - public long getPreparedId() { - return preparedId; - } - /** Returns a reusable FIELD null bitmap buffer, allocated only for nullable executions. */ public ByteBuffer prepareNullBitmapBuffer() { if (nullBitmapBuffer == null) { @@ -122,21 +126,40 @@ public Object getOwner() { return owner; } - public void bind(long id, Object owner, Runnable closeAction) { - this.preparedId = id; + public void bind(long id, Object owner, long sessionId, Runnable closeAction) { this.owner = owner; - this.closeAction = closeAction; + bindConnection(owner, sessionId, id, closeAction); this.closed = false; } + /** Handles belong to a physical session, including its generation after a reconnect. */ + public Long getPreparedId(Object connection, long sessionId) { + ConnectionBinding binding = connectionBindings.get(connection); + return binding != null && binding.sessionId == sessionId ? binding.preparedId : null; + } + + public void bindConnection(Object connection, long sessionId, long id, Runnable closeAction) { + connectionBindings.put(connection, new ConnectionBinding(sessionId, id, closeAction)); + } + + private record ConnectionBinding(long sessionId, long preparedId, Runnable closeAction) {} + public boolean isClosed() { return closed; } @Override public void close() { - closeAction.run(); - closeAction = () -> {}; + if (closed) { + return; + } closed = true; + try { + for (ConnectionBinding binding : connectionBindings.values()) { + binding.closeAction.run(); + } + } finally { + connectionBindings.clear(); + } } } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index f5c6ebd1b809e..11a545da71443 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -1357,8 +1357,7 @@ private SessionConnection getSessionConnection(String deviceId) throws IoTDBConn } } - private SessionConnection getSessionConnection(IDeviceID deviceId) - throws IoTDBConnectionException { + SessionConnection getSessionConnection(IDeviceID deviceId) throws IoTDBConnectionException { TEndPoint endPoint; if (enableRedirection && tableModelDeviceIdToEndpoint != null @@ -1440,7 +1439,7 @@ private void handleRedirection(String deviceId, TEndPoint endpoint) { } } - private void handleRedirection(IDeviceID deviceId, TEndPoint endpoint) { + void handleRedirection(IDeviceID deviceId, TEndPoint endpoint) { if (enableRedirection) { // no need to redirection if (UrlUtils.isWildcardAddress(endpoint.ip)) { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java index f98e58965aeca..28a75d0c30769 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java @@ -781,20 +781,22 @@ protected long prepareNumericRow(TSPrepareNumericRowReq request) protected void executePreparedNumericRow( long id, long timestamp, ByteBuffer values, ByteBuffer tags) - throws IoTDBConnectionException, StatementExecutionException { + throws IoTDBConnectionException, StatementExecutionException, RedirectException { executePreparedNumericRow(id, timestamp, values, tags, null); } protected void executePreparedNumericRow( long id, long timestamp, ByteBuffer values, ByteBuffer tags, ByteBuffer nullBitmap) - throws IoTDBConnectionException, StatementExecutionException { + throws IoTDBConnectionException, StatementExecutionException, RedirectException { TSExecutePreparedNumericRowReq request = new TSExecutePreparedNumericRowReq(sessionId, id, timestamp, values, tags); if (nullBitmap != null) { request.setNullBitmap(nullBitmap); } try { - RpcUtils.verifySuccess(client.executePreparedNumericRow(request)); + // A redirect is a recommendation for later writes: this row has already been accepted. + // Never reconnect/replay a session-local prepared ID after an ambiguous transport failure. + RpcUtils.verifySuccessWithRedirection(client.executePreparedNumericRow(request)); } catch (TException e) { throw new IoTDBConnectionException(e); } @@ -810,6 +812,10 @@ protected void closePreparedNumericRow(long id) throws IoTDBConnectionException } } + long getSessionId() { + return sessionId; + } + private TSStatus insertRecordInternal(TSInsertRecordReq request) throws TException { request.setSessionId(sessionId); return client.insertRecord(request); diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java index da93c37586fbf..9d3e6d8b9bb5e 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java @@ -24,6 +24,7 @@ import org.apache.iotdb.isession.NumericRowPreparedStatement; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.RedirectException; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; import org.apache.iotdb.session.i18n.SessionMessages; @@ -31,6 +32,7 @@ import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.record.Tablet; @@ -38,6 +40,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.Objects; public class TableSession implements ITableSession { @@ -84,18 +87,14 @@ public NumericRowPreparedStatement prepareInsertNumericRow( .setDataTypes(types.stream().map(TSDataType::ordinal).toList()) .setColumnCategories(categories.stream().map(c -> (byte) c.ordinal()).toList()); NumericRowPreparedStatement prepared = - new NumericRowPreparedStatement(tableName, names, types, categories); + new NumericRowPreparedStatement(tableName, names, types, categories, session.getDatabase()); SessionConnection connection = session.getDefaultSessionConnection(); + long id = connection.prepareNumericRow(request); prepared.bind( - connection.prepareNumericRow(request), + id, connection, - () -> { - try { - connection.closePreparedNumericRow(prepared.getPreparedId()); - } catch (IoTDBConnectionException ignored) { - // Closing the session releases all server-side prepared rows. - } - }); + connection.getSessionId(), + closePreparedRow(connection, connection.getSessionId(), id)); return prepared; } @@ -115,7 +114,10 @@ public void executePreparedInsertNumericRow( BitMap nullBitmap) throws StatementExecutionException, IoTDBConnectionException { SessionConnection connection = session.getDefaultSessionConnection(); - if (prepared == null || prepared.isClosed() || prepared.getOwner() != connection) { + if (prepared == null + || prepared.isClosed() + || prepared.getOwner() != connection + || prepared.getPreparedId(connection, connection.getSessionId()) == null) { throw new IoTDBConnectionException( SessionMessages .EXCEPTION_PREPARED_NUMERIC_ROW_IS_CLOSED_OR_BELONGS_TO_ANOTHER_CONNECTION_PREPARE_IT_AGAIN_055CC7BC); @@ -130,6 +132,34 @@ public void executePreparedInsertNumericRow( SessionMessages.EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_COUNT_MISMATCH_9ACD3A0C); } SessionUtils.validateNumericRowNullBitmap(nullBitmap, fieldCount); + // A prepared handle remains bound to its original database after USE. Do not prepare it on + // a redirected session whose database has changed in the meantime. + IDeviceID deviceId = null; + if (session.isEnableRedirection() + && Objects.equals(prepared.getDatabase(), session.getDatabase())) { + String[] segments = new String[tagValues.length + 1]; + segments[0] = prepared.getTableName(); + System.arraycopy(tagValues, 0, segments, 1, tagValues.length); + deviceId = IDeviceID.Factory.DEFAULT_FACTORY.create(segments); + connection = session.getSessionConnection(deviceId); + } + Long id = prepared.getPreparedId(connection, connection.getSessionId()); + if (id == null) { + TSPrepareNumericRowReq request = new TSPrepareNumericRowReq(); + request + .setTableName(prepared.getTableName()) + .setMeasurements(prepared.getColumnNames()) + .setDataTypes(prepared.getDataTypes().stream().map(TSDataType::ordinal).toList()) + .setColumnCategories( + prepared.getColumnCategories().stream().map(c -> (byte) c.ordinal()).toList()); + // Prepare lazily on each destination; IDs must never cross physical sessions. + id = connection.prepareNumericRow(request); + prepared.bindConnection( + connection, + connection.getSessionId(), + id, + closePreparedRow(connection, connection.getSessionId(), id)); + } int tagBytes = 0; for (String tag : tagValues) { tagBytes += ReadWriteIOUtils.sizeToWrite(tag); @@ -158,19 +188,44 @@ public void executePreparedInsertNumericRow( } } valueBuffer.flip(); - if (nullBitmap == null) { - connection.executePreparedNumericRow( - prepared.getPreparedId(), timestamp, valueBuffer, tagBuffer); - } else { - connection.executePreparedNumericRow( - prepared.getPreparedId(), - timestamp, - valueBuffer, - tagBuffer, - SessionUtils.encodeNumericRowNullBitmap(nullBitmap, prepared.prepareNullBitmapBuffer())); + try { + if (nullBitmap == null) { + connection.executePreparedNumericRow(id, timestamp, valueBuffer, tagBuffer); + } else { + connection.executePreparedNumericRow( + id, + timestamp, + valueBuffer, + tagBuffer, + SessionUtils.encodeNumericRowNullBitmap( + nullBitmap, prepared.prepareNullBitmapBuffer())); + } + } catch (RedirectException e) { + // This row succeeded. Update only the route for subsequent rows, without replaying this one. + if (deviceId != null && e.getEndPoint() != null) { + session.handleRedirection(deviceId, e.getEndPoint()); + } + } catch (StatementExecutionException e) { + // A server may have invalidated its handle on schema failure. Do not let another cached + // destination silently revive that logical statement; require an explicit new prepare. + prepared.close(); + throw e; } } + private static Runnable closePreparedRow(SessionConnection connection, long sessionId, long id) { + return () -> { + if (connection.getSessionId() != sessionId) { + return; + } + try { + connection.closePreparedNumericRow(id); + } catch (IoTDBConnectionException ignored) { + // Closing the physical session releases any handles left by a failed close RPC. + } + }; + } + @Override public void executeNonQueryStatement(String sql) throws IoTDBConnectionException, StatementExecutionException { diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowRedirectionTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowRedirectionTest.java new file mode 100644 index 0000000000000..cdbd984aaf74a --- /dev/null +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowRedirectionTest.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.session; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.isession.NumericRowPreparedStatement; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.IClientRPCService; +import org.apache.iotdb.service.rpc.thrift.TSExecutePreparedNumericRowReq; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowResp; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.powermock.reflect.Whitebox; + +import java.util.HashMap; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class PreparedNumericRowRedirectionTest { + private final TEndPoint leader = new TEndPoint("127.0.0.1", 6668); + private Session session; + private TableSession tableSession; + private SessionConnection initial; + private SessionConnection destination; + private IClientRPCService.Iface initialClient; + private IClientRPCService.Iface destinationClient; + + @Before + public void setUp() throws Exception { + session = new Session.Builder().enableRedirection(true).enableAutoFetch(false).build(); + initialClient = mock(IClientRPCService.Iface.class); + destinationClient = mock(IClientRPCService.Iface.class); + initial = connection(initialClient, 11); + destination = connection(destinationClient, 22); + session.setDefaultSessionConnection(initial); + session.tableModelDeviceIdToEndpoint = new HashMap<>(); + session.endPointToSessionConnection = new HashMap<>(); + session.endPointToSessionConnection.put(leader, destination); + tableSession = new TableSession(session); + when(initialClient.prepareNumericRow(any())) + .thenReturn(new TSPrepareNumericRowResp(success()).setPreparedId(101)); + when(destinationClient.prepareNumericRow(any())) + .thenReturn(new TSPrepareNumericRowResp(success()).setPreparedId(202)); + when(initialClient.executePreparedNumericRow(any())) + .thenReturn( + new TSStatus(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) + .setRedirectNode(leader)); + when(destinationClient.executePreparedNumericRow(any())).thenReturn(success()); + when(initialClient.closePreparedNumericRow(anyLong(), anyLong())).thenReturn(success()); + when(destinationClient.closePreparedNumericRow(anyLong(), anyLong())).thenReturn(success()); + } + + // Recommendations acknowledge this row; later rows prepare once on the leader with intact nulls. + @Test + public void testRedirectPreparesPerSessionWithoutReplaying() throws Exception { + NumericRowPreparedStatement prepared = prepare(); + write(prepared, 1, "a"); + verify(destinationClient, never()).executePreparedNumericRow(any()); + write(prepared, 2, "a"); + write(prepared, 3, "a"); + // A different device has no route yet, so it still uses the original handle. + write(prepared, 4, "b"); + verify(initialClient, times(2)).executePreparedNumericRow(any()); + verify(destinationClient, times(1)).prepareNumericRow(any()); + ArgumentCaptor requests = + ArgumentCaptor.forClass(TSExecutePreparedNumericRowReq.class); + verify(destinationClient, times(2)).executePreparedNumericRow(requests.capture()); + for (TSExecutePreparedNumericRowReq request : requests.getAllValues()) { + assertEquals(22, request.getSessionId()); + assertEquals(202, request.getPreparedId()); + assertEquals(2, request.bufferForNullBitmap().get()); + assertEquals(8, request.bufferForValues().remaining()); + assertEquals(42, request.bufferForValues().getLong()); + assertEquals("a", ReadWriteIOUtils.readString(request.bufferForTagValues())); + } + prepared.close(); + prepared.close(); + verify(initialClient).closePreparedNumericRow(11, 101); + verify(destinationClient).closePreparedNumericRow(22, 202); + } + + // Disabling redirection ignores recommendations; USE must not prepare an old row in a new DB. + @Test + public void testDisabledRedirectionAndDatabaseChange() throws Exception { + try (NumericRowPreparedStatement prepared = prepare()) { + session.setEnableRedirection(false); + write(prepared, 1, "a"); + write(prepared, 2, "a"); + assertTrue(session.tableModelDeviceIdToEndpoint.isEmpty()); + session.setEnableRedirection(true); + write(prepared, 3, "a"); + session.changeDatabase("another_database"); + write(prepared, 4, "a"); + verify(initialClient, times(4)).executePreparedNumericRow(any()); + verify(destinationClient, never()).prepareNumericRow(any()); + } + } + + // Reconnecting a destination requires a new ID; reconnecting the owner invalidates the statement. + @Test + public void testReconnectedSessionsNeverReuseOldIds() throws Exception { + NumericRowPreparedStatement prepared = prepare(); + write(prepared, 1, "a"); + write(prepared, 2, "a"); + Whitebox.setInternalState(destination, "sessionId", 23L); + when(destinationClient.prepareNumericRow(any())) + .thenReturn(new TSPrepareNumericRowResp(success()).setPreparedId(303)); + write(prepared, 3, "a"); + assertEquals(Long.valueOf(303), prepared.getPreparedId(destination, 23)); + verify(destinationClient, times(2)).prepareNumericRow(any()); + Whitebox.setInternalState(initial, "sessionId", 12L); + assertThrows(IoTDBConnectionException.class, () -> write(prepared, 4, "a")); + prepared.close(); + verify(initialClient, never()).closePreparedNumericRow(anyLong(), anyLong()); + verify(destinationClient).closePreparedNumericRow(23, 303); + verify(destinationClient, never()).closePreparedNumericRow(22, 202); + } + + // A failed redirected execution is propagated once, and all handles close on a server error. + @Test + public void testFailureDoesNotReplayOrReviveStatement() throws Exception { + NumericRowPreparedStatement prepared = prepare(); + write(prepared, 1, "a"); + when(destinationClient.executePreparedNumericRow(any())) + .thenThrow(new TException("lost response")); + assertThrows(IoTDBConnectionException.class, () -> write(prepared, 2, "a")); + verify(initialClient).executePreparedNumericRow(any()); + verify(destinationClient).executePreparedNumericRow(any()); + assertFalse(prepared.isClosed()); + doReturn( + new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()) + .setMessage("schema error")) + .when(destinationClient) + .executePreparedNumericRow(any()); + assertThrows(StatementExecutionException.class, () -> write(prepared, 3, "a")); + assertTrue(prepared.isClosed()); + verify(initialClient).closePreparedNumericRow(11, 101); + verify(destinationClient).closePreparedNumericRow(22, 202); + assertThrows(IoTDBConnectionException.class, () -> write(prepared, 4, "b")); + verify(initialClient).executePreparedNumericRow(any()); + } + + // A preexisting device route is used immediately, including a device with a null TAG. + @Test + public void testExistingRouteAndNullTag() throws Exception { + session.tableModelDeviceIdToEndpoint.put( + IDeviceID.Factory.DEFAULT_FACTORY.create(new String[] {"t", null}), leader); + try (NumericRowPreparedStatement prepared = prepare()) { + write(prepared, 1, null); + verify(initialClient, never()).executePreparedNumericRow(any()); + verify(destinationClient).executePreparedNumericRow(any()); + } + } + + private NumericRowPreparedStatement prepare() throws Exception { + return tableSession.prepareInsertNumericRow( + "t", + List.of("tag", "v", "n"), + List.of(TSDataType.STRING, TSDataType.INT64, TSDataType.INT64), + List.of(ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD)); + } + + private void write(NumericRowPreparedStatement prepared, long time, String tag) throws Exception { + BitMap nulls = new BitMap(2); + nulls.mark(1); + tableSession.executePreparedInsertNumericRow( + prepared, time, new long[] {42, 99}, new String[] {tag}, nulls); + } + + private static SessionConnection connection(IClientRPCService.Iface client, long sessionId) { + SessionConnection connection = new SessionConnection("table"); + Whitebox.setInternalState(connection, "client", client); + Whitebox.setInternalState(connection, "sessionId", sessionId); + return connection; + } + + private static TSStatus success() { + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } +} From ce6707187dc38676e14c65aa06dd639870b010aa Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 17 Sep 2026 17:33:39 +0800 Subject: [PATCH 14/20] remove unused method --- .../session/PreparedNumericRowBufferTest.java | 12 ++++++------ .../protocol/session/PreparedNumericRowInfo.java | 4 ---- .../apache/iotdb/db/utils/NumericRowValues.java | 14 -------------- .../session/PreparedNumericRowInfoTest.java | 4 ++-- .../plan/parser/NumericTableRowTest.java | 2 +- .../AlignedWritableMemChunkNumericTest.java | 7 +++++-- 6 files changed, 14 insertions(+), 29 deletions(-) diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java index 500e4a7b08801..7a44a16536188 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java @@ -51,14 +51,14 @@ public void testNullBitmapBufferReuse() throws Exception { Session session = mock(Session.class); SessionConnection connection = mock(SessionConnection.class); when(session.getDefaultSessionConnection()).thenReturn(connection); + when(connection.prepareNumericRow(any())).thenReturn(1L); TableSession tableSession = new TableSession(session); NumericRowPreparedStatement prepared = - new NumericRowPreparedStatement( + tableSession.prepareInsertNumericRow( "t", List.of("a", "b", "c"), List.of(TSDataType.INT64, TSDataType.INT64, TSDataType.INT64), List.of(ColumnCategory.FIELD, ColumnCategory.FIELD, ColumnCategory.FIELD)); - prepared.bind(1, connection, () -> {}); List bitmaps = new ArrayList<>(); List payloads = new ArrayList<>(); int[] masks = {2, 7, 0}; @@ -105,9 +105,10 @@ public void testBuffersAcrossTagGrowthShrinkAndRpcFailure() throws Exception { Session session = mock(Session.class); SessionConnection connection = mock(SessionConnection.class); when(session.getDefaultSessionConnection()).thenReturn(connection); + when(connection.prepareNumericRow(any())).thenReturn(1L); TableSession tableSession = new TableSession(session); NumericRowPreparedStatement prepared = - new NumericRowPreparedStatement( + tableSession.prepareInsertNumericRow( "t", List.of("tag", "flag", "count", "value"), List.of(TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.INT32, TSDataType.DOUBLE), @@ -116,7 +117,6 @@ public void testBuffersAcrossTagGrowthShrinkAndRpcFailure() throws Exception { ColumnCategory.FIELD, ColumnCategory.FIELD, ColumnCategory.FIELD)); - prepared.bind(1, connection, () -> {}); List tagBuffers = new ArrayList<>(); List valueBuffers = new ArrayList<>(); String[] tags = {"x", "long-tag-\u4e2d\u6587", "", null, "retry"}; @@ -168,6 +168,7 @@ public void testBuffersAreOwnedByEachStatementWithoutTags() throws Exception { Session session = mock(Session.class); SessionConnection connection = mock(SessionConnection.class); when(session.getDefaultSessionConnection()).thenReturn(connection); + when(connection.prepareNumericRow(any())).thenReturn(0L, 1L); TableSession tableSession = new TableSession(session); List buffers = new ArrayList<>(); doAnswer( @@ -186,9 +187,8 @@ public void testBuffersAreOwnedByEachStatementWithoutTags() throws Exception { anyLong(), anyLong(), any(ByteBuffer.class), any(ByteBuffer.class)); for (int i = 0; i < 2; i++) { NumericRowPreparedStatement prepared = - new NumericRowPreparedStatement( + tableSession.prepareInsertNumericRow( "t", List.of("v"), List.of(TSDataType.INT64), List.of(ColumnCategory.FIELD)); - prepared.bind(i, connection, () -> {}); tableSession.executePreparedInsertNumericRow(prepared, 0, new long[] {42}, new String[0]); } assertNotSame(buffers.get(0), buffers.get(2)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java index 7d3003276eedb..0b4259f2fa96a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfo.java @@ -232,10 +232,6 @@ public Pair refreshTableVersionIfCompatible(TsTable currentTable) { return version; } - public InsertNumericRowStatement bind(long timestamp, ByteBuffer tags, ByteBuffer values) { - return bind(timestamp, tags, values, null); - } - public InsertNumericRowStatement bind( long timestamp, ByteBuffer tags, ByteBuffer values, ByteBuffer nullBitmap) { if (tags == null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/NumericRowValues.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/NumericRowValues.java index 1638f0fd00d8b..c1b3b14bca67c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/NumericRowValues.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/NumericRowValues.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.utils; -import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.tsfile.common.conf.TSFileConfig; @@ -47,11 +46,6 @@ public final class NumericRowValues { private final int size; private int tagCount; - @TestOnly - public NumericRowValues(int size) { - this(0, size); - } - public NumericRowValues(int tagLength, int valueLength) { this.values = new long[valueLength]; this.nulls = new BitSet(tagLength + valueLength); @@ -60,14 +54,6 @@ public NumericRowValues(int tagLength, int valueLength) { this.tagCount = tagLength; } - @TestOnly - public NumericRowValues(long[] values, BitSet nulls) { - this.values = values; - this.nulls = nulls; - this.size = values.length; - this.tagCount = 0; - } - public int size() { return size; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java index 45cb991615f63..cf3533296b1af 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java @@ -235,7 +235,7 @@ public void testIneligibleMetadataHasNoCachedSchema() { req.setDataTypes(Arrays.asList(TSDataType.STRING.ordinal(), TSDataType.INT64.ordinal())); assertNull( new PreparedNumericRowInfo(DATABASE, req) - .bind(1, tagBuffer(1), ByteBuffer.allocate(8)) + .bind(1, tagBuffer(1), ByteBuffer.allocate(8), null) .getPreparedTableVersion()); table.addProp(TreeViewSchema.TREE_PATH_PATTERN, "root.sg.**"); assertNull( @@ -364,6 +364,6 @@ private static ByteBuffer tagBuffer(int count) { private static InsertNumericRowStatement bind(PreparedNumericRowInfo info, int tags) { ByteBuffer values = ByteBuffer.allocate(8); values.putDouble(2.5D).flip(); - return info.bind(1, tagBuffer(tags), values); + return info.bind(1, tagBuffer(tags), values, null); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java index 0541ecd217319..70c39d90072f2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java @@ -248,7 +248,7 @@ public void testBulkTagExpansionKeepsValuesAndNulls() throws Exception { @Test public void testNumericCastsKeepPrimitiveStorage() { // Widen INT32 to DOUBLE in place; reject STRING coercion while accepting an inserted null TAG. - NumericRowValues values = new NumericRowValues(2); + NumericRowValues values = new NumericRowValues(0, 2); values.getValues()[0] = Integer.MIN_VALUE; values.setNull(1); assertTrue(values.cast(0, TSDataType.INT32, TSDataType.DOUBLE)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkNumericTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkNumericTest.java index 8620f17f71bb8..cb8d2a15207f1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkNumericTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkNumericTest.java @@ -30,7 +30,6 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; -import java.util.BitSet; import java.util.Collections; import java.util.List; @@ -122,7 +121,11 @@ public void testMappingResetAndGrowthWithMixedWrites() throws Exception { } private static NumericRowValues row(long... values) { - return new NumericRowValues(values, new BitSet()); + NumericRowValues row = new NumericRowValues(0, values.length); + for (int i = 0; i < values.length; i++) { + row.setRawValue(i, values[i]); + } + return row; } private static int[] mapping(AlignedWritableMemChunk chunk) throws Exception { From c68ef6b1aef983e1f63d973f2211f54354079742 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Thu, 17 Sep 2026 18:40:39 +0800 Subject: [PATCH 15/20] remove unused method --- .../apache/iotdb/db/utils/datastructure/AlignedTVListTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java index a39fc45def75d..22626984637fe 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java @@ -65,7 +65,7 @@ public void testNumericRowsMatchObjectRowsAcrossBlocksAndSorting() { AlignedTVList boxed = AlignedTVList.newAlignedList(types); try { for (int i = ARRAY_SIZE + 3; i >= 0; i--) { - NumericRowValues row = new NumericRowValues(5); + NumericRowValues row = new NumericRowValues(0, 5); row.getValues()[0] = i; row.getValues()[1] = i % 2; row.getValues()[2] = -i; From 5b301e02d30daf1efe699cc6a23907108f74deb2 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 21 Sep 2026 18:26:45 +0800 Subject: [PATCH 16/20] Add perf --- .../it/session/IoTDBTableModelSessionIT.java | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java index db971c7cc020c..df4669db155d7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java @@ -439,16 +439,20 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { final int warmupRounds = 2; final int measuredRounds = 6; final int fieldCount = 99; + final String databasePrefix = "numeric_perf_"; try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { - session.executeNonQueryStatement("CREATE DATABASE numeric_perf"); - session.executeNonQueryStatement("USE numeric_perf"); StringBuilder ddl = new StringBuilder("CREATE TABLE prepared_perf (tag STRING TAG"); for (int i = 0; i < fieldCount; i++) { ddl.append(", f").append(i).append(" DOUBLE FIELD"); } ddl.append(")"); - session.executeNonQueryStatement(ddl.toString()); - session.executeNonQueryStatement(ddl.toString().replace("prepared_perf", "tablet_perf")); + // Give each writer its own database so concurrent writes use independent DataRegions. + for (int i = 0; i < concurrency; i++) { + session.executeNonQueryStatement("CREATE DATABASE " + databasePrefix + i); + session.executeNonQueryStatement("USE " + databasePrefix + i); + session.executeNonQueryStatement(ddl.toString()); + session.executeNonQueryStatement(ddl.toString().replace("prepared_perf", "tablet_perf")); + } } List names = new ArrayList<>(); @@ -474,7 +478,7 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { // Connections, USE, prepare and reusable input allocation are outside every timed round. for (int i = 0; i < concurrency; i++) { sessions[i] = EnvFactory.getEnv().getTableSessionConnection(); - sessions[i].executeNonQueryStatement("USE numeric_perf"); + sessions[i].executeNonQueryStatement("USE " + databasePrefix + i); prepared[i] = sessions[i].prepareInsertNumericRow("prepared_perf", names, types, categories); tablets[i] = new Tablet("tablet_perf", names, types, categories, 1); @@ -551,12 +555,15 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { // Per-device monotonic timestamps span warmup and measurement, so no rows overwrite others. for (int path = 0; path < 2; path++) { String table = path == 0 ? "prepared_perf" : "tablet_perf"; - long expectedRows = Arrays.stream(path == 0 ? preparedRows : tabletRows).sum(); - try (SessionDataSet dataSet = - sessions[0].executeQueryStatement("SELECT count(*) FROM " + table)) { - assertTrue(dataSet.hasNext()); - assertEquals(expectedRows, dataSet.next().getFields().get(0).getLongV()); - assertFalse(dataSet.hasNext()); + long[] rowsByThread = path == 0 ? preparedRows : tabletRows; + long expectedRows = Arrays.stream(rowsByThread).sum(); + for (int i = 0; i < concurrency; i++) { + try (SessionDataSet dataSet = + sessions[i].executeQueryStatement("SELECT count(*) FROM " + table)) { + assertTrue(dataSet.hasNext()); + assertEquals(rowsByThread[i], dataSet.next().getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } } System.out.printf("verified %s: %d rows%n", table, expectedRows); } From 408fdc8f70fe06e73cd7b44027035f4aa42516ac Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 23 Sep 2026 12:21:32 +0800 Subject: [PATCH 17/20] optimize --- .../it/session/IoTDBTableModelSessionIT.java | 72 +++++-- .../dataregion/memtable/AbstractMemTable.java | 29 ++- .../AbstractMemTableNumericRowTest.java | 195 ++++++++++++++++++ 3 files changed, 268 insertions(+), 28 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTableNumericRowTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java index df4669db155d7..3b0313ec006dc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java @@ -435,7 +435,8 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { "Performance benchmark disabled; set iotdb.numeric.row.insert.perf.enabled=true to run", Boolean.getBoolean("iotdb.numeric.row.insert.perf.enabled")); final int concurrency = 16; - // Compare sustained single-row writes after warming both paths; verify all acknowledged rows. + // Compare sustained single-row writes after warming all three paths; verify all acknowledged + // rows. final int warmupRounds = 2; final int measuredRounds = 6; final int fieldCount = 99; @@ -452,6 +453,7 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { session.executeNonQueryStatement("USE " + databasePrefix + i); session.executeNonQueryStatement(ddl.toString()); session.executeNonQueryStatement(ddl.toString().replace("prepared_perf", "tablet_perf")); + session.executeNonQueryStatement(ddl.toString().replace("prepared_perf", "row_perf")); } } @@ -469,10 +471,10 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { ITableSession[] sessions = new ITableSession[concurrency]; NumericRowPreparedStatement[] prepared = new NumericRowPreparedStatement[concurrency]; Tablet[] tablets = new Tablet[concurrency]; + NumericRow[] rows = new NumericRow[concurrency]; String[][] tags = new String[concurrency][]; long[][] numericValues = new long[concurrency][fieldCount]; - long[] preparedRows = new long[concurrency]; - long[] tabletRows = new long[concurrency]; + long[][] pathRows = new long[3][concurrency]; ExecutorService executor = Executors.newFixedThreadPool(concurrency); try { // Connections, USE, prepare and reusable input allocation are outside every timed round. @@ -484,16 +486,18 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { tablets[i] = new Tablet("tablet_perf", names, types, categories, 1); tags[i] = new String[] {"device-" + i}; Arrays.fill(numericValues[i], Double.doubleToRawLongBits(1.0)); + rows[i] = + new NumericRow("row_perf", 0, names, types, categories, numericValues[i], tags[i]); } IntConsumer preparedWrite = threadId -> { try { sessions[threadId].executePreparedInsertNumericRow( prepared[threadId], - preparedRows[threadId], + pathRows[0][threadId], numericValues[threadId], tags[threadId]); - preparedRows[threadId]++; + pathRows[0][threadId]++; } catch (Exception e) { throw new RuntimeException(e); } @@ -502,36 +506,56 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { threadId -> { try { Tablet tablet = tablets[threadId]; - tablet.addTimestamp(0, tabletRows[threadId]); + tablet.addTimestamp(0, pathRows[2][threadId]); tablet.addValue("tag", 0, tags[threadId][0]); for (int i = 0; i < fieldCount; i++) { tablet.addValue(names.get(i + 1), 0, 1.0D); } sessions[threadId].insert(tablet); tablet.reset(); - tabletRows[threadId]++; + pathRows[2][threadId]++; } catch (Exception e) { throw new RuntimeException(e); } }; - double[][] rates = new double[2][measuredRounds]; - long[] totalRows = new long[2]; - long[] totalNanos = new long[2]; + IntConsumer rowWrite = + threadId -> { + try { + rows[threadId] = + new NumericRow( + "row_perf", + pathRows[1][threadId], + names, + types, + categories, + numericValues[threadId], + tags[threadId]); + sessions[threadId].insert(rows[threadId]); + pathRows[1][threadId]++; + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + IntConsumer[] paths = {preparedWrite, rowWrite, tabletWrite}; + String[] pathNames = {"prepared numeric row", "insert NumericRow", "insertTablet batch=1"}; + System.out.printf("configuration: concurrency=%d, tabletBatch=1%n", concurrency); + double[][] rates = new double[paths.length][measuredRounds]; + long[] totalRows = new long[paths.length]; + long[] totalNanos = new long[paths.length]; for (int round = 0; round < warmupRounds + measuredRounds; round++) { boolean warmup = round < warmupRounds; int seconds = warmup ? 3 : 5; - // Alternate AB/BA to distribute shared JVM warmup and background storage work. - for (int position = 0; position < 2; position++) { - int path = (round + position) % 2; + // Rotate all three paths so each occupies every position during measured rounds. + for (int position = 0; position < paths.length; position++) { + int path = (round + position) % paths.length; WritePerformance result = - runConcurrentWrites( - executor, concurrency, seconds, path == 0 ? preparedWrite : tabletWrite); + runConcurrentWrites(executor, concurrency, seconds, paths[path]); System.out.printf( "%s round %d position %d %s: %d rows, %.3f s, %.1f rows/s%n", warmup ? "warmup" : "measured", warmup ? round + 1 : round - warmupRounds + 1, position + 1, - path == 0 ? "prepared numeric row" : "insertTablet batch=1", + pathNames[path], result.rows(), result.nanos() / 1_000_000_000.0, result.rowsPerSecond()); @@ -542,25 +566,31 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { } } } - for (int path = 0; path < 2; path++) { + for (int path = 0; path < paths.length; path++) { Arrays.sort(rates[path]); System.out.printf( "summary %s: median %.1f rows/s, aggregate %.1f rows/s, min %.1f, max %.1f%n", - path == 0 ? "prepared numeric row" : "insertTablet batch=1", + pathNames[path], (rates[path][measuredRounds / 2 - 1] + rates[path][measuredRounds / 2]) / 2, totalRows[path] * 1_000_000_000.0 / totalNanos[path], rates[path][0], rates[path][measuredRounds - 1]); } // Per-device monotonic timestamps span warmup and measurement, so no rows overwrite others. - for (int path = 0; path < 2; path++) { - String table = path == 0 ? "prepared_perf" : "tablet_perf"; - long[] rowsByThread = path == 0 ? preparedRows : tabletRows; + for (int path = 0; path < paths.length; path++) { + String table = + switch (path) { + case 0 -> "prepared_perf"; + case 1 -> "row_perf"; + default -> "tablet_perf"; + }; + long[] rowsByThread = pathRows[path]; long expectedRows = Arrays.stream(rowsByThread).sum(); for (int i = 0; i < concurrency; i++) { try (SessionDataSet dataSet = sessions[i].executeQueryStatement("SELECT count(*) FROM " + table)) { assertTrue(dataSet.hasNext()); + // Each worker writes to its own database, including warm-up rows. assertEquals(rowsByThread[i], dataSet.next().getFields().get(0).getLongV()); assertFalse(dataSet.hasNext()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index bd2b7d5770249..fa8d28d67fd9a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.IFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.DataTypeInconsistentException; import org.apache.iotdb.db.exception.WriteProcessException; @@ -182,12 +183,13 @@ private IWritableMemChunkGroup createMemChunkGroupIfNotExistAndGet( private IWritableMemChunkGroup createAlignedMemChunkGroupIfNotExistAndGet( IDeviceID deviceId, List schemaList) { - List filteredSchemaList = - schemaList.stream().filter(Objects::nonNull).collect(Collectors.toList()); IWritableMemChunkGroup memChunkGroup = memTableMap.computeIfAbsent( deviceId, k -> { + // Existing devices need no filtered copy; new chunks own this mutable schema list. + List filteredSchemaList = + schemaList.stream().filter(Objects::nonNull).collect(Collectors.toList()); seriesNumber += filteredSchemaList.size(); return new AlignedWritableMemChunkGroup(filteredSchemaList, k.isTableModel()); }); @@ -292,12 +294,26 @@ public int insertAlignedRow(AbstractInsertRowNode insertRowNode) { } private int insertAlignedNumericRow(RelationalInsertNumericRowNode row) { - List schemas = new ArrayList<>(); + int measurementCount = row.getMeasurements().length; + int valueCount = row.getValueCount(); + TsTableColumnCategory[] categories = row.getColumnCategories(); + List schemas = new ArrayList<>(measurementCount); + int validValues = 0; int nullPoints = 0; long rowSize = Long.BYTES + Integer.BYTES; - for (int i = 0; i < row.getMeasurements().length; i++) { - if (!row.isValidMeasurement(i, true) || row.isNull(i)) { - if (row.isValidMeasurement(i, true) && i < row.getValueCount() && row.isNull(i)) { + for (int i = 0; i < measurementCount; i++) { + boolean valid = row.isValidMeasurement(i, true); + boolean hasValue = i < valueCount; + // Match numeric-row presence checks, counting valid NULL fields but not missing slots. + if (valid + && hasValue + && categories != null + && i < categories.length + && categories[i] != null) { + validValues++; + } + if (!valid || !hasValue || row.isNull(i)) { + if (valid && hasValue) { nullPoints++; } schemas.add(null); @@ -316,7 +332,6 @@ private int insertAlignedNumericRow(RelationalInsertNumericRowNode row) { .getAlignedMemChunk() .writeAlignedNumericPoints(row.getTime(), row.getNumericValues(), schemas); memSize += rowSize; - int validValues = row.getValidMeasurementNumber(true); int inserted = validValues - (IoTDBDescriptor.getInstance().getConfig().isIncludeNullValueInWriteThroughputMetric() diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTableNumericRowTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTableNumericRowTest.java new file mode 100644 index 0000000000000..098f6ab72e55b --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTableNumericRowTest.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.memtable; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; +import org.apache.iotdb.db.utils.NumericRowValues; +import org.apache.iotdb.db.utils.datastructure.AlignedTVList; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class AbstractMemTableNumericRowTest { + private PrimitiveMemTable memTable; + private boolean previousIncludeNull; + + @Before + public void setUp() { + memTable = new PrimitiveMemTable("numeric_test", "0"); + previousIncludeNull = + IoTDBDescriptor.getInstance().getConfig().isIncludeNullValueInWriteThroughputMetric(); + IoTDBDescriptor.getInstance().getConfig().setIncludeNullValueInWriteThroughputMetric(false); + } + + @After + public void tearDown() { + memTable.release(); + IoTDBDescriptor.getInstance() + .getConfig() + .setIncludeNullValueInWriteThroughputMetric(previousIncludeNull); + } + + @Test + public void testNullAndFailedFieldAccountingWithBothMetricSettings() throws Exception { + // TAGs and failed FIELDs never count; valid NULL FIELDs count only in the enabled metric. + NumericRowValues values = new NumericRowValues(1, 3); + values.setTag(0, "device"); + values.setRawValue(1, Double.doubleToRawLongBits(10)); + values.setNull(2); + RelationalInsertNumericRowNode row = + numericRow(new String[] {"tag", "a", "b", "failed"}, 1, values, 1); + row.markFailedMeasurement(3); + + assertEquals(1, memTable.insertAlignedRow(row)); + assertEquals(2, memTable.getTotalValueCount()); + assertEquals(1, memTable.getNullValueCount()); + assertEquals(1, memTable.getSeriesNumber()); + assertEquals(20, memTable.memSize()); + + IoTDBDescriptor.getInstance().getConfig().setIncludeNullValueInWriteThroughputMetric(true); + row.setTime(2); + assertEquals(2, memTable.insertAlignedRow(row)); + values.setNull(1); + row.setTime(3); + assertEquals(2, memTable.insertAlignedRow(row)); + assertEquals(5, memTable.getTotalPointsNum()); + assertEquals(6, memTable.getTotalValueCount()); + assertEquals(4, memTable.getNullValueCount()); + assertEquals(52, memTable.memSize()); + assertEquals(1, memTable.getSeriesNumber()); + AlignedTVList list = chunk(row).getWorkingTVList(); + assertEquals(10, list.getDoubleByValueIndex(0, 0), 0); + assertTrue(list.isNullValue(2, 0)); + } + + @Test + public void testMissingSlotsAndInvalidMetadataAreNotCounted() throws Exception { + // A valid schema beyond the value buffer is absent, not a NULL point or a new series. + NumericRowValues values = new NumericRowValues(1, 1); + values.setTag(0, "device"); + values.setRawValue(1, Double.doubleToRawLongBits(12)); + RelationalInsertNumericRowNode row = + numericRow(new String[] {"tag", "a", "missing", "invalid"}, 1, values, 1); + row.getMeasurementSchemas()[3] = null; + assertEquals(1, memTable.insertAlignedRow(row)); + assertEquals(1, memTable.getTotalValueCount()); + assertEquals(0, memTable.getNullValueCount()); + assertEquals(1, memTable.getSeriesNumber()); + assertEquals(20, memTable.memSize()); + assertEquals(12, chunk(row).getWorkingTVList().getDoubleByValueIndex(0, 0), 0); + } + + @Test + public void testExistingGroupCanGrowAfterNumericAndTabletWrites() throws Exception { + // Creation filters NULL schemas; a later Tablet materializes that field in the same group. + NumericRowValues values = new NumericRowValues(0, 2); + values.setRawValue(0, Double.doubleToRawLongBits(1)); + values.setNull(1); + RelationalInsertNumericRowNode row = numericRow(new String[] {"a", "b"}, 0, values, 1); + assertEquals(1, memTable.insertAlignedRow(row)); + AlignedWritableMemChunk original = chunk(row); + row.setTime(2); + assertEquals(1, memTable.insertAlignedRow(row)); + RelationalInsertTabletNode tablet = + new RelationalInsertTabletNode( + new PlanNodeId("tablet"), + new PartialPath("t"), + true, + new String[] {"b"}, + new TSDataType[] {TSDataType.DOUBLE}, + new MeasurementSchema[] {new MeasurementSchema("b", TSDataType.DOUBLE)}, + new long[] {3}, + null, + new Object[] {new double[] {30}}, + 1, + new TsTableColumnCategory[] {TsTableColumnCategory.FIELD}); + assertEquals(1, memTable.insertAlignedTablet(tablet, 0, 1, null)); + values.setRawValue(0, Double.doubleToRawLongBits(4)); + values.setRawValue(1, Double.doubleToRawLongBits(40)); + row.setTime(4); + assertEquals(2, memTable.insertAlignedRow(row)); + assertSame(original, chunk(row)); + assertEquals(2, memTable.getSeriesNumber()); + assertEquals(5, memTable.getTotalPointsNum()); + assertEquals(7, memTable.getTotalValueCount()); + assertEquals(2, memTable.getNullValueCount()); + AlignedTVList list = original.getWorkingTVList(); + assertTrue(list.isNullValue(0, 1)); + assertTrue(list.isNullValue(2, 0)); + assertEquals(30, list.getDoubleByValueIndex(2, 1), 0); + assertEquals(4, list.getDoubleByValueIndex(3, 0), 0); + assertEquals(40, list.getDoubleByValueIndex(3, 1), 0); + } + + @Test + public void testEmptyRowDoesNotCreateGroup() throws Exception { + // Empty input leaves counters and the device map untouched. + assertEquals( + 0, memTable.insertAlignedRow(numericRow(new String[0], 0, new NumericRowValues(0, 0), 1))); + assertTrue(memTable.getMemTableMap().isEmpty()); + assertEquals(0, memTable.memSize()); + assertEquals(0, memTable.getTotalValueCount()); + } + + private AlignedWritableMemChunk chunk(RelationalInsertNumericRowNode row) { + return ((AlignedWritableMemChunkGroup) memTable.getMemTableMap().get(row.getDeviceID())) + .getAlignedMemChunk(); + } + + private static RelationalInsertNumericRowNode numericRow( + String[] names, int tags, NumericRowValues values, long time) throws IllegalPathException { + TSDataType[] types = new TSDataType[names.length]; + TsTableColumnCategory[] categories = new TsTableColumnCategory[names.length]; + Arrays.fill(types, TSDataType.DOUBLE); + Arrays.fill(categories, TsTableColumnCategory.FIELD); + MeasurementSchema[] schemas = new MeasurementSchema[names.length]; + for (int i = 0; i < names.length; i++) { + if (i < tags) { + types[i] = TSDataType.STRING; + categories[i] = TsTableColumnCategory.TAG; + } + schemas[i] = new MeasurementSchema(names[i], types[i]); + } + return new RelationalInsertNumericRowNode( + new PlanNodeId("numeric"), + new PartialPath("t"), + names, + types, + schemas, + time, + values, + categories); + } +} From 2794bfc6adcb444d3973f1a00e859384c6f4d3ec Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Wed, 30 Sep 2026 11:09:20 +0800 Subject: [PATCH 18/20] fix multiple issues --- .../manual/basic/IoTDBPipeNumericRowIT.java | 449 ++++++++++++++++++ .../it/session/IoTDBTableModelSessionIT.java | 206 ++++++-- .../it/session/WritePerformanceIoTracker.java | 215 +++++++++ .../iotdb/session/i18n/SessionMessages.java | 1 + .../iotdb/session/i18n/SessionMessages.java | 1 + .../apache/iotdb/session/TableSession.java | 29 ++ .../session/pool/TableSessionWrapper.java | 35 +- .../PreparedNumericRowMetadataTest.java | 218 +++++++++ .../pool/PreparedNumericRowPoolTest.java | 222 +++++++++ .../PipeConvertedInsertRowStatement.java | 35 +- ...tementDataTypeConvertExecutionVisitor.java | 10 + .../thrift/impl/ClientRPCServiceImpl.java | 8 + .../PipeConvertedInsertRowStatementTest.java | 146 ++++++ ...tRPCServiceImplPreparedNumericRowTest.java | 195 ++++++++ .../plan/parser/NumericTableRowTest.java | 61 +++ 15 files changed, 1784 insertions(+), 47 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeNumericRowIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/session/WritePerformanceIoTracker.java create mode 100644 iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowMetadataTest.java create mode 100644 iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/PreparedNumericRowPoolTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImplPreparedNumericRowTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeNumericRowIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeNumericRowIT.java new file mode 100644 index 0000000000000..5211f29077100 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeNumericRowIT.java @@ -0,0 +1,449 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.cluster.RegionRoleType; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.NumericRow; +import org.apache.iotdb.isession.NumericRowPreparedStatement; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.session.TableSessionBuilder; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.LocalDate; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +@RunWith(IoTDBTestRunner.class) +@Category(MultiClusterIT2DualTableManualBasic.class) +public class IoTDBPipeNumericRowIT extends AbstractPipeTableModelDualManualIT { + private static final String DATABASE = "numeric_sync"; + private static final List NAMES = + Arrays.asList("d", "device", "f", "l", "region", "i", "b", "event_date", "event_time"); + private static final List TYPES = + Arrays.asList( + TSDataType.DOUBLE, + TSDataType.STRING, + TSDataType.FLOAT, + TSDataType.INT64, + TSDataType.STRING, + TSDataType.INT32, + TSDataType.BOOLEAN, + TSDataType.DATE, + TSDataType.TIMESTAMP); + private static final List CATEGORIES = + Arrays.asList( + ColumnCategory.FIELD, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + + @Override + @Before + public void setUp() { + MultiEnvFactory.createEnv(2); + senderEnv = MultiEnvFactory.getEnv(0); + receiverEnv = MultiEnvFactory.getEnv(1); + setupConfig(); + senderEnv + .getConfig() + .getCommonConfig() + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) + .setDataRegionGroupExtensionPolicy("CUSTOM") + .setDefaultDataRegionGroupNumPerDatabase(1) + .setDataReplicationFactor(3) + .setSchemaReplicationFactor(3); + senderEnv + .getConfig() + .getDataNodeConfig() + .setMetricReporterType(Collections.singletonList("PROMETHEUS")); + receiverEnv + .getConfig() + .getCommonConfig() + .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + senderEnv.initClusterEnvironment(1, 3); + receiverEnv.initClusterEnvironment(1, 1); + } + + @Test + public void testRealtimeWithoutBatch() throws Exception { + // Unflushed NumericRow and prepared rows must arrive through individual InsertNode requests. + testRealtime(false); + } + + @Test + public void testRealtimeWithBatch() throws Exception { + // Batch transfer must preserve all numeric types, FIELD null bits, and null/empty STRING TAGs. + testRealtime(true); + } + + private void testRealtime(boolean batch) throws Exception { + createTable(); + createPipe(batch, false); + writeRows(0, 32); + assertRowsEventually(receiverEnv, 0, 32); + // The destination must remain readable after its own MemTable has been flushed. + try (ITableSession session = receiverEnv.getTableSessionConnection()) { + session.executeNonQueryStatement("FLUSH"); + } + assertRowsEventually(receiverEnv, 0, 32); + } + + @Test + public void testHistoricalTsFile() throws Exception { + // Start the pipe after flushing so the historical TsFile path also carries numeric-row data. + createTable(); + writeRows(0, 32); + try (ITableSession session = senderEnv.getTableSessionConnection()) { + session.executeNonQueryStatement("FLUSH"); + } + createPipe(true, true); + assertRowsEventually(receiverEnv, 0, 32); + } + + @Test + public void testIoTConsensusReplicaAndRestart() throws Exception { + // Pin writes to the actual DataRegion leader, then stop that leader before querying followers. + // Merely stopping a client entry node can still leave all reads on the original leader. + createTable(); + writeRows(0, 16); + int leader = findDataRegionLeader(); + awaitReplication(leader, 16); + int offlineFollower = (leader + 1) % 3; + senderEnv.shutdownDataNode(offlineFollower); + try { + awaitNodeStatus(offlineFollower, NodeStatus.Unknown); + writeRows(16, 32, leader); + } finally { + senderEnv.startDataNode(offlineFollower); + } + awaitNodeStatus(offlineFollower, NodeStatus.Running); + // Running only confirms node health; IoTConsensus catch-up must finish before losing its + // sender. + awaitReplication(leader, 32); + // Resolve the leader again after restart instead of assuming its assignment is unchanged. + int stoppedLeader = findDataRegionLeader(); + assertEquals( + "The primary that accepted the second write batch must be stopped", leader, stoppedLeader); + senderEnv.shutdownDataNode(stoppedLeader); + try { + assertFalse(senderEnv.getDataNodeWrapper(stoppedLeader).isAlive()); + awaitNodeStatus(stoppedLeader, NodeStatus.Unknown); + for (int follower = 0; follower < 3; follower++) { + if (follower == stoppedLeader) { + continue; + } + assertFalse(senderEnv.getDataNodeWrapper(stoppedLeader).isAlive()); + assertRowsEventually(senderEnv, follower, 32); + assertFalse(senderEnv.getDataNodeWrapper(stoppedLeader).isAlive()); + System.out.printf( + "NumericRow IoTConsensus: stopped primary port=%d, query follower port=%d, verified rows=32%n", + senderEnv.getDataNodeWrapper(stoppedLeader).getPort(), + senderEnv.getDataNodeWrapper(follower).getPort()); + } + } finally { + senderEnv.startDataNode(stoppedLeader); + } + } + + private void awaitReplication(int leader, int expectedWrites) { + String metricsUrl = + "http://" + + senderEnv.getDataNodeWrapper(leader).getIp() + + ":" + + senderEnv.getDataNodeWrapper(leader).getMetricPort() + + "/metrics"; + await() + .atMost(90, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .untilAsserted( + () -> { + String metrics = senderEnv.getUrlContent(metricsUrl, null); + assertNotNull(metrics); + // Require the latest write index too: an old zero-lag sample is not catch-up proof. + // This database has exactly one DataRegion and each numeric row adds one write. + double searchIndex = -1; + double safeIndex = -1; + for (String line : metrics.split("\\R")) { + if (!line.startsWith("iot_consensus{")) { + continue; + } + double value = Double.parseDouble(line.substring(line.lastIndexOf(' ') + 1)); + if (line.contains("type=\"searchIndex\"")) { + searchIndex = value; + } else if (line.contains("type=\"safeIndex\"")) { + safeIndex = value; + } + } + assertTrue( + "Missing or stale searchIndex at " + metricsUrl, searchIndex >= expectedWrites); + assertEquals("Replication is incomplete at " + metricsUrl, searchIndex, safeIndex, 0); + }); + } + + private void awaitNodeStatus(int node, NodeStatus expected) { + // BaseEnv.ensureNodeStatus uses EnvFactory, which cannot be used with MultiCluster + // environments. + await() + .atMost(90, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + TShowClusterResp response = client.showCluster(); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode()); + List matches = + response.getDataNodeList().stream() + .filter( + location -> + location.getClientRpcEndPoint().getPort() + == senderEnv.getDataNodeWrapper(node).getPort()) + .toList(); + assertEquals(1, matches.size()); + assertEquals( + expected.getStatus(), + response.getNodeStatus().get(matches.get(0).getDataNodeId())); + } + }); + } + + private int findDataRegionLeader() throws Exception { + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + TShowRegionResp response = + client.showRegion( + new TShowRegionReq() + .setConsensusGroupType(TConsensusGroupType.DataRegion) + .setDatabases(Collections.singletonList(DATABASE)) + .setIsTableModel(true)); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode()); + // Exactly one three-replica region ensures stopping this leader covers all inserted rows. + List replicas = response.getRegionInfoList(); + assertEquals(3, replicas.size()); + for (TRegionInfo replica : replicas) { + assertEquals(replicas.get(0).getConsensusGroupId(), replica.getConsensusGroupId()); + } + List leaders = + replicas.stream() + .filter(replica -> RegionRoleType.Leader.getRoleType().equals(replica.getRoleType())) + .toList(); + assertEquals(1, leaders.size()); + TRegionInfo leader = leaders.get(0); + for (int node = 0; node < 3; node++) { + if (senderEnv.getDataNodeWrapper(node).getPort() == leader.getClientRpcPort()) { + System.out.printf( + "NumericRow IoTConsensus: region=%d, primary DataNodeId=%d, port=%d%n", + leader.getConsensusGroupId().getId(), + leader.getDataNodeId(), + leader.getClientRpcPort()); + return node; + } + } + throw new AssertionError("No wrapper for DataRegion leader " + leader); + } + } + + private void createTable() throws Exception { + try (ITableSession session = senderEnv.getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE " + DATABASE); + session.executeNonQueryStatement("USE " + DATABASE); + session.executeNonQueryStatement( + "CREATE TABLE readings(region STRING TAG, device STRING TAG, missing STRING TAG, " + + "d DOUBLE FIELD, f FLOAT FIELD, l INT64 FIELD, i INT32 FIELD, b BOOLEAN FIELD, " + + "event_date DATE FIELD, event_time TIMESTAMP FIELD)"); + } + } + + private void createPipe(boolean batch, boolean history) throws Exception { + Map source = new HashMap<>(); + source.put("capture.table", "true"); + source.put("capture.tree", "false"); + source.put("__system.sql-dialect", "table"); + source.put("user", "root"); + source.put("database-name", DATABASE); + source.put("table-name", "readings"); + source.put("source.realtime.mode", "log"); + source.put("source.history.enable", Boolean.toString(history)); + Map sink = new HashMap<>(); + sink.put("sink", "iotdb-thrift-sink"); + sink.put("sink.ip", receiverEnv.getDataNodeWrapper(0).getIp()); + sink.put("sink.port", Integer.toString(receiverEnv.getDataNodeWrapper(0).getPort())); + sink.put("sink.batch.enable", Boolean.toString(batch)); + sink.put("sink.batch.max-delay-seconds", "1"); + sink.put("sink.format", history ? "tsfile" : "tablet"); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("numeric_pipe", sink) + .setExtractorAttributes(source) + .setProcessorAttributes(Collections.emptyMap())) + .getCode()); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("numeric_pipe").getCode()); + } + } + + private void writeRows(int start, int end) throws Exception { + writeRows(start, end, 0); + } + + private void writeRows(int start, int end, int node) throws Exception { + // Pin the client to the selected endpoint; its index is not necessarily the DataRegion leader. + try (ITableSession session = + new TableSessionBuilder() + .nodeUrls( + Collections.singletonList( + senderEnv.getDataNodeWrapper(node).getIpAndPortString())) + .username(SessionConfig.DEFAULT_USER) + .password(SessionConfig.DEFAULT_PASSWORD) + .database(DATABASE) + .enableRedirection(false) + .build(); + NumericRowPreparedStatement prepared = + session.prepareInsertNumericRow("readings", NAMES, TYPES, CATEGORIES)) { + for (int row = start; row < end; row++) { + long[] values = { + Double.doubleToRawLongBits(row + 0.25), + Float.floatToRawIntBits(-row - 0.5f), + Long.MIN_VALUE + row, + Integer.MIN_VALUE + row, + row % 2, + 20260924, + 1000L + row + }; + String[] tags = {device(row), "region"}; + BitMap nulls = new BitMap(7); + if (row % 3 == 0) { + nulls.mark(row % 7); + } + if (row % 2 == 0) { + session.executePreparedInsertNumericRow(prepared, row, values, tags, nulls); + } else { + session.insert( + new NumericRow("readings", row, NAMES, TYPES, CATEGORIES, values, tags, nulls)); + } + } + } + } + + private static String device(int row) { + return row % 3 == 0 ? null : row % 3 == 1 ? "" : "device-\u8bbe\u5907"; + } + + private void assertRowsEventually(BaseEnv env, int node, int count) { + await() + // Restart and leader failover can temporarily reject JDBC connections or query RPCs. + .ignoreExceptionsInstanceOf(SQLException.class) + .atMost(90, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try (Connection connection = + env.getConnection( + env.getDataNodeWrapper(node), + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + "table"); + Statement statement = connection.createStatement()) { + statement.execute("USE " + DATABASE); + try (ResultSet result = + statement.executeQuery( + "SELECT time,region,device,missing,d,f,l,i,b,event_date,event_time FROM readings ORDER BY time")) { + for (int row = 0; row < count; row++) { + assertTrue("Missing row " + row, result.next()); + assertEquals(row, result.getLong(1)); + assertEquals("region", result.getString(2)); + assertEquals(device(row), result.getString(3)); + assertNull(result.getObject(4)); + for (int field = 0; field < 7; field++) { + int column = field + 5; + if (row % 3 == 0 && field == row % 7) { + assertNull(result.getObject(column)); + continue; + } + assertNotNull(result.getObject(column)); + switch (field) { + case 0 -> assertEquals(row + 0.25, result.getDouble(column), 0); + case 1 -> assertEquals(-row - 0.5f, result.getFloat(column), 0); + case 2 -> assertEquals(Long.MIN_VALUE + row, result.getLong(column)); + case 3 -> assertEquals(Integer.MIN_VALUE + row, result.getInt(column)); + case 4 -> assertEquals(row % 2 != 0, result.getBoolean(column)); + case 5 -> + assertEquals( + LocalDate.of(2026, 9, 24), result.getDate(column).toLocalDate()); + case 6 -> assertEquals(1000L + row, result.getLong(column)); + default -> throw new AssertionError(field); + } + } + } + assertFalse(result.next()); + } + } + }); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java index 3b0313ec006dc..e196ff1edf0eb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java @@ -72,6 +72,16 @@ public class IoTDBTableModelSessionIT { @Before public void setUp() throws Exception { + if (Boolean.getBoolean("iotdb.numeric.row.insert.perf.enabled") + && Boolean.getBoolean("iotdb.numeric.row.insert.perf.io.enabled")) { + // Isolate per-round storage accounting from compaction of earlier rounds. + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false); + } EnvFactory.getEnv().initClusterEnvironment(); } @@ -431,10 +441,11 @@ private boolean writeRedirectedNumericRows(ITableSession session, long start) th @Test public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { + // Compare synchronous single-row paths and optionally account for their Linux TCP/storage I/O. Assume.assumeTrue( "Performance benchmark disabled; set iotdb.numeric.row.insert.perf.enabled=true to run", Boolean.getBoolean("iotdb.numeric.row.insert.perf.enabled")); - final int concurrency = 16; + final int concurrency = Integer.getInteger("iotdb.numeric.row.insert.perf.concurrency", 16); // Compare sustained single-row writes after warming all three paths; verify all acknowledged // rows. final int warmupRounds = 2; @@ -497,30 +508,21 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { pathRows[0][threadId], numericValues[threadId], tags[threadId]); - pathRows[0][threadId]++; } catch (Exception e) { throw new RuntimeException(e); } }; - IntConsumer tabletWrite = + IntConsumer tabletPrepare = threadId -> { - try { - Tablet tablet = tablets[threadId]; - tablet.addTimestamp(0, pathRows[2][threadId]); - tablet.addValue("tag", 0, tags[threadId][0]); - for (int i = 0; i < fieldCount; i++) { - tablet.addValue(names.get(i + 1), 0, 1.0D); - } - sessions[threadId].insert(tablet); - tablet.reset(); - pathRows[2][threadId]++; - } catch (Exception e) { - throw new RuntimeException(e); + Tablet tablet = tablets[threadId]; + tablet.addTimestamp(0, pathRows[2][threadId]); + tablet.addValue("tag", 0, tags[threadId][0]); + for (int i = 0; i < fieldCount; i++) { + tablet.addValue(names.get(i + 1), 0, 1.0D); } }; - IntConsumer rowWrite = - threadId -> { - try { + IntConsumer rowPrepare = + threadId -> rows[threadId] = new NumericRow( "row_perf", @@ -530,51 +532,112 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { categories, numericValues[threadId], tags[threadId]); - sessions[threadId].insert(rows[threadId]); - pathRows[1][threadId]++; - } catch (Exception e) { - throw new RuntimeException(e); - } - }; - IntConsumer[] paths = {preparedWrite, rowWrite, tabletWrite}; + IntConsumer[] paths = { + preparedWrite, + threadId -> { + try { + sessions[threadId].insert(rows[threadId]); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, + threadId -> { + try { + sessions[threadId].insert(tablets[threadId]); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }; + IntConsumer[] prepareWrites = {null, rowPrepare, tabletPrepare}; + IntConsumer[] completeWrites = { + threadId -> pathRows[0][threadId]++, + threadId -> pathRows[1][threadId]++, + threadId -> { + tablets[threadId].reset(); + pathRows[2][threadId]++; + } + }; String[] pathNames = {"prepared numeric row", "insert NumericRow", "insertTablet batch=1"}; System.out.printf("configuration: concurrency=%d, tabletBatch=1%n", concurrency); + WritePerformanceIoTracker ioTracker = null; + if (Boolean.getBoolean("iotdb.numeric.row.insert.perf.io.enabled")) { + assertEquals( + "I/O measurement requires one local DataNode", + 1, + EnvFactory.getEnv().getDataNodeWrapperList().size()); + assertEquals( + "I/O measurement requires one local ConfigNode", + 1, + EnvFactory.getEnv().getConfigNodeWrapperList().size()); + ioTracker = + new WritePerformanceIoTracker( + EnvFactory.getEnv().getDataNodeWrapperList().get(0).getPid(), + EnvFactory.getEnv().getConfigNodeWrapperList().get(0).getPid(), + EnvFactory.getEnv().getDataNodeWrapperList().get(0).getPort()); + } double[][] rates = new double[paths.length][measuredRounds]; long[] totalRows = new long[paths.length]; long[] totalNanos = new long[paths.length]; + LongSamples[] requestLatencies = new LongSamples[paths.length]; + for (int path = 0; path < paths.length; path++) { + requestLatencies[path] = new LongSamples(); + } for (int round = 0; round < warmupRounds + measuredRounds; round++) { boolean warmup = round < warmupRounds; int seconds = warmup ? 3 : 5; // Rotate all three paths so each occupies every position during measured rounds. for (int position = 0; position < paths.length; position++) { int path = (round + position) % paths.length; + WritePerformanceIoTracker.Snapshot ioBefore = + ioTracker == null ? null : ioTracker.beforeRound(sessions[0]); WritePerformance result = - runConcurrentWrites(executor, concurrency, seconds, paths[path]); + runConcurrentWrites( + executor, + concurrency, + seconds, + prepareWrites[path], + paths[path], + completeWrites[path]); + if (ioTracker != null) { + System.out.printf( + "io %s round %d position %d %s: rows=%d, %s%n", + warmup ? "warmup" : "measured", + warmup ? round + 1 : round - warmupRounds + 1, + position + 1, + pathNames[path], + result.rows(), + ioTracker.afterRound(ioBefore, sessions[0])); + } System.out.printf( - "%s round %d position %d %s: %d rows, %.3f s, %.1f rows/s%n", + "%s round %d position %d %s: %d rows, %.3f s, %.1f rows/s, API latency %s%n", warmup ? "warmup" : "measured", warmup ? round + 1 : round - warmupRounds + 1, position + 1, pathNames[path], result.rows(), result.nanos() / 1_000_000_000.0, - result.rowsPerSecond()); + result.rowsPerSecond(), + summarizeRequestLatencies(result.requestNanos())); if (!warmup) { rates[path][round - warmupRounds] = result.rowsPerSecond(); totalRows[path] += result.rows(); totalNanos[path] += result.nanos(); + requestLatencies[path].appendAll(result.requestNanos()); } } } for (int path = 0; path < paths.length; path++) { Arrays.sort(rates[path]); System.out.printf( - "summary %s: median %.1f rows/s, aggregate %.1f rows/s, min %.1f, max %.1f%n", + "summary %s: median %.1f rows/s, aggregate %.1f rows/s, min %.1f, max %.1f, " + + "measured API latency %s%n", pathNames[path], (rates[path][measuredRounds / 2 - 1] + rates[path][measuredRounds / 2]) / 2, totalRows[path] * 1_000_000_000.0 / totalNanos[path], rates[path][0], - rates[path][measuredRounds - 1]); + rates[path][measuredRounds - 1], + summarizeRequestLatencies(requestLatencies[path].toArray())); } // Per-device monotonic timestamps span warmup and measurement, so no rows overwrite others. for (int path = 0; path < paths.length; path++) { @@ -619,14 +682,80 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { } } - private record WritePerformance(long rows, long nanos) { + private record WritePerformance(long rows, long nanos, long[] requestNanos) { double rowsPerSecond() { return rows * 1_000_000_000.0 / nanos; } } + private record RequestLatency( + long count, double averageNanos, long p50Nanos, long p95Nanos, long p99Nanos) { + @Override + public String toString() { + return String.format( + "n=%d, avg=%.3f us, p50=%.3f us, p95=%.3f us, p99=%.3f us", + count, + averageNanos / 1_000.0, + p50Nanos / 1_000.0, + p95Nanos / 1_000.0, + p99Nanos / 1_000.0); + } + } + + private static class LongSamples { + private long[] values = new long[1024]; + private int size; + + void add(long value) { + if (size == values.length) { + values = Arrays.copyOf(values, values.length * 2); + } + values[size++] = value; + } + + void appendAll(long[] samples) { + while (values.length - size < samples.length) { + values = Arrays.copyOf(values, values.length * 2); + } + System.arraycopy(samples, 0, values, size, samples.length); + size += samples.length; + } + + long[] toArray() { + return Arrays.copyOf(values, size); + } + } + + private static RequestLatency summarizeRequestLatencies(long[] requestNanos) { + if (requestNanos.length == 0) { + return new RequestLatency(0, 0, 0, 0, 0); + } + long[] sorted = Arrays.copyOf(requestNanos, requestNanos.length); + Arrays.sort(sorted); + double sum = 0; + for (long latency : sorted) { + sum += latency; + } + return new RequestLatency( + sorted.length, + sum / sorted.length, + percentile(sorted, 0.50), + percentile(sorted, 0.95), + percentile(sorted, 0.99)); + } + + private static long percentile(long[] sorted, double quantile) { + return sorted[Math.max(0, (int) Math.ceil(quantile * sorted.length) - 1)]; + } + private static WritePerformance runConcurrentWrites( - ExecutorService executor, int concurrency, int seconds, IntConsumer write) throws Exception { + ExecutorService executor, + int concurrency, + int seconds, + IntConsumer prepare, + IntConsumer write, + IntConsumer complete) + throws Exception { CountDownLatch ready = new CountDownLatch(concurrency); CountDownLatch start = new CountDownLatch(1); AtomicLong deadline = new AtomicLong(); @@ -640,12 +769,19 @@ private static WritePerformance runConcurrentWrites( ready.countDown(); start.await(); long rows = 0; + LongSamples requestNanos = new LongSamples(); while (!Thread.currentThread().isInterrupted() && System.nanoTime() < deadline.get()) { + if (prepare != null) { + prepare.accept(threadId); + } + long requestStart = System.nanoTime(); write.accept(threadId); + requestNanos.add(System.nanoTime() - requestStart); + complete.accept(threadId); rows++; } - return new WritePerformance(rows, System.nanoTime()); + return new WritePerformance(rows, System.nanoTime(), requestNanos.toArray()); })); } assertTrue(ready.await(30, TimeUnit.SECONDS)); @@ -654,13 +790,15 @@ private static WritePerformance runConcurrentWrites( start.countDown(); long rows = 0; long endNanos = startNanos; + LongSamples requestNanos = new LongSamples(); for (Future future : futures) { WritePerformance result = future.get(seconds + 60, TimeUnit.SECONDS); rows += result.rows(); endNanos = Math.max(endNanos, result.nanos()); + requestNanos.appendAll(result.requestNanos()); } // Include completion of in-flight RPCs, but exclude main-thread collection/printing time. - return new WritePerformance(rows, endNanos - startNanos); + return new WritePerformance(rows, endNanos - startNanos, requestNanos.toArray()); } finally { start.countDown(); for (Future future : futures) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/WritePerformanceIoTracker.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/WritePerformanceIoTracker.java new file mode 100644 index 0000000000000..bab6f0ad710c3 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/WritePerformanceIoTracker.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.session; + +import org.apache.iotdb.isession.ITableSession; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** Linux-only, opt-in counters sampled outside the single-row benchmark's timed write loops. */ +final class WritePerformanceIoTracker { + private static final Pattern TCP_COUNTER = + Pattern.compile("\\b(bytes_received|bytes_sent|bytes_retrans):(\\d+)\\b"); + + private final long dataNodePid; + private final long configNodePid; + private final int rpcPort; + + WritePerformanceIoTracker(long dataNodePid, long configNodePid, int rpcPort) throws IOException { + assertTrue("I/O measurement requires Linux /proc", Files.isDirectory(Path.of("/proc"))); + assertTrue("DataNode PID unavailable", dataNodePid > 0); + assertTrue("ConfigNode PID unavailable", configNodePid > 0); + this.dataNodePid = dataNodePid; + this.configNodePid = configNodePid; + this.rpcPort = rpcPort; + // Check readability early; unsupported counters must fail rather than appear as zero I/O. + readDisk(dataNodePid); + readDisk(configNodePid); + System.out.printf( + "io configuration: dataNodePid=%d, configNodePid=%d, rpcPort=%d, " + + "network=server TCP payload counters, disk=/proc/pid/io, " + + "flushBeforeAndAfter=true, compaction=false%n", + dataNodePid, configNodePid, rpcPort); + } + + Snapshot beforeRound(ITableSession session) throws Exception { + // Drain previous MemTables/WAL before the baseline; the following flush is charged to this + // round. + session.executeNonQueryStatement("FLUSH"); + return new Snapshot(readNetwork(), readDisk(dataNodePid), readDisk(configNodePid)); + } + + Result afterRound(Snapshot before, ITableSession session) throws Exception { + // Snapshot TCP before FLUSH so SQL/control traffic is excluded from write-request bytes. + Map network = readNetwork(); + assertEquals( + "RPC connections changed during I/O measurement", + before.network.keySet(), + network.keySet()); + Traffic traffic = new Traffic(0, 0, 0); + for (Map.Entry entry : network.entrySet()) { + traffic = traffic.plus(entry.getValue().minus(before.network.get(entry.getKey()))); + } + Disk activeDataNode = readDisk(dataNodePid).minus(before.dataNode); + long flushStart = System.nanoTime(); + session.executeNonQueryStatement("FLUSH"); + long flushNanos = System.nanoTime() - flushStart; + Disk dataNode = readDisk(dataNodePid).minus(before.dataNode); + Disk configNode = readDisk(configNodePid).minus(before.configNode); + return new Result(traffic, activeDataNode, dataNode, configNode, flushNanos); + } + + private Map readNetwork() throws Exception { + // Count only server endpoints on this test DataNode's RPC port. Loopback RX/TX are not doubled. + Process process = + new ProcessBuilder("ss", "-tinHO", "state", "established", "( sport = :" + rpcPort + " )") + .redirectErrorStream(true) + .start(); + CompletableFuture output = + CompletableFuture.supplyAsync( + () -> { + try { + return new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new CompletionException(e); + } + }); + try { + assertTrue("ss did not finish", process.waitFor(10, TimeUnit.SECONDS)); + String text = output.get(10, TimeUnit.SECONDS); + assertEquals(text, 0, process.exitValue()); + Map result = new HashMap<>(); + for (String line : text.split("\\R")) { + if (line.isBlank()) continue; + String[] fields = line.trim().split("\\s+", 5); + assertTrue("Unexpected ss output: " + line, fields.length == 5); + Map counters = new HashMap<>(); + Matcher matcher = TCP_COUNTER.matcher(fields[4]); + while (matcher.find()) counters.put(matcher.group(1), Long.parseLong(matcher.group(2))); + assertTrue( + "TCP byte counters unavailable: " + line, + counters.containsKey("bytes_received") && counters.containsKey("bytes_sent")); + String connection = fields[2] + "->" + fields[3]; + assertFalse("Duplicate server socket: " + connection, result.containsKey(connection)); + // ss omits zero-valued bytes_retrans; the received/sent counters above are mandatory. + result.put( + connection, + new Traffic( + counters.get("bytes_received"), + counters.get("bytes_sent"), + counters.getOrDefault("bytes_retrans", 0L))); + } + assertFalse("No established test RPC connections", result.isEmpty()); + return result; + } finally { + process.destroyForcibly(); + output.cancel(true); + } + } + + private static Disk readDisk(long pid) throws IOException { + Map counters = new HashMap<>(); + for (String line : Files.readAllLines(Path.of("/proc", Long.toString(pid), "io"))) { + String[] fields = line.split(":\\s*", 2); + counters.put(fields[0], Long.parseLong(fields[1].trim())); + } + assertTrue( + "Process storage byte counters unavailable for PID " + pid, + counters.containsKey("read_bytes") + && counters.containsKey("write_bytes") + && counters.containsKey("cancelled_write_bytes")); + // These are kernel-accounted process storage bytes, not rchar/wchar (which include socket I/O), + // nor physical NAND bytes. Preserve cancelled writes separately instead of hiding them. + return new Disk( + counters.get("read_bytes"), + counters.get("write_bytes"), + counters.get("cancelled_write_bytes")); + } + + record Snapshot(Map network, Disk dataNode, Disk configNode) {} + + private record Traffic(long received, long sent, long retransmitted) { + Traffic minus(Traffic previous) { + assertTrue( + "TCP counters decreased", + received >= previous.received + && sent >= previous.sent + && retransmitted >= previous.retransmitted); + return new Traffic( + received - previous.received, + sent - previous.sent, + retransmitted - previous.retransmitted); + } + + Traffic plus(Traffic other) { + return new Traffic( + received + other.received, sent + other.sent, retransmitted + other.retransmitted); + } + } + + private record Disk(long read, long written, long cancelled) { + Disk minus(Disk previous) { + assertTrue( + "Process I/O counters decreased", + read >= previous.read && written >= previous.written && cancelled >= previous.cancelled); + return new Disk( + read - previous.read, written - previous.written, cancelled - previous.cancelled); + } + } + + record Result( + Traffic traffic, Disk activeDataNode, Disk dataNode, Disk configNode, long flushNanos) { + @Override + public String toString() { + return String.format( + "requestBytes=%d, responseBytes=%d, responseRetransmitBytes=%d, " + + "dnActiveReadBytes=%d, dnActiveWriteBytes=%d, dnActiveCancelledBytes=%d, " + + "dnReadBytes=%d, dnWriteBytes=%d, dnCancelledBytes=%d, " + + "cnReadBytes=%d, cnWriteBytes=%d, cnCancelledBytes=%d, flushNanos=%d", + traffic.received, + traffic.sent, + traffic.retransmitted, + activeDataNode.read, + activeDataNode.written, + activeDataNode.cancelled, + dataNode.read, + dataNode.written, + dataNode.cancelled, + configNode.read, + configNode.written, + configNode.cancelled, + flushNanos); + } + } +} diff --git a/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java b/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java index a1bdf45262ec0..2bcac9214ce50 100644 --- a/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java +++ b/iotdb-client/session/src/main/i18n/en/org/apache/iotdb/session/i18n/SessionMessages.java @@ -296,4 +296,5 @@ private SessionMessages() {} public static final String EXCEPTION_PREPARED_NUMERIC_ROW_PREPARE_RETURNED_NO_STATUS_6FF7BC24 = "Prepared numeric row prepare returned no status"; public static final String EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_COUNT_MISMATCH_9ACD3A0C = "Prepared numeric row value count mismatch"; public static final String EXCEPTION_UNSUPPORTED_NUMERIC_TYPE_06DD3044 = "Unsupported numeric type"; + public static final String EXCEPTION_TABLE_SESSION_IS_CLOSED_77FE359B = "Table session is closed."; } diff --git a/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java b/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java index 260b000e61a6b..aa9c1055fd558 100644 --- a/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java +++ b/iotdb-client/session/src/main/i18n/zh/org/apache/iotdb/session/i18n/SessionMessages.java @@ -295,4 +295,5 @@ private SessionMessages() {} public static final String EXCEPTION_PREPARED_NUMERIC_ROW_PREPARE_RETURNED_NO_STATUS_6FF7BC24 = "预备数值行准备请求未返回状态"; public static final String EXCEPTION_PREPARED_NUMERIC_ROW_VALUE_COUNT_MISMATCH_9ACD3A0C = "预备数值行的值数量不匹配"; public static final String EXCEPTION_UNSUPPORTED_NUMERIC_TYPE_06DD3044 = "不支持的数值类型"; + public static final String EXCEPTION_TABLE_SESSION_IS_CLOSED_77FE359B = "表会话已关闭。"; } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java index 9d3e6d8b9bb5e..c453596c1eef4 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java @@ -67,6 +67,35 @@ public NumericRowPreparedStatement prepareInsertNumericRow( List dataTypes, List columnCategories) throws StatementExecutionException, IoTDBConnectionException { + // Validate before regrouping, which would otherwise discard unsupported categories or + // surplus metadata entries. + if (tableName == null + || columnNames == null + || dataTypes == null + || columnCategories == null + || columnNames.size() != dataTypes.size() + || columnNames.size() != columnCategories.size()) { + throw new IllegalArgumentException( + SessionMessages + .EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233); + } + for (int i = 0; i < columnNames.size(); i++) { + if (columnNames.get(i) == null + || dataTypes.get(i) == null + || columnCategories.get(i) == null) { + throw new IllegalArgumentException( + SessionMessages + .EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233); + } + if ((columnCategories.get(i) != ColumnCategory.TAG + && columnCategories.get(i) != ColumnCategory.FIELD) + || (columnCategories.get(i) == ColumnCategory.TAG + && dataTypes.get(i) != TSDataType.STRING)) { + throw new IllegalArgumentException( + SessionMessages + .EXCEPTION_NUMERIC_ROWS_SUPPORT_NUMERIC_FIELD_COLUMNS_AND_STRING_TAG_COLUMNS_ONLY_87159FFA); + } + } List names = new ArrayList<>(); List types = new ArrayList<>(); List categories = new ArrayList<>(); diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java index 7c950949d0da5..7684beba9abe0 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java @@ -80,6 +80,7 @@ public void insert(Tablet tablet) throws StatementExecutionException, IoTDBConne @Override public void insert(NumericRow row) throws StatementExecutionException, IoTDBConnectionException { + checkOpen(); try { session.insertRelationalRecord(row); } catch (IoTDBConnectionException e) { @@ -97,8 +98,14 @@ public NumericRowPreparedStatement prepareInsertNumericRow( List dataTypes, List columnCategories) throws StatementExecutionException, IoTDBConnectionException { - return tableSession.prepareInsertNumericRow( - tableName, columnNames, dataTypes, columnCategories); + checkOpen(); + try { + return tableSession.prepareInsertNumericRow( + tableName, columnNames, dataTypes, columnCategories); + } catch (IoTDBConnectionException e) { + invalidateSession(); + throw e; + } } @Override @@ -116,17 +123,32 @@ public void executePreparedInsertNumericRow( String[] tagValues, BitMap nullBitmap) throws StatementExecutionException, IoTDBConnectionException { + checkOpen(); try { tableSession.executePreparedInsertNumericRow( prepared, timestamp, values, tagValues, nullBitmap); } catch (IoTDBConnectionException e) { - sessionPool.cleanSessionAndMayThrowConnectionException(session); - closed.set(true); - session = null; + invalidateSession(); throw e; } } + private void checkOpen() throws IoTDBConnectionException { + // The cached delegate still references the physical session after it returns to the pool. + if (closed.get() || session == null) { + throw new IoTDBConnectionException( + SessionMessages.EXCEPTION_TABLE_SESSION_IS_CLOSED_77FE359B); + } + } + + private void invalidateSession() { + Session failedSession = session; + // Invalidate before cleanup so close() cannot return a failed connection to the pool. + closed.set(true); + session = null; + sessionPool.cleanSessionAndMayThrowConnectionException(failedSession); + } + @Override public SessionDataSet executeQueryStatement(String sql) throws StatementExecutionException, IoTDBConnectionException { @@ -168,6 +190,9 @@ public void executeNonQueryStatement(String sql) @Override public void close() throws IoTDBConnectionException { + if (closed.get() || session == null) { + return; + } if (!Objects.equals(session.getSqlDialect(), sessionPool.sqlDialect)) { try { session.executeNonQueryStatement("set sql_dialect=" + sessionPool.sqlDialect); diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowMetadataTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowMetadataTest.java new file mode 100644 index 0000000000000..b47b71a961d03 --- /dev/null +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowMetadataTest.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session; + +import org.apache.iotdb.isession.NumericRowPreparedStatement; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; +import org.apache.iotdb.session.i18n.SessionMessages; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class PreparedNumericRowMetadataTest { + private static final String INVALID_ARGUMENTS = + SessionMessages + .EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233; + private static final String INVALID_CATEGORIES = + SessionMessages + .EXCEPTION_NUMERIC_ROWS_SUPPORT_NUMERIC_FIELD_COLUMNS_AND_STRING_TAG_COLUMNS_ONLY_87159FFA; + + @Test + public void testNullMetadataRejectedBeforeRpc() throws Exception { + assertInvalid( + null, + List.of("v"), + List.of(TSDataType.DOUBLE), + List.of(ColumnCategory.FIELD), + INVALID_ARGUMENTS); + assertInvalid( + "t", null, List.of(TSDataType.DOUBLE), List.of(ColumnCategory.FIELD), INVALID_ARGUMENTS); + assertInvalid("t", List.of("v"), null, List.of(ColumnCategory.FIELD), INVALID_ARGUMENTS); + assertInvalid("t", List.of("v"), List.of(TSDataType.DOUBLE), null, INVALID_ARGUMENTS); + } + + @Test + public void testNullMetadataElementsRejectedBeforeRpc() throws Exception { + assertInvalid( + "t", + Arrays.asList("tag", null), + List.of(TSDataType.STRING, TSDataType.DOUBLE), + List.of(ColumnCategory.TAG, ColumnCategory.FIELD), + INVALID_ARGUMENTS); + assertInvalid( + "t", + List.of("tag", "v"), + Arrays.asList(TSDataType.STRING, null), + List.of(ColumnCategory.TAG, ColumnCategory.FIELD), + INVALID_ARGUMENTS); + assertInvalid( + "t", + List.of("tag", "v"), + List.of(TSDataType.STRING, TSDataType.DOUBLE), + Arrays.asList(ColumnCategory.TAG, null), + INVALID_ARGUMENTS); + } + + @Test + public void testMetadataLengthMismatchRejectedBeforeRpc() throws Exception { + final List names = List.of("a", "b"); + final List types = List.of(TSDataType.DOUBLE, TSDataType.DOUBLE); + final List categories = List.of(ColumnCategory.FIELD, ColumnCategory.FIELD); + assertInvalid("t", names.subList(0, 1), types, categories, INVALID_ARGUMENTS); + assertInvalid("t", names, types.subList(0, 1), categories, INVALID_ARGUMENTS); + assertInvalid("t", names, types, categories.subList(0, 1), INVALID_ARGUMENTS); + assertInvalid( + "t", + names, + List.of(TSDataType.DOUBLE, TSDataType.DOUBLE, TSDataType.DOUBLE), + categories, + INVALID_ARGUMENTS); + assertInvalid( + "t", + names, + types, + List.of(ColumnCategory.FIELD, ColumnCategory.FIELD, ColumnCategory.FIELD), + INVALID_ARGUMENTS); + } + + @Test + public void testAttributeColumnRejectedBeforeRpc() throws Exception { + assertInvalid( + "t", + List.of("tag", "attr", "v"), + List.of(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE), + List.of(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD), + INVALID_CATEGORIES); + } + + @Test + public void testNonStringTagRejectedBeforeRpc() throws Exception { + for (TSDataType type : List.of(TSDataType.INT64, TSDataType.TEXT)) { + assertInvalid( + "t", + List.of("tag", "v"), + List.of(type, TSDataType.DOUBLE), + List.of(ColumnCategory.TAG, ColumnCategory.FIELD), + INVALID_CATEGORIES); + } + } + + @Test + public void testMixedColumnOrderPreservesMetadataAndValues() throws Exception { + final Session session = mock(Session.class); + final SessionConnection connection = mock(SessionConnection.class); + when(session.getDefaultSessionConnection()).thenReturn(connection); + when(session.getDatabase()).thenReturn("db"); + when(connection.getSessionId()).thenReturn(10L); + when(connection.prepareNumericRow(any())).thenReturn(42L); + final TableSession tableSession = new TableSession(session); + final List names = List.of("v1", "tag1", "v2", "tag2"); + final List types = + List.of(TSDataType.DOUBLE, TSDataType.STRING, TSDataType.INT32, TSDataType.STRING); + final List categories = + List.of(ColumnCategory.FIELD, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.TAG); + final NumericRowPreparedStatement prepared = + tableSession.prepareInsertNumericRow("t", names, types, categories); + final ArgumentCaptor request = + ArgumentCaptor.forClass(TSPrepareNumericRowReq.class); + verify(connection).prepareNumericRow(request.capture()); + assertMetadata(prepared, request.getValue()); + doAnswer( + invocation -> { + final ByteBuffer values = invocation.getArgument(2); + final ByteBuffer tags = invocation.getArgument(3); + assertEquals("a", ReadWriteIOUtils.readString(tags)); + assertEquals("b", ReadWriteIOUtils.readString(tags)); + assertFalse(tags.hasRemaining()); + assertEquals(12.5, values.getDouble(), 0.0); + assertEquals(7, values.getInt()); + assertFalse(values.hasRemaining()); + return null; + }) + .when(connection) + .executePreparedNumericRow(eq(42L), eq(123L), any(ByteBuffer.class), any(ByteBuffer.class)); + tableSession.executePreparedInsertNumericRow( + prepared, 123L, new long[] {Double.doubleToRawLongBits(12.5), 7}, new String[] {"a", "b"}); + verify(connection) + .executePreparedNumericRow(eq(42L), eq(123L), any(ByteBuffer.class), any(ByteBuffer.class)); + assertEquals(List.of("v1", "tag1", "v2", "tag2"), names); + assertEquals( + List.of(TSDataType.DOUBLE, TSDataType.STRING, TSDataType.INT32, TSDataType.STRING), types); + assertEquals( + List.of(ColumnCategory.FIELD, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.TAG), + categories); + } + + private static void assertMetadata( + NumericRowPreparedStatement prepared, TSPrepareNumericRowReq request) { + assertEquals("t", prepared.getTableName()); + assertEquals("db", prepared.getDatabase()); + assertEquals(2, prepared.getTagCount()); + assertEquals(List.of("tag1", "tag2", "v1", "v2"), prepared.getColumnNames()); + assertEquals( + List.of(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE, TSDataType.INT32), + prepared.getDataTypes()); + assertEquals( + List.of(ColumnCategory.TAG, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD), + prepared.getColumnCategories()); + assertEquals("t", request.getTableName()); + assertEquals(prepared.getColumnNames(), request.getMeasurements()); + assertEquals( + prepared.getDataTypes().stream().map(TSDataType::ordinal).toList(), request.getDataTypes()); + assertEquals( + prepared.getColumnCategories().stream().map(c -> (byte) c.ordinal()).toList(), + request.getColumnCategories()); + } + + private static void assertInvalid( + String table, + List names, + List types, + List categories, + String message) + throws Exception { + final Session session = mock(Session.class); + try { + new TableSession(session).prepareInsertNumericRow(table, names, types, categories); + fail("Invalid metadata must be rejected before preparing a statement"); + } catch (IllegalArgumentException e) { + assertEquals(message, e.getMessage()); + } + verifyNoMoreInteractions(session); + } +} diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/PreparedNumericRowPoolTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/PreparedNumericRowPoolTest.java new file mode 100644 index 0000000000000..f65f761c7b940 --- /dev/null +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/PreparedNumericRowPoolTest.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.pool; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.isession.NumericRow; +import org.apache.iotdb.isession.NumericRowPreparedStatement; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.IClientRPCService; +import org.apache.iotdb.service.rpc.thrift.TSExecutePreparedNumericRowReq; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowResp; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.session.SessionConnection; +import org.apache.iotdb.session.i18n.SessionMessages; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.powermock.reflect.Whitebox; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +public class PreparedNumericRowPoolTest { + private Session session; + private SessionPool pool; + private TableSessionWrapper wrapper; + private IClientRPCService.Iface client; + + @Before + public void setUp() throws Exception { + session = new Session.Builder().enableRedirection(false).enableAutoFetch(false).build(); + client = mock(IClientRPCService.Iface.class); + SessionConnection connection = new SessionConnection("table"); + Whitebox.setInternalState(connection, "client", client); + Whitebox.setInternalState(connection, "sessionId", 11L); + Whitebox.setInternalState(session, "defaultSessionConnection", connection); + pool = mock(SessionPool.class); + pool.sqlDialect = session.getSqlDialect(); + wrapper = new TableSessionWrapper(session, pool); + when(client.prepareNumericRow(any())) + .thenReturn(new TSPrepareNumericRowResp(success()).setPreparedId(101)) + .thenReturn(new TSPrepareNumericRowResp(success()).setPreparedId(102)); + when(client.executePreparedNumericRow(any())).thenReturn(success()); + when(client.closePreparedNumericRow(anyLong(), anyLong())).thenReturn(success()); + } + + @Test + public void testClosedWrapperCannotUseReborrowedSession() throws Exception { + NumericRowPreparedStatement oldPrepared = prepare(wrapper); + wrapper.close(); + // A new borrower owns the same physical connection; the previous wrapper must stay closed. + try (TableSessionWrapper next = new TableSessionWrapper(session, pool); + NumericRowPreparedStatement nextPrepared = prepare(next)) { + clearInvocations(client); + IoTDBConnectionException error = + assertThrows(IoTDBConnectionException.class, () -> prepare(wrapper)); + assertEquals(SessionMessages.EXCEPTION_TABLE_SESSION_IS_CLOSED_77FE359B, error.getMessage()); + assertThrows( + IoTDBConnectionException.class, + () -> + wrapper.executePreparedInsertNumericRow( + oldPrepared, 1L, new long[] {42}, new String[] {"a"})); + assertThrows( + IoTDBConnectionException.class, + () -> + wrapper.executePreparedInsertNumericRow( + oldPrepared, 1L, new long[] {42}, new String[] {"a"}, new BitMap(1))); + verifyZeroInteractions(client); + verify(pool, never()).cleanSessionAndMayThrowConnectionException(any()); + + next.executePreparedInsertNumericRow(nextPrepared, 2L, new long[] {43}, new String[] {"a"}); + ArgumentCaptor request = + ArgumentCaptor.forClass(TSExecutePreparedNumericRowReq.class); + verify(client).executePreparedNumericRow(request.capture()); + assertEquals(102L, request.getValue().getPreparedId()); + } + wrapper.close(); + verify(pool, times(2)).putBack(session); + } + + @Test + public void testClosedWrapperCannotInsertNumericRow() throws Exception { + wrapper.close(); + + IoTDBConnectionException error = + assertThrows( + IoTDBConnectionException.class, + () -> + wrapper.insert( + new NumericRow( + "t", + 1L, + List.of("tag", "v"), + List.of(TSDataType.STRING, TSDataType.INT64), + List.of(ColumnCategory.TAG, ColumnCategory.FIELD), + new long[] {42L}, + new String[] {"a"}))); + + assertEquals(SessionMessages.EXCEPTION_TABLE_SESSION_IS_CLOSED_77FE359B, error.getMessage()); + verifyZeroInteractions(client); + } + + @Test + public void testPrepareTransportFailureDiscardsSession() throws Exception { + TException transportFailure = new TException("transport closed"); + when(client.prepareNumericRow(any())).thenThrow(transportFailure); + IoTDBConnectionException error = + assertThrows( + IoTDBConnectionException.class, + () -> { + try (TableSessionWrapper borrowed = wrapper) { + prepare(borrowed); + } + }); + assertSame(transportFailure, error.getCause()); + // Automatic close must neither mask the transport failure nor return the broken session. + assertEquals(0, error.getSuppressed().length); + assertThrows(IoTDBConnectionException.class, () -> prepare(wrapper)); + wrapper.close(); + verify(pool).cleanSessionAndMayThrowConnectionException(session); + verify(pool, never()).putBack(any()); + verify(client).prepareNumericRow(any()); + } + + @Test + public void testExecuteTransportFailureDiscardsSession() throws Exception { + NumericRowPreparedStatement prepared = prepare(wrapper); + TException transportFailure = new TException("transport closed"); + when(client.executePreparedNumericRow(any())).thenThrow(transportFailure); + IoTDBConnectionException error = + assertThrows( + IoTDBConnectionException.class, + () -> { + try (TableSessionWrapper borrowed = wrapper) { + borrowed.executePreparedInsertNumericRow( + prepared, 1L, new long[] {42}, new String[] {"a"}); + } + }); + assertSame(transportFailure, error.getCause()); + assertEquals(0, error.getSuppressed().length); + assertThrows( + IoTDBConnectionException.class, + () -> + wrapper.executePreparedInsertNumericRow( + prepared, 2L, new long[] {43}, new String[] {"a"})); + verify(pool).cleanSessionAndMayThrowConnectionException(session); + verify(pool, never()).putBack(any()); + verify(client).executePreparedNumericRow(any()); + } + + @Test + public void testPrepareStatementErrorPreservesHealthySession() throws Exception { + when(client.prepareNumericRow(any())) + .thenReturn( + new TSPrepareNumericRowResp( + new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()) + .setMessage("invalid prepared metadata"))); + try (TableSessionWrapper borrowed = wrapper) { + assertThrows(StatementExecutionException.class, () -> prepare(borrowed)); + } + verify(pool, never()).cleanSessionAndMayThrowConnectionException(any()); + verify(pool).putBack(session); + } + + @Test + public void testCloseReturnsSessionOnlyOnce() throws Exception { + wrapper.close(); + wrapper.close(); + verify(pool).putBack(session); + verify(pool, never()).cleanSessionAndMayThrowConnectionException(any()); + verifyZeroInteractions(client); + } + + private static NumericRowPreparedStatement prepare(TableSessionWrapper borrowed) + throws Exception { + return borrowed.prepareInsertNumericRow( + "t", + List.of("tag", "v"), + List.of(TSDataType.STRING, TSDataType.INT64), + List.of(ColumnCategory.TAG, ColumnCategory.FIELD)); + } + + private static TSStatus success() { + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java index 99edef690b5cf..e83d2afc53b1c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java @@ -26,6 +26,8 @@ import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.receiver.transform.converter.ValueConverter; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.tsfile.annotations.TableModel; @@ -45,6 +47,31 @@ public class PipeConvertedInsertRowStatement extends InsertRowStatement { LoggerFactory.getLogger(PipeConvertedInsertRowStatement.class); public PipeConvertedInsertRowStatement(final InsertRowStatement insertRowStatement) { + this((InsertBaseStatement) insertRowStatement); + time = insertRowStatement.getTime(); + values = insertRowStatement.getValues(); + isNeedInferType = insertRowStatement.isNeedInferType(); + deviceID = insertRowStatement.getRawTableDeviceID(); + removeAllFailedMeasurementMarks(); + } + + public PipeConvertedInsertRowStatement(final InsertNumericRowStatement insertRowStatement) { + this((InsertBaseStatement) insertRowStatement); + time = insertRowStatement.getTime(); + // Box only at the Pipe conversion boundary: the receiver may require non-numeric FIELDs. + values = new Object[insertRowStatement.getValueCount()]; + for (int i = 0; i < values.length; i++) { + values[i] = insertRowStatement.getFirstValueOfIndex(i); + } + isNeedInferType = false; + // Schema analysis and failed-column recovery must not mutate the original numeric row. + columnCategories = columnCategories == null ? null : columnCategories.clone(); + tagColumnIndices = null; + attrColumnIndices = null; + removeAllFailedMeasurementMarks(); + } + + private PipeConvertedInsertRowStatement(final InsertBaseStatement insertRowStatement) { super(); // Statement isDebug = insertRowStatement.isDebug(); @@ -59,12 +86,6 @@ public PipeConvertedInsertRowStatement(final InsertRowStatement insertRowStateme attrColumnIndices = insertRowStatement.getAttrColumnIndices(); writeToTable = insertRowStatement.isWriteToTable(); databaseName = insertRowStatement.getDatabaseName().orElse(null); - // InsertRowStatement - time = insertRowStatement.getTime(); - values = insertRowStatement.getValues(); - isNeedInferType = insertRowStatement.isNeedInferType(); - deviceID = insertRowStatement.getRawTableDeviceID(); - // To ensure that the measurement remains unchanged during the WAL writing process, the array // needs to be copied before the failed Measurement mark can be deleted. final MeasurementSchema[] measurementSchemas = insertRowStatement.getMeasurementSchemas(); @@ -87,8 +108,6 @@ public PipeConvertedInsertRowStatement(final InsertRowStatement insertRowStateme if (failedMeasurementIndex2Info != null) { this.failedMeasurementIndex2Info = new HashMap<>(failedMeasurementIndex2Info); } - - removeAllFailedMeasurementMarks(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java index 5a80da6f6e6e1..d76428d481382 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.StatementNode; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; @@ -221,6 +222,15 @@ public Optional visitInsertRow( new PipeConvertedInsertRowStatement(insertRowStatement), statusAndDatabaseName.getRight()); } + @Override + public Optional visitInsertNumericRow( + final InsertNumericRowStatement insertNumericRowStatement, + final Pair statusAndDatabaseName) { + return tryExecute( + new PipeConvertedInsertRowStatement(insertNumericRowStatement), + statusAndDatabaseName.getRight()); + } + @Override public Optional visitInsertRows( final InsertRowsStatement insertRowsStatement, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 7435e901d2fdd..d96df20b784d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -2614,6 +2614,7 @@ public TSStatus executePreparedNumericRow(TSExecutePreparedNumericRowReq req) { IClientSession session = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); if (!SESSION_MANAGER.checkLogin(session)) return getNotLoggedInStatus(); InsertNumericRowStatement statement = null; + OperationQuota quota = null; try { if (req.getSessionId() != session.getId()) return RpcUtils.getStatus(TSStatusCode.NOT_LOGIN); statement = @@ -2624,6 +2625,8 @@ public TSStatus executePreparedNumericRow(TSExecutePreparedNumericRowReq req) { ByteBuffer.wrap(req.getTagValues()), ByteBuffer.wrap(req.getValues()), req.bufferForNullBitmap()); + quota = + DataNodeThrottleQuotaManager.getInstance().checkQuota(session.getUsername(), statement); long queryId = SESSION_MANAGER.requestQueryId(); return COORDINATOR.executeForTableModel( statement, @@ -2635,6 +2638,8 @@ public TSStatus executePreparedNumericRow(TSExecutePreparedNumericRowReq req) { metadata, config.getConnectionTimeoutInMS()) .status; + } catch (IoTDBException e) { + return onIoTDBException(e, OperationType.INSERT_RECORD, e.getErrorCode()); } catch (Exception e) { return onQueryException( e, OperationType.INSERT_RECORD.getName(), TSStatusCode.EXECUTE_STATEMENT_ERROR); @@ -2643,6 +2648,9 @@ public TSStatus executePreparedNumericRow(TSExecutePreparedNumericRowReq req) { // Remove only this session's invalid handle, releasing its reservation exactly once. session.removePreparedNumericRow(req.getPreparedId()); } + if (quota != null) { + quota.close(); + } } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatementTest.java index 8c563a84b1db4..a58a05af70ad5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatementTest.java @@ -19,15 +19,26 @@ package org.apache.iotdb.db.pipe.receiver.transform.statement; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.db.pipe.receiver.visitor.PipeTableStatementDataTypeConvertExecutionVisitor; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; +import org.apache.iotdb.db.utils.NumericRowValues; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; import org.junit.Test; import java.time.ZoneId; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; public class PipeConvertedInsertRowStatementTest { @@ -52,4 +63,139 @@ public void testTransferTypeKeepsNullValue() throws Exception { Assert.assertNull(convertedStatement.getValues()[0]); Assert.assertFalse(convertedStatement.isNeedInferType()); } + + @Test + public void testNumericRowDispatchConvertsNumericAndStringFields() throws Exception { + final InsertNumericRowStatement original = numericRow(); + final long[] originalValues = original.getValues().clone(); + final AtomicInteger executions = new AtomicInteger(); + final TSStatus success = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + final PipeTableStatementDataTypeConvertExecutionVisitor visitor = + new PipeTableStatementDataTypeConvertExecutionVisitor( + (statement, database) -> { + executions.incrementAndGet(); + Assert.assertEquals("db", database); + Assert.assertTrue(statement instanceof PipeConvertedInsertRowStatement); + final PipeConvertedInsertRowStatement converted = + (PipeConvertedInsertRowStatement) statement; + Assert.assertTrue(converted.isForceTypeConversion()); + Assert.assertFalse(converted.isNeedInferType()); + Assert.assertTrue(converted.isAligned()); + Assert.assertTrue(converted.isWriteToTable()); + Assert.assertEquals(original.getTime(), converted.getTime()); + Assert.assertEquals(original.getDatabaseName(), converted.getDatabaseName()); + Assert.assertEquals(original.getTableDeviceID(), converted.getTableDeviceID()); + Assert.assertEquals(original.getFirstValueOfIndex(0), converted.getValues()[0]); + Assert.assertTrue(converted.checkAndCastDataType(1, TSDataType.INT32)); + Assert.assertTrue(converted.checkAndCastDataType(2, TSDataType.STRING)); + Assert.assertTrue(converted.checkAndCastDataType(3, TSDataType.STRING)); + Assert.assertEquals(12, converted.getValues()[1]); + Assert.assertEquals( + new Binary("8.5", TSFileConfig.STRING_CHARSET), converted.getValues()[2]); + Assert.assertNull(converted.getValues()[3]); + Assert.assertArrayEquals( + new TSDataType[] { + TSDataType.STRING, TSDataType.INT32, TSDataType.STRING, TSDataType.STRING + }, + converted.getDataTypes()); + return success; + }); + + Assert.assertEquals(success, visitor.process(original, new Pair<>(success, "db")).orElse(null)); + Assert.assertEquals(1, executions.get()); + Assert.assertArrayEquals(originalValues, original.getValues()); + Assert.assertArrayEquals( + new TSDataType[] { + TSDataType.STRING, TSDataType.DOUBLE, TSDataType.DOUBLE, TSDataType.INT64 + }, + original.getDataTypes()); + Assert.assertTrue(original.getNumericValues().isNull(3)); + } + + @Test + public void testNumericRowRetryRestoresFailuresOnlyInCopy() throws Exception { + final InsertNumericRowStatement original = numericRow(); + original.markFailedMeasurement(0, new IllegalArgumentException("tag failure")); + original.markFailedMeasurement(1, new IllegalArgumentException("field failure")); + final long[] failedValues = original.getValues().clone(); + final PipeConvertedInsertRowStatement converted = new PipeConvertedInsertRowStatement(original); + + Assert.assertFalse(converted.hasFailedMeasurements()); + Assert.assertEquals(List.of(0), converted.getTagColumnIndices()); + Assert.assertEquals("tag", converted.getMeasurements()[0]); + Assert.assertEquals("count", converted.getMeasurements()[1]); + Assert.assertEquals(new Binary("d1", TSFileConfig.STRING_CHARSET), converted.getValues()[0]); + Assert.assertEquals(12.5, converted.getValues()[1]); + Assert.assertTrue(converted.checkAndCastDataType(1, TSDataType.INT32)); + Assert.assertEquals(12, converted.getValues()[1]); + + Assert.assertEquals(2, original.getFailedMeasurementNumber()); + Assert.assertNotSame( + original.getFailedMeasurementInfoMap(), converted.getFailedMeasurementInfoMap()); + Assert.assertNull(original.getMeasurements()[0]); + Assert.assertNull(original.getMeasurements()[1]); + Assert.assertNull(original.getDataTypes()[1]); + Assert.assertTrue(original.getNumericValues().isNull(0)); + Assert.assertTrue(original.getNumericValues().isNull(1)); + Assert.assertArrayEquals(failedValues, original.getValues()); + Assert.assertEquals(12.5, original.getFailedMeasurementInfoMap().get(1).getValue()); + } + + @Test + public void testNumericRowConversionOwnsMutableMetadataAndPreservesNulls() throws Exception { + final InsertNumericRowStatement original = numericRow(); + original.getNumericValues().setTag(0, null); + final PipeConvertedInsertRowStatement converted = new PipeConvertedInsertRowStatement(original); + Assert.assertNull(converted.getValues()[0]); + Assert.assertNull(converted.getValues()[3]); + Assert.assertEquals(original.getTableDeviceID(), converted.getTableDeviceID()); + Assert.assertNotSame(original.getMeasurements(), converted.getMeasurements()); + Assert.assertNotSame(original.getDataTypes(), converted.getDataTypes()); + Assert.assertNotSame(original.getMeasurementSchemas(), converted.getMeasurementSchemas()); + Assert.assertNotSame(original.getColumnCategories(), converted.getColumnCategories()); + converted.swapColumn(1, 2); + converted.setColumnCategory(TsTableColumnCategory.ATTRIBUTE, 1); + converted.markFailedMeasurement(2, new IllegalArgumentException("conversion failure")); + + Assert.assertArrayEquals( + new String[] {"tag", "count", "text", "nullable"}, original.getMeasurements()); + Assert.assertEquals(TsTableColumnCategory.FIELD, original.getColumnCategory(1)); + Assert.assertEquals("count", original.getMeasurementSchemas()[1].getMeasurementName()); + Assert.assertEquals(12.5, original.getFirstValueOfIndex(1)); + Assert.assertEquals(8.5, original.getFirstValueOfIndex(2)); + Assert.assertFalse(original.hasFailedMeasurements()); + } + + private static InsertNumericRowStatement numericRow() throws Exception { + final InsertNumericRowStatement statement = new InsertNumericRowStatement(); + statement.setDevicePath(new PartialPath("t")); + statement.setDatabaseName("db"); + statement.setTime(123L); + statement.setMeasurements(new String[] {"tag", "count", "text", "nullable"}); + statement.setDataTypes( + new TSDataType[] { + TSDataType.STRING, TSDataType.DOUBLE, TSDataType.DOUBLE, TSDataType.INT64 + }); + statement.setColumnCategories( + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD, + TsTableColumnCategory.FIELD + }); + statement.setMeasurementSchemas( + new MeasurementSchema[] { + new MeasurementSchema("tag", TSDataType.STRING), + new MeasurementSchema("count", TSDataType.DOUBLE), + new MeasurementSchema("text", TSDataType.DOUBLE), + new MeasurementSchema("nullable", TSDataType.INT64) + }); + final NumericRowValues values = new NumericRowValues(1, 3); + values.setTag(0, "d1"); + values.setObject(1, TSDataType.DOUBLE, 12.5); + values.setObject(2, TSDataType.DOUBLE, 8.5); + values.setNull(3); + statement.setNumericValues(values); + return statement; + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImplPreparedNumericRowTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImplPreparedNumericRowTest.java new file mode 100644 index 0000000000000..917bcfa4d7642 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImplPreparedNumericRowTest.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.protocol.thrift.impl; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.RpcThrottlingException; +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.protocol.session.PreparedNumericRowInfo; +import org.apache.iotdb.db.protocol.session.SessionManager; +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.plan.Coordinator; +import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; +import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeThrottleQuotaManager; +import org.apache.iotdb.db.storageengine.rescon.quotas.OperationQuota; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TSExecutePreparedNumericRowReq; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InOrder; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; + +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*", "javax.management.*"}) +@PrepareForTest({ + ClientRPCServiceImpl.class, + DataNodeThrottleQuotaManager.class, + PreparedNumericRowInfo.class +}) +public class ClientRPCServiceImplPreparedNumericRowTest { + private ClientRPCServiceImpl service; + private int originalDataNodeId; + private Coordinator coordinator; + private Coordinator originalCoordinator; + private SessionManager originalSessionManager; + private IClientSession session; + private PreparedNumericRowInfo prepared; + private InsertNumericRowStatement statement; + private DataNodeThrottleQuotaManager quotaManager; + private OperationQuota quota; + private TSExecutePreparedNumericRowReq request; + private TSStatus success; + + @Before + public void setUp() throws Exception { + // Coordinator initialization requires a DataNode ID even when query execution is mocked. + originalDataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0); + // Exercise the actual RPC method while isolating query execution and per-user quota state. + service = mock(ClientRPCServiceImpl.class, CALLS_REAL_METHODS); + originalCoordinator = Whitebox.getInternalState(ClientRPCServiceImpl.class, "COORDINATOR"); + originalSessionManager = + Whitebox.getInternalState(ClientRPCServiceImpl.class, "SESSION_MANAGER"); + coordinator = mock(Coordinator.class); + SessionManager sessions = mock(SessionManager.class); + Whitebox.setInternalState(ClientRPCServiceImpl.class, "COORDINATOR", coordinator); + Whitebox.setInternalState(ClientRPCServiceImpl.class, "SESSION_MANAGER", sessions); + session = mock(IClientSession.class); + prepared = PowerMockito.mock(PreparedNumericRowInfo.class); + statement = new InsertNumericRowStatement(); + quotaManager = mock(DataNodeThrottleQuotaManager.class); + quota = mock(OperationQuota.class); + PowerMockito.mockStatic(DataNodeThrottleQuotaManager.class); + when(DataNodeThrottleQuotaManager.getInstance()).thenReturn(quotaManager); + when(sessions.getCurrSessionAndUpdateIdleTime()).thenReturn(session); + when(sessions.checkLogin(session)).thenReturn(true); + when(session.getId()).thenReturn(11L); + when(session.getUsername()).thenReturn("quota-user"); + when(session.getPreparedNumericRow(101L)).thenReturn(prepared); + when(prepared.bind(anyLong(), any(), any(), any())).thenReturn(statement); + when(quotaManager.checkQuota("quota-user", statement)).thenReturn(quota); + success = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + when(coordinator.executeForTableModel( + same(statement), any(), same(session), anyLong(), any(), eq(""), any(), anyLong())) + .thenReturn(new ExecutionResult(new QueryId("prepared_quota"), success)); + request = + new TSExecutePreparedNumericRowReq( + 11L, 101L, 1L, ByteBuffer.allocate(0), ByteBuffer.allocate(0)); + } + + @After + public void tearDown() { + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(originalDataNodeId); + Whitebox.setInternalState(ClientRPCServiceImpl.class, "COORDINATOR", originalCoordinator); + Whitebox.setInternalState( + ClientRPCServiceImpl.class, "SESSION_MANAGER", originalSessionManager); + } + + @Test + public void testChecksQuotaBeforeExecutionAndClosesOnSuccess() throws Exception { + assertSame(success, service.executePreparedNumericRow(request)); + InOrder order = inOrder(quotaManager, coordinator, quota); + order.verify(quotaManager).checkQuota("quota-user", statement); + order + .verify(coordinator) + .executeForTableModel( + same(statement), any(), same(session), anyLong(), any(), eq(""), any(), anyLong()); + order.verify(quota).close(); + verify(session, never()).removePreparedNumericRow(anyLong()); + } + + @Test + public void testQuotaRejectionDoesNotExecuteOrInvalidatePreparedRow() throws Exception { + int code = TSStatusCode.NUM_WRITE_REQUESTS_EXCEEDED.getStatusCode(); + when(quotaManager.checkQuota("quota-user", statement)) + .thenThrow(new RpcThrottlingException("write request quota exceeded", code)); + TSStatus status = service.executePreparedNumericRow(request); + assertEquals(code, status.getCode()); + // Standard RPC handling adds an operation prefix while retaining the quota failure detail. + assertTrue(status.getMessage().endsWith("write request quota exceeded")); + verifyZeroInteractions(coordinator, quota); + verify(session, never()).removePreparedNumericRow(anyLong()); + } + + @Test + public void testExecutionExceptionClosesQuota() throws Exception { + when(coordinator.executeForTableModel( + same(statement), any(), same(session), anyLong(), any(), eq(""), any(), anyLong())) + .thenThrow(new IllegalStateException("executor failure")); + TSStatus status = service.executePreparedNumericRow(request); + assertNotEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + verify(quota).close(); + verify(session, never()).removePreparedNumericRow(anyLong()); + } + + @Test + public void testSchemaFailureClosesQuotaAndRemovesPreparedRow() throws Exception { + TSStatus schemaFailure = new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()); + when(coordinator.executeForTableModel( + same(statement), any(), same(session), anyLong(), any(), eq(""), any(), anyLong())) + .thenAnswer( + invocation -> { + statement.markTableSchemaValidationFailed(); + return new ExecutionResult(new QueryId("prepared_quota"), schemaFailure); + }); + assertSame(schemaFailure, service.executePreparedNumericRow(request)); + verify(quotaManager).checkQuota("quota-user", statement); + verify(quota).close(); + verify(session).removePreparedNumericRow(101L); + } + + @Test + public void testInvalidPayloadDoesNotConsumeQuota() throws Exception { + when(prepared.bind(anyLong(), any(), any(), any())) + .thenThrow(new SemanticException("invalid numeric payload")); + TSStatus status = service.executePreparedNumericRow(request); + assertNotEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + verifyZeroInteractions(quotaManager, coordinator, quota); + verify(session, never()).removePreparedNumericRow(anyLong()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java index 70c39d90072f2..fbb69b4be0442 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java @@ -20,10 +20,18 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.consensus.index.impl.IoTProgressIndex; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.commons.request.IConsensusRequest; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; +import org.apache.iotdb.consensus.common.request.BatchIndexedConsensusRequest; +import org.apache.iotdb.consensus.common.request.ByteBufferConsensusRequest; +import org.apache.iotdb.consensus.common.request.DeserializedBatchIndexedConsensusRequest; +import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest; +import org.apache.iotdb.consensus.common.request.IoTConsensusRequest; +import org.apache.iotdb.db.consensus.statemachine.dataregion.IoTConsensusDataRegionStateMachine; import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTablePatternParser; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletBatchReqV2; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; @@ -35,6 +43,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.memtable.AlignedWritableMemChunkGroup; import org.apache.iotdb.db.storageengine.dataregion.memtable.PrimitiveMemTable; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; @@ -299,6 +308,58 @@ public void testWalEnvelopeRetainsTypeAndFrozenIndex() throws Exception { } } + @Test + public void testIoTConsensusBatchFromMemoryAndWal() throws Exception { + // The real state machine must accept both live PlanNode bytes and WAL catch-up envelopes, + // attach the sender progress index, and dispatch each numeric row to DataRegion.insert. + RelationalInsertNumericRowNode original = nodeWithNullTag(); + original.markFailedMeasurement(2); + original.setSearchIndex(42); + original.setLastFragment(true); + WALInfoEntry entry = new WALInfoEntry(7, original); + ByteBuffer wal = ByteBuffer.allocate(entry.serializedSize()); + entry.serialize(new WALByteBufferForTest(wal)); + wal.flip(); + BatchIndexedConsensusRequest batch = new BatchIndexedConsensusRequest(3); + batch.add( + new IndexedConsensusRequest( + -1, + 41, + Collections.singletonList( + new ByteBufferConsensusRequest(original.serializeToByteBuffer())))); + batch.add( + new IndexedConsensusRequest( + -1, 42, Collections.singletonList(new IoTConsensusRequest(wal)))); + DataRegion region = Mockito.mock(DataRegion.class); + IoTConsensusDataRegionStateMachine machine = new IoTConsensusDataRegionStateMachine(region); + DeserializedBatchIndexedConsensusRequest decoded = + (DeserializedBatchIndexedConsensusRequest) machine.deserializeRequest(batch); + assertEquals(2, decoded.getInsertNodes().size()); + for (int i = 0; i < 2; i++) { + IConsensusRequest request = decoded.getInsertNodes().get(i); + assertTrue(request instanceof RelationalInsertNumericRowNode); + RelationalInsertNumericRowNode row = (RelationalInsertNumericRowNode) request; + assertEquals(original.getDeviceID(), row.getDeviceID()); + assertEquals(original.getTime(), row.getTime()); + assertEquals(original.getValidMeasurementNumber(false), row.getMeasurements().length); + assertEquals( + ((RelationalInsertNumericRowNode) decoded.getInsertNodes().get(0)).getNumericValues(), + row.getNumericValues()); + assertEquals(new IoTProgressIndex(3, 41L + i), row.getProgressIndex()); + assertObjectStorageAbsent(RelationalInsertNumericRowNode.class, row); + } + TSStatus status = machine.write(decoded); + assertEquals(2, status.getSubStatusSize()); + for (int i = 0; i < 2; i++) { + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getSubStatus().get(i).getCode()); + RelationalInsertNumericRowNode row = + (RelationalInsertNumericRowNode) decoded.getInsertNodes().get(i); + Mockito.verify(region).insert(Mockito.same(row)); + Mockito.verify(region).insertSeparatorToWAL(Mockito.same(row)); + } + } + @Test public void testPipeBatchRetainsIndependentNumericStatements() throws Exception { RelationalInsertNumericRowNode node = nodeWithNullTag(); From d29c9c420a32bd20e6e1480e7313c8d040c1eb07 Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 12 Oct 2026 11:37:31 +0800 Subject: [PATCH 19/20] Finalize prepared numeric row API and performance coverage Remove the non-prepared numeric row API and its obsolete tests. Extend the opt-in single-row benchmark with partial-column workloads, latency, write-stage timing and process resource sampling. --- .../env/cluster/config/MppDataNodeConfig.java | 6 + .../remote/config/RemoteDataNodeConfig.java | 5 + .../iotdb/itbase/env/DataNodeConfig.java | 2 + .../manual/basic/IoTDBPipeNumericRowIT.java | 10 +- .../IoTDBNumericRowInsertPerformanceIT.java | 538 ----------------- .../it/session/IoTDBTableModelSessionIT.java | 540 ++++++++++++++---- .../WritePerformanceResourceTracker.java | 234 ++++++++ .../apache/iotdb/isession/ITableSession.java | 29 +- .../org/apache/iotdb/isession/NumericRow.java | 122 ---- .../isession/NumericRowPreparedStatement.java | 15 +- .../org/apache/iotdb/session/Session.java | 85 --- .../apache/iotdb/session/TableSession.java | 6 - .../session/pool/TableSessionWrapper.java | 14 - .../iotdb/session/util/SessionUtils.java | 103 ---- .../session/PreparedNumericRowBufferTest.java | 56 ++ .../org/apache/iotdb/session/SessionTest.java | 104 ---- .../pool/PreparedNumericRowPoolTest.java | 23 - .../iotdb/session/util/SessionUtilsTest.java | 202 +------ .../plan/parser/StatementGenerator.java | 37 +- .../crud/InsertNumericRowStatement.java | 65 --- .../session/PreparedNumericRowInfoTest.java | 50 +- .../plan/parser/NumericTableRowTest.java | 128 ++--- .../src/main/thrift/client.thrift | 8 +- 23 files changed, 819 insertions(+), 1563 deletions(-) delete mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/performance/IoTDBNumericRowInsertPerformanceIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/session/WritePerformanceResourceTracker.java delete mode 100644 iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRow.java diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java index 8c5f56ad9f1f0..cfa55537dcb70 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java @@ -56,6 +56,12 @@ public DataNodeConfig setMetricReporterType(List metricReporterTypes) { return this; } + @Override + public DataNodeConfig setMetricPrometheusReporterAsyncUpdate(boolean asyncUpdate) { + properties.setProperty("prometheus_reporter_async_update", String.valueOf(asyncUpdate)); + return this; + } + @Override public DataNodeConfig setMetricPrometheusReporterUsername(String username) { properties.setProperty("metric_prometheus_reporter_username", username); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java index c97e2b5065af6..f754b71ebe14a 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java @@ -28,6 +28,11 @@ public DataNodeConfig setMetricReporterType(List metricReporterTypes) { return this; } + @Override + public DataNodeConfig setMetricPrometheusReporterAsyncUpdate(boolean asyncUpdate) { + return this; + } + @Override public DataNodeConfig setMetricPrometheusReporterUsername(String username) { return this; diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java index fb969c778ea8f..c820e1325e953 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java @@ -25,6 +25,8 @@ public interface DataNodeConfig { DataNodeConfig setMetricReporterType(List metricReporterTypes); + DataNodeConfig setMetricPrometheusReporterAsyncUpdate(boolean asyncUpdate); + DataNodeConfig setMetricPrometheusReporterUsername(String username); DataNodeConfig setMetricPrometheusReporterPassword(String password); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeNumericRowIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeNumericRowIT.java index 5211f29077100..cc2e614b43a86 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeNumericRowIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeNumericRowIT.java @@ -31,7 +31,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.isession.ITableSession; -import org.apache.iotdb.isession.NumericRow; import org.apache.iotdb.isession.NumericRowPreparedStatement; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.MultiEnvFactory; @@ -127,7 +126,7 @@ public void setUp() { @Test public void testRealtimeWithoutBatch() throws Exception { - // Unflushed NumericRow and prepared rows must arrive through individual InsertNode requests. + // Unflushed prepared rows must arrive through individual InsertNode requests. testRealtime(false); } @@ -381,12 +380,7 @@ private void writeRows(int start, int end, int node) throws Exception { if (row % 3 == 0) { nulls.mark(row % 7); } - if (row % 2 == 0) { - session.executePreparedInsertNumericRow(prepared, row, values, tags, nulls); - } else { - session.insert( - new NumericRow("readings", row, NAMES, TYPES, CATEGORIES, values, tags, nulls)); - } + session.executePreparedInsertNumericRow(prepared, row, values, tags, nulls); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/performance/IoTDBNumericRowInsertPerformanceIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/performance/IoTDBNumericRowInsertPerformanceIT.java deleted file mode 100644 index e2d6a481743eb..0000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/performance/IoTDBNumericRowInsertPerformanceIT.java +++ /dev/null @@ -1,538 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.relational.it.performance; - -import org.apache.iotdb.isession.ITableSession; -import org.apache.iotdb.isession.NumericRow; -import org.apache.iotdb.isession.SessionDataSet; -import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; - -import org.apache.tsfile.enums.ColumnCategory; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.write.record.Tablet; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Properties; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -/** - * Manual end-to-end benchmark for the numeric table-row API and relational {@link Tablet} writes. - * - *

Enable this test with {@code -Diotdb.numeric.row.insert.perf.enabled=true}. The timed region - * includes the concurrent Session calls and server write confirmations. Schema creation and - * client-side input preparation are outside the timed region. By default 16 client sessions write - * one row per request; set {@code iotdb.numeric.row.insert.perf.client.count} or {@code - * iotdb.numeric.row.insert.perf.tablet.row.count} to vary concurrency or Tablet batching. - * - *

Results are written to {@code target/numeric-row-insert-performance.properties}. Percentiles - * describe elapsed time per complete round, not individual RPC latency. See the adjacent README for - * reproducible commands and interpretation. - */ -@RunWith(IoTDBTestRunner.class) -@Category(TableLocalStandaloneIT.class) -public class IoTDBNumericRowInsertPerformanceIT { - - private static final String PROPERTY_PREFIX = "iotdb.numeric.row.insert.perf."; - private static final String ENABLED_PROPERTY = PROPERTY_PREFIX + "enabled"; - private static final String ROW_COUNT_PROPERTY = PROPERTY_PREFIX + "row.count"; - private static final String COLUMN_COUNT_PROPERTY = PROPERTY_PREFIX + "column.count"; - private static final String CLIENT_COUNT_PROPERTY = PROPERTY_PREFIX + "client.count"; - private static final String TABLET_ROW_COUNT_PROPERTY = PROPERTY_PREFIX + "tablet.row.count"; - private static final String WARMUP_ROUND_COUNT_PROPERTY = PROPERTY_PREFIX + "warmup.round.count"; - private static final String ROUND_COUNT_PROPERTY = PROPERTY_PREFIX + "round.count"; - private static final String RESULT_PATH_PROPERTY = PROPERTY_PREFIX + "result.path"; - - private static final int ROW_COUNT = Integer.getInteger(ROW_COUNT_PROPERTY, 10_000); - private static final int COLUMN_COUNT = Integer.getInteger(COLUMN_COUNT_PROPERTY, 100); - private static final int CLIENT_COUNT = Integer.getInteger(CLIENT_COUNT_PROPERTY, 16); - private static final int TABLET_ROW_COUNT = Integer.getInteger(TABLET_ROW_COUNT_PROPERTY, 1); - private static final int WARMUP_ROUND_COUNT = Integer.getInteger(WARMUP_ROUND_COUNT_PROPERTY, 1); - private static final int ROUND_COUNT = Integer.getInteger(ROUND_COUNT_PROPERTY, 5); - - private static final String DATABASE = "numeric_row_insert_performance"; - private static final List TYPE_SEQUENCE = - Arrays.asList( - TSDataType.BOOLEAN, - TSDataType.INT32, - TSDataType.INT64, - TSDataType.FLOAT, - TSDataType.DOUBLE); - private static List columnNames; - private static List dataTypes; - private static List columnCategories; - - private static boolean environmentInitialized; - - @BeforeClass - public static void setUp() throws Exception { - Assume.assumeTrue(Boolean.getBoolean(ENABLED_PROPERTY)); - // Validate before allocating inputs or starting nodes, including when a count is negative. - assertValidConfiguration(); - columnNames = createColumnNames(); - dataTypes = createDataTypes(); - columnCategories = Collections.nCopies(COLUMN_COUNT, ColumnCategory.FIELD); - environmentInitialized = true; - EnvFactory.getEnv().initClusterEnvironment(); - } - - @AfterClass - public static void tearDown() throws Exception { - if (environmentInitialized) { - EnvFactory.getEnv().cleanClusterEnvironment(); - } - } - - @Test - // Compare identical, prebuilt numeric data; verify every field count outside the timed calls. - public void testNumericRowInsertPerformanceAgainstTablet() throws Exception { - final long[] numericRowElapsedNanos = new long[ROUND_COUNT]; - final long[] tabletElapsedNanos = new long[ROUND_COUNT]; - - final List sessions = new ArrayList<>(CLIENT_COUNT); - final ExecutorService executor = Executors.newFixedThreadPool(CLIENT_COUNT); - try { - final ITableSession setupSession = EnvFactory.getEnv().getTableSessionConnection(); - setupSession.executeNonQueryStatement("CREATE DATABASE " + DATABASE); - setupSession.executeNonQueryStatement("USE " + DATABASE); - sessions.add(setupSession); - for (int client = 1; client < CLIENT_COUNT; client++) { - final ITableSession session = EnvFactory.getEnv().getTableSessionConnection(); - session.executeNonQueryStatement("USE " + DATABASE); - sessions.add(session); - } - - final int totalRoundCount = WARMUP_ROUND_COUNT + ROUND_COUNT; - for (int round = 0; round < totalRoundCount; round++) { - final List numericRowWorkloads = - prepareNumericRowWorkloads(setupSession, round); - final List tabletWorkloads = prepareTabletWorkloads(setupSession, round); - - final long numericRowNanos; - final long tabletNanos; - // Alternate execution order so one path does not consistently receive a warmer server. - if ((round & 1) == 0) { - numericRowNanos = executeNumericRowWorkload(sessions, numericRowWorkloads, executor); - tabletNanos = executeTabletWorkload(sessions, tabletWorkloads, executor); - } else { - tabletNanos = executeTabletWorkload(sessions, tabletWorkloads, executor); - numericRowNanos = executeNumericRowWorkload(sessions, numericRowWorkloads, executor); - } - for (NumericRowWorkload workload : numericRowWorkloads) { - assertRowCount(setupSession, workload.tableName); - } - for (TabletWorkload workload : tabletWorkloads) { - assertRowCount(setupSession, workload.tableName); - } - - if (round >= WARMUP_ROUND_COUNT) { - final int measuredRound = round - WARMUP_ROUND_COUNT; - numericRowElapsedNanos[measuredRound] = numericRowNanos; - tabletElapsedNanos[measuredRound] = tabletNanos; - } - } - } finally { - executor.shutdownNow(); - for (ITableSession session : sessions) { - session.close(); - } - } - - final BenchmarkResult result = new BenchmarkResult(numericRowElapsedNanos, tabletElapsedNanos); - writeResult(result); - } - - private static List prepareNumericRowWorkloads( - ITableSession session, int round) throws Exception { - final List workloads = new ArrayList<>(CLIENT_COUNT); - for (int client = 0; client < CLIENT_COUNT; client++) { - final String tableName = "numeric_row_" + round + "_" + client; - createTable(session, tableName); - final long startTimestamp = ((long) round * CLIENT_COUNT + client) * ROW_COUNT; - final NumericRow[] rows = new NumericRow[ROW_COUNT]; - final String[] tagValues = new String[0]; - for (int row = 0; row < ROW_COUNT; row++) { - rows[row] = - new NumericRow( - tableName, - startTimestamp + row, - columnNames, - dataTypes, - columnCategories, - createRawValues(startTimestamp + row), - tagValues); - } - workloads.add(new NumericRowWorkload(tableName, rows)); - } - return workloads; - } - - private static List prepareTabletWorkloads(ITableSession session, int round) - throws Exception { - final List workloads = new ArrayList<>(CLIENT_COUNT); - for (int client = 0; client < CLIENT_COUNT; client++) { - final String tableName = "tablet_" + round + "_" + client; - createTable(session, tableName); - final long startTimestamp = ((long) round * CLIENT_COUNT + client) * ROW_COUNT; - final List tablets = new ArrayList<>((ROW_COUNT - 1) / TABLET_ROW_COUNT + 1); - for (int rowStart = 0; rowStart < ROW_COUNT; ) { - final int rowCount = Math.min(TABLET_ROW_COUNT, ROW_COUNT - rowStart); - final Tablet tablet = - new Tablet(tableName, columnNames, dataTypes, columnCategories, rowCount); - for (int row = 0; row < rowCount; row++) { - final int rowIndex = tablet.getRowSize(); - final long timestamp = startTimestamp + rowStart + row; - final long[] values = createRawValues(timestamp); - tablet.addTimestamp(rowIndex, timestamp); - for (int column = 0; column < COLUMN_COUNT; column++) { - tablet.addValue( - columnNames.get(column), - rowIndex, - toTabletValue(dataTypes.get(column), values[column])); - } - } - tablets.add(tablet); - rowStart += rowCount; - } - workloads.add(new TabletWorkload(tableName, tablets)); - } - return workloads; - } - - private static long executeNumericRowWorkload( - List sessions, List workloads, ExecutorService executor) - throws Exception { - return executeConcurrently( - sessions, - workloads, - executor, - (session, workload) -> { - for (NumericRow row : workload.rows) { - session.insert(row); - } - }); - } - - private static long executeTabletWorkload( - List sessions, List workloads, ExecutorService executor) - throws Exception { - return executeConcurrently( - sessions, - workloads, - executor, - (session, workload) -> { - for (Tablet tablet : workload.tablets) { - session.insert(tablet); - } - }); - } - - private static long executeConcurrently( - List sessions, - List workloads, - ExecutorService executor, - WorkloadWriter writer) - throws Exception { - final CountDownLatch ready = new CountDownLatch(CLIENT_COUNT); - final CountDownLatch start = new CountDownLatch(1); - final List> futures = new ArrayList<>(CLIENT_COUNT); - for (int client = 0; client < CLIENT_COUNT; client++) { - final ITableSession session = sessions.get(client); - final T workload = workloads.get(client); - futures.add( - executor.submit( - () -> { - ready.countDown(); - start.await(); - writer.write(session, workload); - return null; - })); - } - ready.await(); - final long startNanos = System.nanoTime(); - start.countDown(); - for (Future future : futures) { - future.get(); - } - return System.nanoTime() - startNanos; - } - - @FunctionalInterface - private interface WorkloadWriter { - - void write(ITableSession session, T workload) throws Exception; - } - - private static void createTable(ITableSession session, String tableName) throws Exception { - final StringBuilder statement = new StringBuilder("CREATE TABLE "); - statement.append(tableName).append('('); - for (int column = 0; column < COLUMN_COUNT; column++) { - if (column > 0) { - statement.append(','); - } - statement - .append(columnNames.get(column)) - .append(' ') - .append(dataTypes.get(column)) - .append(" FIELD"); - } - statement.append(')'); - session.executeNonQueryStatement(statement.toString()); - } - - private static void assertRowCount(ITableSession session, String tableName) throws Exception { - final List counts = new ArrayList<>(COLUMN_COUNT); - for (String columnName : columnNames) { - counts.add("COUNT(" + columnName + ")"); - } - try (SessionDataSet dataSet = - session.executeQueryStatement( - "SELECT " + String.join(",", counts) + " FROM " + tableName)) { - Assert.assertTrue(dataSet.hasNext()); - for (var field : dataSet.next().getFields()) { - Assert.assertEquals(ROW_COUNT, field.getLongV()); - } - Assert.assertFalse(dataSet.hasNext()); - } - } - - private static long[] createRawValues(long timestamp) { - final long[] values = new long[COLUMN_COUNT]; - for (int column = 0; column < COLUMN_COUNT; column++) { - final TSDataType type = dataTypes.get(column); - // Floating-point values occupy raw IEEE 754 bits in the numeric-row API. - values[column] = - switch (type) { - case BOOLEAN -> (timestamp + column) & 1; - case INT32 -> (int) (timestamp + column); - case INT64 -> timestamp + column; - case FLOAT -> Float.floatToRawIntBits((float) timestamp + column / 10.0F); - case DOUBLE -> Double.doubleToRawLongBits(timestamp + column / 10.0D); - default -> throw new AssertionError(type); - }; - } - return values; - } - - private static Object toTabletValue(TSDataType type, long value) { - return switch (type) { - case BOOLEAN -> value != 0; - case INT32 -> (int) value; - case INT64 -> value; - case FLOAT -> Float.intBitsToFloat((int) value); - case DOUBLE -> Double.longBitsToDouble(value); - default -> throw new AssertionError(type); - }; - } - - private static List createColumnNames() { - final List columnNames = new ArrayList<>(COLUMN_COUNT); - for (int column = 0; column < COLUMN_COUNT; column++) { - columnNames.add("v" + column); - } - return Collections.unmodifiableList(columnNames); - } - - private static List createDataTypes() { - final List dataTypes = new ArrayList<>(COLUMN_COUNT); - for (int column = 0; column < COLUMN_COUNT; column++) { - dataTypes.add(TYPE_SEQUENCE.get(column % TYPE_SEQUENCE.size())); - } - return Collections.unmodifiableList(dataTypes); - } - - private static void assertValidConfiguration() { - Assert.assertTrue(ROW_COUNT > 0); - Assert.assertTrue(COLUMN_COUNT > 0); - Assert.assertTrue(CLIENT_COUNT > 0); - Assert.assertTrue(TABLET_ROW_COUNT > 0); - Assert.assertTrue(WARMUP_ROUND_COUNT >= 0); - Assert.assertTrue(ROUND_COUNT > 0); - Assert.assertTrue((long) WARMUP_ROUND_COUNT + ROUND_COUNT <= Integer.MAX_VALUE); - } - - private static void writeResult(BenchmarkResult result) throws IOException { - final Properties properties = new Properties(); - properties.setProperty("format.version", "2"); - properties.setProperty("java.version", System.getProperty("java.version")); - properties.setProperty("os.name", System.getProperty("os.name")); - properties.setProperty("os.arch", System.getProperty("os.arch")); - properties.setProperty( - "client.available.processors", - Integer.toString(Runtime.getRuntime().availableProcessors())); - properties.setProperty( - "client.max.heap.bytes", Long.toString(Runtime.getRuntime().maxMemory())); - properties.setProperty("client.count", Integer.toString(CLIENT_COUNT)); - properties.setProperty("row.count", Integer.toString(ROW_COUNT)); - properties.setProperty("column.count", Integer.toString(COLUMN_COUNT)); - properties.setProperty("tablet.row.count", Integer.toString(TABLET_ROW_COUNT)); - properties.setProperty("warmup.round.count", Integer.toString(WARMUP_ROUND_COUNT)); - properties.setProperty("round.count", Integer.toString(ROUND_COUNT)); - properties.setProperty("column.types", join(dataTypes)); - properties.setProperty( - "numeric.row.requests.per.round", Integer.toString(CLIENT_COUNT * ROW_COUNT)); - properties.setProperty( - "insert.tablet.requests.per.round", - Integer.toString(CLIENT_COUNT * ((ROW_COUNT - 1) / TABLET_ROW_COUNT + 1))); - properties.setProperty("numeric.row.round.elapsed.nanos", join(result.numericRowElapsedNanos)); - properties.setProperty("insert.tablet.round.elapsed.nanos", join(result.tabletElapsedNanos)); - properties.setProperty( - "numeric.row.round.p50.nanos", Long.toString(result.numericRowP50Nanos())); - properties.setProperty( - "numeric.row.round.p95.nanos", Long.toString(result.numericRowP95Nanos())); - properties.setProperty("numeric.row.rows.per.second", format(result.numericRowRowsPerSecond())); - properties.setProperty( - "numeric.row.values.per.second", format(result.numericRowValuesPerSecond())); - properties.setProperty("insert.tablet.round.p50.nanos", Long.toString(result.tabletP50Nanos())); - properties.setProperty("insert.tablet.round.p95.nanos", Long.toString(result.tabletP95Nanos())); - properties.setProperty("insert.tablet.rows.per.second", format(result.tabletRowsPerSecond())); - properties.setProperty( - "insert.tablet.values.per.second", format(result.tabletValuesPerSecond())); - - final Path resultPath = - Paths.get( - System.getProperty( - RESULT_PATH_PROPERTY, "target/numeric-row-insert-performance.properties")); - final Path parent = resultPath.getParent(); - if (parent != null) { - Files.createDirectories(parent); - } - try (OutputStream outputStream = Files.newOutputStream(resultPath)) { - properties.store(outputStream, null); - } - } - - private static String join(List values) { - final StringBuilder builder = new StringBuilder(); - for (int index = 0; index < values.size(); index++) { - if (index > 0) { - builder.append(','); - } - builder.append(values.get(index)); - } - return builder.toString(); - } - - private static String join(long[] values) { - final StringBuilder builder = new StringBuilder(); - for (int index = 0; index < values.length; index++) { - if (index > 0) { - builder.append(','); - } - builder.append(values[index]); - } - return builder.toString(); - } - - private static String format(double value) { - return String.format(Locale.ROOT, "%.2f", value); - } - - private static long percentile(long[] latencies, double percentile) { - final long[] sorted = latencies.clone(); - Arrays.sort(sorted); - return sorted[(int) Math.ceil(percentile * sorted.length) - 1]; - } - - private static final class NumericRowWorkload { - - private final String tableName; - private final NumericRow[] rows; - - private NumericRowWorkload(String tableName, NumericRow[] rows) { - this.tableName = tableName; - this.rows = rows; - } - } - - private static final class TabletWorkload { - - private final String tableName; - private final List tablets; - - private TabletWorkload(String tableName, List tablets) { - this.tableName = tableName; - this.tablets = tablets; - } - } - - private static final class BenchmarkResult { - - private final long[] numericRowElapsedNanos; - private final long[] tabletElapsedNanos; - - private BenchmarkResult(long[] numericRowElapsedNanos, long[] tabletElapsedNanos) { - this.numericRowElapsedNanos = numericRowElapsedNanos; - this.tabletElapsedNanos = tabletElapsedNanos; - } - - private long numericRowP50Nanos() { - return percentile(numericRowElapsedNanos, 0.5D); - } - - private long numericRowP95Nanos() { - return percentile(numericRowElapsedNanos, 0.95D); - } - - private double numericRowRowsPerSecond() { - return CLIENT_COUNT * ROW_COUNT * 1_000_000_000.0D / numericRowP50Nanos(); - } - - private double numericRowValuesPerSecond() { - return numericRowRowsPerSecond() * COLUMN_COUNT; - } - - private long tabletP50Nanos() { - return percentile(tabletElapsedNanos, 0.5D); - } - - private long tabletP95Nanos() { - return percentile(tabletElapsedNanos, 0.95D); - } - - private double tabletRowsPerSecond() { - return CLIENT_COUNT * ROW_COUNT * 1_000_000_000.0D / tabletP50Nanos(); - } - - private double tabletValuesPerSecond() { - return tabletRowsPerSecond() * COLUMN_COUNT; - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java index e196ff1edf0eb..6f6ee44f129a0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBTableModelSessionIT.java @@ -20,7 +20,6 @@ package org.apache.iotdb.relational.it.session; import org.apache.iotdb.isession.ITableSession; -import org.apache.iotdb.isession.NumericRow; import org.apache.iotdb.isession.NumericRowPreparedStatement; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.isession.pool.ITableSessionPool; @@ -28,8 +27,10 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.TableClusterIT; import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.jdbc.Config; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.UrlUtils; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; @@ -48,8 +49,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.SplittableRandom; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -57,6 +60,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.IntConsumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showTablesColumnHeaders; import static org.junit.Assert.assertEquals; @@ -72,15 +77,26 @@ public class IoTDBTableModelSessionIT { @Before public void setUp() throws Exception { - if (Boolean.getBoolean("iotdb.numeric.row.insert.perf.enabled") - && Boolean.getBoolean("iotdb.numeric.row.insert.perf.io.enabled")) { - // Isolate per-round storage accounting from compaction of earlier rounds. - EnvFactory.getEnv() - .getConfig() - .getCommonConfig() - .setEnableSeqSpaceCompaction(false) - .setEnableUnseqSpaceCompaction(false) - .setEnableCrossSpaceCompaction(false); + if (Boolean.getBoolean("iotdb.numeric.row.insert.perf.enabled")) { + // A saturated single DataNode can delay ConfigNode heartbeats while the benchmark is running. + // Keep the test focused on write-path throughput instead of fencing the node for that delay. + EnvFactory.getEnv().getConfig().getConfigNodeConfig().setMetadataLeaseFenceMs(120_000); + if (Boolean.getBoolean("iotdb.numeric.row.insert.perf.breakdown.enabled")) { + EnvFactory.getEnv() + .getConfig() + .getDataNodeConfig() + .setMetricReporterType(Collections.singletonList("PROMETHEUS")) + .setMetricPrometheusReporterAsyncUpdate(false); + } + if (Boolean.getBoolean("iotdb.numeric.row.insert.perf.io.enabled")) { + // Isolate per-round storage accounting from compaction of earlier rounds. + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false); + } } EnvFactory.getEnv().initClusterEnvironment(); } @@ -91,7 +107,7 @@ public void tearDown() throws Exception { } @Test - public void testNumericRowInsert() throws Exception { + public void testPreparedNumericRowInsert() throws Exception { // Exercise direct and pooled table sessions, schema reordering and missing STRING columns. // The same values must remain readable from MemTable and after flushing to TsFile. try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -120,26 +136,20 @@ public void testNumericRowInsert() throws Exception { 20260910, 1234 }; - session.insert( - new NumericRow( - "readings", - 1, - names, - types, - Collections.nCopies(types.size(), ColumnCategory.FIELD), - values, - new String[0])); + try (NumericRowPreparedStatement rowStatement = + session.prepareInsertNumericRow( + "readings", names, types, Collections.nCopies(types.size(), ColumnCategory.FIELD))) { + session.executePreparedInsertNumericRow(rowStatement, 1, values, new String[0]); + } try (ITableSessionPool pool = EnvFactory.getEnv().getTableSessionPool(1, "numeric_rows"); - ITableSession pooledSession = pool.getSession()) { - pooledSession.insert( - new NumericRow( - "readings", - 2, - names, - types, - Collections.nCopies(types.size(), ColumnCategory.FIELD), - values, - new String[0])); + ITableSession pooledSession = pool.getSession(); + NumericRowPreparedStatement pooledRow = + pooledSession.prepareInsertNumericRow( + "readings", + names, + types, + Collections.nCopies(types.size(), ColumnCategory.FIELD))) { + pooledSession.executePreparedInsertNumericRow(pooledRow, 2, values, new String[0]); } assertNumericRows(session); session.executeNonQueryStatement("FLUSH"); @@ -148,7 +158,7 @@ public void testNumericRowInsert() throws Exception { } @Test - public void testNumericRowInsertWithTags() throws Exception { + public void testPreparedNumericRowInsertWithTags() throws Exception { // Different TAGs at the same timestamp must stay separate, with identical behavior after flush. try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { session.executeNonQueryStatement("CREATE DATABASE numeric_tag_rows"); @@ -162,45 +172,29 @@ public void testNumericRowInsertWithTags() throws Exception { List categories = Arrays.asList( ColumnCategory.FIELD, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.TAG); - session.insert( - new NumericRow( - "readings", - 1, - names, - types, - categories, - new long[] {1, 10}, - new String[] {"设备甲😀", "华北"})); + try (NumericRowPreparedStatement rowStatement = + session.prepareInsertNumericRow("readings", names, types, categories)) { + session.executePreparedInsertNumericRow( + rowStatement, 1, new long[] {1, 10}, new String[] {"设备甲😀", "华北"}); + } try (ITableSessionPool pool = EnvFactory.getEnv().getTableSessionPool(1, "numeric_tag_rows"); ITableSession pooled = pool.getSession()) { - pooled.insert( - new NumericRow( - "readings", - 1, - names, - types, - categories, - new long[] {0, 20}, - new String[] {"设备乙", "华北"})); - } - session.insert( - new NumericRow( - "readings", - 2, - names, - types, - categories, - new long[] {1, 30}, - new String[] {"", null})); - session.insert( - new NumericRow( - "readings", - 2, - names, - types, - categories, - new long[] {0, 40}, - new String[] {null, null})); + try (NumericRowPreparedStatement rowStatement = + pooled.prepareInsertNumericRow("readings", names, types, categories)) { + pooled.executePreparedInsertNumericRow( + rowStatement, 1, new long[] {0, 20}, new String[] {"设备乙", "华北"}); + } + } + try (NumericRowPreparedStatement rowStatement = + session.prepareInsertNumericRow("readings", names, types, categories)) { + session.executePreparedInsertNumericRow( + rowStatement, 2, new long[] {1, 30}, new String[] {"", null}); + } + try (NumericRowPreparedStatement rowStatement = + session.prepareInsertNumericRow("readings", names, types, categories)) { + session.executePreparedInsertNumericRow( + rowStatement, 2, new long[] {0, 40}, new String[] {null, null}); + } assertNumericTagRows(session); session.executeNonQueryStatement("FLUSH"); assertNumericTagRows(session); @@ -278,17 +272,16 @@ public void testNumericRowNullBitmap() throws Exception { nulls.mark(7); nulls.mark(8); String[] tags = {"device"}; - session.insert(new NumericRow("readings", 1, names, types, categories, values, tags, nulls)); try (NumericRowPreparedStatement prepared = session.prepareInsertNumericRow("readings", names, types, categories)) { + session.executePreparedInsertNumericRow(prepared, 1, values, tags, nulls); session.executePreparedInsertNumericRow(prepared, 2, values, tags, nulls); session.executePreparedInsertNumericRow(prepared, 3, values, tags); // All-null table rows retain their timestamp/TAGs; null slots must never become zeros. BitMap allNulls = new BitMap(9); allNulls.markAll(); session.executePreparedInsertNumericRow(prepared, 3, values, tags, allNulls); - session.insert( - new NumericRow("readings", 10, names, types, categories, values, tags, allNulls)); + session.executePreparedInsertNumericRow(prepared, 10, values, tags, allNulls); session.executePreparedInsertNumericRow(prepared, 11, values, tags, allNulls); session.executeNonQueryStatement("ALTER TABLE readings ADD COLUMN tag2 STRING TAG"); session.executePreparedInsertNumericRow(prepared, 4, values, tags, nulls); @@ -297,7 +290,7 @@ public void testNumericRowNullBitmap() throws Exception { ITableSession pooled = pool.getSession(); NumericRowPreparedStatement prepared = pooled.prepareInsertNumericRow("readings", names, types, categories)) { - pooled.insert(new NumericRow("readings", 5, names, types, categories, values, tags, nulls)); + pooled.executePreparedInsertNumericRow(prepared, 5, values, tags, nulls); pooled.executePreparedInsertNumericRow(prepared, 6, values, tags, nulls); } assertNumericNullRows(session); @@ -441,17 +434,64 @@ private boolean writeRedirectedNumericRows(ITableSession session, long start) th @Test public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { + compareNumericRowWritePerformance(false); + } + + @Test + public void testPreparedNumericRowAndTabletSparseWritePerformance() throws Exception { + compareNumericRowWritePerformance(true); + } + + @Test + public void testPreparedNumericRowAndTabletWriteRatioPerformance() throws Exception { + Assume.assumeTrue( + "Performance benchmark disabled; set iotdb.numeric.row.insert.perf.enabled=true to run", + Boolean.getBoolean("iotdb.numeric.row.insert.perf.enabled")); + final int fieldCount = Integer.getInteger("iotdb.numeric.row.insert.perf.fields", 99); + boolean first = true; + for (int ratio : configuredSparseWriteRatios()) { + if (!first) { + // Recreate the instance so earlier ratios cannot contribute flush or compaction work. + EnvFactory.getEnv().cleanClusterEnvironment(); + EnvFactory.getEnv() + .setTestMethodName("testPreparedNumericRowAndTabletWriteRatioPerformance_" + ratio); + setUp(); + } + first = false; + int writtenFields = Math.max(1, (fieldCount * ratio + 99) / 100); + compareNumericRowWritePerformance(true, writtenFields, "sparse-" + ratio + "%"); + } + } + + private void compareNumericRowWritePerformance(boolean sparse) throws Exception { + int writtenFields = + sparse ? Integer.getInteger("iotdb.numeric.row.insert.perf.sparse.fields", 10) : 99; + compareNumericRowWritePerformance(sparse, writtenFields, sparse ? "sparse" : "full-row"); + } + + private void compareNumericRowWritePerformance( + boolean sparse, int writtenFields, String scenarioName) throws Exception { // Compare synchronous single-row paths and optionally account for their Linux TCP/storage I/O. Assume.assumeTrue( "Performance benchmark disabled; set iotdb.numeric.row.insert.perf.enabled=true to run", Boolean.getBoolean("iotdb.numeric.row.insert.perf.enabled")); final int concurrency = Integer.getInteger("iotdb.numeric.row.insert.perf.concurrency", 16); - // Compare sustained single-row writes after warming all three paths; verify all acknowledged + // Compare sustained single-row writes after warming both paths; verify all acknowledged // rows. final int warmupRounds = 2; final int measuredRounds = 6; - final int fieldCount = 99; - final String databasePrefix = "numeric_perf_"; + final int fieldCount = Integer.getInteger("iotdb.numeric.row.insert.perf.fields", 99); + final int deviceCount = + sparse ? Integer.getInteger("iotdb.numeric.row.insert.perf.sparse.devices", 100) : 1; + final long seed = Long.getLong("iotdb.numeric.row.insert.perf.sparse.seed", 20261008L); + assertTrue("Concurrency and device count must be positive", concurrency > 0 && deviceCount > 0); + assertTrue( + "Written FIELD count must be positive and within the table schema", + writtenFields > 0 && writtenFields <= fieldCount); + final String databasePrefix = + (sparse ? "numeric_sparse_perf_" : "numeric_perf_") + + scenarioName.replace('%', 'p').replace('-', '_') + + "_"; try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { StringBuilder ddl = new StringBuilder("CREATE TABLE prepared_perf (tag STRING TAG"); for (int i = 0; i < fieldCount; i++) { @@ -464,7 +504,6 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { session.executeNonQueryStatement("USE " + databasePrefix + i); session.executeNonQueryStatement(ddl.toString()); session.executeNonQueryStatement(ddl.toString().replace("prepared_perf", "tablet_perf")); - session.executeNonQueryStatement(ddl.toString().replace("prepared_perf", "row_perf")); } } @@ -482,11 +521,12 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { ITableSession[] sessions = new ITableSession[concurrency]; NumericRowPreparedStatement[] prepared = new NumericRowPreparedStatement[concurrency]; Tablet[] tablets = new Tablet[concurrency]; - NumericRow[] rows = new NumericRow[concurrency]; String[][] tags = new String[concurrency][]; long[][] numericValues = new long[concurrency][fieldCount]; - long[][] pathRows = new long[3][concurrency]; + long[][] pathRows = new long[2][concurrency]; + SparseNumericRowInput[][] sparseRows = new SparseNumericRowInput[2][concurrency]; ExecutorService executor = Executors.newFixedThreadPool(concurrency); + WritePerformanceResourceTracker resourceTracker = null; try { // Connections, USE, prepare and reusable input allocation are outside every timed round. for (int i = 0; i < concurrency; i++) { @@ -497,17 +537,32 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { tablets[i] = new Tablet("tablet_perf", names, types, categories, 1); tags[i] = new String[] {"device-" + i}; Arrays.fill(numericValues[i], Double.doubleToRawLongBits(1.0)); - rows[i] = - new NumericRow("row_perf", 0, names, types, categories, numericValues[i], tags[i]); + if (sparse) { + // Independent generators replay the same device/column sequence for each path. + for (int path = 0; path < sparseRows.length; path++) { + sparseRows[path][i] = + new SparseNumericRowInput(deviceCount, fieldCount, writtenFields, seed + i); + } + } } IntConsumer preparedWrite = threadId -> { try { - sessions[threadId].executePreparedInsertNumericRow( - prepared[threadId], - pathRows[0][threadId], - numericValues[threadId], - tags[threadId]); + if (sparse) { + SparseNumericRowInput input = sparseRows[0][threadId]; + sessions[threadId].executePreparedInsertNumericRow( + prepared[threadId], + pathRows[0][threadId], + numericValues[threadId], + input.tagValues, + input.nulls); + } else { + sessions[threadId].executePreparedInsertNumericRow( + prepared[threadId], + pathRows[0][threadId], + numericValues[threadId], + tags[threadId]); + } } catch (Exception e) { throw new RuntimeException(e); } @@ -515,32 +570,22 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { IntConsumer tabletPrepare = threadId -> { Tablet tablet = tablets[threadId]; - tablet.addTimestamp(0, pathRows[2][threadId]); - tablet.addValue("tag", 0, tags[threadId][0]); + tablet.addTimestamp(0, pathRows[1][threadId]); + SparseNumericRowInput input = sparseRows[1][threadId]; + if (sparse) { + input.next(); + } + tablet.addValue("tag", 0, sparse ? input.tagValues[0] : tags[threadId][0]); for (int i = 0; i < fieldCount; i++) { - tablet.addValue(names.get(i + 1), 0, 1.0D); + if (sparse && input.nulls.isMarked(i)) { + tablet.addValue(names.get(i + 1), 0, null); + } else { + tablet.addValue(names.get(i + 1), 0, 1.0D); + } } }; - IntConsumer rowPrepare = - threadId -> - rows[threadId] = - new NumericRow( - "row_perf", - pathRows[1][threadId], - names, - types, - categories, - numericValues[threadId], - tags[threadId]); IntConsumer[] paths = { preparedWrite, - threadId -> { - try { - sessions[threadId].insert(rows[threadId]); - } catch (Exception e) { - throw new RuntimeException(e); - } - }, threadId -> { try { sessions[threadId].insert(tablets[threadId]); @@ -549,17 +594,31 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { } } }; - IntConsumer[] prepareWrites = {null, rowPrepare, tabletPrepare}; + IntConsumer[] prepareWrites = { + sparse ? threadId -> sparseRows[0][threadId].next() : null, tabletPrepare + }; IntConsumer[] completeWrites = { - threadId -> pathRows[0][threadId]++, - threadId -> pathRows[1][threadId]++, threadId -> { + if (sparse) sparseRows[0][threadId].acknowledge(); + pathRows[0][threadId]++; + }, + threadId -> { + if (sparse) sparseRows[1][threadId].acknowledge(); tablets[threadId].reset(); - pathRows[2][threadId]++; + pathRows[1][threadId]++; } }; - String[] pathNames = {"prepared numeric row", "insert NumericRow", "insertTablet batch=1"}; - System.out.printf("configuration: concurrency=%d, tabletBatch=1%n", concurrency); + String[] pathNames = {"prepared numeric row", "insertTablet batch=1"}; + System.out.printf( + "configuration: scenario=%s, concurrency=%d, tabletBatch=1, devicesPerWriter=%d, " + + "totalFields=%d, writtenFields=%d, actualWriteRatio=%.2f%%, seed=%d%n", + scenarioName, + concurrency, + deviceCount, + fieldCount, + writtenFields, + writtenFields * 100.0 / fieldCount, + seed); WritePerformanceIoTracker ioTracker = null; if (Boolean.getBoolean("iotdb.numeric.row.insert.perf.io.enabled")) { assertEquals( @@ -576,29 +635,46 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { EnvFactory.getEnv().getConfigNodeWrapperList().get(0).getPid(), EnvFactory.getEnv().getDataNodeWrapperList().get(0).getPort()); } + resourceTracker = + new WritePerformanceResourceTracker( + EnvFactory.getEnv().getDataNodeWrapperList().get(0).getPid()); double[][] rates = new double[paths.length][measuredRounds]; long[] totalRows = new long[paths.length]; long[] totalNanos = new long[paths.length]; + double[] peakCpuPercent = new double[paths.length]; + Arrays.fill(peakCpuPercent, Double.NaN); + long[] peakResidentBytes = new long[paths.length]; LongSamples[] requestLatencies = new LongSamples[paths.length]; for (int path = 0; path < paths.length; path++) { requestLatencies[path] = new LongSamples(); } + Map previousMetrics = + Boolean.getBoolean("iotdb.numeric.row.insert.perf.breakdown.enabled") + ? readPerformanceMetrics() + : Collections.emptyMap(); for (int round = 0; round < warmupRounds + measuredRounds; round++) { boolean warmup = round < warmupRounds; int seconds = warmup ? 3 : 5; - // Rotate all three paths so each occupies every position during measured rounds. + // Rotate both paths so each occupies every position during measured rounds. for (int position = 0; position < paths.length; position++) { int path = (round + position) % paths.length; WritePerformanceIoTracker.Snapshot ioBefore = ioTracker == null ? null : ioTracker.beforeRound(sessions[0]); - WritePerformance result = - runConcurrentWrites( - executor, - concurrency, - seconds, - prepareWrites[path], - paths[path], - completeWrites[path]); + resourceTracker.beginRound(); + WritePerformance result = null; + WritePerformanceResourceTracker.RoundPeak resourcePeak; + try { + result = + runConcurrentWrites( + executor, + concurrency, + seconds, + prepareWrites[path], + paths[path], + completeWrites[path]); + } finally { + resourcePeak = resourceTracker.endRound(); + } if (ioTracker != null) { System.out.printf( "io %s round %d position %d %s: rows=%d, %s%n", @@ -619,11 +695,37 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { result.nanos() / 1_000_000_000.0, result.rowsPerSecond(), summarizeRequestLatencies(result.requestNanos())); + System.out.printf( + "resource %s round %d position %d %s: %s%n", + warmup ? "warmup" : "measured", + warmup ? round + 1 : round - warmupRounds + 1, + position + 1, + pathNames[path], + resourcePeak.format()); + if (Boolean.getBoolean("iotdb.numeric.row.insert.perf.breakdown.enabled")) { + Map currentMetrics = readPerformanceMetrics(); + printPerformanceBreakdown( + warmup ? "warmup" : "measured", + warmup ? round + 1 : round - warmupRounds + 1, + position + 1, + pathNames[path], + previousMetrics, + currentMetrics); + previousMetrics = currentMetrics; + } if (!warmup) { rates[path][round - warmupRounds] = result.rowsPerSecond(); totalRows[path] += result.rows(); totalNanos[path] += result.nanos(); requestLatencies[path].appendAll(result.requestNanos()); + if (!Double.isNaN(resourcePeak.peakCpuPercent())) { + peakCpuPercent[path] = + Double.isNaN(peakCpuPercent[path]) + ? resourcePeak.peakCpuPercent() + : Math.max(peakCpuPercent[path], resourcePeak.peakCpuPercent()); + } + peakResidentBytes[path] = + Math.max(peakResidentBytes[path], resourcePeak.peakResidentBytes()); } } } @@ -631,20 +733,26 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { Arrays.sort(rates[path]); System.out.printf( "summary %s: median %.1f rows/s, aggregate %.1f rows/s, min %.1f, max %.1f, " - + "measured API latency %s%n", + + "measured API latency %s, peak DataNode CPU %s, peak DataNode RSS %s, " + + "scenario=%s, writtenFields=%d/%d, nonNullFieldsPerSecond=%.1f%n", pathNames[path], (rates[path][measuredRounds / 2 - 1] + rates[path][measuredRounds / 2]) / 2, totalRows[path] * 1_000_000_000.0 / totalNanos[path], rates[path][0], rates[path][measuredRounds - 1], - summarizeRequestLatencies(requestLatencies[path].toArray())); + summarizeRequestLatencies(requestLatencies[path].toArray()), + formatPeakCpu(peakCpuPercent[path]), + formatPeakMemory(peakResidentBytes[path]), + scenarioName, + writtenFields, + fieldCount, + totalRows[path] * 1_000_000_000.0 / totalNanos[path] * writtenFields); } // Per-device monotonic timestamps span warmup and measurement, so no rows overwrite others. for (int path = 0; path < paths.length; path++) { String table = switch (path) { case 0 -> "prepared_perf"; - case 1 -> "row_perf"; default -> "tablet_perf"; }; long[] rowsByThread = pathRows[path]; @@ -658,15 +766,30 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { assertFalse(dataSet.hasNext()); } } - System.out.printf("verified %s: %d rows%n", table, expectedRows); + if (sparse) { + // Check counts for every device and FIELD; stale Tablet bits must not fill omitted cells. + for (int i = 0; i < concurrency; i++) { + sparseRows[path][i].verify(sessions[i], table); + } + } + System.out.printf( + "verified %s: %d rows, %d non-null FIELD values%n", + table, expectedRows, expectedRows * writtenFields); } } finally { executor.shutdownNow(); try { assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES)); } finally { - // Closing each session also releases its server-side prepared metadata. Exception closeFailure = null; + if (resourceTracker != null) { + try { + resourceTracker.close(); + } catch (Exception e) { + closeFailure = e; + } + } + // Closing each session also releases its server-side prepared metadata. for (ITableSession session : sessions) { if (session != null) { try { @@ -682,6 +805,171 @@ public void testPreparedNumericRowAndTabletWritePerformance() throws Exception { } } + private static final Pattern PERFORMANCE_TIMER_SUM_PATTERN = + Pattern.compile( + "^(performance_overview[^\\s{]+)_sum\\{[^}]*stage=\\\"([^\\\"]+)\\\"[^}]*\\}\\s+([-+0-9.eE]+)$"); + private static final Pattern PERFORMANCE_TIMER_COUNT_PATTERN = + Pattern.compile( + "^(performance_overview[^\\s{]+)_count\\{[^}]*stage=\\\"([^\\\"]+)\\\"[^}]*\\}\\s+([-+0-9.eE]+)$"); + + private static Map readPerformanceMetrics() { + Map values = new LinkedHashMap<>(); + for (var dataNode : EnvFactory.getEnv().getDataNodeWrapperList()) { + String content = + EnvFactory.getEnv() + .getUrlContent( + Config.IOTDB_HTTP_URL_PREFIX + + UrlUtils.formatTEndPointIpv4AndIpv6Url( + dataNode.getIp(), dataNode.getMetricPort()) + + "/metrics", + null); + if (content == null) { + continue; + } + for (String line : content.split("\\n")) { + Matcher sumMatcher = PERFORMANCE_TIMER_SUM_PATTERN.matcher(line.trim()); + if (sumMatcher.matches()) { + values + .computeIfAbsent( + sumMatcher.group(1) + "." + sumMatcher.group(2), key -> new double[2])[0] += + Double.parseDouble(sumMatcher.group(3)); + continue; + } + Matcher countMatcher = PERFORMANCE_TIMER_COUNT_PATTERN.matcher(line.trim()); + if (countMatcher.matches()) { + values + .computeIfAbsent( + countMatcher.group(1) + "." + countMatcher.group(2), key -> new double[2])[ + 1] += + Double.parseDouble(countMatcher.group(3)); + } + } + } + Map result = new LinkedHashMap<>(); + values.forEach((key, value) -> result.put(key, new MetricSample(value[0], value[1]))); + return result; + } + + private static void printPerformanceBreakdown( + String phase, + int round, + int position, + String path, + Map previous, + Map current) { + StringBuilder output = + new StringBuilder( + String.format("breakdown %s round %d position %d %s:", phase, round, position, path)); + for (Map.Entry entry : current.entrySet()) { + MetricSample before = previous.getOrDefault(entry.getKey(), MetricSample.ZERO); + double count = entry.getValue().count() - before.count(); + double seconds = entry.getValue().seconds() - before.seconds(); + if (count > 0 && seconds >= 0) { + output.append( + String.format( + " %s count=%.0f total=%.3f ms avg=%.3f us;", + entry.getKey(), count, seconds * 1000, seconds * 1_000_000 / count)); + } + } + System.out.println(output); + } + + private record MetricSample(double seconds, double count) { + private static final MetricSample ZERO = new MetricSample(0, 0); + } + + private static int[] configuredSparseWriteRatios() { + String configured = + System.getProperty("iotdb.numeric.row.insert.perf.sparse.ratios", "10,50,100"); + String[] tokens = configured.split(",", -1); + int[] ratios = new int[tokens.length]; + for (int i = 0; i < tokens.length; i++) { + ratios[i] = Integer.parseInt(tokens[i].trim()); + assertTrue( + "Sparse write ratios must be between 1 and 100", ratios[i] > 0 && ratios[i] <= 100); + } + return ratios; + } + + /** Reproducible random device and distinct FIELD selection, with no per-request allocations. */ + private static final class SparseNumericRowInput { + private final SplittableRandom random; + private final String[] devices; + private final String[] tagValues = new String[1]; + private final int[] columns; + private final BitMap nulls; + private final int writtenFields; + private final long[][] expected; + private int device; + + private SparseNumericRowInput(int deviceCount, int fieldCount, int writtenFields, long seed) { + random = new SplittableRandom(seed); + devices = new String[deviceCount]; + for (int i = 0; i < deviceCount; i++) devices[i] = "device-" + i; + columns = new int[fieldCount]; + for (int i = 0; i < fieldCount; i++) columns[i] = i; + nulls = new BitMap(fieldCount); + this.writtenFields = writtenFields; + // Slot zero counts rows, followed by one counter for each FIELD. + expected = new long[deviceCount][fieldCount + 1]; + } + + private void next() { + device = random.nextInt(devices.length); + tagValues[0] = devices[device]; + nulls.markAll(); + // Partial Fisher-Yates sampling gives exactly writtenFields distinct columns per request. + for (int i = 0; i < writtenFields; i++) { + int other = random.nextInt(i, columns.length); + int column = columns[other]; + columns[other] = columns[i]; + columns[i] = column; + nulls.unmark(column); + } + } + + private void acknowledge() { + expected[device][0]++; + for (int i = 0; i < writtenFields; i++) expected[device][columns[i] + 1]++; + } + + private void verify(ITableSession session, String table) throws Exception { + StringBuilder query = new StringBuilder("SELECT tag, count(*)"); + for (int i = 0; i < columns.length; i++) query.append(", count(f").append(i).append(')'); + query.append(" FROM ").append(table).append(" GROUP BY tag"); + boolean[] seen = new boolean[devices.length]; + try (SessionDataSet result = session.executeQueryStatement(query.toString())) { + while (result.hasNext()) { + RowRecord row = result.next(); + int index = + Integer.parseInt( + row.getFields().get(0).getStringValue().substring("device-".length())); + assertTrue("Unexpected device", index >= 0 && index < devices.length); + assertEquals(devices[index], row.getFields().get(0).getStringValue()); + assertFalse("Duplicate device", seen[index]); + seen[index] = true; + for (int i = 0; i <= columns.length; i++) { + assertEquals( + table + "/" + devices[index] + "/count column " + i, + expected[index][i], + row.getFields().get(i + 1).getLongV()); + } + } + } + for (int i = 0; i < devices.length; i++) assertEquals(expected[i][0] > 0, seen[i]); + } + } + + private static String formatPeakCpu(double peakCpuPercent) { + return Double.isNaN(peakCpuPercent) ? "unavailable" : String.format("%.1f%%", peakCpuPercent); + } + + private static String formatPeakMemory(long peakResidentBytes) { + return peakResidentBytes <= 0 + ? "unavailable" + : String.format("%.1f MiB", peakResidentBytes / 1024.0 / 1024.0); + } + private record WritePerformance(long rows, long nanos, long[] requestNanos) { double rowsPerSecond() { return rows * 1_000_000_000.0 / nanos; diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/WritePerformanceResourceTracker.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/WritePerformanceResourceTracker.java new file mode 100644 index 0000000000000..64a680875af7c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/WritePerformanceResourceTracker.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iotdb.relational.it.session; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +/** Samples the DataNode process resource usage while one benchmark round is running. */ +final class WritePerformanceResourceTracker implements AutoCloseable { + private static final Path PROC = Path.of("/proc"); + private static final long SAMPLE_INTERVAL_MILLIS = 100; + + private final long dataNodePid; + private final long clockTicksPerSecond; + private final Thread sampler; + private final AtomicReference currentRound = new AtomicReference<>(); + private final AtomicReference samplingFailure = new AtomicReference<>(); + private final CountDownLatch stopped = new CountDownLatch(1); + + WritePerformanceResourceTracker(long dataNodePid) throws IOException { + this.dataNodePid = dataNodePid; + if (!isSupported()) { + clockTicksPerSecond = 0; + sampler = null; + System.out.println("resource configuration: DataNode CPU/memory sampling unavailable"); + return; + } + if (dataNodePid <= 0) { + throw new IOException("DataNode PID unavailable: " + dataNodePid); + } + clockTicksPerSecond = readClockTicksPerSecond(); + readProcessSample(); + sampler = new Thread(this::sampleLoop, "numeric-row-resource-sampler"); + sampler.setDaemon(true); + sampler.start(); + System.out.printf( + "resource configuration: dataNodePid=%d, sampleIntervalMs=%d, cpu=process CPU, " + + "memory=resident set size%n", + dataNodePid, SAMPLE_INTERVAL_MILLIS); + } + + boolean isSupported() { + return Files.isDirectory(PROC); + } + + void beginRound() throws IOException { + checkFailure(); + if (!isSupported()) { + return; + } + Round round = new Round(); + round.sample(); + currentRound.set(round); + } + + RoundPeak endRound() throws IOException { + if (!isSupported()) { + return RoundPeak.UNAVAILABLE; + } + Round round = currentRound.getAndSet(null); + if (round == null) { + throw new IllegalStateException("Resource round was not started"); + } + RoundPeak peak = round.finish(); + checkFailure(); + return peak; + } + + @Override + public void close() throws Exception { + if (sampler == null) { + return; + } + sampler.interrupt(); + if (!stopped.await(5, TimeUnit.SECONDS)) { + throw new IllegalStateException("Resource sampler did not stop"); + } + checkFailure(); + } + + private void sampleLoop() { + try { + while (!Thread.currentThread().isInterrupted()) { + Round round = currentRound.get(); + if (round != null) { + round.sample(); + } + Thread.sleep(SAMPLE_INTERVAL_MILLIS); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Throwable t) { + samplingFailure.compareAndSet(null, t); + } finally { + stopped.countDown(); + } + } + + private void checkFailure() throws IOException { + Throwable failure = samplingFailure.get(); + if (failure == null) { + return; + } + if (failure instanceof IOException ioException) { + throw ioException; + } + throw new IOException("Failed to sample DataNode resources", failure); + } + + private ProcessSample readProcessSample() throws IOException { + Path processDirectory = PROC.resolve(Long.toString(dataNodePid)); + String stat = Files.readString(processDirectory.resolve("stat")); + int closingParenthesis = stat.lastIndexOf(") "); + if (closingParenthesis < 0) { + throw new IOException("Malformed /proc stat for DataNode PID " + dataNodePid); + } + String[] fields = stat.substring(closingParenthesis + 2).trim().split("\\s+"); + if (fields.length <= 12) { + throw new IOException("Incomplete /proc stat for DataNode PID " + dataNodePid); + } + long cpuTicks = Long.parseLong(fields[11]) + Long.parseLong(fields[12]); + long residentBytes = 0; + for (String line : Files.readAllLines(processDirectory.resolve("status"))) { + if (line.startsWith("VmRSS:")) { + residentBytes = Long.parseLong(line.replaceAll("[^0-9]", "").trim()) * 1024; + break; + } + } + if (residentBytes <= 0) { + throw new IOException("DataNode RSS unavailable for PID " + dataNodePid); + } + return new ProcessSample(System.nanoTime(), cpuTicks, residentBytes); + } + + private static long readClockTicksPerSecond() throws IOException { + Process process = new ProcessBuilder("getconf", "CLK_TCK").redirectErrorStream(true).start(); + try { + if (!process.waitFor(5, TimeUnit.SECONDS)) { + throw new IOException("getconf CLK_TCK timed out"); + } + String output = + new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8).trim(); + if (process.exitValue() != 0) { + throw new IOException("getconf CLK_TCK failed: " + output); + } + long result = Long.parseLong(output); + if (result <= 0) { + throw new IOException("Invalid CLK_TCK: " + result); + } + return result; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while reading CLK_TCK", e); + } finally { + process.destroyForcibly(); + } + } + + private final class Round { + private long lastCpuTicks; + private long lastSampleNanos; + private double peakCpuPercent = Double.NaN; + private long peakResidentBytes; + private boolean active = true; + + private synchronized void sample() throws IOException { + if (!active) { + return; + } + // Read and apply samples under the same lock so the final sample cannot overtake the sampler. + ProcessSample sample = readProcessSample(); + peakResidentBytes = Math.max(peakResidentBytes, sample.residentBytes); + long elapsedNanos = sample.nanoTime - lastSampleNanos; + if (lastSampleNanos != 0) { + // Keep the CPU baseline for short intervals: tick quantization can inflate a tail sample. + if (elapsedNanos < TimeUnit.MILLISECONDS.toNanos(SAMPLE_INTERVAL_MILLIS)) { + return; + } + long cpuTicks = sample.cpuTicks - lastCpuTicks; + double cpuPercent = + cpuTicks * 100.0 / clockTicksPerSecond / (elapsedNanos / 1_000_000_000.0); + peakCpuPercent = + Double.isNaN(peakCpuPercent) ? cpuPercent : Math.max(peakCpuPercent, cpuPercent); + } + lastCpuTicks = sample.cpuTicks; + lastSampleNanos = sample.nanoTime; + } + + private synchronized RoundPeak finish() throws IOException { + try { + sample(); + return new RoundPeak(peakCpuPercent, peakResidentBytes); + } finally { + active = false; + } + } + } + + private record ProcessSample(long nanoTime, long cpuTicks, long residentBytes) {} + + record RoundPeak(double peakCpuPercent, long peakResidentBytes) { + private static final RoundPeak UNAVAILABLE = new RoundPeak(Double.NaN, -1); + + String format() { + if (this == UNAVAILABLE) { + return "peakDataNodeCpu=unavailable, peakDataNodeRss=unavailable"; + } + return String.format( + "peakDataNodeCpu=%s, peakDataNodeRss=%.1f MiB", + Double.isNaN(peakCpuPercent) ? "unavailable" : String.format("%.1f%%", peakCpuPercent), + peakResidentBytes / 1024.0 / 1024.0); + } + } +} diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java index 1cfb0b86eedf0..7fa1df842f8ad 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java @@ -52,17 +52,12 @@ public interface ITableSession extends AutoCloseable { void insert(Tablet tablet) throws StatementExecutionException, IoTDBConnectionException; /** - * Inserts one table-model {@link NumericRow} with numeric FIELDs and optional STRING TAGs. + * Prepares repeated single-row writes with STRING TAGs and BOOLEAN, INT32, INT64, FLOAT, DOUBLE, + * DATE or TIMESTAMP FIELDs. ATTRIBUTE columns and other FIELD types are unsupported. Column + * names, types and categories must have matching lengths and order. * - *

This API requires a server supporting numeric row requests. The row's metadata and value - * arrays must not be modified until this call returns. - * - * @param row the numeric row to insert - * @throws StatementExecutionException if an error occurs while executing the statement - * @throws IoTDBConnectionException if there is an issue with the IoTDB connection + *

The returned statement belongs to this session and is not thread-safe. Close it after use. */ - void insert(NumericRow row) throws StatementExecutionException, IoTDBConnectionException; - NumericRowPreparedStatement prepareInsertNumericRow( String tableName, List columnNames, @@ -71,10 +66,18 @@ NumericRowPreparedStatement prepareInsertNumericRow( throws StatementExecutionException, IoTDBConnectionException; /** - * Executes a prepared row, following write redirection recommendations for subsequent rows. - * Destination handles are prepared lazily and released when the statement is closed. An execution - * error reported by the server closes the statement; prepare it again before reuse. Transport - * failures are propagated without replaying a write whose outcome is unknown. + * Executes one row using FIELD and TAG arrays in their respective occurrence order in the + * prepared column list. TAGs may be null or empty. + * + *

BOOLEAN uses zero for false and nonzero for true. INT32 and DATE use the low 32 bits (DATE + * is yyyyMMdd); INT64 and TIMESTAMP use the long value. FLOAT uses {@link + * Float#floatToRawIntBits(float)} in the low 32 bits and DOUBLE uses {@link + * Double#doubleToRawLongBits(double)}. Do not modify input arrays during execution. + * + *

Follows write redirection recommendations for subsequent rows. Destination handles are + * prepared lazily and released when the statement is closed. An execution error reported by the + * server closes the statement; prepare it again before reuse. Transport failures are propagated + * without replaying a write whose outcome is unknown. */ void executePreparedInsertNumericRow( NumericRowPreparedStatement prepared, long timestamp, long[] values, String[] tagValues) diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRow.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRow.java deleted file mode 100644 index 040cb7632ba18..0000000000000 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRow.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.isession; - -import org.apache.tsfile.enums.ColumnCategory; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.utils.BitMap; - -import java.util.List; - -/** - * A table-model row with numeric FIELDs in {@code long[]} and STRING TAGs in {@code String[]}. - * - *

Column names, types and categories describe all supplied columns in the same order. The value - * arrays contain only their respective FIELD or TAG values, in occurrence order in the column list. - * TAGs may be null or empty; missing TAG columns are null. An optional null bitmap marks null - * FIELDs by their index in {@code values}; marked slots retain their position but are not sent. - * ATTRIBUTE columns and nonnumeric FIELDs are not supported. - * - *

BOOLEAN uses zero for false and nonzero for true. INT32 and DATE use the low 32 bits (DATE is - * encoded as yyyyMMdd); INT64 and TIMESTAMP use the long value. FLOAT uses {@link - * Float#floatToRawIntBits(float)} in the low 32 bits and DOUBLE uses {@link - * Double#doubleToRawLongBits(double)}. - * - *

Metadata and arrays are retained without copying or boxing. Do not modify them while {@link - * ITableSession#insert(NumericRow)} is in progress. Arguments are validated when inserting the row. - */ -public final class NumericRow { - private final String tableName; - private final long timestamp; - private final List columnNames; - private final List dataTypes; - private final List columnCategories; - private final long[] values; - private final String[] tagValues; - private final BitMap nullBitmap; - - /** Creates a row with numeric FIELDs and optional STRING TAGs in separate arrays. */ - public NumericRow( - String tableName, - long timestamp, - List columnNames, - List dataTypes, - List columnCategories, - long[] values, - String[] tagValues) { - this(tableName, timestamp, columnNames, dataTypes, columnCategories, values, tagValues, null); - } - - /** - * Creates a row with an optional FIELD-only null bitmap, sized exactly to {@code values.length}. - * A marked bit denotes null. The bitmap is retained and must not change during insertion. - * Requires a server supporting numeric row null bitmaps when the bitmap is supplied. - */ - public NumericRow( - String tableName, - long timestamp, - List columnNames, - List dataTypes, - List columnCategories, - long[] values, - String[] tagValues, - BitMap nullBitmap) { - this.tableName = tableName; - this.timestamp = timestamp; - this.columnNames = columnNames; - this.dataTypes = dataTypes; - this.columnCategories = columnCategories; - this.values = values; - this.tagValues = tagValues; - this.nullBitmap = nullBitmap; - } - - public String getTableName() { - return tableName; - } - - public long getTimestamp() { - return timestamp; - } - - public List getColumnNames() { - return columnNames; - } - - public List getDataTypes() { - return dataTypes; - } - - public List getColumnCategories() { - return columnCategories; - } - - public long[] getValues() { - return values; - } - - public String[] getTagValues() { - return tagValues; - } - - public BitMap getNullBitmap() { - return nullBitmap; - } -} diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java index 2fa9ee2125754..077cc946f249d 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/NumericRowPreparedStatement.java @@ -2,8 +2,21 @@ * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding copyright ownership. + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ + package org.apache.iotdb.isession; import org.apache.iotdb.isession.i18n.ISessionMessages; diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index 11a545da71443..57754c22b109b 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -23,7 +23,6 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.isession.INodeSupplier; import org.apache.iotdb.isession.ISession; -import org.apache.iotdb.isession.NumericRow; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.isession.template.Template; @@ -67,7 +66,6 @@ import org.apache.thrift.TException; import org.apache.tsfile.common.conf.TSFileDescriptor; -import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; @@ -2874,89 +2872,6 @@ public void insertRelationalTablet(Tablet tablet) } } - /** Inserts numeric FIELDs and STRING TAGs without constructing a boxed row. */ - public void insertRelationalRecord(NumericRow row) - throws IoTDBConnectionException, StatementExecutionException { - if (row == null || row.getTableName() == null) { - throw new IllegalArgumentException( - SessionMessages - .EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233); - } - String tableName = row.getTableName(); - long timestamp = row.getTimestamp(); - List measurements = row.getColumnNames(); - List types = row.getDataTypes(); - List columnCategories = row.getColumnCategories(); - long[] values = row.getValues(); - String[] tagValues = row.getTagValues(); - if (measurements == null - || types == null - || columnCategories == null - || values == null - || tagValues == null - || measurements.size() != types.size() - || measurements.size() != columnCategories.size()) { - throw new IllegalArgumentException( - SessionMessages - .EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233); - } - // Keep wire metadata aligned with the compact payload: TAG columns precede FIELD columns. - List rpcMeasurements = new ArrayList<>(measurements.size()); - List rpcTypes = new ArrayList<>(types.size()); - List rpcColumnCategories = new ArrayList<>(columnCategories.size()); - for (int pass = 0; pass < 2; pass++) { - ColumnCategory category = pass == 0 ? ColumnCategory.TAG : ColumnCategory.FIELD; - for (int i = 0; i < columnCategories.size(); i++) { - if (columnCategories.get(i) == category) { - rpcMeasurements.add(measurements.get(i)); - rpcTypes.add(types.get(i)); - rpcColumnCategories.add(category); - } - } - } - if (rpcMeasurements.size() != measurements.size()) { - throw new IllegalArgumentException( - SessionMessages - .EXCEPTION_NUMERIC_ROWS_SUPPORT_NUMERIC_FIELD_COLUMNS_AND_STRING_TAG_COLUMNS_ONLY_87159FFA); - } - ByteBuffer valueBuffer = - SessionUtils.getValueBuffer( - rpcTypes, values, rpcMeasurements, rpcColumnCategories, tagValues, row.getNullBitmap()); - TSInsertRecordReq request = new TSInsertRecordReq(); - request.setPrefixPath(tableName); - request.setTimestamp(timestamp); - request.setMeasurements(rpcMeasurements); - request.setValues(valueBuffer); - request.setIsWriteToTable(true); - request.setIsNumericRow(true); - if (row.getNullBitmap() != null) { - request.setNullBitmap(SessionUtils.encodeNumericRowNullBitmap(row.getNullBitmap(), null)); - } - request.setIsAligned(true); - request.setColumnCategoryies( - rpcColumnCategories.stream() - .map(category -> (byte) category.ordinal()) - .collect(Collectors.toList())); - String[] deviceSegments = new String[tagValues.length + 1]; - deviceSegments[0] = tableName; - System.arraycopy(tagValues, 0, deviceSegments, 1, tagValues.length); - IDeviceID deviceId = IDeviceID.Factory.DEFAULT_FACTORY.create(deviceSegments); - try { - getSessionConnection(deviceId).insertRecord(request); - } catch (RedirectException e) { - handleRedirection(deviceId, e.getEndPoint()); - } catch (IoTDBConnectionException e) { - if (!enableRedirection || tableModelDeviceIdToEndpoint.remove(deviceId) == null) { - throw e; - } - try { - getDefaultSessionConnection().insertRecord(request); - } catch (RedirectException redirect) { - handleRedirection(deviceId, redirect.getEndPoint()); - } - } - } - private void insertRelationalTabletWithLeaderCache(Tablet tablet) throws IoTDBConnectionException, StatementExecutionException { Map relationalTabletGroup = new HashMap<>(); diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java index c453596c1eef4..6d7ddc1f64c07 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java @@ -20,7 +20,6 @@ package org.apache.iotdb.session; import org.apache.iotdb.isession.ITableSession; -import org.apache.iotdb.isession.NumericRow; import org.apache.iotdb.isession.NumericRowPreparedStatement; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; @@ -55,11 +54,6 @@ public void insert(Tablet tablet) throws StatementExecutionException, IoTDBConne session.insertRelationalTablet(tablet); } - @Override - public void insert(NumericRow row) throws StatementExecutionException, IoTDBConnectionException { - session.insertRelationalRecord(row); - } - @Override public NumericRowPreparedStatement prepareInsertNumericRow( String tableName, diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java index 7684beba9abe0..331a078ae9e54 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java @@ -20,7 +20,6 @@ package org.apache.iotdb.session.pool; import org.apache.iotdb.isession.ITableSession; -import org.apache.iotdb.isession.NumericRow; import org.apache.iotdb.isession.NumericRowPreparedStatement; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; @@ -78,19 +77,6 @@ public void insert(Tablet tablet) throws StatementExecutionException, IoTDBConne } } - @Override - public void insert(NumericRow row) throws StatementExecutionException, IoTDBConnectionException { - checkOpen(); - try { - session.insertRelationalRecord(row); - } catch (IoTDBConnectionException e) { - sessionPool.cleanSessionAndMayThrowConnectionException(session); - closed.set(true); - session = null; - throw e; - } - } - @Override public NumericRowPreparedStatement prepareInsertNumericRow( String tableName, diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java index fd1dd00cc3862..3c3168243346d 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java @@ -163,109 +163,6 @@ public static ByteBuffer getValueBuffer( return buffer; } - /** Encodes separate, densely packed FIELD and TAG arrays in the metadata column order. */ - public static ByteBuffer getValueBuffer( - List types, - long[] values, - List measurements, - List categories, - String[] tagValues) - throws IoTDBConnectionException { - return getValueBuffer(types, values, measurements, categories, tagValues, null); - } - - /** Null FIELD slots retain their indices but consume no payload bytes. */ - public static ByteBuffer getValueBuffer( - List types, - long[] values, - List measurements, - List categories, - String[] tagValues, - BitMap nullBitmap) - throws IoTDBConnectionException { - if (types == null - || values == null - || measurements == null - || categories == null - || tagValues == null - || types.size() != measurements.size() - || types.size() != categories.size()) { - throw invalidNumericRowArguments(); - } - validateNumericRowNullBitmap(nullBitmap, values.length); - int fieldCount = 0; - int tagCount = 0; - int length = types.size(); - for (int i = 0; i < types.size(); i++) { - TSDataType type = types.get(i); - if (categories.get(i) == ColumnCategory.TAG && type == TSDataType.STRING) { - if (tagCount >= tagValues.length) { - throw invalidNumericRowArguments(); - } - length += ReadWriteIOUtils.sizeToWrite(tagValues[tagCount++]); - } else if (categories.get(i) == ColumnCategory.FIELD && type != null) { - if (fieldCount >= values.length) { - throw invalidNumericRowArguments(); - } - int valueSize = - switch (type) { - case BOOLEAN -> Byte.BYTES; - case INT32, DATE, FLOAT -> Integer.BYTES; - case INT64, TIMESTAMP, DOUBLE -> Long.BYTES; - default -> throw new IoTDBConnectionException(MSG_UNSUPPORTED_DATA_TYPE + type); - }; - if (nullBitmap == null || !nullBitmap.isMarked(fieldCount)) { - length += valueSize; - } - fieldCount++; - } else { - throw new IllegalArgumentException( - SessionMessages - .EXCEPTION_NUMERIC_ROWS_SUPPORT_NUMERIC_FIELD_COLUMNS_AND_STRING_TAG_COLUMNS_ONLY_87159FFA); - } - } - if (fieldCount != values.length || tagCount != tagValues.length) { - throw invalidNumericRowArguments(); - } - ByteBuffer buffer = ByteBuffer.allocate(length); - int fieldIndex = 0; - int tagIndex = 0; - for (TSDataType type : types) { - ReadWriteIOUtils.write(type, buffer); - } - tagIndex = 0; - for (int i = 0; i < types.size(); i++) { - TSDataType type = types.get(i); - if (categories.get(i) == ColumnCategory.TAG && type == TSDataType.STRING) { - ReadWriteIOUtils.write(tagValues[tagIndex++], buffer); - } - } - fieldIndex = 0; - for (int i = 0; i < types.size(); i++) { - TSDataType type = types.get(i); - if (categories.get(i) == ColumnCategory.FIELD) { - long value = values[fieldIndex++]; - if (nullBitmap != null && nullBitmap.isMarked(fieldIndex - 1)) { - continue; - } - switch (type) { - case BOOLEAN -> buffer.put((byte) (value == 0 ? 0 : 1)); - case INT32, DATE, FLOAT -> buffer.putInt((int) value); - case INT64, TIMESTAMP, DOUBLE -> buffer.putLong(value); - default -> throw new IoTDBConnectionException(MSG_UNSUPPORTED_DATA_TYPE + type); - } - } - } - buffer.flip(); - return buffer; - } - - private static IllegalArgumentException invalidNumericRowArguments() { - return new IllegalArgumentException( - SessionMessages - .EXCEPTION_NUMERIC_ROW_ARGUMENTS_MUST_BE_NON_NULL_COLUMN_METADATA_LENGTHS_AND_PER_CATEGORY_VALUE_COUNTS_MUST_MATCH_EB12B233); - } - public static void validateNumericRowNullBitmap(BitMap nullBitmap, int fieldCount) { if (nullBitmap != null && nullBitmap.getSize() != fieldCount) { throw new IllegalArgumentException( diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java index 7a44a16536188..6e889f56875f0 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/PreparedNumericRowBufferTest.java @@ -30,6 +30,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static org.junit.Assert.assertEquals; @@ -42,9 +43,64 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class PreparedNumericRowBufferTest { + @Test + public void testAllNumericTypesPreserveRawBits() throws Exception { + Session session = mock(Session.class); + SessionConnection connection = mock(SessionConnection.class); + when(session.getDefaultSessionConnection()).thenReturn(connection); + when(connection.prepareNumericRow(any())).thenReturn(1L); + TableSession tableSession = new TableSession(session); + NumericRowPreparedStatement prepared = + tableSession.prepareInsertNumericRow( + "t", + List.of("b", "i", "l", "f", "d", "date", "ts"), + List.of( + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.DATE, + TSDataType.TIMESTAMP), + Collections.nCopies(7, ColumnCategory.FIELD)); + doAnswer( + invocation -> { + ByteBuffer values = invocation.getArgument(2); + assertEquals(37, values.remaining()); + assertEquals(1, values.get()); + assertEquals(Integer.MIN_VALUE, values.getInt()); + assertEquals(Long.MIN_VALUE, values.getLong()); + assertEquals(0x7fc01234, values.getInt()); + assertEquals(0x7ff8000000001234L, values.getLong()); + assertEquals(20260910, values.getInt()); + assertEquals(Long.MAX_VALUE, values.getLong()); + assertFalse(values.hasRemaining()); + return null; + }) + .when(connection) + .executePreparedNumericRow(eq(1L), anyLong(), any(ByteBuffer.class), any(ByteBuffer.class)); + // High bits of 32-bit fields are ignored; noncanonical NaNs must survive without normalization. + tableSession.executePreparedInsertNumericRow( + prepared, + 42, + new long[] { + -1, + 0x1234567880000000L, + Long.MIN_VALUE, + 0x123456787fc01234L, + 0x7ff8000000001234L, + 0x1234567800000000L | 20260910, + Long.MAX_VALUE + }, + new String[0]); + verify(connection) + .executePreparedNumericRow(eq(1L), eq(42L), any(ByteBuffer.class), any(ByteBuffer.class)); + } + // Reusing the statement after sparse/all-null rows must reset both payload and bitmap cursors. @Test public void testNullBitmapBufferReuse() throws Exception { diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java index 9f79ee200d91f..e4d2dbbb41dc8 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionTest.java @@ -22,18 +22,13 @@ import org.apache.iotdb.common.rpc.thrift.TAggregationType; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.isession.ISession; -import org.apache.iotdb.isession.ITableSession; -import org.apache.iotdb.isession.NumericRow; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.isession.util.Version; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.service.rpc.thrift.TSInsertRecordReq; import org.apache.iotdb.service.rpc.thrift.TSQueryTemplateResp; -import org.apache.iotdb.session.util.SessionUtils; import org.apache.tsfile.common.conf.TSFileConfig; -import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -46,7 +41,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; @@ -140,104 +134,6 @@ public void testBuildSession() { .build(); } - @Test - public void testNumericTableRowRequest() throws Exception { - // A BOOLEAN/DOUBLE row must carry the table RPC flags and explicit FIELD categories. - ITableSession tableSession = new TableSession((Session) session); - List names = Arrays.asList("enabled", "temperature"); - List types = Arrays.asList(TSDataType.BOOLEAN, TSDataType.DOUBLE); - tableSession.insert( - new NumericRow( - "readings", - 123L, - names, - types, - Collections.nCopies(types.size(), ColumnCategory.FIELD), - new long[] {1L, Double.doubleToRawLongBits(-0.0)}, - new String[0])); - ArgumentCaptor captor = ArgumentCaptor.forClass(TSInsertRecordReq.class); - Mockito.verify(sessionConnection).insertRecord(captor.capture()); - TSInsertRecordReq request = captor.getValue(); - Assert.assertTrue(request.isIsNumericRow()); - Assert.assertTrue(request.isIsWriteToTable()); - Assert.assertTrue(request.isIsAligned()); - assertEquals("readings", request.getPrefixPath()); - assertEquals(123L, request.getTimestamp()); - assertEquals(names, request.getMeasurements()); - assertEquals( - Collections.nCopies(2, (byte) ColumnCategory.FIELD.ordinal()), - request.getColumnCategoryies()); - } - - @Test - public void testNumericTableRowRejectsNonFieldColumnsBeforeRpc() throws Exception { - // Numeric TAG values are invalid and must fail locally without sending a request. - ITableSession tableSession = new TableSession((Session) session); - Assert.assertThrows( - IllegalArgumentException.class, () -> tableSession.insert((NumericRow) null)); - Assert.assertThrows( - IllegalArgumentException.class, - () -> - tableSession.insert( - new NumericRow( - "readings", - 1L, - Collections.singletonList("tag"), - Collections.singletonList(TSDataType.INT64), - Collections.singletonList(ColumnCategory.TAG), - new long[] {1}, - new String[0]))); - Mockito.verify(sessionConnection, Mockito.never()).insertRecord(any(TSInsertRecordReq.class)); - } - - @Test - public void testNumericTableRowWithTagsRequest() throws Exception { - // The NumericRow entry point must forward the TAG payload and metadata with numeric flags. - ITableSession tableSession = new TableSession((Session) session); - List names = Arrays.asList("enabled", "tag", "value", "region"); - List types = - Arrays.asList(TSDataType.BOOLEAN, TSDataType.STRING, TSDataType.DOUBLE, TSDataType.STRING); - List categories = - Arrays.asList( - ColumnCategory.FIELD, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.TAG); - List rpcNames = Arrays.asList("tag", "region", "enabled", "value"); - List rpcTypes = - Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.DOUBLE); - List rpcCategories = - Arrays.asList( - ColumnCategory.TAG, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD); - tableSession.insert( - new NumericRow( - "readings", - 123L, - names, - types, - categories, - new long[] {1, Double.doubleToRawLongBits(2.5)}, - new String[] {"device", "cn"})); - ArgumentCaptor captor = ArgumentCaptor.forClass(TSInsertRecordReq.class); - Mockito.verify(sessionConnection).insertRecord(captor.capture()); - TSInsertRecordReq request = captor.getValue(); - Assert.assertTrue(request.isIsNumericRow()); - Assert.assertTrue(request.isIsWriteToTable()); - assertEquals( - Arrays.asList( - (byte) ColumnCategory.TAG.ordinal(), - (byte) ColumnCategory.TAG.ordinal(), - (byte) ColumnCategory.FIELD.ordinal(), - (byte) ColumnCategory.FIELD.ordinal()), - request.getColumnCategoryies()); - assertEquals(rpcNames, request.getMeasurements()); - assertEquals( - SessionUtils.getValueBuffer( - rpcTypes, - new long[] {1, Double.doubleToRawLongBits(2.5)}, - rpcNames, - rpcCategories, - new String[] {"device", "cn"}), - request.bufferForValues()); - } - @Test public void testTimeZone() throws IoTDBConnectionException, StatementExecutionException { String timeZone = session.getTimeZone(); diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/PreparedNumericRowPoolTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/PreparedNumericRowPoolTest.java index f65f761c7b940..7a356747001b7 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/PreparedNumericRowPoolTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/pool/PreparedNumericRowPoolTest.java @@ -20,7 +20,6 @@ package org.apache.iotdb.session.pool; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.isession.NumericRow; import org.apache.iotdb.isession.NumericRowPreparedStatement; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; @@ -114,28 +113,6 @@ public void testClosedWrapperCannotUseReborrowedSession() throws Exception { verify(pool, times(2)).putBack(session); } - @Test - public void testClosedWrapperCannotInsertNumericRow() throws Exception { - wrapper.close(); - - IoTDBConnectionException error = - assertThrows( - IoTDBConnectionException.class, - () -> - wrapper.insert( - new NumericRow( - "t", - 1L, - List.of("tag", "v"), - List.of(TSDataType.STRING, TSDataType.INT64), - List.of(ColumnCategory.TAG, ColumnCategory.FIELD), - new long[] {42L}, - new String[] {"a"}))); - - assertEquals(SessionMessages.EXCEPTION_TABLE_SESSION_IS_CLOSED_77FE359B, error.getMessage()); - verifyZeroInteractions(client); - } - @Test public void testPrepareTransportFailureDiscardsSession() throws Exception { TException transportFailure = new TException("transport closed"); diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java index 632fbbda8fbf1..977c6113be906 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/util/SessionUtilsTest.java @@ -28,7 +28,6 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; -import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -46,38 +45,8 @@ public class SessionUtilsTest { - // Null FIELDs are omitted without shifting indices around interleaved TAG metadata or NaN bits. @Test - public void testNumericRowNullBitmapEncoding() throws Exception { - BitMap nulls = new BitMap(9); - nulls.mark(0); - nulls.mark(7); - nulls.mark(8); - List types = new ArrayList<>(Collections.nCopies(9, TSDataType.DOUBLE)); - List categories = new ArrayList<>(Collections.nCopies(9, ColumnCategory.FIELD)); - types.add(3, TSDataType.STRING); - categories.add(3, ColumnCategory.TAG); - long[] values = new long[9]; - Arrays.fill(values, 0x7ff8000000001234L); - ByteBuffer payload = - SessionUtils.getValueBuffer( - types, values, Collections.nCopies(10, "v"), categories, new String[] {"tag"}, nulls); - for (TSDataType type : types) { - Assert.assertEquals(type.serialize(), payload.get()); - } - Assert.assertEquals("tag", ReadWriteIOUtils.readString(payload)); - for (int i = 0; i < 6; i++) { - Assert.assertEquals(0x7ff8000000001234L, payload.getLong()); - } - Assert.assertFalse(payload.hasRemaining()); - Assert.assertEquals( - ByteBuffer.wrap(new byte[] {(byte) 0x81, 1}), - SessionUtils.encodeNumericRowNullBitmap(nulls, null)); - } - - // markAll may set padding in BitMap; the wire representation uses exactly ceil(FIELDs/8) bytes. - @Test - public void testNumericRowNullBitmapBoundaries() throws Exception { + public void testNumericRowNullBitmapBoundaries() { for (int size : new int[] {1, 7, 8, 9, 16}) { BitMap nulls = new BitMap(size); nulls.markAll(); @@ -87,177 +56,10 @@ public void testNumericRowNullBitmapBoundaries() throws Exception { Assert.assertEquals( size % 8 == 0 ? 255 : (1 << (size % 8)) - 1, bitmap.get(bitmap.limit() - 1) & 255); Assert.assertArrayEquals(original, nulls.getByteArray()); - ByteBuffer payload = - SessionUtils.getValueBuffer( - Collections.nCopies(size, TSDataType.INT64), - new long[size], - Collections.nCopies(size, "v"), - Collections.nCopies(size, ColumnCategory.FIELD), - new String[0], - nulls); - Assert.assertEquals(size, payload.remaining()); } assertThrows( IllegalArgumentException.class, - () -> - SessionUtils.getValueBuffer( - List.of(TSDataType.INT64), - new long[1], - List.of("v"), - List.of(ColumnCategory.FIELD), - new String[0], - new BitMap(2))); - } - - @Test - public void testNumericRowEncodingPreservesRawBits() throws IoTDBConnectionException { - List types = - Arrays.asList( - TSDataType.BOOLEAN, - TSDataType.BOOLEAN, - TSDataType.INT32, - TSDataType.INT64, - TSDataType.FLOAT, - TSDataType.DOUBLE, - TSDataType.DATE, - TSDataType.TIMESTAMP); - // Noncanonical NaN payloads must survive encoding without a float/double round trip. - long[] values = { - 0, - -7, - Integer.MIN_VALUE, - Long.MIN_VALUE, - 0x7fc01234, - 0x7ff8000000001234L, - 20260910, - Long.MAX_VALUE - }; - ByteBuffer actual = - SessionUtils.getValueBuffer( - types, - values, - Collections.nCopies(8, "s"), - Collections.nCopies(8, ColumnCategory.FIELD), - new String[0]); - ByteBuffer expected = ByteBuffer.allocate(46); - for (TSDataType type : types) { - expected.put(type.serialize()); - } - expected.put((byte) 0); - expected.put((byte) 1); - expected.putInt(Integer.MIN_VALUE); - expected.putLong(Long.MIN_VALUE); - expected.putInt(0x7fc01234); - expected.putLong(0x7ff8000000001234L); - expected.putInt(20260910); - expected.putLong(Long.MAX_VALUE); - expected.flip(); - Assert.assertEquals(expected, actual); - } - - @Test - public void testNumericRowRejectsInvalidValues() { - // Reject mismatched lengths, null types and every unsupported type before encoding. - assertThrows( - IllegalArgumentException.class, - () -> - SessionUtils.getValueBuffer( - Collections.singletonList(TSDataType.INT64), - new long[0], - Collections.singletonList("s"), - Collections.singletonList(ColumnCategory.FIELD), - new String[0])); - assertThrows( - IllegalArgumentException.class, - () -> - SessionUtils.getValueBuffer( - Collections.singletonList(null), - new long[] {1}, - Collections.singletonList("s"), - Collections.singletonList(ColumnCategory.FIELD), - new String[0])); - for (TSDataType type : - Arrays.asList( - TSDataType.TEXT, - TSDataType.STRING, - TSDataType.BLOB, - TSDataType.OBJECT, - TSDataType.VECTOR, - TSDataType.UNKNOWN)) { - assertThrows( - IoTDBConnectionException.class, - () -> - SessionUtils.getValueBuffer( - Collections.singletonList(type), - new long[] {1}, - Collections.singletonList("s"), - Collections.singletonList(ColumnCategory.FIELD), - new String[0])); - } - } - - @Test - public void testNumericRowWithSeparateTags() throws Exception { - // TAG strings, including null and empty, must not consume slots from the compact long[]. - List types = - Arrays.asList( - TSDataType.STRING, - TSDataType.DOUBLE, - TSDataType.STRING, - TSDataType.BOOLEAN, - TSDataType.STRING); - List categories = - Arrays.asList( - ColumnCategory.TAG, - ColumnCategory.FIELD, - ColumnCategory.TAG, - ColumnCategory.FIELD, - ColumnCategory.TAG); - ByteBuffer buffer = - SessionUtils.getValueBuffer( - types, - new long[] {0x7ff8000000001234L, -1}, - Arrays.asList("a", "d", "b", "enabled", "c"), - categories, - new String[] {"设备😀", null, ""}); - for (TSDataType type : types) { - Assert.assertEquals(type.serialize(), buffer.get()); - } - Assert.assertEquals("设备😀", ReadWriteIOUtils.readString(buffer)); - Assert.assertNull(ReadWriteIOUtils.readString(buffer)); - Assert.assertEquals("", ReadWriteIOUtils.readString(buffer)); - Assert.assertEquals(0x7ff8000000001234L, buffer.getLong()); - Assert.assertEquals(1, buffer.get()); - Assert.assertFalse(buffer.hasRemaining()); - } - - @Test - public void testNumericRowRejectsInvalidTagArrays() { - // Mismatched per-category counts and ATTRIBUTE values must fail before sending the RPC. - List types = Arrays.asList(TSDataType.STRING, TSDataType.INT64); - List names = Arrays.asList("tag", "field"); - List categories = Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD); - for (String[] tags : Arrays.asList(new String[0], new String[] {"a", "b"})) { - assertThrows( - IllegalArgumentException.class, - () -> SessionUtils.getValueBuffer(types, new long[] {1}, names, categories, tags)); - } - assertThrows( - IllegalArgumentException.class, - () -> - SessionUtils.getValueBuffer(types, new long[0], names, categories, new String[] {"a"})); - assertThrows( - IllegalArgumentException.class, - () -> SessionUtils.getValueBuffer(types, new long[] {1}, names, categories, null)); - assertThrows( - IllegalArgumentException.class, - () -> - SessionUtils.getValueBuffer( - types, - new long[] {1}, - names, - Arrays.asList(ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD), - new String[] {"a"})); + () -> SessionUtils.validateNumericRowNullBitmap(new BitMap(2), 1)); } @Test diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java index eb8cebad679ad..874f8ffa02e9a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java @@ -49,7 +49,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.DeleteDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; -import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; @@ -72,7 +71,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.UnsetSchemaTemplateStatement; import org.apache.iotdb.db.schemaengine.schemaregion.utils.MetaFormatUtils; import org.apache.iotdb.db.schemaengine.template.TemplateQueryType; -import org.apache.iotdb.db.utils.NumericRowValues; import org.apache.iotdb.db.utils.QueryDataSetUtils; import org.apache.iotdb.db.utils.TabletDecoder; import org.apache.iotdb.mpp.rpc.thrift.TFetchTimeseriesReq; @@ -285,10 +283,7 @@ public static InsertBaseStatement createStatement(TSInsertRecordReq insertRecord throws IllegalPathException, QueryProcessException { final long startTime = System.nanoTime(); // construct insert statement - InsertBaseStatement insertStatement = - insertRecordReq.isIsNumericRow() - ? new InsertNumericRowStatement() - : new InsertRowStatement(); + InsertRowStatement insertStatement = new InsertRowStatement(); insertStatement.setDevicePath( insertRecordReq.isIsWriteToTable() ? new PartialPath(insertRecordReq.getPrefixPath(), false) @@ -297,20 +292,8 @@ public static InsertBaseStatement createStatement(TSInsertRecordReq insertRecord insertStatement.setMeasurements(insertRecordReq.getMeasurements().toArray(new String[0])); insertStatement.setAligned(insertRecordReq.isAligned); - if (insertRecordReq.isIsNumericRow()) { - if (!insertRecordReq.isIsWriteToTable()) { - throw new IllegalArgumentException( - DataNodeQueryMessages - .QUERY_EXCEPTION_MISSING_OR_INVALID_COLUMN_CATEGORIES_FOR_TABLE_INSERTION_5DF990B9); - } - InsertNumericRowStatement numeric = (InsertNumericRowStatement) insertStatement; - numeric.setTime(insertRecordReq.getTimestamp()); - insertStatement.setAligned(true); - } else { - InsertRowStatement row = (InsertRowStatement) insertStatement; - row.setTime(insertRecordReq.getTimestamp()); - row.fillValues(insertRecordReq.values); - } + insertStatement.setTime(insertRecordReq.getTimestamp()); + insertStatement.fillValues(insertRecordReq.values); if (insertRecordReq.isIsWriteToTable()) { insertStatement.setWriteToTable(insertRecordReq.isIsWriteToTable()); if (!insertRecordReq.isSetColumnCategoryies() @@ -327,20 +310,6 @@ public static InsertBaseStatement createStatement(TSInsertRecordReq insertRecord ColumnCategory.values()[insertRecordReq.getColumnCategoryies().get(i).intValue()]); } insertStatement.setColumnCategories(columnCategories); - if (insertRecordReq.isIsNumericRow()) { - ((InsertNumericRowStatement) insertStatement) - .fillValues(insertRecordReq.values, insertRecordReq.bufferForNullBitmap()); - for (int i = 0; i < columnCategories.length; i++) { - TSDataType type = insertStatement.getDataType(i); - if (!(columnCategories[i] == TsTableColumnCategory.TAG && type == TSDataType.STRING) - && !(columnCategories[i] == TsTableColumnCategory.FIELD - && NumericRowValues.supports(type))) { - throw new IllegalArgumentException( - DataNodeQueryMessages - .QUERY_EXCEPTION_MISSING_OR_INVALID_COLUMN_CATEGORIES_FOR_TABLE_INSERTION_5DF990B9); - } - } - } } PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return insertStatement; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java index 62970448eb401..46b70fea2c67d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertNumericRowStatement.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.queryengine.plan.statement.crud; -import org.apache.iotdb.calc.exception.QueryProcessException; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.path.PartialPath; @@ -42,9 +41,7 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; -import org.apache.tsfile.write.UnSupportedDataTypeException; -import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -117,68 +114,6 @@ public int getValueCount() { return numericValues == null ? 0 : numericValues.size(); } - /** RPC cells carry typed primitive FIELD bits or a length-prefixed STRING TAG. */ - public void fillValues(ByteBuffer buffer) throws QueryProcessException { - fillValues(buffer, null); - } - - public void fillValues(ByteBuffer buffer, ByteBuffer nullBitmap) throws QueryProcessException { - dataTypes = new TSDataType[measurements.length]; - for (int i = 0; i < dataTypes.length; i++) { - byte type = buffer.get(); - if (type < 0 || type >= TSDataType.values().length) { - throw new QueryProcessException(DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE + type); - } - dataTypes[i] = TSDataType.values()[type]; - if (columnCategories[i] == TsTableColumnCategory.TAG && dataTypes[i] != TSDataType.STRING) { - throw new IllegalArgumentException( - DataNodeQueryMessages - .QUERY_EXCEPTION_MISSING_OR_INVALID_COLUMN_CATEGORIES_FOR_TABLE_INSERTION_5DF990B9); - } - if (columnCategories[i] == TsTableColumnCategory.FIELD - && !NumericRowValues.supports(dataTypes[i])) { - if (dataTypes[i] == TSDataType.STRING) { - throw new IllegalArgumentException( - DataNodeQueryMessages - .QUERY_EXCEPTION_MISSING_OR_INVALID_COLUMN_CATEGORIES_FOR_TABLE_INSERTION_5DF990B9); - } - throw new UnSupportedDataTypeException( - DataNodeQueryMessages.UNSUPPORTED_DATA_TYPE + dataTypes[i]); - } - } - int tagCount = countTagColumns(columnCategories); - NumericRowValues.validateNullBitmap(nullBitmap, measurements.length - tagCount); - numericValues = new NumericRowValues(tagCount, measurements.length - tagCount); - ColumnCategory[] categories = new ColumnCategory[measurements.length]; - for (int i = 0; i < measurements.length; i++) { - categories[i] = - columnCategories[i] == TsTableColumnCategory.TAG - ? ColumnCategory.TAG - : ColumnCategory.FIELD; - } - numericValues.setColumnCategories(categories); - for (int i = 0; i < dataTypes.length; i++) { - if (columnCategories[i] == TsTableColumnCategory.TAG) { - numericValues.read(i, dataTypes[i], buffer); - } - } - for (int i = 0; i < dataTypes.length; i++) { - if (columnCategories[i] == TsTableColumnCategory.FIELD) { - if (NumericRowValues.isNullField(nullBitmap, i - tagCount)) { - numericValues.setNull(i); - } else { - numericValues.read(i, dataTypes[i], buffer); - } - } - } - if (buffer.hasRemaining()) { - throw new IllegalArgumentException( - String.format( - DataNodeQueryMessages.EXCEPTION_NUMERIC_ROW_PAYLOAD_HAS_ARG_TRAILING_BYTES_89E46A34, - buffer.remaining())); - } - } - @Override public R accept(StatementVisitor visitor, C context) { return visitor.visitInsertNumericRow(this, context); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java index cf3533296b1af..9fc462abfff87 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/session/PreparedNumericRowInfoTest.java @@ -30,14 +30,12 @@ import org.apache.iotdb.db.queryengine.plan.analyze.lock.DataNodeSchemaLockManager; import org.apache.iotdb.db.queryengine.plan.analyze.lock.SchemaLockType; import org.apache.iotdb.db.queryengine.plan.analyze.schema.SchemaValidator; -import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertNumericRow; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertNumericRowStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; -import org.apache.iotdb.service.rpc.thrift.TSInsertRecordReq; import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; import org.apache.tsfile.enums.ColumnCategory; @@ -289,25 +287,6 @@ public void testNullBitmapDecodingAndValidation() throws Exception { InsertNumericRowStatement prepared = info.bind(1, tagBuffer(1), values, bitmap); assertEquals(1, bitmap.position()); assertEquals(0, values.position()); - ByteBuffer payload = ByteBuffer.allocate(10 + 5 + values.remaining()); - payload.put(TSDataType.STRING.serialize()); - for (int i = 0; i < 9; i++) { - payload.put(TSDataType.DOUBLE.serialize()); - } - payload.put(tagBuffer(1)).put(values.duplicate()).flip(); - TSInsertRecordReq insertReq = new TSInsertRecordReq(); - insertReq - .setPrefixPath("t") - .setTimestamp(1) - .setMeasurements(req.getMeasurements()) - .setColumnCategoryies(req.getColumnCategories()) - .setIsNumericRow(true) - .setIsWriteToTable(true) - .setValues(payload) - .setNullBitmap(bitmap); - InsertNumericRowStatement direct = - (InsertNumericRowStatement) StatementGenerator.createStatement(insertReq); - assertEquals(prepared.getNumericValues(), direct.getNumericValues()); assertTrue(prepared.getNumericValues().isNull(1)); assertTrue(prepared.getNumericValues().isNull(8)); assertTrue(prepared.getNumericValues().isNull(9)); @@ -317,17 +296,11 @@ public void testNullBitmapDecodingAndValidation() throws Exception { assertThrows( IllegalArgumentException.class, () -> info.bind(1, tagBuffer(1), values, ByteBuffer.wrap(bad))); - insertReq.setValues(payload.duplicate().rewind()).setNullBitmap(bad); - assertThrows( - IllegalArgumentException.class, () -> StatementGenerator.createStatement(insertReq)); } assertThrows( SemanticException.class, () -> info.bind(1, tagBuffer(1), ByteBuffer.allocate(9 * 8), bitmap)); ByteBuffer allNulls = ByteBuffer.wrap(new byte[] {(byte) 255, 1}); - insertReq.setValues(payload.duplicate().rewind()).setNullBitmap(allNulls); - assertThrows( - IllegalArgumentException.class, () -> StatementGenerator.createStatement(insertReq)); InsertNumericRowStatement allNull = info.bind(1, tagBuffer(1), ByteBuffer.allocate(0), allNulls); for (int i = 1; i <= 9; i++) { @@ -335,6 +308,29 @@ public void testNullBitmapDecodingAndValidation() throws Exception { } } + @Test + public void testUnsupportedColumnTypesRejectedDuringPrepare() { + for (TSDataType type : + new TSDataType[] { + TSDataType.TEXT, + TSDataType.STRING, + TSDataType.BLOB, + TSDataType.OBJECT, + TSDataType.VECTOR, + TSDataType.UNKNOWN + }) { + TSPrepareNumericRowReq req = request("v"); + req.getDataTypes().set(0, type.ordinal()); + assertThrows(SemanticException.class, () -> new PreparedNumericRowInfo(DATABASE, req)); + } + TSPrepareNumericRowReq req = request("tag", "v"); + req.getDataTypes().set(0, TSDataType.INT64.ordinal()); + assertThrows(SemanticException.class, () -> new PreparedNumericRowInfo(DATABASE, req)); + req.getDataTypes().set(0, TSDataType.STRING.ordinal()); + req.getColumnCategories().set(0, (byte) ColumnCategory.ATTRIBUTE.ordinal()); + assertThrows(SemanticException.class, () -> new PreparedNumericRowInfo(DATABASE, req)); + } + private static void install(TsTable table) { DataNodeTableCache.getInstance().preUpdateTable(DATABASE, table, null); DataNodeTableCache.getInstance().commitUpdateTable(DATABASE, "t", null); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java index fbb69b4be0442..cc85949aaad7b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/NumericTableRowTest.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.consensus.statemachine.dataregion.IoTConsensusDataRegionStateMachine; import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTablePatternParser; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletBatchReqV2; +import org.apache.iotdb.db.protocol.session.PreparedNumericRowInfo; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertNumericRowNode; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; @@ -53,8 +54,7 @@ import org.apache.iotdb.db.utils.NumericRowValues; import org.apache.iotdb.db.utils.datastructure.AlignedTVList; import org.apache.iotdb.rpc.TSStatusCode; -import org.apache.iotdb.service.rpc.thrift.TSInsertRecordReq; -import org.apache.iotdb.session.util.SessionUtils; +import org.apache.iotdb.service.rpc.thrift.TSPrepareNumericRowReq; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.ColumnCategory; @@ -62,7 +62,6 @@ import org.apache.tsfile.read.common.type.TypeFactory; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Test; @@ -81,10 +80,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -/** Exercises one numeric row from the client codec through schema edits, WAL and memtable. */ +/** Exercises one numeric row from prepared binding through schema edits, WAL and memtable. */ public class NumericTableRowTest { private static final List NAMES = Arrays.asList("b", "i", "l", "f", "d", "date", "ts"); private static final List TYPES = @@ -100,30 +98,32 @@ public class NumericTableRowTest { 1, Integer.MIN_VALUE, Long.MIN_VALUE, 0x7fc01234, 0x7ff8000000001234L, 20260910, Long.MAX_VALUE }; - private TSInsertRecordReq request() throws Exception { - return new TSInsertRecordReq() - .setPrefixPath("numeric_table") - .setTimestamp(42) - .setMeasurements(NAMES) - .setValues( - SessionUtils.getValueBuffer( - TYPES, - BITS, - NAMES, - Collections.nCopies(NAMES.size(), ColumnCategory.FIELD), - new String[0])) - .setIsAligned(true) - .setIsWriteToTable(true) - .setIsNumericRow(true) - .setColumnCategoryies( - Collections.nCopies(NAMES.size(), (byte) ColumnCategory.FIELD.ordinal())); + private InsertNumericRowStatement preparedRow() { + TSPrepareNumericRowReq req = + new TSPrepareNumericRowReq() + .setTableName("numeric_table") + .setMeasurements(NAMES) + .setDataTypes(TYPES.stream().map(TSDataType::ordinal).toList()) + .setColumnCategories( + Collections.nCopies(NAMES.size(), (byte) ColumnCategory.FIELD.ordinal())); + ByteBuffer values = ByteBuffer.allocate(37); + values + .put((byte) BITS[0]) + .putInt((int) BITS[1]) + .putLong(BITS[2]) + .putInt((int) BITS[3]) + .putLong(BITS[4]) + .putInt((int) BITS[5]) + .putLong(BITS[6]) + .flip(); + return new PreparedNumericRowInfo("numeric_test", req) + .bind(42, ByteBuffer.allocate(0), values, null); } @Test - public void testRpcDecodeAndSchemaChangesRetainPrimitiveStorage() throws Exception { + public void testPreparedBindAndSchemaChangesRetainPrimitiveStorage() throws Exception { // Missing TAG insertion, column swaps and partial failure recovery must retain long[] values. - InsertNumericRowStatement statement = - (InsertNumericRowStatement) StatementGenerator.createStatement(request()); + InsertNumericRowStatement statement = preparedRow(); assertArrayEquals(BITS, statement.getNumericValues().getValues()); assertObjectStorageAbsent(InsertNumericRowStatement.class, statement); assertTrue(statement.isWriteToTable()); @@ -149,18 +149,6 @@ public void testRpcDecodeAndSchemaChangesRetainPrimitiveStorage() throws Excepti assertEquals("numeric_table", statement.getTableDeviceID().getTableName()); } - @Test - public void testRpcRejectsNumericTreeAndNonStringTagRequests() throws Exception { - // Raw RPC callers must use STRING TAGs and numeric FIELDs in the table model. - TSInsertRecordReq tree = request().setIsWriteToTable(false); - assertThrows(IllegalArgumentException.class, () -> StatementGenerator.createStatement(tree)); - TSInsertRecordReq tag = - request() - .setColumnCategoryies( - Collections.nCopies(NAMES.size(), (byte) ColumnCategory.TAG.ordinal())); - assertThrows(IllegalArgumentException.class, () -> StatementGenerator.createStatement(tag)); - } - @Test public void testStringTagsSurviveSchemaChanges() throws Exception { // The client places TAG metadata before FIELD metadata while retaining dense value arrays. @@ -170,24 +158,20 @@ public void testStringTagsSurviveSchemaChanges() throws Exception { List categories = Arrays.asList( ColumnCategory.TAG, ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD); - TSInsertRecordReq req = - request() + TSPrepareNumericRowReq req = + new TSPrepareNumericRowReq() + .setTableName("numeric_table") .setMeasurements(names) - .setValues( - SessionUtils.getValueBuffer( - types, - new long[] {1, Long.MIN_VALUE}, - names, - categories, - new String[] {"设备😀", ""})) - .setColumnCategoryies( - Arrays.asList( - (byte) ColumnCategory.TAG.ordinal(), - (byte) ColumnCategory.TAG.ordinal(), - (byte) ColumnCategory.FIELD.ordinal(), - (byte) ColumnCategory.FIELD.ordinal())); + .setDataTypes(types.stream().map(TSDataType::ordinal).toList()) + .setColumnCategories(categories.stream().map(c -> (byte) c.ordinal()).toList()); + ByteBuffer tags = ByteBuffer.allocate(64); + ReadWriteIOUtils.write("设备😀", tags); + ReadWriteIOUtils.write("", tags); + tags.flip(); + ByteBuffer values = ByteBuffer.allocate(9); + values.put((byte) 1).putLong(Long.MIN_VALUE).flip(); InsertNumericRowStatement statement = - (InsertNumericRowStatement) StatementGenerator.createStatement(req); + new PreparedNumericRowInfo("numeric_test", req).bind(42, tags, values, null); assertEquals("设备😀", statement.getNumericValues().getTag(0)); assertEquals("", statement.getNumericValues().getTag(1)); assertArrayEquals(new long[] {1, Long.MIN_VALUE}, statement.getValues()); @@ -241,8 +225,7 @@ public void testStringTagsSurviveWalConsensusAndTabletAdapters() throws Exceptio @Test public void testBulkTagExpansionKeepsValuesAndNulls() throws Exception { // A reversed schema mapping with two missing TAGs must preserve numeric bits and null slots. - InsertNumericRowStatement statement = - (InsertNumericRowStatement) StatementGenerator.createStatement(request()); + InsertNumericRowStatement statement = preparedRow(); statement.rebuildArraysAfterExpansion( new int[] {-1, -1, 6, 5, 4, 3, 2, 1, 0}, new String[] {"tag1", "tag2", "ts", "date", "d", "f", "l", "i", "b"}); @@ -384,8 +367,7 @@ public void testPipeBatchRetainsIndependentNumericStatements() throws Exception @Test public void testNumericRowRedirectUsesSingleEndpoint() throws Exception { - InsertNumericRowStatement statement = - (InsertNumericRowStatement) StatementGenerator.createStatement(request()); + InsertNumericRowStatement statement = preparedRow(); Analysis analysis = new Analysis(new InsertNumericRow(statement, null), Collections.emptyMap()); TEndPoint local = new TEndPoint("127.0.0.1", 6667); TEndPoint leader = new TEndPoint("127.0.0.1", 6668); @@ -489,35 +471,6 @@ public Object getValue(int index) { } } - @Test - public void testRpcRejectsNonNumericTypes() throws Exception { - // Raw RPC callers cannot smuggle unsupported values into numeric FIELD storage. - for (TSDataType type : - Arrays.asList( - TSDataType.TEXT, - TSDataType.BLOB, - TSDataType.OBJECT, - TSDataType.VECTOR, - TSDataType.UNKNOWN)) { - TSInsertRecordReq invalid = - request().setValues(ByteBuffer.wrap(new byte[] {type.serialize()})); - assertThrows( - UnSupportedDataTypeException.class, () -> StatementGenerator.createStatement(invalid)); - } - // STRING has a valid wire representation for TAGs, but must still be rejected as a FIELD. - ByteBuffer stringValue = ByteBuffer.allocate(32); - ReadWriteIOUtils.write(TSDataType.STRING, stringValue); - ReadWriteIOUtils.write("invalid", stringValue); - stringValue.flip(); - TSInsertRecordReq stringField = - request() - .setMeasurements(Collections.singletonList("s")) - .setColumnCategoryies(Collections.singletonList((byte) ColumnCategory.FIELD.ordinal())) - .setValues(stringValue); - assertThrows( - IllegalArgumentException.class, () -> StatementGenerator.createStatement(stringField)); - } - @Test public void testTabletAdaptersDoNotMaterializeObjectRows() throws Exception { // Pipe and subscription must preserve values and null TAGs without boxing the entire row. @@ -548,8 +501,7 @@ public void testTabletAdaptersDoNotMaterializeObjectRows() throws Exception { } private RelationalInsertNumericRowNode nodeWithNullTag() throws Exception { - InsertNumericRowStatement statement = - (InsertNumericRowStatement) StatementGenerator.createStatement(request()); + InsertNumericRowStatement statement = preparedRow(); statement.insertColumn( 0, new ColumnSchema( diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift index edcf8834804fc..b01ef1a294652 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift @@ -248,11 +248,6 @@ struct TSInsertRecordReq { 6: optional bool isAligned 7: optional bool isWriteToTable 8: optional list columnCategoryies - // Numeric table rows retain primitive storage throughout the server write path. - 9: optional bool isNumericRow - // Numeric FIELDs only, in FIELD occurrence order: bit i (LSB first) means null. - // Exactly ceil(FIELD count / 8) bytes, unused high bits zero; null FIELD payloads are omitted. - 10: optional binary nullBitmap } struct TSInsertStringRecordReq { @@ -738,6 +733,7 @@ struct TSExecutePreparedNumericRowReq { 4: required binary values // Length-prefixed UTF-8 TAGs; length -1 represents a null TAG. 5: required binary tagValues - // Same FIELD-only bitmap and omitted-null-payload encoding as TSInsertRecordReq. + // FIELD-only bitmap: bit i (LSB first) marks FIELD i as null; null values omit payload bytes. + // Exactly ceil(FIELD count / 8) bytes, with unused high bits zero; absent means no nulls. 6: optional binary nullBitmap } From c22f833ad3e979bc435bdc4653e9188637d109bb Mon Sep 17 00:00:00 2001 From: Tian Jiang Date: Mon, 12 Oct 2026 12:19:13 +0800 Subject: [PATCH 20/20] Declare PowerMock reflection dependency for DataNode tests --- iotdb-core/datanode/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml index 48bc0490611dd..658861c66a9bf 100644 --- a/iotdb-core/datanode/pom.xml +++ b/iotdb-core/datanode/pom.xml @@ -279,6 +279,11 @@ powermock-core test + + org.powermock + powermock-reflect + test + org.powermock powermock-module-junit4