From 4c96388169eb31af69ac5da7d4d5d74972dbd618 Mon Sep 17 00:00:00 2001 From: Vladislav Pyatkov Date: Mon, 6 Jul 2026 22:06:38 +0300 Subject: [PATCH] IGNITE-28864 Add technical columns to Calcite engin SQL scan --- .../query/calcite/prepare/IgnitePlanner.java | 2 +- .../calcite/prepare/IgniteSqlValidator.java | 79 +++- .../schema/CacheTableDescriptorImpl.java | 116 +++++- .../query/calcite/util/IgniteResource.java | 4 + .../integration/TechnicalColumnsScanTest.java | 375 ++++++++++++++++++ .../testsuites/IntegrationTestSuite.java | 2 + .../query/QueryTypeDescriptorImpl.java | 4 +- .../internal/processors/query/QueryUtils.java | 22 + .../schema/management/SchemaManager.java | 3 +- 9 files changed, 578 insertions(+), 29 deletions(-) create mode 100644 modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgnitePlanner.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgnitePlanner.java index 553ca3ba89a94..99b7d33f10627 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgnitePlanner.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgnitePlanner.java @@ -365,7 +365,7 @@ private static boolean isAsCall(SqlNode node) { } CalciteCatalogReader catalogReader = this.catalogReader.withSchemaPath(schemaPath); - SqlValidator validator = new IgniteSqlValidator(operatorTbl, catalogReader, typeFactory, validatorCfg, ctx.parameters()); + IgniteSqlValidator validator = new IgniteSqlValidator(operatorTbl, catalogReader, typeFactory, validatorCfg, ctx.parameters()); SqlToRelConverter sqlToRelConverter = sqlToRelConverter(validator, catalogReader, sqlToRelConverterCfg); RelRoot root = sqlToRelConverter.convertQuery(sqlNode, true, false); root = root.withRel(sqlToRelConverter.decorrelate(sqlNode, root.rel)); diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java index 1c7f00da26537..91567b9cb5529 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java @@ -136,16 +136,32 @@ public IgniteSqlValidator( nullType = typeFactory.createSqlType(SqlTypeName.NULL); } + /** {@inheritDoc} */ @Override public void validateInsert(SqlInsert insert) { validateTableModify(insert.getTargetTable()); if (insert.getTargetColumnList() == null) insert.setOperand(3, inferColumnList(insert)); + else + validateInsertTargets(insert); super.validateInsert(insert); } + /** + * Validates insert target columns to ensure they do not contain any technical columns. + * + * @param insert Insert statement. + */ + private void validateInsertTargets(SqlInsert insert) { + for (SqlNode node : insert.getTargetColumnList()) { + SqlIdentifier id = (SqlIdentifier)node; + + validateTechnicalColumnDmlTarget(id); + } + } + /** {@inheritDoc} */ @Override public void validateUpdate(SqlUpdate call) { validateTableModify(call.getTargetTable()); @@ -198,10 +214,19 @@ private void validateTableModify(SqlNode table) { SqlIdentifier alias = call.getAlias() != null ? call.getAlias() : new SqlIdentifier(deriveAlias(targetTable, 0), SqlParserPos.ZERO); - table.unwrap(IgniteTable.class).descriptor().selectForUpdateRowType((IgniteTypeFactory)typeFactory) - .getFieldNames().stream() - .map(name -> alias.plus(name, SqlParserPos.ZERO)) - .forEach(selectList::add); + RelDataType updateRowType = table.unwrap(IgniteTable.class).descriptor() + .selectForUpdateRowType(typeFactory()); + + for (RelDataTypeField field : updateRowType.getFieldList()) { + if (QueryUtils.isTechnicalFieldName(field.getName())) { + selectList.add(SqlValidatorUtil.addAlias( + SqlLiteral.createNull(SqlParserPos.ZERO), + "__IGNITE_DML" + field.getName() + )); + } + else + selectList.add(alias.plus(field.getName(), SqlParserPos.ZERO)); + } int ordinal = 0; // Force unique aliases to avoid a duplicate for Y with SET X=Y @@ -247,6 +272,7 @@ private void validateTableModify(SqlNode table) { super.validateSelect(select, targetRowType); } + /** {@inheritDoc} */ @Override protected void validateNamespace(SqlValidatorNamespace namespace, RelDataType targetRowType) { SqlValidatorTable table = namespace.getTable(); @@ -294,7 +320,7 @@ else if (n instanceof SqlDynamicParam) { if (call.getKind() == SqlKind.AS) { final String alias = deriveAlias(call, 0); - if (isSystemFieldName(alias)) + if (QueryUtils.isReservedFieldName(alias)) throw newValidationError(call, IgniteResource.INSTANCE.illegalAlias(alias)); } else if (call.getKind() == SqlKind.CAST) { @@ -419,18 +445,26 @@ private SqlNode rewriteTableToQuery(SqlNode from) { SelectScope scope, boolean includeSysVars ) { - if (!includeSysVars && exp.getKind() == SqlKind.IDENTIFIER && isSystemFieldName(deriveAlias(exp, 0))) { - SqlQualified qualified = scope.fullyQualify((SqlIdentifier)exp); + if (!includeSysVars && exp.getKind() == SqlKind.IDENTIFIER) { + String alias = deriveAlias(exp, 0); - if (qualified.namespace == null) + // Technical columns are never exposed via SELECT *. + if (QueryUtils.isTechnicalFieldNameIgnoreCase(alias)) return; - if (qualified.namespace.getTable() != null) { - // If child is table and has only system fields, expand star to these fields. - // Otherwise, expand star to non-system fields only. - for (RelDataTypeField fld : qualified.namespace.getRowType().getFieldList()) { - if (!isSystemField(fld)) - return; + if (QueryUtils.isReservedFieldName(alias)) { + SqlQualified qualified = scope.fullyQualify((SqlIdentifier)exp); + + if (qualified.namespace == null) + return; + + if (qualified.namespace.getTable() != null) { + // If child is table and has only system fields, expand star to these fields. + // Otherwise, expand star to non-system fields only. + for (RelDataTypeField fld : qualified.namespace.getRowType().getFieldList()) { + if (!isSystemField(fld)) + return; + } } } } @@ -440,7 +474,7 @@ private SqlNode rewriteTableToQuery(SqlNode from) { /** {@inheritDoc} */ @Override public boolean isSystemField(RelDataTypeField field) { - return isSystemFieldName(field.getName()); + return QueryUtils.isReservedFieldName(field.getName()); } /** */ @@ -524,6 +558,7 @@ private void validateUpdateFields(SqlUpdate call) { for (SqlNode node : call.getTargetColumnList()) { SqlIdentifier id = (SqlIdentifier)node; + validateTechnicalColumnDmlTarget(id); RelDataTypeField target = SqlValidatorUtil.getTargetField( baseType, typeFactory(), id, getCatalogReader(), relOptTable); @@ -566,12 +601,6 @@ private IgniteTypeFactory typeFactory() { return (IgniteTypeFactory)typeFactory; } - /** */ - private boolean isSystemFieldName(String alias) { - return QueryUtils.KEY_FIELD_NAME.equalsIgnoreCase(alias) - || QueryUtils.VAL_FIELD_NAME.equalsIgnoreCase(alias); - } - /** {@inheritDoc} */ @Override public RelDataType deriveType(SqlValidatorScope scope, SqlNode expr) { if (expr instanceof SqlDynamicParam) { @@ -584,6 +613,14 @@ private boolean isSystemFieldName(String alias) { return super.deriveType(scope, expr); } + /** */ + private void validateTechnicalColumnDmlTarget(SqlIdentifier id) { + String fieldName = id.names.get(id.names.size() - 1); + + if (QueryUtils.isTechnicalFieldNameIgnoreCase(fieldName)) + throw newValidationError(id, IgniteResource.INSTANCE.cannotModifyTechnicalColumn(id.toString())); + } + /** @return A derived type or {@code null} if unable to determine. */ @Nullable private RelDataType deriveDynamicParameterType(SqlDynamicParam node, RelDataType nullValType) { RelDataType type = getValidatedNodeTypeIfKnown(node); diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java index 2cd164bc81570..e3a8085baa61c 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java @@ -53,6 +53,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.query.GridQueryProperty; import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor; import org.apache.ignite.internal.processors.query.QueryUtils; @@ -123,7 +124,7 @@ public CacheTableDescriptorImpl(GridCacheContextInfo cacheInfo, GridQueryT Set fields = this.typeDesc.fields().keySet(); - List descriptors = new ArrayList<>(fields.size() + 2); + List descriptors = new ArrayList<>(fields.size() + 4); // A _key/_val field is virtual in case there is an alias or a property(es) mapped to the _key/_val field. BitSet virtualFields = new BitSet(); @@ -177,6 +178,28 @@ else if (Objects.equals(field, typeDesc.valueFieldAlias())) { } } + virtualFields.set(descriptors.size()); + descriptors.add(new TechnicalDescriptor(QueryUtils.VER_FIELD_NAME, GridCacheVersion.class, descriptors.size()) { + @Override public Object value(ExecutionContext ectx, GridCacheContext cctx, CacheDataRow src) + throws IgniteCheckedException { + GridCacheVersion ver = src.version(); + + if (ver != null) + return ver; + + CacheDataRow row = cctx.offheap().read(cctx, src.key()); + + return row == null ? null : row.version(); + } + }); + + virtualFields.set(descriptors.size()); + descriptors.add(new TechnicalDescriptor(QueryUtils.SRC_FIELD_NAME, Integer.class, descriptors.size()) { + @Override public Object value(ExecutionContext ectx, GridCacheContext cctx, CacheDataRow src) { + return cctx.cacheId(); + } + }); + Map descriptorsMap = U.newHashMap(descriptors.size()); for (CacheColumnDescriptor descriptor : descriptors) descriptorsMap.put(descriptor.name(), descriptor); @@ -278,6 +301,9 @@ else if (affFields.isEmpty()) @Override public boolean isUpdateAllowed(RelOptTable tbl, int colIdx) { final CacheColumnDescriptor desc = descriptors[colIdx]; + if (QueryUtils.isTechnicalFieldName(desc.name())) + return false; + return !desc.key() && (desc.field() || QueryUtils.isSqlType(desc.storageType())); } @@ -382,9 +408,12 @@ private Object insertVal(Row row, ExecutionContext ectx) throws Ignit for (int i = 2; i < descriptors.length; i++) { final CacheColumnDescriptor desc = descriptors[i]; + if (!desc.field() || desc.key()) + continue; + Object fieldVal = hnd.get(i, row); - if (desc.field() && !desc.key() && fieldVal != null) + if (fieldVal != null) desc.set(val, TypeUtils.fromInternal(ectx, fieldVal, desc.storageType())); } } @@ -442,14 +471,18 @@ private ModifyTuple mergeTuple(Row row, List updateColList, Execut int rowColumnsCnt = hnd.columnCount(row); - if (rowColumnsCnt == descriptors.length) + // An empty update column list unambiguously means there is no WHEN MATCHED clause at all (a MERGE + // statement always has at least one WHEN clause), so the row can only originate from the INSERT + // section. Note: the row width alone can't be used to detect this case, since, depending on the + // number of updated columns, it may coincide with the width of a WHEN MATCHED-only row. + if (updateColList.isEmpty()) return insertTuple(row, ectx); // Only WHEN NOT MATCHED clause in MERGE. else if (rowColumnsCnt == descriptors.length + updateColList.size()) return updateTuple(row, updateColList, 0, ectx); // Only WHEN MATCHED clause in MERGE. else { // Both WHEN MATCHED and WHEN NOT MATCHED clauses in MERGE. - assert rowColumnsCnt == descriptors.length * 2 + updateColList.size() : "Unexpected columns count: " + - rowColumnsCnt; + assert rowColumnsCnt == 2 * descriptors.length + updateColList.size() : "Unexpected columns count: " + + rowColumnsCnt; int updateOffset = descriptors.length; // Offset of fields for update statement. @@ -805,6 +838,79 @@ private FieldDescriptor(GridQueryProperty desc, int fieldIdx) { } } + /** */ + private abstract static class TechnicalDescriptor implements CacheColumnDescriptor { + /** */ + private final String name; + + /** */ + private final Class storageType; + + /** */ + private final int fieldIdx; + + /** */ + private volatile RelDataType logicalType; + + /** */ + private TechnicalDescriptor(String name, Class storageType, int fieldIdx) { + this.name = name; + this.storageType = storageType; + this.fieldIdx = fieldIdx; + } + + /** {@inheritDoc} */ + @Override public boolean field() { + return false; + } + + /** {@inheritDoc} */ + @Override public boolean key() { + return false; + } + + /** {@inheritDoc} */ + @Override public boolean hasDefaultValue() { + return false; + } + + /** {@inheritDoc} */ + @Override public Object defaultValue() { + throw new AssertionError(); + } + + /** {@inheritDoc} */ + @Override public String name() { + return name; + } + + /** {@inheritDoc} */ + @Override public int fieldIndex() { + return fieldIdx; + } + + /** {@inheritDoc} */ + @Override public RelDataType logicalType(IgniteTypeFactory f) { + if (logicalType == null) { + logicalType = storageType == GridCacheVersion.class + ? f.createTypeWithNullability(f.createJavaType(storageType), true) + : TypeUtils.sqlType(f, storageType, PRECISION_NOT_SPECIFIED, SCALE_NOT_SPECIFIED, true); + } + + return logicalType; + } + + /** {@inheritDoc} */ + @Override public Class storageType() { + return storageType; + } + + /** {@inheritDoc} */ + @Override public void set(Object dst, Object val) { + throw new AssertionError(); + } + } + /** {@inheritDoc} */ @Override public GridQueryTypeDescriptor typeDescription() { return typeDesc; diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteResource.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteResource.java index df74d56a91638..e7cf8d3a62bf4 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteResource.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/util/IgniteResource.java @@ -35,6 +35,10 @@ public interface IgniteResource { @Resources.BaseMessage("Cannot update field \"{0}\". You cannot update key, key fields or val field in case the val is a complex type.") Resources.ExInst cannotUpdateField(String field); + /** */ + @Resources.BaseMessage("Cannot modify technical column \"{0}\".") + Resources.ExInst cannotModifyTechnicalColumn(String field); + /** */ @Resources.BaseMessage("Illegal aggregate function. {0} is unsupported at the moment.") Resources.ExInst unsupportedAggregationFunction(String a0); diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java new file mode 100644 index 0000000000000..c8f1fed7bc09b --- /dev/null +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/TechnicalColumnsScanTest.java @@ -0,0 +1,375 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite.integration; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import javax.cache.CacheException; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.calcite.CalciteQueryEngineConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.SqlConfiguration; +import org.apache.ignite.configuration.TransactionConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.processors.query.IgniteSQLException; +import org.apache.ignite.internal.processors.query.QueryUtils; +import org.apache.ignite.internal.processors.query.calcite.CalciteQueryProcessor; +import org.apache.ignite.internal.processors.query.calcite.exec.ArrayRowHandler; +import org.apache.ignite.internal.processors.query.calcite.exec.ExecutionContext; +import org.apache.ignite.internal.processors.query.calcite.exec.tracker.NoOpIoTracker; +import org.apache.ignite.internal.processors.query.calcite.exec.tracker.NoOpMemoryTracker; +import org.apache.ignite.internal.processors.query.calcite.metadata.ColocationGroup; +import org.apache.ignite.internal.processors.query.calcite.metadata.FragmentDescription; +import org.apache.ignite.internal.processors.query.calcite.metadata.FragmentMapping; +import org.apache.ignite.internal.processors.query.calcite.prepare.BaseQueryContext; +import org.apache.ignite.internal.processors.query.calcite.prepare.MappingQueryContext; +import org.apache.ignite.internal.processors.query.calcite.schema.ColumnDescriptor; +import org.apache.ignite.internal.processors.query.calcite.schema.IgniteCacheTable; +import org.apache.ignite.internal.processors.query.calcite.schema.IgniteIndex; +import org.apache.ignite.internal.processors.query.calcite.util.Commons; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; +import org.junit.Test; + +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED; + +/** Tests technical columns returned by direct table and index scans. */ +public class TechnicalColumnsScanTest extends GridCommonAbstractTest { + /** */ + private IgniteEx node; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setSqlConfiguration(new SqlConfiguration().setQueryEnginesConfiguration( + new CalciteQueryEngineConfiguration().setDefault(true))) + .setTransactionConfiguration(new TransactionConfiguration().setTxAwareQueriesEnabled(true)); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + node = startGrid(0); + + awaitPartitionMapExchange(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + super.afterTest(); + } + + /** */ + @Test + public void testTableScanReturnsTechnicalColumns() throws Exception { + createAndPopulatePersonTable(); + + IgniteCacheTable tbl = personTable(); + ScanContext scanCtx = scanContext(tbl); + + assertTechnicalColumns(tbl.scan(scanCtx.ectx, scanCtx.grp, requiredColumns(tbl)), tbl); + } + + /** */ + @Test + public void testIndexScanReturnsTechnicalColumns() throws Exception { + createAndPopulatePersonTable(); + + IgniteCacheTable tbl = personTable(); + IgniteIndex idx = tbl.getIndex("AGE_IDX"); + + assertNotNull(idx); + + ScanContext scanCtx = scanContext(tbl); + + assertTechnicalColumns(idx.scan(scanCtx.ectx, scanCtx.grp, null, requiredColumns(tbl)), tbl); + } + + /** Verifies that an explicit SQL query returns technical columns. */ + @Test + public void testExplicitSelectReturnsTechnicalColumns() throws Exception { + createAndPopulatePersonTable(); + + List> rows = sql("SELECT id, _ver, _src FROM Person"); + + assertEquals(30, rows.size()); + + Integer expSrc = personTable().descriptor().cacheInfo().cacheId(); + + for (List row : rows) { + assertEquals(3, row.size()); + assertTrue(row.get(0) instanceof Integer); + assertTrue("Unexpected _VER value: " + row.get(1), row.get(1) instanceof GridCacheVersion); + assertEquals(expSrc, row.get(2)); + } + } + + /** */ + @Test + public void testCannotCreateTableWithTechnicalColumnNames() { + assertTechnicalColumnCreateForbidden("CREATE TABLE PersonVer (id INT PRIMARY KEY, _ver INT)", + QueryUtils.VER_FIELD_NAME); + assertTechnicalColumnCreateForbidden("CREATE TABLE PersonSrc (id INT PRIMARY KEY, _src INT)", + QueryUtils.SRC_FIELD_NAME); + } + + /** */ + @Test + public void testCannotAddTechnicalColumnNames() throws Exception { + createAndPopulatePersonTable(); + + assertTechnicalColumnAddForbidden("ALTER TABLE Person ADD COLUMN _ver INT", + QueryUtils.VER_FIELD_NAME); + assertTechnicalColumnAddForbidden("ALTER TABLE Person ADD COLUMN _src INT", + QueryUtils.SRC_FIELD_NAME); + } + + /** */ + @Test + public void testTechnicalColumnsAreHiddenFromSelectStar() throws Exception { + createAndPopulatePersonTable(); + + List> rows = sql("SELECT * FROM Person"); + + assertEquals(30, rows.size()); + assertEquals(3, rows.get(0).size()); + + } + + /** */ + @Test + public void testTechnicalColumnsCannotBeDmlTargets() throws Exception { + createAndPopulatePersonTable(); + + assertTechnicalColumnDmlTargetForbidden( + "INSERT INTO Person (id, name, age, _ver) VALUES (1, 'Ann', 21, NULL)"); + assertTechnicalColumnDmlTargetForbidden( + "INSERT INTO Person (id, name, age, _src) VALUES (1, 'Ann', 21, NULL)"); + assertTechnicalColumnDmlTargetForbidden("UPDATE Person SET _ver = NULL"); + assertTechnicalColumnDmlTargetForbidden("UPDATE Person SET _src = NULL"); + } + + /** */ + private void assertTechnicalColumnCreateForbidden(String qry, String colName) { + String expMsg = "Name '" + colName + "' is reserved and cannot be used as a field name"; + + try { + sql(qry); + + fail("Exception is expected"); + } + catch (CacheException e) { + assertTrue("Expected reserved field name exception was not found: " + e, + hasCauseOrSuppressed(e, IgniteCheckedException.class, expMsg)); + } + } + + /** */ + private boolean hasCauseOrSuppressed(Throwable t, Class cls, String msg) { + if (t == null) + return false; + + if (cls.isAssignableFrom(t.getClass()) && t.getMessage() != null && t.getMessage().contains(msg)) + return true; + + if (hasCauseOrSuppressed(t.getCause(), cls, msg)) + return true; + + for (Throwable suppressed : t.getSuppressed()) { + if (hasCauseOrSuppressed(suppressed, cls, msg)) + return true; + } + + return false; + } + + /** */ + private void assertTechnicalColumnAddForbidden(String qry, String colName) { + GridTestUtils.assertThrowsAnyCause(log, () -> sql(qry), IgniteSQLException.class, + "Column already exists: " + colName); + } + + /** */ + private void assertTechnicalColumnDmlTargetForbidden(String qry) { + GridTestUtils.assertThrowsAnyCause(log, () -> sql(qry), IgniteSQLException.class, + "Cannot modify technical column"); + } + + /** */ + private void createAndPopulatePersonTable() throws Exception { + sql("CREATE TABLE Person (id INT PRIMARY KEY, name VARCHAR, age INT) WITH atomicity=TRANSACTIONAL"); + sql("CREATE INDEX age_idx ON Person(age)"); + + try (Transaction tx = node.transactions().txStart(PESSIMISTIC, READ_COMMITTED)) { + for (int i = 1; i <= 30; i++) + sql("INSERT INTO Person(id, name, age) VALUES (?, ?, ?)", i, personName(i), 20 + i); + + tx.commit(); + } + + awaitPartitionMapExchange(); + } + + /** */ + private void assertTechnicalColumns(Iterable rowsIterable, IgniteCacheTable tbl) throws Exception { + List rows = materialize(rowsIterable); + Set ids = new HashSet<>(); + Integer expSrc = tbl.descriptor().cacheInfo().cacheId(); + + assertEquals(30, rows.size()); + + for (Object[] row : rows) { + assertEquals(3, row.length); + assertTrue(row[0] instanceof Integer); + assertTrue("Unexpected _VER value [val=" + row[1] + ", cls=" + + (row[1] == null ? null : row[1].getClass()) + ']', row[1] instanceof GridCacheVersion); + assertEquals(expSrc, row[2]); + + ids.add((Integer)row[0]); + } + + for (int i = 1; i <= 30; i++) + assertTrue("Missing id: " + i, ids.contains(i)); + } + + /** */ + private List materialize(Iterable rowsIterable) throws Exception { + List rows = new ArrayList<>(); + + try { + for (Object[] row : rowsIterable) + rows.add(row); + } + finally { + if (rowsIterable instanceof AutoCloseable) + ((AutoCloseable)rowsIterable).close(); + } + + return rows; + } + + /** */ + private ImmutableBitSet requiredColumns(IgniteCacheTable tbl) { + return ImmutableBitSet.of( + columnIndex(tbl, "ID"), + columnIndex(tbl, QueryUtils.VER_FIELD_NAME), + columnIndex(tbl, QueryUtils.SRC_FIELD_NAME) + ); + } + + /** */ + private int columnIndex(IgniteCacheTable tbl, String name) { + ColumnDescriptor desc = tbl.descriptor().columnDescriptor(name); + + assertNotNull(name, desc); + + return desc.fieldIndex(); + } + + /** */ + private ScanContext scanContext(IgniteCacheTable tbl) { + UUID nodeId = node.localNode().id(); + AffinityTopologyVersion topVer = node.context().cache().context().exchange().readyAffinityVersion(); + BaseQueryContext qctx = BaseQueryContext.builder().logger(log).build(); + + ExecutionContext ectx = new ExecutionContext<>( + qctx, + null, + null, + UUID.randomUUID(), + nodeId, + nodeId, + topVer, + new FragmentDescription(0, FragmentMapping.create(nodeId), null, Collections.emptyMap()), + ArrayRowHandler.INSTANCE, + NoOpMemoryTracker.INSTANCE, + NoOpIoTracker.INSTANCE, + 0, + Collections.emptyMap(), + null + ); + + ColocationGroup grp = tbl.colocationGroup(new MappingQueryContext(qctx, nodeId, topVer, null)).finalizeMapping(); + + return new ScanContext(ectx, grp); + } + + /** */ + private IgniteCacheTable personTable() { + CalciteQueryProcessor qryProc = Commons.lookupComponent(node.context(), CalciteQueryProcessor.class); + + return (IgniteCacheTable)qryProc.schemaHolder().schema(QueryUtils.DFLT_SCHEMA).getTable("PERSON"); + } + + /** */ + private List> sql(String sql, Object... args) { + return node.context().query().querySqlFields(new SqlFieldsQuery(sql).setSchema("PUBLIC").setArgs(args), true).getAll(); + } + + /** */ + private String personName(int id) { + switch (id) { + case 1: + return "Alice"; + + case 2: + return "Bob"; + + case 3: + return "Ann"; + + case 4: + return "Carl"; + + case 5: + return "Alex"; + + case 6: + return "Diana"; + + default: + return "Person" + id; + } + } + + /** */ + private static class ScanContext { + /** */ + private final ExecutionContext ectx; + + /** */ + private final ColocationGroup grp; + + /** */ + private ScanContext(ExecutionContext ectx, ColocationGroup grp) { + this.ectx = ectx; + this.grp = grp; + } + } +} diff --git a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java index b965adf193dc1..968dac021e784 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java +++ b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java @@ -78,6 +78,7 @@ import org.apache.ignite.internal.processors.query.calcite.integration.SystemViewsIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.TableDdlIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.TableDmlIntegrationTest; +import org.apache.ignite.internal.processors.query.calcite.integration.TechnicalColumnsScanTest; import org.apache.ignite.internal.processors.query.calcite.integration.TimeoutIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.UnnestIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.UnstableTopologyIntegrationTest; @@ -185,6 +186,7 @@ TxThreadLockingTest.class, SelectByKeyFieldTest.class, WindowIntegrationTest.class, + TechnicalColumnsScanTest.class, }) public class IntegrationTestSuite { } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryTypeDescriptorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryTypeDescriptorImpl.java index 7b917e69a4d30..b752dea5bcc52 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryTypeDescriptorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryTypeDescriptorImpl.java @@ -411,7 +411,9 @@ else if (F.isEmpty(keyFieldAlias()) || !usedProps.contains(keyFieldAlias())) * @return {@code True} if exists. */ public boolean hasField(String field) { - return props.containsKey(field) || QueryUtils.VAL_FIELD_NAME.equalsIgnoreCase(field); + return props.containsKey(field) + || QueryUtils.VAL_FIELD_NAME.equalsIgnoreCase(field) + || QueryUtils.isTechnicalFieldNameIgnoreCase(field); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java index 21a16ba4d6151..b8a8449f2483e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java @@ -111,6 +111,12 @@ public class QueryUtils { /** Field name for value. */ public static final String VAL_FIELD_NAME = CommonUtils.VAL_FIELD_NAME; + /** Field name for row version. */ + public static final String VER_FIELD_NAME = "_VER"; + + /** Field name for row source. */ + public static final String SRC_FIELD_NAME = "_SRC"; + /** Well-known template name for PARTITIONED cache. */ public static final String TEMPLATE_PARTITIONED = "PARTITIONED"; @@ -1476,6 +1482,22 @@ public static SchemaOperationException validateDropColumn(GridQueryTypeDescripto return null; } + /** */ + public static boolean isReservedFieldName(String fieldName) { + return KEY_FIELD_NAME.equalsIgnoreCase(fieldName) || VAL_FIELD_NAME.equalsIgnoreCase(fieldName) || + isTechnicalFieldNameIgnoreCase(fieldName); + } + + /** */ + public static boolean isTechnicalFieldName(String fieldName) { + return VER_FIELD_NAME.equals(fieldName) || SRC_FIELD_NAME.equals(fieldName); + } + + /** */ + public static boolean isTechnicalFieldNameIgnoreCase(String fieldName) { + return VER_FIELD_NAME.equalsIgnoreCase(fieldName) || SRC_FIELD_NAME.equalsIgnoreCase(fieldName); + } + /** * Returns true if the exception is triggered by query cancel. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/management/SchemaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/management/SchemaManager.java index 750ff09dee523..09039ae51add4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/management/SchemaManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/management/SchemaManager.java @@ -86,6 +86,7 @@ import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName; import static org.apache.ignite.internal.processors.query.QueryUtils.KEY_FIELD_NAME; import static org.apache.ignite.internal.processors.query.QueryUtils.VAL_FIELD_NAME; +import static org.apache.ignite.internal.processors.query.QueryUtils.isReservedFieldName; import static org.apache.ignite.internal.processors.query.QueryUtils.matches; /** @@ -411,7 +412,7 @@ private static void validateTypeDescriptor(GridQueryTypeDescriptor type) throws String ptrn = "Name ''{0}'' is reserved and cannot be used as a field name [type=" + type.name() + "]"; for (String name : names) { - if (name.equalsIgnoreCase(KEY_FIELD_NAME) || name.equalsIgnoreCase(VAL_FIELD_NAME)) + if (isReservedFieldName(name)) throw new IgniteCheckedException(MessageFormat.format(ptrn, name)); } }