From 352d98a2ddf3d810e616a8d2b3689ce4d075fcad Mon Sep 17 00:00:00 2001 From: pernyf Date: Wed, 18 Nov 2026 12:26:19 +0100 Subject: [PATCH 1/3] Resolve cast-style CONVERT columns against the visible schema SqlParser.analyzeConvertFunction assumed that an inner expression spelling a type name was the target type, so CONVERT(VARCHAR, int8) extracted a phantom VARCHAR column whenever int8 was a real column named after a type. Mirror CastExpressions.asCast with a schema-column predicate so extraction agrees with evaluation, and thread the predicate through ColumnsUsed, collectQualifiedColumns, and their validation call sites. When both arguments name visible columns the legacy heuristic is retained, keeping the ambiguous-CONVERT error from the evaluator. --- .../alipsa/jparq/JParqPreparedStatement.java | 98 ++++---- .../java/se/alipsa/jparq/JParqResultSet.java | 14 +- .../se/alipsa/jparq/engine/ColumnsUsed.java | 62 ++++- .../jparq/engine/MaterializedNames.java | 31 +++ .../se/alipsa/jparq/engine/SqlParser.java | 92 +++++++- .../functions/ConvertTypeNameColumnTest.java | 212 ++++++++++++++++++ 6 files changed, 444 insertions(+), 65 deletions(-) create mode 100644 src/test/java/jparq/functions/ConvertTypeNameColumnTest.java diff --git a/src/main/java/se/alipsa/jparq/JParqPreparedStatement.java b/src/main/java/se/alipsa/jparq/JParqPreparedStatement.java index 22282d03..dc342d6b 100644 --- a/src/main/java/se/alipsa/jparq/JParqPreparedStatement.java +++ b/src/main/java/se/alipsa/jparq/JParqPreparedStatement.java @@ -55,6 +55,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -2358,6 +2359,7 @@ private List> applySetOrdering(List> rows, List schemaColumns = MaterializedNames.lookupKeys(labelIndexes.keySet()); List resolved = new ArrayList<>(orderBy.size()); boolean hasExpressions = false; for (SqlParser.SetOrder order : orderBy) { @@ -2375,7 +2377,7 @@ private List> applySetOrdering(List> rows, List expressionColumns = ColumnsUsed.inWhere(order.expression()); + Set expressionColumns = ColumnsUsed.inWhere(order.expression(), schemaColumns); for (String column : expressionColumns) { if (column == null) { continue; @@ -3170,7 +3172,8 @@ private void validateJoinConditions(List tables) thr qualified.put(JParqUtil.normalizeQualifier(qualifier), columns); } Set missing = new LinkedHashSet<>(); - for (Column column : ColumnsUsed.referencesInScope(table.joinCondition())) { + Predicate schemaColumns = MaterializedNames.lookupKeys(available); + for (Column column : ColumnsUsed.referencesInScope(table.joinCondition(), schemaColumns)) { String referenceQualifier = column.getTable() == null ? null : column.getTable().getName(); Set visible = referenceQualifier == null || referenceQualifier.isBlank() ? available @@ -3248,6 +3251,7 @@ private void validateJoinOutputColumns(JoinRecordReader reader) throws SQLExcept final Schema schema = reader.schema(); validateAmbiguousColumns(parsedSelect, schema, reader.qualifierColumnMapping()); final Set required = new LinkedHashSet<>(); + Predicate schemaColumns = MaterializedNames.visibleColumns(schema); List expressions = new ArrayList<>(parsedSelect.expressions()); expressions.add(parsedSelect.where()); expressions.addAll(parsedSelect.groupByExpressions()); @@ -3255,14 +3259,14 @@ private void validateJoinOutputColumns(JoinRecordReader reader) throws SQLExcept expressions.add(key.expression()); } for (Expression expression : expressions) { - addColumns(required, ColumnsUsed.inScope(expression)); + addColumns(required, ColumnsUsed.inScope(expression, schemaColumns)); } if (tableReferences.stream().anyMatch(ref -> !ref.joinGroup().isEmpty())) { Map> qualifierMapping = reader.qualifierColumnMapping(); List qualifiedExpressions = new ArrayList<>(expressions); qualifiedExpressions.add(parsedSelect.having()); for (Expression expression : qualifiedExpressions) { - for (Column column : ColumnsUsed.referencesInScope(expression)) { + for (Column column : ColumnsUsed.referencesInScope(expression, schemaColumns)) { String qualifier = column.getTable() == null ? null : column.getTable().getName(); validateGroupedQualifier(qualifierMapping, qualifier, column.getColumnName()); } @@ -3277,7 +3281,7 @@ private void validateJoinOutputColumns(JoinRecordReader reader) throws SQLExcept addColumn(required, key.column()); } } - addColumns(required, ColumnsUsed.inScope(parsedSelect.having())); + addColumns(required, ColumnsUsed.inScope(parsedSelect.having(), schemaColumns)); validateRequiredColumns(schema, required, null, parsedSelect.labels(), parsedSelect.columnNames(), referenced); } @@ -3326,7 +3330,7 @@ private void validateJoinTableColumns(Schema schema, SqlParser.TableReference re if (schema == null || ref == null) { return; } - Set required = collectColumnsForTable(ref); + Set required = collectColumnsForTable(ref, MaterializedNames.visibleColumns(schema)); if (ref.usingColumns() != null) { required.addAll(ref.usingColumns()); } @@ -3341,9 +3345,11 @@ private void validateJoinTableColumns(Schema schema, SqlParser.TableReference re * * @param ref * table reference being validated + * @param schemaColumns + * predicate identifying visible source column names * @return mutable set of referenced column names (empty if none) */ - private Set collectColumnsForTable(SqlParser.TableReference ref) { + private Set collectColumnsForTable(SqlParser.TableReference ref, Predicate schemaColumns) { if (parsedSelect == null || ref == null) { return new LinkedHashSet<>(); } @@ -3354,10 +3360,12 @@ private Set collectColumnsForTable(SqlParser.TableReference ref) { boolean singleTable = !parsedSelect.hasMultipleTables(); List primaryQualifier = singleTable ? qualifierList(ref, true) : List.of(); Set required = new LinkedHashSet<>(); - collectColumnReferencesFromList(parsedSelect.expressions(), qualifiers, primaryQualifier, required); - collectColumnReferencesFromExpression(parsedSelect.where(), qualifiers, primaryQualifier, required); + collectColumnReferencesFromList(parsedSelect.expressions(), qualifiers, primaryQualifier, required, null, + schemaColumns); + collectColumnReferencesFromExpression(parsedSelect.where(), qualifiers, primaryQualifier, required, null, + schemaColumns); for (Expression group : parsedSelect.groupByExpressions()) { - collectColumnReferencesFromExpression(group, qualifiers, primaryQualifier, required); + collectColumnReferencesFromExpression(group, qualifiers, primaryQualifier, required, null, schemaColumns); } for (SqlParser.OrderKey key : parsedSelect.orderBy()) { if (shouldIncludeOrderKey(key, qualifiers, singleTable)) { @@ -3369,7 +3377,8 @@ private Set collectColumnsForTable(SqlParser.TableReference ref) { addColumn(required, key.column()); } } - collectColumnReferencesFromExpression(ref.joinCondition(), qualifiers, primaryQualifier, required); + collectColumnReferencesFromExpression(ref.joinCondition(), qualifiers, primaryQualifier, required, null, + schemaColumns); return required; } @@ -3386,19 +3395,19 @@ private Set collectColumnsForTable(SqlParser.TableReference ref) { * ambiguous) * @param target * set receiving the column names + * @param referenced + * set receiving bound column names, or null + * @param schemaColumns + * predicate identifying visible source column names */ private void collectColumnReferencesFromList(List expressions, List qualifiers, - List primaryQualifier, Set target) { - collectColumnReferencesFromList(expressions, qualifiers, primaryQualifier, target, null); - } - - private void collectColumnReferencesFromList(List expressions, List qualifiers, - List primaryQualifier, Set target, Set referenced) { + List primaryQualifier, Set target, Set referenced, Predicate schemaColumns) { if (expressions == null || expressions.isEmpty()) { return; } for (Expression expression : expressions) { - collectColumnReferencesFromExpression(expression, qualifiers, primaryQualifier, target, referenced); + collectColumnReferencesFromExpression(expression, qualifiers, primaryQualifier, target, referenced, + schemaColumns); } } @@ -3413,22 +3422,21 @@ private void collectColumnReferencesFromList(List expressions, List< * single qualifier used when unqualified columns should be included * @param target * set receiving the column names + * @param referenced + * set receiving bound column names, or null + * @param schemaColumns + * predicate identifying visible source column names */ private void collectColumnReferencesFromExpression(Expression expression, List qualifiers, - List primaryQualifier, Set target) { - collectColumnReferencesFromExpression(expression, qualifiers, primaryQualifier, target, null); - } - - private void collectColumnReferencesFromExpression(Expression expression, List qualifiers, - List primaryQualifier, Set target, Set referenced) { + List primaryQualifier, Set target, Set referenced, Predicate schemaColumns) { if (expression == null || target == null) { return; } - Set collected = SqlParser.collectQualifiedColumns(expression, qualifiers, false); + Set collected = SqlParser.collectQualifiedColumns(expression, qualifiers, false, schemaColumns); addColumns(target, collected); addColumns(referenced, collected); if (primaryQualifier != null && !primaryQualifier.isEmpty()) { - Set primaryCollected = SqlParser.collectQualifiedColumns(expression, primaryQualifier); + Set primaryCollected = SqlParser.collectQualifiedColumns(expression, primaryQualifier, schemaColumns); addColumns(target, primaryCollected); addColumns(referenced, primaryCollected); } @@ -6022,7 +6030,7 @@ private Set validateSelectColumns(SqlParser.Select select, Schema schema AggregateFunctions.AggregatePlan aggregatePlan, Map ctes, Configuration configuration) throws SQLException { validateAmbiguousColumns(select, schema, Map.of()); - ColumnRequirement requirement = requiredColumns(select, aggregatePlan); + ColumnRequirement requirement = requiredColumns(select, aggregatePlan, MaterializedNames.visibleColumns(schema)); collectSubqueryDependencies(select, schema, requirement.required(), new SubqueryExecutor(stmt.getConn(), null, table -> subqueryTableColumns(table, ctes, configuration))); SqlParser.TableReference baseRef = select.tableReferences().isEmpty() ? null : select.tableReferences().getFirst(); @@ -6048,11 +6056,12 @@ private void validateAmbiguousColumns(SqlParser.Select select, Schema schema, Map> qualifiers) { Map index = MaterializedNames.lookupIndex(schema); Map> qualified = ColumnMappingUtil.normaliseQualifierMapping(qualifiers); + Predicate schemaColumns = MaterializedNames.lookupKeys(index.keySet()); List expressions = new ArrayList<>(select.expressions()); expressions.add(select.where()); expressions.addAll(select.groupByExpressions()); for (Expression expression : expressions) { - for (Column column : ColumnsUsed.referencesInScope(expression)) { + for (Column column : ColumnsUsed.referencesInScope(expression, schemaColumns)) { String qualifier = column.getTable() == null ? null : column.getTable().getName(); validateAmbiguousColumn(column.getColumnName(), qualifier, index, qualified); } @@ -6061,7 +6070,7 @@ private void validateAmbiguousColumns(SqlParser.Select select, Schema schema, orders.addAll(select.preOrderBy()); for (SqlParser.OrderKey order : orders) { if (order.expression() != null) { - for (Column column : ColumnsUsed.referencesInScope(order.expression())) { + for (Column column : ColumnsUsed.referencesInScope(order.expression(), schemaColumns)) { String qualifier = column.getTable() == null ? null : column.getTable().getName(); validateAmbiguousColumn(column.getColumnName(), qualifier, index, qualified); } @@ -6166,7 +6175,8 @@ private void validatePredicateSubqueries(SqlParser.Select select, Map required, Set referenced) { } - private ColumnRequirement requiredColumns(SqlParser.Select select, AggregateFunctions.AggregatePlan aggregatePlan) { + private ColumnRequirement requiredColumns(SqlParser.Select select, AggregateFunctions.AggregatePlan aggregatePlan, + Predicate schemaColumns) { if (select == null) { return new ColumnRequirement(Set.of(), Set.of()); } @@ -6180,7 +6190,7 @@ private ColumnRequirement requiredColumns(SqlParser.Select select, AggregateFunc } } } - Set whereColumns = ColumnsUsed.inWhere(select.where()); + Set whereColumns = ColumnsUsed.inWhere(select.where(), schemaColumns); addColumns(needed, whereColumns); addColumns(referenced, whereColumns); SqlParser.TableReference baseRef = select.tableReferences() == null || select.tableReferences().isEmpty() @@ -6188,21 +6198,25 @@ private ColumnRequirement requiredColumns(SqlParser.Select select, AggregateFunc : select.tableReferences().getFirst(); List qualifiers = qualifierList(baseRef, false); List primaryQualifier = qualifierList(baseRef, true); - collectColumnReferencesFromList(select.expressions(), qualifiers, primaryQualifier, needed, referenced); - collectColumnReferencesFromExpression(select.where(), qualifiers, primaryQualifier, needed, referenced); - collectColumnReferencesFromExpression(select.having(), qualifiers, primaryQualifier, needed); + collectColumnReferencesFromList(select.expressions(), qualifiers, primaryQualifier, needed, referenced, + schemaColumns); + collectColumnReferencesFromExpression(select.where(), qualifiers, primaryQualifier, needed, referenced, + schemaColumns); + collectColumnReferencesFromExpression(select.having(), qualifiers, primaryQualifier, needed, null, schemaColumns); for (SqlParser.OrderKey key : select.orderBy()) { if (key.expression() != null) { - addColumns(needed, ColumnsUsed.inWhere(key.expression())); - collectColumnReferencesFromExpression(key.expression(), qualifiers, primaryQualifier, needed, referenced); + addColumns(needed, ColumnsUsed.inWhere(key.expression(), schemaColumns)); + collectColumnReferencesFromExpression(key.expression(), qualifiers, primaryQualifier, needed, referenced, + schemaColumns); } else if (!isOrdinalReference(key.column()) && !isGroupingFunctionReference(key.column())) { addColumn(needed, key.column()); } } for (SqlParser.OrderKey key : select.preOrderBy()) { if (key.expression() != null) { - addColumns(needed, ColumnsUsed.inWhere(key.expression())); - collectColumnReferencesFromExpression(key.expression(), qualifiers, primaryQualifier, needed, referenced); + addColumns(needed, ColumnsUsed.inWhere(key.expression(), schemaColumns)); + collectColumnReferencesFromExpression(key.expression(), qualifiers, primaryQualifier, needed, referenced, + schemaColumns); } else if (!isOrdinalReference(key.column()) && !isGroupingFunctionReference(key.column())) { addColumn(needed, key.column()); } @@ -6210,22 +6224,22 @@ private ColumnRequirement requiredColumns(SqlParser.Select select, AggregateFunc WindowPlan windowPlan = WindowFunctions.plan(select.expressions()); if (windowPlan != null) { for (Expression input : windowPlan.inputExpressions()) { - collectColumnReferencesFromExpression(input, qualifiers, primaryQualifier, needed, referenced); - addColumns(needed, ColumnsUsed.inWhere(input)); + collectColumnReferencesFromExpression(input, qualifiers, primaryQualifier, needed, referenced, schemaColumns); + addColumns(needed, ColumnsUsed.inWhere(input, schemaColumns)); } } if (aggregatePlan != null) { for (AggregateFunctions.AggregateSpec spec : aggregatePlan.specs()) { if (!spec.countStar()) { for (Expression arg : spec.arguments()) { - Set argColumns = ColumnsUsed.inWhere(arg); + Set argColumns = ColumnsUsed.inWhere(arg, schemaColumns); addColumns(needed, argColumns); addColumns(referenced, argColumns); } } } for (AggregateFunctions.GroupExpression groupExpr : aggregatePlan.groupExpressions()) { - Set groupColumns = ColumnsUsed.inWhere(groupExpr.expression()); + Set groupColumns = ColumnsUsed.inWhere(groupExpr.expression(), schemaColumns); addColumns(needed, groupColumns); addColumns(referenced, groupColumns); } diff --git a/src/main/java/se/alipsa/jparq/JParqResultSet.java b/src/main/java/se/alipsa/jparq/JParqResultSet.java index 2cc255cd..acfccbc1 100644 --- a/src/main/java/se/alipsa/jparq/JParqResultSet.java +++ b/src/main/java/se/alipsa/jparq/JParqResultSet.java @@ -26,6 +26,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiFunction; +import java.util.function.Predicate; import javax.sql.rowset.serial.SerialBlob; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.operators.conditional.AndExpression; @@ -372,12 +373,13 @@ public JParqResultSet(RecordReader reader, SqlParser.Select select, String table this.columnOrder.addAll(proj); } Set requiredColumns = new LinkedHashSet<>(proj); - requiredColumns.addAll(ColumnsUsed.inWhere(effectiveResidual)); - requiredColumns.addAll(ColumnsUsed.inWhere(select.having())); - requiredColumns.addAll(SqlParser.collectQualifiedColumns(effectiveResidual, queryQualifiers)); - requiredColumns.addAll(SqlParser.collectQualifiedColumns(select.having(), queryQualifiers)); + Predicate schemaColumns = MaterializedNames.visibleColumns(schema); + requiredColumns.addAll(ColumnsUsed.inWhere(effectiveResidual, schemaColumns)); + requiredColumns.addAll(ColumnsUsed.inWhere(select.having(), schemaColumns)); + requiredColumns.addAll(SqlParser.collectQualifiedColumns(effectiveResidual, queryQualifiers, schemaColumns)); + requiredColumns.addAll(SqlParser.collectQualifiedColumns(select.having(), queryQualifiers, schemaColumns)); for (Expression expression : selectExpressions) { - requiredColumns.addAll(SqlParser.collectQualifiedColumns(expression, queryQualifiers)); + requiredColumns.addAll(SqlParser.collectQualifiedColumns(expression, queryQualifiers, schemaColumns)); } if (select.orderBy() != null) { for (SqlParser.OrderKey orderKey : select.orderBy()) { @@ -388,7 +390,7 @@ public JParqResultSet(RecordReader reader, SqlParser.Select select, String table } if (windowPlan != null) { for (Expression input : windowPlan.inputExpressions()) { - requiredColumns.addAll(SqlParser.collectQualifiedColumns(input, queryQualifiers)); + requiredColumns.addAll(SqlParser.collectQualifiedColumns(input, queryQualifiers, schemaColumns)); } } if (schema != null) { diff --git a/src/main/java/se/alipsa/jparq/engine/ColumnsUsed.java b/src/main/java/se/alipsa/jparq/engine/ColumnsUsed.java index 832de315..c6d62d35 100644 --- a/src/main/java/se/alipsa/jparq/engine/ColumnsUsed.java +++ b/src/main/java/se/alipsa/jparq/engine/ColumnsUsed.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Predicate; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.TranscodingFunction; import net.sf.jsqlparser.schema.Column; @@ -42,7 +43,21 @@ private ColumnsUsed() { * @return set of column names referenced by the expression (empty if none) */ public static Set inWhere(Expression where) { - return collect(where, true); + return inWhere(where, name -> false); + } + + /** + * Extract all column names referenced in a SQL {@code WHERE} expression, using + * visible source column names to resolve cast-style CONVERT arguments. + * + * @param where + * the expression (may be null) + * @param schemaColumns + * predicate identifying visible source column names + * @return set of column names referenced by the expression (empty if none) + */ + public static Set inWhere(Expression where, Predicate schemaColumns) { + return collect(where, true, schemaColumns); } /** @@ -53,7 +68,21 @@ public static Set inWhere(Expression where) { * @return column names belonging to the current query scope */ public static Set inScope(Expression expression) { - return collect(expression, false); + return inScope(expression, name -> false); + } + + /** + * Collects column names in an expression without entering nested query scopes, + * using visible source column names to resolve cast-style CONVERT arguments. + * + * @param expression + * expression to inspect, or null + * @param schemaColumns + * predicate identifying visible source column names + * @return column names belonging to the current query scope + */ + public static Set inScope(Expression expression, Predicate schemaColumns) { + return collect(expression, false, schemaColumns); } /** @@ -64,8 +93,22 @@ public static Set inScope(Expression expression) { * @return column nodes without references inside nested queries */ public static List referencesInScope(Expression expression) { + return referencesInScope(expression, name -> false); + } + + /** + * Retains table qualifiers while collecting references in the current scope, + * using visible source column names to resolve cast-style CONVERT arguments. + * + * @param expression + * expression to inspect, or null + * @param schemaColumns + * predicate identifying visible source column names + * @return column nodes without references inside nested queries + */ + public static List referencesInScope(Expression expression, Predicate schemaColumns) { List columns = new ArrayList<>(); - walk(expression, false, columns::add); + walk(expression, false, schemaColumns, columns::add); return List.copyOf(columns); } @@ -76,11 +119,13 @@ public static List referencesInScope(Expression expression) { * expression to inspect * @param includeSubqueries * whether nested query expressions should be visited + * @param schemaColumns + * predicate identifying visible source column names * @return referenced column names */ - private static Set collect(Expression where, boolean includeSubqueries) { + private static Set collect(Expression where, boolean includeSubqueries, Predicate schemaColumns) { LinkedHashSet cols = new LinkedHashSet<>(); - walk(where, includeSubqueries, column -> cols.add(column.getColumnName())); + walk(where, includeSubqueries, schemaColumns, column -> cols.add(column.getColumnName())); return cols; } @@ -91,10 +136,13 @@ private static Set collect(Expression where, boolean includeSubqueries) * expression to inspect * @param includeSubqueries * whether nested expressions should be visited + * @param schemaColumns + * predicate identifying visible source column names * @param consumer * receiver for each column reference */ - private static void walk(Expression where, boolean includeSubqueries, Consumer consumer) { + private static void walk(Expression where, boolean includeSubqueries, Predicate schemaColumns, + Consumer consumer) { if (where == null) { return; } @@ -119,7 +167,7 @@ public Void visit(Column column, S context) { @Override public Void visit(TranscodingFunction convert, S context) { - SqlParser.ConvertProcessingResult result = SqlParser.analyzeConvertFunction(convert); + SqlParser.ConvertProcessingResult result = SqlParser.analyzeConvertFunction(convert, schemaColumns); if (result.syntheticColumnName() != null) { consumer.accept(new Column(result.syntheticColumnName())); return null; diff --git a/src/main/java/se/alipsa/jparq/engine/MaterializedNames.java b/src/main/java/se/alipsa/jparq/engine/MaterializedNames.java index c1f8a4b1..c2ef184d 100644 --- a/src/main/java/se/alipsa/jparq/engine/MaterializedNames.java +++ b/src/main/java/se/alipsa/jparq/engine/MaterializedNames.java @@ -1,8 +1,10 @@ package se.alipsa.jparq.engine; +import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import java.util.regex.Pattern; import org.apache.avro.Schema; @@ -99,6 +101,35 @@ public static Map lookupIndex(Schema schema) { return Map.copyOf(index); } + /** + * Builds a predicate identifying visible SQL column names of a schema. + * + * @param schema + * source schema, or null for an unavailable schema + * @return predicate matching visible SQL column names + */ + public static Predicate visibleColumns(Schema schema) { + return lookupKeys(lookupIndex(schema).keySet()); + } + + /** + * Builds a predicate matching SQL names against a collection of lookup keys. + * + * @param keys + * SQL lookup keys of visible column names, or null + * @return predicate matching the supplied lookup keys + */ + public static Predicate lookupKeys(Collection keys) { + if (keys == null || keys.isEmpty()) { + return name -> false; + } + Set known = Set.copyOf(keys); + return name -> { + String key = Identifier.lookupKey(name); + return key != null && known.contains(key); + }; + } + /** * Prevents an unresolved SQL identifier from addressing a generated internal * name. diff --git a/src/main/java/se/alipsa/jparq/engine/SqlParser.java b/src/main/java/se/alipsa/jparq/engine/SqlParser.java index 5f8224cb..476e6532 100644 --- a/src/main/java/se/alipsa/jparq/engine/SqlParser.java +++ b/src/main/java/se/alipsa/jparq/engine/SqlParser.java @@ -14,6 +14,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -89,10 +90,12 @@ record ConvertProcessingResult(boolean processInner, String syntheticColumnName) * *

* This method handles the special case where CONVERT is used with a type name - * as the first argument (e.g., CONVERT(VARCHAR, column_name)). In this case, - * JSqlParser swaps the arguments, putting the type name where the column would - * normally be. This method detects that pattern and returns a result indicating - * the actual column name to use. + * in place of a column name. JSqlParser assigns the first argument to + * {@code getColDataType()} and the second to {@code getExpression()}, so this + * method detects the pattern where the arguments are in the value-first order + * and returns a result indicating the actual column name to use. Without schema + * knowledge, an inner expression spelling a type name is assumed to be the + * target type. * * @param convert * the transcoding function to analyze (may be null) @@ -100,6 +103,28 @@ record ConvertProcessingResult(boolean processInner, String syntheticColumnName) * synthetic column name */ static ConvertProcessingResult analyzeConvertFunction(TranscodingFunction convert) { + return analyzeConvertFunction(convert, name -> false); + } + + /** + * Analyzes a {@link TranscodingFunction} for column extraction using visible + * source column names. + * + *

+ * The resolution mirrors {@link CastExpressions#asCast(Expression, Predicate)} + * so that column extraction agrees with evaluation: when both arguments spell + * type names, a {@code Column} operand naming a visible source column keeps its + * value role instead of being treated as the target type. When the arguments + * are ambiguous, the type-name heuristic without schema knowledge is retained. + * + * @param convert + * the transcoding function to analyze (may be null) + * @param schemaColumns + * predicate identifying visible source column names + * @return a result indicating whether to process the inner expression and/or a + * synthetic column name + */ + static ConvertProcessingResult analyzeConvertFunction(TranscodingFunction convert, Predicate schemaColumns) { if (convert == null) { return new ConvertProcessingResult(false, null); } @@ -108,8 +133,17 @@ static ConvertProcessingResult analyzeConvertFunction(TranscodingFunction conver } Expression inner = convert.getExpression(); String typeCandidate = convert.getColDataType() == null ? null : convert.getColDataType().getDataType(); - if (inner != null && isConvertTypeName(inner.toString()) && typeCandidate != null - && (inner instanceof Column || !isConvertTypeName(typeCandidate))) { + if (inner == null || typeCandidate == null) { + return new ConvertProcessingResult(true, null); + } + CastExpression cast = CastExpressions.asCast(convert, schemaColumns); + if (cast != null) { + if (cast.getLeftExpression() != inner) { + return new ConvertProcessingResult(false, cast.getLeftExpression().toString()); + } + return new ConvertProcessingResult(true, null); + } + if (isConvertTypeName(inner.toString()) && (inner instanceof Column || !isConvertTypeName(typeCandidate))) { return new ConvertProcessingResult(false, typeCandidate); } return new ConvertProcessingResult(true, null); @@ -2768,8 +2802,9 @@ private static String orderingAliasLabel(String name, Map al */ private static Expression resolveOrderingAliases(Expression expression, Map aliases, Set ambiguous, Set inputs) { + Predicate schemaColumns = MaterializedNames.lookupKeys(inputs); Map replacements = new java.util.IdentityHashMap<>(); - for (Column column : ColumnsUsed.referencesInScope(expression)) { + for (Column column : ColumnsUsed.referencesInScope(expression, schemaColumns)) { String input = Identifier.lookupKey(column.getColumnName()); String alternative = ColumnMappingUtil.alternativeColumnKey(input); if (extractQualifier(column) != null || inputs.contains(input) @@ -3081,7 +3116,24 @@ public Void visit(ParenthesedSelect subSelect, S context) { * @return a set of column names that reference one of the qualifiers */ public static Set collectQualifiedColumns(Expression expr, List qualifiers) { - return collectQualifiedColumns(expr, qualifiers, true); + return collectQualifiedColumns(expr, qualifiers, true, name -> false); + } + + /** + * Collects references to supplied qualifiers using visible source column names + * to resolve cast-style CONVERT arguments. + * + * @param expr + * the expression to inspect + * @param qualifiers + * table names or aliases to match + * @param schemaColumns + * predicate identifying visible source column names + * @return matching column names + */ + public static Set collectQualifiedColumns(Expression expr, List qualifiers, + Predicate schemaColumns) { + return collectQualifiedColumns(expr, qualifiers, true, schemaColumns); } /** @@ -3089,7 +3141,7 @@ public static Set collectQualifiedColumns(Expression expr, List * names. * * @param expr - * expression to inspect + * the expression to inspect * @param qualifiers * table names or aliases to match * @param includeUnqualified @@ -3098,6 +3150,26 @@ public static Set collectQualifiedColumns(Expression expr, List */ public static Set collectQualifiedColumns(Expression expr, List qualifiers, boolean includeUnqualified) { + return collectQualifiedColumns(expr, qualifiers, includeUnqualified, name -> false); + } + + /** + * Collects references to supplied qualifiers, optionally including unqualified + * names, using visible source column names to resolve cast-style CONVERT + * arguments. + * + * @param expr + * the expression to inspect + * @param qualifiers + * table names or aliases to match + * @param includeUnqualified + * whether a single qualifier also owns unqualified references + * @param schemaColumns + * predicate identifying visible source column names + * @return matching column names + */ + public static Set collectQualifiedColumns(Expression expr, List qualifiers, + boolean includeUnqualified, Predicate schemaColumns) { if (expr == null || qualifiers == null || qualifiers.isEmpty()) { return Set.of(); } @@ -3161,7 +3233,7 @@ public Void visit(net.sf.jsqlparser.statement.select.Select select, S contex @Override public Void visit(TranscodingFunction convert, S context) { - ConvertProcessingResult result = analyzeConvertFunction(convert); + ConvertProcessingResult result = analyzeConvertFunction(convert, schemaColumns); if (result.syntheticColumnName() != null) { Column synthetic = new Column(result.syntheticColumnName()); synthetic.accept(this, context); diff --git a/src/test/java/jparq/functions/ConvertTypeNameColumnTest.java b/src/test/java/jparq/functions/ConvertTypeNameColumnTest.java new file mode 100644 index 00000000..9a0c584e --- /dev/null +++ b/src/test/java/jparq/functions/ConvertTypeNameColumnTest.java @@ -0,0 +1,212 @@ +package jparq.functions; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Map; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.avro.AvroParquetWriter; +import org.apache.parquet.hadoop.util.HadoopOutputFile; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +/** + * Resolves cast-style CONVERT against physical columns whose names spell SQL + * type names, so column extraction agrees with evaluation. + */ +class ConvertTypeNameColumnTest { + + private static final Schema CONV_SCHEMA = SchemaBuilder.record("conv").fields().requiredInt("id").requiredLong("int8") + .requiredInt("int4").requiredDouble("float8").requiredBoolean("bool").requiredString("text") + .requiredString("note").endRecord(); + + private static final Schema AMBIGUOUS_SCHEMA = SchemaBuilder.record("amb").fields().requiredInt("id") + .requiredString("varchar").requiredLong("int8").endRecord(); + + @TempDir + static Path directory; + + /** + * Writes fixture tables containing columns named after SQL type names. + * + * @throws Exception + * if writing the fixture fails + */ + @BeforeAll + static void writeFixture() throws Exception { + var configuration = new Configuration(false); + List> convRows = List.of( + Map.of("id", 1, "int8", 10L, "int4", 100, "float8", 1.5d, "bool", true, "text", "ten", "note", "first"), + Map.of("id", 2, "int8", 20L, "int4", 200, "float8", 2.5d, "bool", false, "text", "twenty", "note", "second")); + writeTable("conv.parquet", CONV_SCHEMA, configuration, convRows); + List> ambiguousRows = List.of(Map.of("id", 1, "varchar", "ten", "int8", 10L)); + writeTable("amb.parquet", AMBIGUOUS_SCHEMA, configuration, ambiguousRows); + } + + private static void writeTable(String name, Schema schema, Configuration configuration, + List> rows) throws Exception { + var output = HadoopOutputFile.fromPath(new org.apache.hadoop.fs.Path(directory.resolve(name).toUri()), + configuration); + try (var writer = AvroParquetWriter.builder(output).withSchema(schema).withConf(configuration) + .build()) { + for (Map row : rows) { + var record = new GenericData.Record(schema); + row.forEach(record::put); + writer.write(record); + } + } + } + + /** + * Converts physical columns named after SQL type names in both argument orders. + * + * @param predicatePushdown + * whether predicate pushdown is enabled + * @param projectionPushdown + * whether projection pushdown is enabled + * @throws Exception + * if query execution fails + */ + @ParameterizedTest + @CsvSource({ + "true,true", "true,false", "false,true", "false,false" + }) + void convertsTypeNamedColumns(boolean predicatePushdown, boolean projectionPushdown) throws Exception { + String url = "jdbc:jparq:" + directory + "?predicatePushdown=" + predicatePushdown + "&projectionPushdown=" + + projectionPushdown; + try (var connection = DriverManager.getConnection(url); var statement = connection.createStatement()) { + for (String column : List.of("int8", "int4", "float8")) { + for (String type : List.of("VARCHAR", "DOUBLE")) { + assertConvertMatchesCast(statement, column, type); + } + } + for (String column : List.of("bool", "text")) { + assertConvertMatchesCast(statement, column, "VARCHAR"); + } + // The regression from issue 324: the type position must not be read as a + // column when the operand is a physical column named after a type. + String sql = "SELECT CONVERT(VARCHAR, int8) AS v, CONVERT(DOUBLE, int8) AS d, CONVERT(VARCHAR, note) AS n " + + "FROM conv ORDER BY id"; + try (var result = statement.executeQuery(sql)) { + assertTrue(result.next(), sql); + assertEquals("10", result.getObject(1), sql); + assertEquals(10.0d, result.getObject(2), sql); + assertEquals("first", result.getObject(3), sql); + assertTrue(result.next(), sql); + assertEquals("20", result.getObject(1), sql); + assertEquals(20.0d, result.getObject(2), sql); + assertEquals("second", result.getObject(3), sql); + assertFalse(result.next(), sql); + } + } + } + + /** + * Compares both CONVERT argument orders against the equivalent CAST. + * + * @param statement + * statement used for the comparison queries + * @param column + * physical column named after a SQL type + * @param type + * cast target type + * @throws Exception + * if query execution fails + */ + private static void assertConvertMatchesCast(Statement statement, String column, String type) throws Exception { + for (String arguments : List.of(type + ", " + column, column + ", " + type)) { + String sql = "SELECT CONVERT(" + arguments + ") AS converted, CAST(" + column + " AS " + type + + ") AS casted, id FROM conv ORDER BY id"; + try (var result = statement.executeQuery(sql)) { + for (int row = 1; row <= 2; row++) { + assertTrue(result.next(), sql); + assertEquals(result.getObject(2), result.getObject(1), + sql + " row " + row + ": CONVERT should behave like CAST"); + } + assertFalse(result.next(), sql); + } + } + } + + /** + * Extracts type-named columns referenced through cast-style CONVERT in WHERE, + * GROUP BY, and ORDER BY clauses. + * + * @param predicatePushdown + * whether predicate pushdown is enabled + * @param projectionPushdown + * whether projection pushdown is enabled + * @throws Exception + * if query execution fails + */ + @ParameterizedTest + @CsvSource({ + "true,true", "true,false", "false,true", "false,false" + }) + void extractsTypeNamedColumnsFromClauses(boolean predicatePushdown, boolean projectionPushdown) throws Exception { + String url = "jdbc:jparq:" + directory + "?predicatePushdown=" + predicatePushdown + "&projectionPushdown=" + + projectionPushdown; + try (var connection = DriverManager.getConnection(url); var statement = connection.createStatement()) { + try (var result = statement.executeQuery("SELECT id FROM conv WHERE CONVERT(VARCHAR, int8) = '10' ORDER BY id")) { + assertTrue(result.next()); + assertEquals(1, result.getObject(1)); + assertFalse(result.next()); + } + try (var result = statement.executeQuery("SELECT CONVERT(VARCHAR, int8) AS v, COUNT(*) AS c FROM conv " + + "GROUP BY CONVERT(VARCHAR, int8) ORDER BY v")) { + assertTrue(result.next()); + assertEquals("10", result.getObject(1)); + assertEquals(1L, result.getObject(2)); + assertTrue(result.next()); + assertEquals("20", result.getObject(1)); + assertEquals(1L, result.getObject(2)); + assertFalse(result.next()); + } + try (var result = statement.executeQuery("SELECT id FROM conv ORDER BY CONVERT(VARCHAR, int8) DESC")) { + assertTrue(result.next()); + assertEquals(2, result.getObject(1)); + assertTrue(result.next()); + assertEquals(1, result.getObject(1)); + assertFalse(result.next()); + } + try (var result = statement.executeQuery( + "SELECT a.note FROM conv a JOIN amb b ON a.text = b.varchar AND a.int8 = CONVERT(BIGINT, b.int8) " + + "ORDER BY a.id")) { + assertTrue(result.next()); + assertEquals("first", result.getObject(1)); + assertFalse(result.next()); + } + } + } + + /** + * Reports ambiguous CONVERT arguments instead of a phantom missing column when + * both type positions name physical columns. + * + * @throws Exception + * if connection creation fails + */ + @Test + void rejectsAmbiguousTypeNamedColumnsDuringPreparation() throws Exception { + try (var connection = DriverManager.getConnection("jdbc:jparq:" + directory)) { + SQLException error = assertThrows(SQLException.class, + () -> connection.prepareStatement("SELECT CONVERT(VARCHAR, int8) FROM amb")); + assertEquals("42000", error.getSQLState()); + assertTrue(error.getMessage().contains("Ambiguous CONVERT arguments"), error.getMessage()); + } + } +} From 4a1cf4e77d4d21a973b652e91b3633d96d71e138 Mon Sep 17 00:00:00 2001 From: pernyf Date: Wed, 18 Nov 2026 12:26:30 +0100 Subject: [PATCH 2/3] Ignore .DS_Store files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index dc5df759..7ac8c8ee 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ /src/test/resources/acmeh2.trace.db /src/test/resources/datasets.trace.db /.m2/ +.DS_Store From 129b4a8f717fa1ae43602db901a6b8ec4139bd04 Mon Sep 17 00:00:00 2001 From: pernyf Date: Wed, 18 Nov 2026 13:31:36 +0100 Subject: [PATCH 3/3] Declare the CONVERT schema predicate adjacent to its first use --- src/main/java/se/alipsa/jparq/JParqPreparedStatement.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/se/alipsa/jparq/JParqPreparedStatement.java b/src/main/java/se/alipsa/jparq/JParqPreparedStatement.java index dc342d6b..49ee4b22 100644 --- a/src/main/java/se/alipsa/jparq/JParqPreparedStatement.java +++ b/src/main/java/se/alipsa/jparq/JParqPreparedStatement.java @@ -3251,13 +3251,13 @@ private void validateJoinOutputColumns(JoinRecordReader reader) throws SQLExcept final Schema schema = reader.schema(); validateAmbiguousColumns(parsedSelect, schema, reader.qualifierColumnMapping()); final Set required = new LinkedHashSet<>(); - Predicate schemaColumns = MaterializedNames.visibleColumns(schema); List expressions = new ArrayList<>(parsedSelect.expressions()); expressions.add(parsedSelect.where()); expressions.addAll(parsedSelect.groupByExpressions()); for (SqlParser.OrderKey key : parsedSelect.orderBy()) { expressions.add(key.expression()); } + Predicate schemaColumns = MaterializedNames.visibleColumns(schema); for (Expression expression : expressions) { addColumns(required, ColumnsUsed.inScope(expression, schemaColumns)); }