diff --git a/.gitignore b/.gitignore index dc5df759..7ac8c8ee 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ /src/test/resources/acmeh2.trace.db /src/test/resources/datasets.trace.db /.m2/ +.DS_Store diff --git a/src/main/java/se/alipsa/jparq/JParqPreparedStatement.java b/src/main/java/se/alipsa/jparq/JParqPreparedStatement.java index 22282d03..49ee4b22 100644 --- a/src/main/java/se/alipsa/jparq/JParqPreparedStatement.java +++ b/src/main/java/se/alipsa/jparq/JParqPreparedStatement.java @@ -55,6 +55,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -2358,6 +2359,7 @@ private List> applySetOrdering(List> rows, List schemaColumns = MaterializedNames.lookupKeys(labelIndexes.keySet()); List resolved = new ArrayList<>(orderBy.size()); boolean hasExpressions = false; for (SqlParser.SetOrder order : orderBy) { @@ -2375,7 +2377,7 @@ private List> applySetOrdering(List> rows, List expressionColumns = ColumnsUsed.inWhere(order.expression()); + Set expressionColumns = ColumnsUsed.inWhere(order.expression(), schemaColumns); for (String column : expressionColumns) { if (column == null) { continue; @@ -3170,7 +3172,8 @@ private void validateJoinConditions(List tables) thr qualified.put(JParqUtil.normalizeQualifier(qualifier), columns); } Set missing = new LinkedHashSet<>(); - for (Column column : ColumnsUsed.referencesInScope(table.joinCondition())) { + Predicate schemaColumns = MaterializedNames.lookupKeys(available); + for (Column column : ColumnsUsed.referencesInScope(table.joinCondition(), schemaColumns)) { String referenceQualifier = column.getTable() == null ? null : column.getTable().getName(); Set visible = referenceQualifier == null || referenceQualifier.isBlank() ? available @@ -3254,15 +3257,16 @@ private void validateJoinOutputColumns(JoinRecordReader reader) throws SQLExcept for (SqlParser.OrderKey key : parsedSelect.orderBy()) { expressions.add(key.expression()); } + Predicate schemaColumns = MaterializedNames.visibleColumns(schema); for (Expression expression : expressions) { - addColumns(required, ColumnsUsed.inScope(expression)); + addColumns(required, ColumnsUsed.inScope(expression, schemaColumns)); } if (tableReferences.stream().anyMatch(ref -> !ref.joinGroup().isEmpty())) { Map> qualifierMapping = reader.qualifierColumnMapping(); List qualifiedExpressions = new ArrayList<>(expressions); qualifiedExpressions.add(parsedSelect.having()); for (Expression expression : qualifiedExpressions) { - for (Column column : ColumnsUsed.referencesInScope(expression)) { + for (Column column : ColumnsUsed.referencesInScope(expression, schemaColumns)) { String qualifier = column.getTable() == null ? null : column.getTable().getName(); validateGroupedQualifier(qualifierMapping, qualifier, column.getColumnName()); } @@ -3277,7 +3281,7 @@ private void validateJoinOutputColumns(JoinRecordReader reader) throws SQLExcept addColumn(required, key.column()); } } - addColumns(required, ColumnsUsed.inScope(parsedSelect.having())); + addColumns(required, ColumnsUsed.inScope(parsedSelect.having(), schemaColumns)); validateRequiredColumns(schema, required, null, parsedSelect.labels(), parsedSelect.columnNames(), referenced); } @@ -3326,7 +3330,7 @@ private void validateJoinTableColumns(Schema schema, SqlParser.TableReference re if (schema == null || ref == null) { return; } - Set required = collectColumnsForTable(ref); + Set required = collectColumnsForTable(ref, MaterializedNames.visibleColumns(schema)); if (ref.usingColumns() != null) { required.addAll(ref.usingColumns()); } @@ -3341,9 +3345,11 @@ private void validateJoinTableColumns(Schema schema, SqlParser.TableReference re * * @param ref * table reference being validated + * @param schemaColumns + * predicate identifying visible source column names * @return mutable set of referenced column names (empty if none) */ - private Set collectColumnsForTable(SqlParser.TableReference ref) { + private Set collectColumnsForTable(SqlParser.TableReference ref, Predicate schemaColumns) { if (parsedSelect == null || ref == null) { return new LinkedHashSet<>(); } @@ -3354,10 +3360,12 @@ private Set collectColumnsForTable(SqlParser.TableReference ref) { boolean singleTable = !parsedSelect.hasMultipleTables(); List primaryQualifier = singleTable ? qualifierList(ref, true) : List.of(); Set required = new LinkedHashSet<>(); - collectColumnReferencesFromList(parsedSelect.expressions(), qualifiers, primaryQualifier, required); - collectColumnReferencesFromExpression(parsedSelect.where(), qualifiers, primaryQualifier, required); + collectColumnReferencesFromList(parsedSelect.expressions(), qualifiers, primaryQualifier, required, null, + schemaColumns); + collectColumnReferencesFromExpression(parsedSelect.where(), qualifiers, primaryQualifier, required, null, + schemaColumns); for (Expression group : parsedSelect.groupByExpressions()) { - collectColumnReferencesFromExpression(group, qualifiers, primaryQualifier, required); + collectColumnReferencesFromExpression(group, qualifiers, primaryQualifier, required, null, schemaColumns); } for (SqlParser.OrderKey key : parsedSelect.orderBy()) { if (shouldIncludeOrderKey(key, qualifiers, singleTable)) { @@ -3369,7 +3377,8 @@ private Set collectColumnsForTable(SqlParser.TableReference ref) { addColumn(required, key.column()); } } - collectColumnReferencesFromExpression(ref.joinCondition(), qualifiers, primaryQualifier, required); + collectColumnReferencesFromExpression(ref.joinCondition(), qualifiers, primaryQualifier, required, null, + schemaColumns); return required; } @@ -3386,19 +3395,19 @@ private Set collectColumnsForTable(SqlParser.TableReference ref) { * ambiguous) * @param target * set receiving the column names + * @param referenced + * set receiving bound column names, or null + * @param schemaColumns + * predicate identifying visible source column names */ private void collectColumnReferencesFromList(List expressions, List qualifiers, - List primaryQualifier, Set target) { - collectColumnReferencesFromList(expressions, qualifiers, primaryQualifier, target, null); - } - - private void collectColumnReferencesFromList(List expressions, List qualifiers, - List primaryQualifier, Set target, Set referenced) { + List primaryQualifier, Set target, Set referenced, Predicate schemaColumns) { if (expressions == null || expressions.isEmpty()) { return; } for (Expression expression : expressions) { - collectColumnReferencesFromExpression(expression, qualifiers, primaryQualifier, target, referenced); + collectColumnReferencesFromExpression(expression, qualifiers, primaryQualifier, target, referenced, + schemaColumns); } } @@ -3413,22 +3422,21 @@ private void collectColumnReferencesFromList(List expressions, List< * single qualifier used when unqualified columns should be included * @param target * set receiving the column names + * @param referenced + * set receiving bound column names, or null + * @param schemaColumns + * predicate identifying visible source column names */ private void collectColumnReferencesFromExpression(Expression expression, List qualifiers, - List primaryQualifier, Set target) { - collectColumnReferencesFromExpression(expression, qualifiers, primaryQualifier, target, null); - } - - private void collectColumnReferencesFromExpression(Expression expression, List qualifiers, - List primaryQualifier, Set target, Set referenced) { + List primaryQualifier, Set target, Set referenced, Predicate schemaColumns) { if (expression == null || target == null) { return; } - Set collected = SqlParser.collectQualifiedColumns(expression, qualifiers, false); + Set collected = SqlParser.collectQualifiedColumns(expression, qualifiers, false, schemaColumns); addColumns(target, collected); addColumns(referenced, collected); if (primaryQualifier != null && !primaryQualifier.isEmpty()) { - Set primaryCollected = SqlParser.collectQualifiedColumns(expression, primaryQualifier); + Set primaryCollected = SqlParser.collectQualifiedColumns(expression, primaryQualifier, schemaColumns); addColumns(target, primaryCollected); addColumns(referenced, primaryCollected); } @@ -6022,7 +6030,7 @@ private Set validateSelectColumns(SqlParser.Select select, Schema schema AggregateFunctions.AggregatePlan aggregatePlan, Map ctes, Configuration configuration) throws SQLException { validateAmbiguousColumns(select, schema, Map.of()); - ColumnRequirement requirement = requiredColumns(select, aggregatePlan); + ColumnRequirement requirement = requiredColumns(select, aggregatePlan, MaterializedNames.visibleColumns(schema)); collectSubqueryDependencies(select, schema, requirement.required(), new SubqueryExecutor(stmt.getConn(), null, table -> subqueryTableColumns(table, ctes, configuration))); SqlParser.TableReference baseRef = select.tableReferences().isEmpty() ? null : select.tableReferences().getFirst(); @@ -6048,11 +6056,12 @@ private void validateAmbiguousColumns(SqlParser.Select select, Schema schema, Map> qualifiers) { Map index = MaterializedNames.lookupIndex(schema); Map> qualified = ColumnMappingUtil.normaliseQualifierMapping(qualifiers); + Predicate schemaColumns = MaterializedNames.lookupKeys(index.keySet()); List expressions = new ArrayList<>(select.expressions()); expressions.add(select.where()); expressions.addAll(select.groupByExpressions()); for (Expression expression : expressions) { - for (Column column : ColumnsUsed.referencesInScope(expression)) { + for (Column column : ColumnsUsed.referencesInScope(expression, schemaColumns)) { String qualifier = column.getTable() == null ? null : column.getTable().getName(); validateAmbiguousColumn(column.getColumnName(), qualifier, index, qualified); } @@ -6061,7 +6070,7 @@ private void validateAmbiguousColumns(SqlParser.Select select, Schema schema, orders.addAll(select.preOrderBy()); for (SqlParser.OrderKey order : orders) { if (order.expression() != null) { - for (Column column : ColumnsUsed.referencesInScope(order.expression())) { + for (Column column : ColumnsUsed.referencesInScope(order.expression(), schemaColumns)) { String qualifier = column.getTable() == null ? null : column.getTable().getName(); validateAmbiguousColumn(column.getColumnName(), qualifier, index, qualified); } @@ -6166,7 +6175,8 @@ private void validatePredicateSubqueries(SqlParser.Select select, Map required, Set referenced) { } - private ColumnRequirement requiredColumns(SqlParser.Select select, AggregateFunctions.AggregatePlan aggregatePlan) { + private ColumnRequirement requiredColumns(SqlParser.Select select, AggregateFunctions.AggregatePlan aggregatePlan, + Predicate schemaColumns) { if (select == null) { return new ColumnRequirement(Set.of(), Set.of()); } @@ -6180,7 +6190,7 @@ private ColumnRequirement requiredColumns(SqlParser.Select select, AggregateFunc } } } - Set whereColumns = ColumnsUsed.inWhere(select.where()); + Set whereColumns = ColumnsUsed.inWhere(select.where(), schemaColumns); addColumns(needed, whereColumns); addColumns(referenced, whereColumns); SqlParser.TableReference baseRef = select.tableReferences() == null || select.tableReferences().isEmpty() @@ -6188,21 +6198,25 @@ private ColumnRequirement requiredColumns(SqlParser.Select select, AggregateFunc : select.tableReferences().getFirst(); List qualifiers = qualifierList(baseRef, false); List primaryQualifier = qualifierList(baseRef, true); - collectColumnReferencesFromList(select.expressions(), qualifiers, primaryQualifier, needed, referenced); - collectColumnReferencesFromExpression(select.where(), qualifiers, primaryQualifier, needed, referenced); - collectColumnReferencesFromExpression(select.having(), qualifiers, primaryQualifier, needed); + collectColumnReferencesFromList(select.expressions(), qualifiers, primaryQualifier, needed, referenced, + schemaColumns); + collectColumnReferencesFromExpression(select.where(), qualifiers, primaryQualifier, needed, referenced, + schemaColumns); + collectColumnReferencesFromExpression(select.having(), qualifiers, primaryQualifier, needed, null, schemaColumns); for (SqlParser.OrderKey key : select.orderBy()) { if (key.expression() != null) { - addColumns(needed, ColumnsUsed.inWhere(key.expression())); - collectColumnReferencesFromExpression(key.expression(), qualifiers, primaryQualifier, needed, referenced); + addColumns(needed, ColumnsUsed.inWhere(key.expression(), schemaColumns)); + collectColumnReferencesFromExpression(key.expression(), qualifiers, primaryQualifier, needed, referenced, + schemaColumns); } else if (!isOrdinalReference(key.column()) && !isGroupingFunctionReference(key.column())) { addColumn(needed, key.column()); } } for (SqlParser.OrderKey key : select.preOrderBy()) { if (key.expression() != null) { - addColumns(needed, ColumnsUsed.inWhere(key.expression())); - collectColumnReferencesFromExpression(key.expression(), qualifiers, primaryQualifier, needed, referenced); + addColumns(needed, ColumnsUsed.inWhere(key.expression(), schemaColumns)); + collectColumnReferencesFromExpression(key.expression(), qualifiers, primaryQualifier, needed, referenced, + schemaColumns); } else if (!isOrdinalReference(key.column()) && !isGroupingFunctionReference(key.column())) { addColumn(needed, key.column()); } @@ -6210,22 +6224,22 @@ private ColumnRequirement requiredColumns(SqlParser.Select select, AggregateFunc WindowPlan windowPlan = WindowFunctions.plan(select.expressions()); if (windowPlan != null) { for (Expression input : windowPlan.inputExpressions()) { - collectColumnReferencesFromExpression(input, qualifiers, primaryQualifier, needed, referenced); - addColumns(needed, ColumnsUsed.inWhere(input)); + collectColumnReferencesFromExpression(input, qualifiers, primaryQualifier, needed, referenced, schemaColumns); + addColumns(needed, ColumnsUsed.inWhere(input, schemaColumns)); } } if (aggregatePlan != null) { for (AggregateFunctions.AggregateSpec spec : aggregatePlan.specs()) { if (!spec.countStar()) { for (Expression arg : spec.arguments()) { - Set argColumns = ColumnsUsed.inWhere(arg); + Set argColumns = ColumnsUsed.inWhere(arg, schemaColumns); addColumns(needed, argColumns); addColumns(referenced, argColumns); } } } for (AggregateFunctions.GroupExpression groupExpr : aggregatePlan.groupExpressions()) { - Set groupColumns = ColumnsUsed.inWhere(groupExpr.expression()); + Set groupColumns = ColumnsUsed.inWhere(groupExpr.expression(), schemaColumns); addColumns(needed, groupColumns); addColumns(referenced, groupColumns); } diff --git a/src/main/java/se/alipsa/jparq/JParqResultSet.java b/src/main/java/se/alipsa/jparq/JParqResultSet.java index 2cc255cd..acfccbc1 100644 --- a/src/main/java/se/alipsa/jparq/JParqResultSet.java +++ b/src/main/java/se/alipsa/jparq/JParqResultSet.java @@ -26,6 +26,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiFunction; +import java.util.function.Predicate; import javax.sql.rowset.serial.SerialBlob; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.operators.conditional.AndExpression; @@ -372,12 +373,13 @@ public JParqResultSet(RecordReader reader, SqlParser.Select select, String table this.columnOrder.addAll(proj); } Set requiredColumns = new LinkedHashSet<>(proj); - requiredColumns.addAll(ColumnsUsed.inWhere(effectiveResidual)); - requiredColumns.addAll(ColumnsUsed.inWhere(select.having())); - requiredColumns.addAll(SqlParser.collectQualifiedColumns(effectiveResidual, queryQualifiers)); - requiredColumns.addAll(SqlParser.collectQualifiedColumns(select.having(), queryQualifiers)); + Predicate schemaColumns = MaterializedNames.visibleColumns(schema); + requiredColumns.addAll(ColumnsUsed.inWhere(effectiveResidual, schemaColumns)); + requiredColumns.addAll(ColumnsUsed.inWhere(select.having(), schemaColumns)); + requiredColumns.addAll(SqlParser.collectQualifiedColumns(effectiveResidual, queryQualifiers, schemaColumns)); + requiredColumns.addAll(SqlParser.collectQualifiedColumns(select.having(), queryQualifiers, schemaColumns)); for (Expression expression : selectExpressions) { - requiredColumns.addAll(SqlParser.collectQualifiedColumns(expression, queryQualifiers)); + requiredColumns.addAll(SqlParser.collectQualifiedColumns(expression, queryQualifiers, schemaColumns)); } if (select.orderBy() != null) { for (SqlParser.OrderKey orderKey : select.orderBy()) { @@ -388,7 +390,7 @@ public JParqResultSet(RecordReader reader, SqlParser.Select select, String table } if (windowPlan != null) { for (Expression input : windowPlan.inputExpressions()) { - requiredColumns.addAll(SqlParser.collectQualifiedColumns(input, queryQualifiers)); + requiredColumns.addAll(SqlParser.collectQualifiedColumns(input, queryQualifiers, schemaColumns)); } } if (schema != null) { diff --git a/src/main/java/se/alipsa/jparq/engine/ColumnsUsed.java b/src/main/java/se/alipsa/jparq/engine/ColumnsUsed.java index 832de315..c6d62d35 100644 --- a/src/main/java/se/alipsa/jparq/engine/ColumnsUsed.java +++ b/src/main/java/se/alipsa/jparq/engine/ColumnsUsed.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Predicate; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.TranscodingFunction; import net.sf.jsqlparser.schema.Column; @@ -42,7 +43,21 @@ private ColumnsUsed() { * @return set of column names referenced by the expression (empty if none) */ public static Set inWhere(Expression where) { - return collect(where, true); + return inWhere(where, name -> false); + } + + /** + * Extract all column names referenced in a SQL {@code WHERE} expression, using + * visible source column names to resolve cast-style CONVERT arguments. + * + * @param where + * the expression (may be null) + * @param schemaColumns + * predicate identifying visible source column names + * @return set of column names referenced by the expression (empty if none) + */ + public static Set inWhere(Expression where, Predicate schemaColumns) { + return collect(where, true, schemaColumns); } /** @@ -53,7 +68,21 @@ public static Set inWhere(Expression where) { * @return column names belonging to the current query scope */ public static Set inScope(Expression expression) { - return collect(expression, false); + return inScope(expression, name -> false); + } + + /** + * Collects column names in an expression without entering nested query scopes, + * using visible source column names to resolve cast-style CONVERT arguments. + * + * @param expression + * expression to inspect, or null + * @param schemaColumns + * predicate identifying visible source column names + * @return column names belonging to the current query scope + */ + public static Set inScope(Expression expression, Predicate schemaColumns) { + return collect(expression, false, schemaColumns); } /** @@ -64,8 +93,22 @@ public static Set inScope(Expression expression) { * @return column nodes without references inside nested queries */ public static List referencesInScope(Expression expression) { + return referencesInScope(expression, name -> false); + } + + /** + * Retains table qualifiers while collecting references in the current scope, + * using visible source column names to resolve cast-style CONVERT arguments. + * + * @param expression + * expression to inspect, or null + * @param schemaColumns + * predicate identifying visible source column names + * @return column nodes without references inside nested queries + */ + public static List referencesInScope(Expression expression, Predicate schemaColumns) { List columns = new ArrayList<>(); - walk(expression, false, columns::add); + walk(expression, false, schemaColumns, columns::add); return List.copyOf(columns); } @@ -76,11 +119,13 @@ public static List referencesInScope(Expression expression) { * expression to inspect * @param includeSubqueries * whether nested query expressions should be visited + * @param schemaColumns + * predicate identifying visible source column names * @return referenced column names */ - private static Set collect(Expression where, boolean includeSubqueries) { + private static Set collect(Expression where, boolean includeSubqueries, Predicate schemaColumns) { LinkedHashSet cols = new LinkedHashSet<>(); - walk(where, includeSubqueries, column -> cols.add(column.getColumnName())); + walk(where, includeSubqueries, schemaColumns, column -> cols.add(column.getColumnName())); return cols; } @@ -91,10 +136,13 @@ private static Set collect(Expression where, boolean includeSubqueries) * expression to inspect * @param includeSubqueries * whether nested expressions should be visited + * @param schemaColumns + * predicate identifying visible source column names * @param consumer * receiver for each column reference */ - private static void walk(Expression where, boolean includeSubqueries, Consumer consumer) { + private static void walk(Expression where, boolean includeSubqueries, Predicate schemaColumns, + Consumer consumer) { if (where == null) { return; } @@ -119,7 +167,7 @@ public Void visit(Column column, S context) { @Override public Void visit(TranscodingFunction convert, S context) { - SqlParser.ConvertProcessingResult result = SqlParser.analyzeConvertFunction(convert); + SqlParser.ConvertProcessingResult result = SqlParser.analyzeConvertFunction(convert, schemaColumns); if (result.syntheticColumnName() != null) { consumer.accept(new Column(result.syntheticColumnName())); return null; diff --git a/src/main/java/se/alipsa/jparq/engine/MaterializedNames.java b/src/main/java/se/alipsa/jparq/engine/MaterializedNames.java index c1f8a4b1..c2ef184d 100644 --- a/src/main/java/se/alipsa/jparq/engine/MaterializedNames.java +++ b/src/main/java/se/alipsa/jparq/engine/MaterializedNames.java @@ -1,8 +1,10 @@ package se.alipsa.jparq.engine; +import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import java.util.regex.Pattern; import org.apache.avro.Schema; @@ -99,6 +101,35 @@ public static Map lookupIndex(Schema schema) { return Map.copyOf(index); } + /** + * Builds a predicate identifying visible SQL column names of a schema. + * + * @param schema + * source schema, or null for an unavailable schema + * @return predicate matching visible SQL column names + */ + public static Predicate visibleColumns(Schema schema) { + return lookupKeys(lookupIndex(schema).keySet()); + } + + /** + * Builds a predicate matching SQL names against a collection of lookup keys. + * + * @param keys + * SQL lookup keys of visible column names, or null + * @return predicate matching the supplied lookup keys + */ + public static Predicate lookupKeys(Collection keys) { + if (keys == null || keys.isEmpty()) { + return name -> false; + } + Set known = Set.copyOf(keys); + return name -> { + String key = Identifier.lookupKey(name); + return key != null && known.contains(key); + }; + } + /** * Prevents an unresolved SQL identifier from addressing a generated internal * name. diff --git a/src/main/java/se/alipsa/jparq/engine/SqlParser.java b/src/main/java/se/alipsa/jparq/engine/SqlParser.java index 5f8224cb..476e6532 100644 --- a/src/main/java/se/alipsa/jparq/engine/SqlParser.java +++ b/src/main/java/se/alipsa/jparq/engine/SqlParser.java @@ -14,6 +14,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -89,10 +90,12 @@ record ConvertProcessingResult(boolean processInner, String syntheticColumnName) * *

* This method handles the special case where CONVERT is used with a type name - * as the first argument (e.g., CONVERT(VARCHAR, column_name)). In this case, - * JSqlParser swaps the arguments, putting the type name where the column would - * normally be. This method detects that pattern and returns a result indicating - * the actual column name to use. + * in place of a column name. JSqlParser assigns the first argument to + * {@code getColDataType()} and the second to {@code getExpression()}, so this + * method detects the pattern where the arguments are in the value-first order + * and returns a result indicating the actual column name to use. Without schema + * knowledge, an inner expression spelling a type name is assumed to be the + * target type. * * @param convert * the transcoding function to analyze (may be null) @@ -100,6 +103,28 @@ record ConvertProcessingResult(boolean processInner, String syntheticColumnName) * synthetic column name */ static ConvertProcessingResult analyzeConvertFunction(TranscodingFunction convert) { + return analyzeConvertFunction(convert, name -> false); + } + + /** + * Analyzes a {@link TranscodingFunction} for column extraction using visible + * source column names. + * + *

+ * The resolution mirrors {@link CastExpressions#asCast(Expression, Predicate)} + * so that column extraction agrees with evaluation: when both arguments spell + * type names, a {@code Column} operand naming a visible source column keeps its + * value role instead of being treated as the target type. When the arguments + * are ambiguous, the type-name heuristic without schema knowledge is retained. + * + * @param convert + * the transcoding function to analyze (may be null) + * @param schemaColumns + * predicate identifying visible source column names + * @return a result indicating whether to process the inner expression and/or a + * synthetic column name + */ + static ConvertProcessingResult analyzeConvertFunction(TranscodingFunction convert, Predicate schemaColumns) { if (convert == null) { return new ConvertProcessingResult(false, null); } @@ -108,8 +133,17 @@ static ConvertProcessingResult analyzeConvertFunction(TranscodingFunction conver } Expression inner = convert.getExpression(); String typeCandidate = convert.getColDataType() == null ? null : convert.getColDataType().getDataType(); - if (inner != null && isConvertTypeName(inner.toString()) && typeCandidate != null - && (inner instanceof Column || !isConvertTypeName(typeCandidate))) { + if (inner == null || typeCandidate == null) { + return new ConvertProcessingResult(true, null); + } + CastExpression cast = CastExpressions.asCast(convert, schemaColumns); + if (cast != null) { + if (cast.getLeftExpression() != inner) { + return new ConvertProcessingResult(false, cast.getLeftExpression().toString()); + } + return new ConvertProcessingResult(true, null); + } + if (isConvertTypeName(inner.toString()) && (inner instanceof Column || !isConvertTypeName(typeCandidate))) { return new ConvertProcessingResult(false, typeCandidate); } return new ConvertProcessingResult(true, null); @@ -2768,8 +2802,9 @@ private static String orderingAliasLabel(String name, Map al */ private static Expression resolveOrderingAliases(Expression expression, Map aliases, Set ambiguous, Set inputs) { + Predicate schemaColumns = MaterializedNames.lookupKeys(inputs); Map replacements = new java.util.IdentityHashMap<>(); - for (Column column : ColumnsUsed.referencesInScope(expression)) { + for (Column column : ColumnsUsed.referencesInScope(expression, schemaColumns)) { String input = Identifier.lookupKey(column.getColumnName()); String alternative = ColumnMappingUtil.alternativeColumnKey(input); if (extractQualifier(column) != null || inputs.contains(input) @@ -3081,7 +3116,24 @@ public Void visit(ParenthesedSelect subSelect, S context) { * @return a set of column names that reference one of the qualifiers */ public static Set collectQualifiedColumns(Expression expr, List qualifiers) { - return collectQualifiedColumns(expr, qualifiers, true); + return collectQualifiedColumns(expr, qualifiers, true, name -> false); + } + + /** + * Collects references to supplied qualifiers using visible source column names + * to resolve cast-style CONVERT arguments. + * + * @param expr + * the expression to inspect + * @param qualifiers + * table names or aliases to match + * @param schemaColumns + * predicate identifying visible source column names + * @return matching column names + */ + public static Set collectQualifiedColumns(Expression expr, List qualifiers, + Predicate schemaColumns) { + return collectQualifiedColumns(expr, qualifiers, true, schemaColumns); } /** @@ -3089,7 +3141,7 @@ public static Set collectQualifiedColumns(Expression expr, List * names. * * @param expr - * expression to inspect + * the expression to inspect * @param qualifiers * table names or aliases to match * @param includeUnqualified @@ -3098,6 +3150,26 @@ public static Set collectQualifiedColumns(Expression expr, List */ public static Set collectQualifiedColumns(Expression expr, List qualifiers, boolean includeUnqualified) { + return collectQualifiedColumns(expr, qualifiers, includeUnqualified, name -> false); + } + + /** + * Collects references to supplied qualifiers, optionally including unqualified + * names, using visible source column names to resolve cast-style CONVERT + * arguments. + * + * @param expr + * the expression to inspect + * @param qualifiers + * table names or aliases to match + * @param includeUnqualified + * whether a single qualifier also owns unqualified references + * @param schemaColumns + * predicate identifying visible source column names + * @return matching column names + */ + public static Set collectQualifiedColumns(Expression expr, List qualifiers, + boolean includeUnqualified, Predicate schemaColumns) { if (expr == null || qualifiers == null || qualifiers.isEmpty()) { return Set.of(); } @@ -3161,7 +3233,7 @@ public Void visit(net.sf.jsqlparser.statement.select.Select select, S contex @Override public Void visit(TranscodingFunction convert, S context) { - ConvertProcessingResult result = analyzeConvertFunction(convert); + ConvertProcessingResult result = analyzeConvertFunction(convert, schemaColumns); if (result.syntheticColumnName() != null) { Column synthetic = new Column(result.syntheticColumnName()); synthetic.accept(this, context); diff --git a/src/test/java/jparq/functions/ConvertTypeNameColumnTest.java b/src/test/java/jparq/functions/ConvertTypeNameColumnTest.java new file mode 100644 index 00000000..9a0c584e --- /dev/null +++ b/src/test/java/jparq/functions/ConvertTypeNameColumnTest.java @@ -0,0 +1,212 @@ +package jparq.functions; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Map; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.avro.AvroParquetWriter; +import org.apache.parquet.hadoop.util.HadoopOutputFile; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +/** + * Resolves cast-style CONVERT against physical columns whose names spell SQL + * type names, so column extraction agrees with evaluation. + */ +class ConvertTypeNameColumnTest { + + private static final Schema CONV_SCHEMA = SchemaBuilder.record("conv").fields().requiredInt("id").requiredLong("int8") + .requiredInt("int4").requiredDouble("float8").requiredBoolean("bool").requiredString("text") + .requiredString("note").endRecord(); + + private static final Schema AMBIGUOUS_SCHEMA = SchemaBuilder.record("amb").fields().requiredInt("id") + .requiredString("varchar").requiredLong("int8").endRecord(); + + @TempDir + static Path directory; + + /** + * Writes fixture tables containing columns named after SQL type names. + * + * @throws Exception + * if writing the fixture fails + */ + @BeforeAll + static void writeFixture() throws Exception { + var configuration = new Configuration(false); + List> convRows = List.of( + Map.of("id", 1, "int8", 10L, "int4", 100, "float8", 1.5d, "bool", true, "text", "ten", "note", "first"), + Map.of("id", 2, "int8", 20L, "int4", 200, "float8", 2.5d, "bool", false, "text", "twenty", "note", "second")); + writeTable("conv.parquet", CONV_SCHEMA, configuration, convRows); + List> ambiguousRows = List.of(Map.of("id", 1, "varchar", "ten", "int8", 10L)); + writeTable("amb.parquet", AMBIGUOUS_SCHEMA, configuration, ambiguousRows); + } + + private static void writeTable(String name, Schema schema, Configuration configuration, + List> rows) throws Exception { + var output = HadoopOutputFile.fromPath(new org.apache.hadoop.fs.Path(directory.resolve(name).toUri()), + configuration); + try (var writer = AvroParquetWriter.builder(output).withSchema(schema).withConf(configuration) + .build()) { + for (Map row : rows) { + var record = new GenericData.Record(schema); + row.forEach(record::put); + writer.write(record); + } + } + } + + /** + * Converts physical columns named after SQL type names in both argument orders. + * + * @param predicatePushdown + * whether predicate pushdown is enabled + * @param projectionPushdown + * whether projection pushdown is enabled + * @throws Exception + * if query execution fails + */ + @ParameterizedTest + @CsvSource({ + "true,true", "true,false", "false,true", "false,false" + }) + void convertsTypeNamedColumns(boolean predicatePushdown, boolean projectionPushdown) throws Exception { + String url = "jdbc:jparq:" + directory + "?predicatePushdown=" + predicatePushdown + "&projectionPushdown=" + + projectionPushdown; + try (var connection = DriverManager.getConnection(url); var statement = connection.createStatement()) { + for (String column : List.of("int8", "int4", "float8")) { + for (String type : List.of("VARCHAR", "DOUBLE")) { + assertConvertMatchesCast(statement, column, type); + } + } + for (String column : List.of("bool", "text")) { + assertConvertMatchesCast(statement, column, "VARCHAR"); + } + // The regression from issue 324: the type position must not be read as a + // column when the operand is a physical column named after a type. + String sql = "SELECT CONVERT(VARCHAR, int8) AS v, CONVERT(DOUBLE, int8) AS d, CONVERT(VARCHAR, note) AS n " + + "FROM conv ORDER BY id"; + try (var result = statement.executeQuery(sql)) { + assertTrue(result.next(), sql); + assertEquals("10", result.getObject(1), sql); + assertEquals(10.0d, result.getObject(2), sql); + assertEquals("first", result.getObject(3), sql); + assertTrue(result.next(), sql); + assertEquals("20", result.getObject(1), sql); + assertEquals(20.0d, result.getObject(2), sql); + assertEquals("second", result.getObject(3), sql); + assertFalse(result.next(), sql); + } + } + } + + /** + * Compares both CONVERT argument orders against the equivalent CAST. + * + * @param statement + * statement used for the comparison queries + * @param column + * physical column named after a SQL type + * @param type + * cast target type + * @throws Exception + * if query execution fails + */ + private static void assertConvertMatchesCast(Statement statement, String column, String type) throws Exception { + for (String arguments : List.of(type + ", " + column, column + ", " + type)) { + String sql = "SELECT CONVERT(" + arguments + ") AS converted, CAST(" + column + " AS " + type + + ") AS casted, id FROM conv ORDER BY id"; + try (var result = statement.executeQuery(sql)) { + for (int row = 1; row <= 2; row++) { + assertTrue(result.next(), sql); + assertEquals(result.getObject(2), result.getObject(1), + sql + " row " + row + ": CONVERT should behave like CAST"); + } + assertFalse(result.next(), sql); + } + } + } + + /** + * Extracts type-named columns referenced through cast-style CONVERT in WHERE, + * GROUP BY, and ORDER BY clauses. + * + * @param predicatePushdown + * whether predicate pushdown is enabled + * @param projectionPushdown + * whether projection pushdown is enabled + * @throws Exception + * if query execution fails + */ + @ParameterizedTest + @CsvSource({ + "true,true", "true,false", "false,true", "false,false" + }) + void extractsTypeNamedColumnsFromClauses(boolean predicatePushdown, boolean projectionPushdown) throws Exception { + String url = "jdbc:jparq:" + directory + "?predicatePushdown=" + predicatePushdown + "&projectionPushdown=" + + projectionPushdown; + try (var connection = DriverManager.getConnection(url); var statement = connection.createStatement()) { + try (var result = statement.executeQuery("SELECT id FROM conv WHERE CONVERT(VARCHAR, int8) = '10' ORDER BY id")) { + assertTrue(result.next()); + assertEquals(1, result.getObject(1)); + assertFalse(result.next()); + } + try (var result = statement.executeQuery("SELECT CONVERT(VARCHAR, int8) AS v, COUNT(*) AS c FROM conv " + + "GROUP BY CONVERT(VARCHAR, int8) ORDER BY v")) { + assertTrue(result.next()); + assertEquals("10", result.getObject(1)); + assertEquals(1L, result.getObject(2)); + assertTrue(result.next()); + assertEquals("20", result.getObject(1)); + assertEquals(1L, result.getObject(2)); + assertFalse(result.next()); + } + try (var result = statement.executeQuery("SELECT id FROM conv ORDER BY CONVERT(VARCHAR, int8) DESC")) { + assertTrue(result.next()); + assertEquals(2, result.getObject(1)); + assertTrue(result.next()); + assertEquals(1, result.getObject(1)); + assertFalse(result.next()); + } + try (var result = statement.executeQuery( + "SELECT a.note FROM conv a JOIN amb b ON a.text = b.varchar AND a.int8 = CONVERT(BIGINT, b.int8) " + + "ORDER BY a.id")) { + assertTrue(result.next()); + assertEquals("first", result.getObject(1)); + assertFalse(result.next()); + } + } + } + + /** + * Reports ambiguous CONVERT arguments instead of a phantom missing column when + * both type positions name physical columns. + * + * @throws Exception + * if connection creation fails + */ + @Test + void rejectsAmbiguousTypeNamedColumnsDuringPreparation() throws Exception { + try (var connection = DriverManager.getConnection("jdbc:jparq:" + directory)) { + SQLException error = assertThrows(SQLException.class, + () -> connection.prepareStatement("SELECT CONVERT(VARCHAR, int8) FROM amb")); + assertEquals("42000", error.getSQLState()); + assertTrue(error.getMessage().contains("Ambiguous CONVERT arguments"), error.getMessage()); + } + } +}