Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@
/src/test/resources/acmeh2.trace.db
/src/test/resources/datasets.trace.db
/.m2/
.DS_Store
98 changes: 56 additions & 42 deletions src/main/java/se/alipsa/jparq/JParqPreparedStatement.java
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -2358,6 +2359,7 @@ private List<List<Object>> applySetOrdering(List<List<Object>> rows, List<String
labelIndexes.put(key, i);
}
}
Predicate<String> schemaColumns = MaterializedNames.lookupKeys(labelIndexes.keySet());
List<ResolvedSetOrder> resolved = new ArrayList<>(orderBy.size());
boolean hasExpressions = false;
for (SqlParser.SetOrder order : orderBy) {
Expand All @@ -2375,7 +2377,7 @@ private List<List<Object>> applySetOrdering(List<List<Object>> rows, List<String
}
index = mapped;
} else {
Set<String> expressionColumns = ColumnsUsed.inWhere(order.expression());
Set<String> expressionColumns = ColumnsUsed.inWhere(order.expression(), schemaColumns);
for (String column : expressionColumns) {
if (column == null) {
continue;
Expand Down Expand Up @@ -3170,7 +3172,8 @@ private void validateJoinConditions(List<JoinRecordReader.JoinTable> tables) thr
qualified.put(JParqUtil.normalizeQualifier(qualifier), columns);
}
Set<String> missing = new LinkedHashSet<>();
for (Column column : ColumnsUsed.referencesInScope(table.joinCondition())) {
Predicate<String> schemaColumns = MaterializedNames.lookupKeys(available);
for (Column column : ColumnsUsed.referencesInScope(table.joinCondition(), schemaColumns)) {
String referenceQualifier = column.getTable() == null ? null : column.getTable().getName();
Set<String> visible = referenceQualifier == null || referenceQualifier.isBlank()
? available
Expand Down Expand Up @@ -3254,15 +3257,16 @@ private void validateJoinOutputColumns(JoinRecordReader reader) throws SQLExcept
for (SqlParser.OrderKey key : parsedSelect.orderBy()) {
expressions.add(key.expression());
}
Predicate<String> schemaColumns = MaterializedNames.visibleColumns(schema);
for (Expression expression : expressions) {
addColumns(required, ColumnsUsed.inScope(expression));
addColumns(required, ColumnsUsed.inScope(expression, schemaColumns));
}
if (tableReferences.stream().anyMatch(ref -> !ref.joinGroup().isEmpty())) {
Map<String, Map<String, String>> qualifierMapping = reader.qualifierColumnMapping();
List<Expression> qualifiedExpressions = new ArrayList<>(expressions);
qualifiedExpressions.add(parsedSelect.having());
for (Expression expression : qualifiedExpressions) {
for (Column column : ColumnsUsed.referencesInScope(expression)) {
for (Column column : ColumnsUsed.referencesInScope(expression, schemaColumns)) {
String qualifier = column.getTable() == null ? null : column.getTable().getName();
validateGroupedQualifier(qualifierMapping, qualifier, column.getColumnName());
}
Expand All @@ -3277,7 +3281,7 @@ private void validateJoinOutputColumns(JoinRecordReader reader) throws SQLExcept
addColumn(required, key.column());
}
}
addColumns(required, ColumnsUsed.inScope(parsedSelect.having()));
addColumns(required, ColumnsUsed.inScope(parsedSelect.having(), schemaColumns));
validateRequiredColumns(schema, required, null, parsedSelect.labels(), parsedSelect.columnNames(), referenced);
}

Expand Down Expand Up @@ -3326,7 +3330,7 @@ private void validateJoinTableColumns(Schema schema, SqlParser.TableReference re
if (schema == null || ref == null) {
return;
}
Set<String> required = collectColumnsForTable(ref);
Set<String> required = collectColumnsForTable(ref, MaterializedNames.visibleColumns(schema));
if (ref.usingColumns() != null) {
required.addAll(ref.usingColumns());
}
Expand All @@ -3341,9 +3345,11 @@ private void validateJoinTableColumns(Schema schema, SqlParser.TableReference re
*
* @param ref
* table reference being validated
* @param schemaColumns
* predicate identifying visible source column names
* @return mutable set of referenced column names (empty if none)
*/
private Set<String> collectColumnsForTable(SqlParser.TableReference ref) {
private Set<String> collectColumnsForTable(SqlParser.TableReference ref, Predicate<String> schemaColumns) {
if (parsedSelect == null || ref == null) {
return new LinkedHashSet<>();
}
Expand All @@ -3354,10 +3360,12 @@ private Set<String> collectColumnsForTable(SqlParser.TableReference ref) {
boolean singleTable = !parsedSelect.hasMultipleTables();
List<String> primaryQualifier = singleTable ? qualifierList(ref, true) : List.of();
Set<String> required = new LinkedHashSet<>();
collectColumnReferencesFromList(parsedSelect.expressions(), qualifiers, primaryQualifier, required);
collectColumnReferencesFromExpression(parsedSelect.where(), qualifiers, primaryQualifier, required);
collectColumnReferencesFromList(parsedSelect.expressions(), qualifiers, primaryQualifier, required, null,
schemaColumns);
collectColumnReferencesFromExpression(parsedSelect.where(), qualifiers, primaryQualifier, required, null,
schemaColumns);
for (Expression group : parsedSelect.groupByExpressions()) {
collectColumnReferencesFromExpression(group, qualifiers, primaryQualifier, required);
collectColumnReferencesFromExpression(group, qualifiers, primaryQualifier, required, null, schemaColumns);
}
for (SqlParser.OrderKey key : parsedSelect.orderBy()) {
if (shouldIncludeOrderKey(key, qualifiers, singleTable)) {
Expand All @@ -3369,7 +3377,8 @@ private Set<String> collectColumnsForTable(SqlParser.TableReference ref) {
addColumn(required, key.column());
}
}
collectColumnReferencesFromExpression(ref.joinCondition(), qualifiers, primaryQualifier, required);
collectColumnReferencesFromExpression(ref.joinCondition(), qualifiers, primaryQualifier, required, null,
schemaColumns);
return required;
}

Expand All @@ -3386,19 +3395,19 @@ private Set<String> collectColumnsForTable(SqlParser.TableReference ref) {
* ambiguous)
* @param target
* set receiving the column names
* @param referenced
* set receiving bound column names, or null
* @param schemaColumns
* predicate identifying visible source column names
*/
private void collectColumnReferencesFromList(List<Expression> expressions, List<String> qualifiers,
List<String> primaryQualifier, Set<String> target) {
collectColumnReferencesFromList(expressions, qualifiers, primaryQualifier, target, null);
}

private void collectColumnReferencesFromList(List<Expression> expressions, List<String> qualifiers,
List<String> primaryQualifier, Set<String> target, Set<String> referenced) {
List<String> primaryQualifier, Set<String> target, Set<String> referenced, Predicate<String> schemaColumns) {
if (expressions == null || expressions.isEmpty()) {
return;
}
for (Expression expression : expressions) {
collectColumnReferencesFromExpression(expression, qualifiers, primaryQualifier, target, referenced);
collectColumnReferencesFromExpression(expression, qualifiers, primaryQualifier, target, referenced,
schemaColumns);
}
}

Expand All @@ -3413,22 +3422,21 @@ private void collectColumnReferencesFromList(List<Expression> expressions, List<
* single qualifier used when unqualified columns should be included
* @param target
* set receiving the column names
* @param referenced
* set receiving bound column names, or null
* @param schemaColumns
* predicate identifying visible source column names
*/
private void collectColumnReferencesFromExpression(Expression expression, List<String> qualifiers,
List<String> primaryQualifier, Set<String> target) {
collectColumnReferencesFromExpression(expression, qualifiers, primaryQualifier, target, null);
}

private void collectColumnReferencesFromExpression(Expression expression, List<String> qualifiers,
List<String> primaryQualifier, Set<String> target, Set<String> referenced) {
List<String> primaryQualifier, Set<String> target, Set<String> referenced, Predicate<String> schemaColumns) {
if (expression == null || target == null) {
return;
}
Set<String> collected = SqlParser.collectQualifiedColumns(expression, qualifiers, false);
Set<String> collected = SqlParser.collectQualifiedColumns(expression, qualifiers, false, schemaColumns);
addColumns(target, collected);
addColumns(referenced, collected);
if (primaryQualifier != null && !primaryQualifier.isEmpty()) {
Set<String> primaryCollected = SqlParser.collectQualifiedColumns(expression, primaryQualifier);
Set<String> primaryCollected = SqlParser.collectQualifiedColumns(expression, primaryQualifier, schemaColumns);
addColumns(target, primaryCollected);
addColumns(referenced, primaryCollected);
}
Expand Down Expand Up @@ -6022,7 +6030,7 @@ private Set<String> validateSelectColumns(SqlParser.Select select, Schema schema
AggregateFunctions.AggregatePlan aggregatePlan, Map<Identifier, CteResult> ctes, Configuration configuration)
throws SQLException {
validateAmbiguousColumns(select, schema, Map.of());
ColumnRequirement requirement = requiredColumns(select, aggregatePlan);
ColumnRequirement requirement = requiredColumns(select, aggregatePlan, MaterializedNames.visibleColumns(schema));
collectSubqueryDependencies(select, schema, requirement.required(),
new SubqueryExecutor(stmt.getConn(), null, table -> subqueryTableColumns(table, ctes, configuration)));
SqlParser.TableReference baseRef = select.tableReferences().isEmpty() ? null : select.tableReferences().getFirst();
Expand All @@ -6048,11 +6056,12 @@ private void validateAmbiguousColumns(SqlParser.Select select, Schema schema,
Map<String, Map<String, String>> qualifiers) {
Map<String, String> index = MaterializedNames.lookupIndex(schema);
Map<String, Map<String, String>> qualified = ColumnMappingUtil.normaliseQualifierMapping(qualifiers);
Predicate<String> schemaColumns = MaterializedNames.lookupKeys(index.keySet());
List<Expression> expressions = new ArrayList<>(select.expressions());
expressions.add(select.where());
expressions.addAll(select.groupByExpressions());
for (Expression expression : expressions) {
for (Column column : ColumnsUsed.referencesInScope(expression)) {
for (Column column : ColumnsUsed.referencesInScope(expression, schemaColumns)) {
String qualifier = column.getTable() == null ? null : column.getTable().getName();
validateAmbiguousColumn(column.getColumnName(), qualifier, index, qualified);
}
Expand All @@ -6061,7 +6070,7 @@ private void validateAmbiguousColumns(SqlParser.Select select, Schema schema,
orders.addAll(select.preOrderBy());
for (SqlParser.OrderKey order : orders) {
if (order.expression() != null) {
for (Column column : ColumnsUsed.referencesInScope(order.expression())) {
for (Column column : ColumnsUsed.referencesInScope(order.expression(), schemaColumns)) {
String qualifier = column.getTable() == null ? null : column.getTable().getName();
validateAmbiguousColumn(column.getColumnName(), qualifier, index, qualified);
}
Expand Down Expand Up @@ -6166,7 +6175,8 @@ private void validatePredicateSubqueries(SqlParser.Select select, Map<Identifier
private record ColumnRequirement(Set<String> required, Set<String> referenced) {
}

private ColumnRequirement requiredColumns(SqlParser.Select select, AggregateFunctions.AggregatePlan aggregatePlan) {
private ColumnRequirement requiredColumns(SqlParser.Select select, AggregateFunctions.AggregatePlan aggregatePlan,
Predicate<String> schemaColumns) {
if (select == null) {
return new ColumnRequirement(Set.of(), Set.of());
}
Expand All @@ -6180,52 +6190,56 @@ private ColumnRequirement requiredColumns(SqlParser.Select select, AggregateFunc
}
}
}
Set<String> whereColumns = ColumnsUsed.inWhere(select.where());
Set<String> whereColumns = ColumnsUsed.inWhere(select.where(), schemaColumns);
addColumns(needed, whereColumns);
addColumns(referenced, whereColumns);
SqlParser.TableReference baseRef = select.tableReferences() == null || select.tableReferences().isEmpty()
? null
: select.tableReferences().getFirst();
List<String> qualifiers = qualifierList(baseRef, false);
List<String> primaryQualifier = qualifierList(baseRef, true);
collectColumnReferencesFromList(select.expressions(), qualifiers, primaryQualifier, needed, referenced);
collectColumnReferencesFromExpression(select.where(), qualifiers, primaryQualifier, needed, referenced);
collectColumnReferencesFromExpression(select.having(), qualifiers, primaryQualifier, needed);
collectColumnReferencesFromList(select.expressions(), qualifiers, primaryQualifier, needed, referenced,
schemaColumns);
collectColumnReferencesFromExpression(select.where(), qualifiers, primaryQualifier, needed, referenced,
schemaColumns);
collectColumnReferencesFromExpression(select.having(), qualifiers, primaryQualifier, needed, null, schemaColumns);
for (SqlParser.OrderKey key : select.orderBy()) {
if (key.expression() != null) {
addColumns(needed, ColumnsUsed.inWhere(key.expression()));
collectColumnReferencesFromExpression(key.expression(), qualifiers, primaryQualifier, needed, referenced);
addColumns(needed, ColumnsUsed.inWhere(key.expression(), schemaColumns));
collectColumnReferencesFromExpression(key.expression(), qualifiers, primaryQualifier, needed, referenced,
schemaColumns);
} else if (!isOrdinalReference(key.column()) && !isGroupingFunctionReference(key.column())) {
addColumn(needed, key.column());
}
}
for (SqlParser.OrderKey key : select.preOrderBy()) {
if (key.expression() != null) {
addColumns(needed, ColumnsUsed.inWhere(key.expression()));
collectColumnReferencesFromExpression(key.expression(), qualifiers, primaryQualifier, needed, referenced);
addColumns(needed, ColumnsUsed.inWhere(key.expression(), schemaColumns));
collectColumnReferencesFromExpression(key.expression(), qualifiers, primaryQualifier, needed, referenced,
schemaColumns);
} else if (!isOrdinalReference(key.column()) && !isGroupingFunctionReference(key.column())) {
addColumn(needed, key.column());
}
}
WindowPlan windowPlan = WindowFunctions.plan(select.expressions());
if (windowPlan != null) {
for (Expression input : windowPlan.inputExpressions()) {
collectColumnReferencesFromExpression(input, qualifiers, primaryQualifier, needed, referenced);
addColumns(needed, ColumnsUsed.inWhere(input));
collectColumnReferencesFromExpression(input, qualifiers, primaryQualifier, needed, referenced, schemaColumns);
addColumns(needed, ColumnsUsed.inWhere(input, schemaColumns));
}
}
if (aggregatePlan != null) {
for (AggregateFunctions.AggregateSpec spec : aggregatePlan.specs()) {
if (!spec.countStar()) {
for (Expression arg : spec.arguments()) {
Set<String> argColumns = ColumnsUsed.inWhere(arg);
Set<String> argColumns = ColumnsUsed.inWhere(arg, schemaColumns);
addColumns(needed, argColumns);
addColumns(referenced, argColumns);
}
}
}
for (AggregateFunctions.GroupExpression groupExpr : aggregatePlan.groupExpressions()) {
Set<String> groupColumns = ColumnsUsed.inWhere(groupExpr.expression());
Set<String> groupColumns = ColumnsUsed.inWhere(groupExpr.expression(), schemaColumns);
addColumns(needed, groupColumns);
addColumns(referenced, groupColumns);
}
Expand Down
14 changes: 8 additions & 6 deletions src/main/java/se/alipsa/jparq/JParqResultSet.java
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BiFunction;
import java.util.function.Predicate;
import javax.sql.rowset.serial.SerialBlob;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.operators.conditional.AndExpression;
Expand Down Expand Up @@ -372,12 +373,13 @@ public JParqResultSet(RecordReader reader, SqlParser.Select select, String table
this.columnOrder.addAll(proj);
}
Set<String> requiredColumns = new LinkedHashSet<>(proj);
requiredColumns.addAll(ColumnsUsed.inWhere(effectiveResidual));
requiredColumns.addAll(ColumnsUsed.inWhere(select.having()));
requiredColumns.addAll(SqlParser.collectQualifiedColumns(effectiveResidual, queryQualifiers));
requiredColumns.addAll(SqlParser.collectQualifiedColumns(select.having(), queryQualifiers));
Predicate<String> schemaColumns = MaterializedNames.visibleColumns(schema);
requiredColumns.addAll(ColumnsUsed.inWhere(effectiveResidual, schemaColumns));
requiredColumns.addAll(ColumnsUsed.inWhere(select.having(), schemaColumns));
requiredColumns.addAll(SqlParser.collectQualifiedColumns(effectiveResidual, queryQualifiers, schemaColumns));
requiredColumns.addAll(SqlParser.collectQualifiedColumns(select.having(), queryQualifiers, schemaColumns));
for (Expression expression : selectExpressions) {
requiredColumns.addAll(SqlParser.collectQualifiedColumns(expression, queryQualifiers));
requiredColumns.addAll(SqlParser.collectQualifiedColumns(expression, queryQualifiers, schemaColumns));
}
if (select.orderBy() != null) {
for (SqlParser.OrderKey orderKey : select.orderBy()) {
Expand All @@ -388,7 +390,7 @@ public JParqResultSet(RecordReader reader, SqlParser.Select select, String table
}
if (windowPlan != null) {
for (Expression input : windowPlan.inputExpressions()) {
requiredColumns.addAll(SqlParser.collectQualifiedColumns(input, queryQualifiers));
requiredColumns.addAll(SqlParser.collectQualifiedColumns(input, queryQualifiers, schemaColumns));
}
}
if (schema != null) {
Expand Down
Loading
Loading