Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import org.apache.spark.sql.catalyst.analysis.CheckViews
import org.apache.spark.sql.catalyst.analysis.ResolveBranch
import org.apache.spark.sql.catalyst.analysis.ResolveReferencedByViews
import org.apache.spark.sql.catalyst.analysis.ResolveViews
import org.apache.spark.sql.catalyst.optimizer.InstallNestedDynamicPruning
import org.apache.spark.sql.catalyst.optimizer.ReplaceStaticInvoke
import org.apache.spark.sql.catalyst.parser.extensions.IcebergSparkSqlExtensionsParser
import org.apache.spark.sql.execution.datasources.v2.ExtendedDataSourceV2Strategy
Expand All @@ -38,6 +39,8 @@ class IcebergSparkSessionExtensions extends (SparkSessionExtensions => Unit) {
extensions.injectResolutionRule { spark => ResolveViews(spark) }
extensions.injectPostHocResolutionRule { spark => ResolveBranch(spark) }
extensions.injectCheckRule(_ => CheckViews)
// keeps dynamic pruning subqueries from nesting into each other
extensions.injectPostHocResolutionRule { spark => InstallNestedDynamicPruning(spark) }

// optimizer extensions
extensions.injectOptimizerRule { _ => ReplaceStaticInvoke }
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.iceberg.spark.extensions;

import static org.assertj.core.api.Assertions.assertThat;

import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.iceberg.ParameterizedTestExtension;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.catalyst.optimizer.NestedDynamicPruning;
import org.apache.spark.sql.internal.SQLConf;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.extension.ExtendWith;

@ExtendWith(ParameterizedTestExtension.class)
public class TestNestedDynamicPruning extends ExtensionsTestBase {

@AfterEach
public void removeTables() {
sql("DROP TABLE IF EXISTS %s", tableName);
spark.catalog().dropTempView("dim");
}

@TestTemplate
public void testExtensionsInstallTheRule() {
sql("SELECT 1");

assertThat(NestedDynamicPruning.installed(spark)).isTrue();
}

@TestTemplate
public void testSelfJoinsDoNotNestPruningSubqueries() {
sql("CREATE TABLE %s (id BIGINT, key BIGINT) USING iceberg PARTITIONED BY (key)", tableName);
sql("INSERT INTO %s SELECT id, id %% 10 FROM range(0, 1000)", tableName);
// files keep the optimizer from folding the dimension into a constant
String dimLocation = temp.resolve("dim").toString();
spark
.range(1, 3)
.selectExpr("id AS key", "IF(id = 1, 'one', 'two') AS name")
.write()
.parquet(dimLocation);
spark.read().parquet(dimLocation).createOrReplaceTempView("dim");

// each level joins the level below with itself, which nests pruning subqueries without the
// rule: the plan of this query then holds 121 pruning filters
String query =
String.format(
"WITH l1 AS (SELECT f.id, f.key FROM %s f JOIN dim d ON f.key = d.key AND d.name = 'one'), "
+ "l2 AS (SELECT a.id, a.key FROM l1 a JOIN l1 b ON a.key = b.key AND a.id = b.id), "
+ "l3 AS (SELECT a.id, a.key FROM l2 a JOIN l2 b ON a.key = b.key AND a.id = b.id), "
+ "l4 AS (SELECT a.id, a.key FROM l3 a JOIN l3 b ON a.key = b.key AND a.id = b.id), "
+ "l5 AS (SELECT a.id, a.key FROM l4 a JOIN l4 b ON a.key = b.key AND a.id = b.id) "
+ "SELECT count(*), sum(id) FROM l5",
tableName);

List<Row> plan = spark.sql("EXPLAIN EXTENDED " + query).collectAsList();
int pruningFilters =
StringUtils.countMatches(plan.get(0).getString(0), "dynamicpruningexpression");
assertThat(pruningFilters)
.as("Joins on the partition column should prune without nesting")
.isBetween(1, 20);

String pruning = SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED().key();
spark.conf().set(pruning, "false");
List<Object[]> expected;
try {
expected = sql(query);
} finally {
spark.conf().unset(pruning);
}

assertEquals("Should have expected rows", expected, sql(query));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.spark.sql.catalyst.optimizer

import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.expressions.And
import org.apache.spark.sql.catalyst.expressions.DynamicPruningSubquery
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.expressions.PredicateHelper
import org.apache.spark.sql.catalyst.plans.logical.Filter
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.rules.Rule

/**
* Drops each dynamic pruning filter whose filtering plan holds dynamic pruning filters of its own.
*
* Spark adds one pruning subquery per join that can prune a scan, holding the plan of the join's
* other side. When that side holds pruned scans too, as a common table expression joined with
* itself does, each subquery copies their subqueries, and the physical plan grows exponentially
* with the depth of the joins. A filtering side that big is no broadcast dimension either, so its
* pruning would not run.
*
* A dynamic pruning filter only skips rows the join drops anyway, so dropping the filter leaves
* the result as it is.
*/
object NestedDynamicPruning extends Rule[LogicalPlan] with PredicateHelper {

/**
* Adds the rule to the user provided optimizations of `session`, once.
*
* No session extension point runs after Spark's own dynamic pruning, but the user provided
* optimizations do, and Spark reads them each time it optimizes a query.
*/
def install(session: SparkSession): Unit = {
val experimental = session.experimental
experimental.synchronized {
if (!experimental.extraOptimizations.contains(this)) {
experimental.extraOptimizations = experimental.extraOptimizations :+ this
}
}
}

/** Whether `session` runs the rule. */
def installed(session: SparkSession): Boolean =
session.experimental.extraOptimizations.contains(this)

override def apply(plan: LogicalPlan): LogicalPlan =
plan.transformUp {
case Filter(condition, child) if condition.exists(isNested) =>
splitConjunctivePredicates(condition)
.filterNot(isNested)
.reduceOption(And)
.map(Filter(_, child))
.getOrElse(child)
}

private def isNested(expression: Expression): Boolean =
expression match {
case subquery: DynamicPruningSubquery =>
subquery.buildQuery.exists(
_.expressions.exists(_.exists(_.isInstanceOf[DynamicPruningSubquery])))
case _ => false
}
}

/**
* The analyzer rule through which the session extensions install [[NestedDynamicPruning]]. Spark
* builds the analyzer of a session before it optimizes a query of that session, so the rule is
* in place in time. The rule itself changes no plan.
*/
case class InstallNestedDynamicPruning(session: SparkSession) extends Rule[LogicalPlan] {
NestedDynamicPruning.install(session)

override def apply(plan: LogicalPlan): LogicalPlan = plan
}
Loading