From a5022a6f9541e5cab6f2b9fde594c6e1df2b204c Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Wed, 18 Nov 2026 15:35:34 +0000 Subject: [PATCH] Spark 4.2: Drop dynamic pruning filters that nest pruning filters Spark adds one pruning subquery per join that can prune a scan, holding the plan of the join's other side. When a common table expression joins with itself, each subquery copies the subqueries of that side, and the plan grows by a factor of three per level of joins: five levels on a partition column give 121 pruning filters. The Iceberg session extensions now install NestedDynamicPruning, which drops each pruning filter whose filtering plan holds pruning filters of its own. A pruning filter only skips rows that the join drops, so the result does not change. Signed-off-by: Robert Kruszewski --- .../IcebergSparkSessionExtensions.scala | 3 + .../extensions/TestNestedDynamicPruning.java | 92 +++++++++++++++++++ .../optimizer/NestedDynamicPruning.scala | 91 ++++++++++++++++++ 3 files changed, 186 insertions(+) create mode 100644 spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestNestedDynamicPruning.java create mode 100644 spark/v4.2/spark/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedDynamicPruning.scala diff --git a/spark/v4.2/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala b/spark/v4.2/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala index f8532b12963f..e7162501755a 100644 --- a/spark/v4.2/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala +++ b/spark/v4.2/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala @@ -23,6 +23,7 @@ import org.apache.spark.sql.catalyst.analysis.CheckViews import org.apache.spark.sql.catalyst.analysis.ResolveBranch import org.apache.spark.sql.catalyst.analysis.ResolveReferencedByViews import org.apache.spark.sql.catalyst.analysis.ResolveViews +import org.apache.spark.sql.catalyst.optimizer.InstallNestedDynamicPruning import org.apache.spark.sql.catalyst.optimizer.ReplaceStaticInvoke import org.apache.spark.sql.catalyst.parser.extensions.IcebergSparkSqlExtensionsParser import org.apache.spark.sql.execution.datasources.v2.ExtendedDataSourceV2Strategy @@ -38,6 +39,8 @@ class IcebergSparkSessionExtensions extends (SparkSessionExtensions => Unit) { extensions.injectResolutionRule { spark => ResolveViews(spark) } extensions.injectPostHocResolutionRule { spark => ResolveBranch(spark) } extensions.injectCheckRule(_ => CheckViews) + // keeps dynamic pruning subqueries from nesting into each other + extensions.injectPostHocResolutionRule { spark => InstallNestedDynamicPruning(spark) } // optimizer extensions extensions.injectOptimizerRule { _ => ReplaceStaticInvoke } diff --git a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestNestedDynamicPruning.java b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestNestedDynamicPruning.java new file mode 100644 index 000000000000..dd344e3db4ca --- /dev/null +++ b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestNestedDynamicPruning.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.extensions; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.iceberg.ParameterizedTestExtension; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.catalyst.optimizer.NestedDynamicPruning; +import org.apache.spark.sql.internal.SQLConf; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(ParameterizedTestExtension.class) +public class TestNestedDynamicPruning extends ExtensionsTestBase { + + @AfterEach + public void removeTables() { + sql("DROP TABLE IF EXISTS %s", tableName); + spark.catalog().dropTempView("dim"); + } + + @TestTemplate + public void testExtensionsInstallTheRule() { + sql("SELECT 1"); + + assertThat(NestedDynamicPruning.installed(spark)).isTrue(); + } + + @TestTemplate + public void testSelfJoinsDoNotNestPruningSubqueries() { + sql("CREATE TABLE %s (id BIGINT, key BIGINT) USING iceberg PARTITIONED BY (key)", tableName); + sql("INSERT INTO %s SELECT id, id %% 10 FROM range(0, 1000)", tableName); + // files keep the optimizer from folding the dimension into a constant + String dimLocation = temp.resolve("dim").toString(); + spark + .range(1, 3) + .selectExpr("id AS key", "IF(id = 1, 'one', 'two') AS name") + .write() + .parquet(dimLocation); + spark.read().parquet(dimLocation).createOrReplaceTempView("dim"); + + // each level joins the level below with itself, which nests pruning subqueries without the + // rule: the plan of this query then holds 121 pruning filters + String query = + String.format( + "WITH l1 AS (SELECT f.id, f.key FROM %s f JOIN dim d ON f.key = d.key AND d.name = 'one'), " + + "l2 AS (SELECT a.id, a.key FROM l1 a JOIN l1 b ON a.key = b.key AND a.id = b.id), " + + "l3 AS (SELECT a.id, a.key FROM l2 a JOIN l2 b ON a.key = b.key AND a.id = b.id), " + + "l4 AS (SELECT a.id, a.key FROM l3 a JOIN l3 b ON a.key = b.key AND a.id = b.id), " + + "l5 AS (SELECT a.id, a.key FROM l4 a JOIN l4 b ON a.key = b.key AND a.id = b.id) " + + "SELECT count(*), sum(id) FROM l5", + tableName); + + List plan = spark.sql("EXPLAIN EXTENDED " + query).collectAsList(); + int pruningFilters = + StringUtils.countMatches(plan.get(0).getString(0), "dynamicpruningexpression"); + assertThat(pruningFilters) + .as("Joins on the partition column should prune without nesting") + .isBetween(1, 20); + + String pruning = SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED().key(); + spark.conf().set(pruning, "false"); + List expected; + try { + expected = sql(query); + } finally { + spark.conf().unset(pruning); + } + + assertEquals("Should have expected rows", expected, sql(query)); + } +} diff --git a/spark/v4.2/spark/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedDynamicPruning.scala b/spark/v4.2/spark/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedDynamicPruning.scala new file mode 100644 index 000000000000..23e3ffca8f90 --- /dev/null +++ b/spark/v4.2/spark/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedDynamicPruning.scala @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.catalyst.optimizer + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.expressions.And +import org.apache.spark.sql.catalyst.expressions.DynamicPruningSubquery +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.expressions.PredicateHelper +import org.apache.spark.sql.catalyst.plans.logical.Filter +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.rules.Rule + +/** + * Drops each dynamic pruning filter whose filtering plan holds dynamic pruning filters of its own. + * + * Spark adds one pruning subquery per join that can prune a scan, holding the plan of the join's + * other side. When that side holds pruned scans too, as a common table expression joined with + * itself does, each subquery copies their subqueries, and the physical plan grows exponentially + * with the depth of the joins. A filtering side that big is no broadcast dimension either, so its + * pruning would not run. + * + * A dynamic pruning filter only skips rows the join drops anyway, so dropping the filter leaves + * the result as it is. + */ +object NestedDynamicPruning extends Rule[LogicalPlan] with PredicateHelper { + + /** + * Adds the rule to the user provided optimizations of `session`, once. + * + * No session extension point runs after Spark's own dynamic pruning, but the user provided + * optimizations do, and Spark reads them each time it optimizes a query. + */ + def install(session: SparkSession): Unit = { + val experimental = session.experimental + experimental.synchronized { + if (!experimental.extraOptimizations.contains(this)) { + experimental.extraOptimizations = experimental.extraOptimizations :+ this + } + } + } + + /** Whether `session` runs the rule. */ + def installed(session: SparkSession): Boolean = + session.experimental.extraOptimizations.contains(this) + + override def apply(plan: LogicalPlan): LogicalPlan = + plan.transformUp { + case Filter(condition, child) if condition.exists(isNested) => + splitConjunctivePredicates(condition) + .filterNot(isNested) + .reduceOption(And) + .map(Filter(_, child)) + .getOrElse(child) + } + + private def isNested(expression: Expression): Boolean = + expression match { + case subquery: DynamicPruningSubquery => + subquery.buildQuery.exists( + _.expressions.exists(_.exists(_.isInstanceOf[DynamicPruningSubquery]))) + case _ => false + } +} + +/** + * The analyzer rule through which the session extensions install [[NestedDynamicPruning]]. Spark + * builds the analyzer of a session before it optimizes a query of that session, so the rule is + * in place in time. The rule itself changes no plan. + */ +case class InstallNestedDynamicPruning(session: SparkSession) extends Rule[LogicalPlan] { + NestedDynamicPruning.install(session) + + override def apply(plan: LogicalPlan): LogicalPlan = plan +}