From 08297c502b7173c4ef378925f8e199b3b215c8b0 Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 18 Nov 2026 12:46:39 -0800 Subject: [PATCH 1/2] feat: add DataFrame helpers getFallbackReasons and planAccelerated to ExtendedExplainInfo `getFallbackReasons(df)` returns the fallback reasons of a DataFrame's executed plan. `planAccelerated(df)` is true when Comet is enabled for the session, using the same check as the Comet planner rules, and the plan has no fallback reasons. --- .../latest/understanding-comet-plans.md | 5 ++++ .../apache/comet/ExtendedExplainInfo.scala | 24 +++++++++++++++++- .../apache/comet/exec/CometExecSuite.scala | 25 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/docs/source/user-guide/latest/understanding-comet-plans.md b/docs/source/user-guide/latest/understanding-comet-plans.md index 9e7597597fe..a73000c837c 100644 --- a/docs/source/user-guide/latest/understanding-comet-plans.md +++ b/docs/source/user-guide/latest/understanding-comet-plans.md @@ -249,6 +249,11 @@ val info = new ExtendedExplainInfo() // Sorted, deduplicated list of fallback reasons across the whole plan. val reasons: Seq[String] = info.getFallbackReasons(plan) +// The same for a DataFrame's executed plan. It does not run the query, so for +// an adaptive query, run it first to get the reasons for the final plan. +val dfReasons: Seq[String] = info.getFallbackReasons(df) +// True when Comet is enabled for the session and the plan has no fallback reasons. +val accelerated: Boolean = info.planAccelerated(df) // Sorted, deduplicated names of the expressions Comet accelerated, split by how // they run. Structural nodes (attribute references, literals, aliases) are not diff --git a/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala b/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala index 2d10c851720..06157d99207 100644 --- a/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala +++ b/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala @@ -23,7 +23,7 @@ import java.util.{Collections, IdentityHashMap, Locale} import scala.collection.mutable -import org.apache.spark.sql.ExtendedExplainGenerator +import org.apache.spark.sql.{DataFrame, ExtendedExplainGenerator} import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference, Expression, Literal, ScalaUDF} import org.apache.spark.sql.catalyst.trees.{TreeNode, TreeNodeTag} import org.apache.spark.sql.comet.{CometColumnarToRowExec, CometEmptyRelationExec, CometInMemoryTableScanExec, CometPlan, CometSparkToColumnarExec} @@ -64,6 +64,28 @@ class ExtendedExplainInfo extends ExtendedExplainGenerator { fallbackReasons(plan).toSeq.sorted } + /** + * The fallback reasons in the executed plan of `df`, as returned by `getFallbackReasons(plan)`. + * This does not run the query. For an adaptive query that has not run yet, the executed plan is + * the initial plan, so run the query first (for example with `collect()`) to get the reasons + * for the final plan and its query stages. + */ + def getFallbackReasons(df: DataFrame): Seq[String] = { + getFallbackReasons(df.queryExecution.executedPlan) + } + + /** + * Whether Comet is enabled for the session of `df` and reports no fallback reasons for its + * executed plan. Enabled means the check the Comet planner rules make, so this is false when + * Comet disabled itself, for example because Spark is not in off-heap mode or the Comet shuffle + * manager is not set, even if `spark.comet.enabled` is true. Like `getFallbackReasons(df)`, + * this does not run the query. + */ + def planAccelerated(df: DataFrame): Boolean = { + val conf = df.queryExecution.sparkSession.sessionState.conf + CometSparkSessionExtensions.isCometLoaded(conf) && getFallbackReasons(df).isEmpty + } + /** * Names of the expressions in `plan` that Comet lowered to native DataFusion expressions, * sorted alphabetically. Names are the expression's `prettyName` (the UDF name for a diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index 8e78885e6ce..44caf7e5409 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -239,6 +239,31 @@ class CometExecSuite extends CometTestBase { } } + test("getFallbackReasons and planAccelerated for a DataFrame") { + withParquetTable((0 until 1000).map(i => (i, i + 1)), "tbl") { + val info = new ExtendedExplainInfo() + + val fallback = + sql("SELECT * FROM tbl").sample(withReplacement = true, fraction = 0.3, seed = 42) + fallback.collect() + val reasons = info.getFallbackReasons(fallback) + assert(reasons.exists(_.contains("Sampling with replacement is not supported")), reasons) + assert(reasons == info.getFallbackReasons(fallback.queryExecution.executedPlan)) + assert(!info.planAccelerated(fallback)) + + val native = sql("SELECT _1 + 1 FROM tbl WHERE _2 > 10") + checkSparkAnswerAndOperator(native) + assert(info.getFallbackReasons(native).isEmpty, info.getFallbackReasons(native)) + assert(info.planAccelerated(native)) + + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + val sparkOnly = sql("SELECT _1 + 1 FROM tbl WHERE _2 > 10") + sparkOnly.collect() + assert(!info.planAccelerated(sparkOnly)) + } + } + } + test("sample falls back to Spark when disabled") { withSQLConf(CometConf.COMET_EXEC_SAMPLE_ENABLED.key -> "false") { withParquetTable((0 until 1000).map(i => (i, i + 1)), "tbl") { From 63b16354359b2aeccc566f76906aab32576adfc2 Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 19 Nov 2026 08:39:32 -0800 Subject: [PATCH 2/2] Rename planAccelerated to isFullyAccelerated and document it The old name read as "some of the plan is accelerated". The method returns true only when Comet is enabled and the plan has no fallback reasons, so a plan with partial fallback is not fully accelerated. Add a user guide subsection that describes the check. --- .../latest/understanding-comet-plans.md | 26 ++++++++++++++++++- .../apache/comet/ExtendedExplainInfo.scala | 13 +++++----- .../apache/comet/exec/CometExecSuite.scala | 8 +++--- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/docs/source/user-guide/latest/understanding-comet-plans.md b/docs/source/user-guide/latest/understanding-comet-plans.md index a73000c837c..a52b77ec48d 100644 --- a/docs/source/user-guide/latest/understanding-comet-plans.md +++ b/docs/source/user-guide/latest/understanding-comet-plans.md @@ -253,7 +253,7 @@ val reasons: Seq[String] = info.getFallbackReasons(plan) // an adaptive query, run it first to get the reasons for the final plan. val dfReasons: Seq[String] = info.getFallbackReasons(df) // True when Comet is enabled for the session and the plan has no fallback reasons. -val accelerated: Boolean = info.planAccelerated(df) +val fullyAccelerated: Boolean = info.isFullyAccelerated(df) // Sorted, deduplicated names of the expressions Comet accelerated, split by how // they run. Structural nodes (attribute references, literals, aliases) are not @@ -291,6 +291,30 @@ Project [COMET: from_unixtime(eventTime#5L, yyyy-MM-dd HH:mm:ss, Some(GMT)) is n Comet accelerated 1 out of 2 eligible operators (50%). Final plan contains 1 transitions between Spark and Comet. Accelerated expressions: 0 native, 0 codegen dispatch. ``` +### Checking That a Query Is Fully Accelerated + +`isFullyAccelerated(df)` returns `true` when both of the following hold: + +- Comet is enabled for the DataFrame's session. This is the same check Comet's + planner rules make, so it is `false` when Comet disables itself, for example + because Spark is not in off-heap mode or the Comet shuffle manager is not set, + even if `spark.comet.enabled` is `true`. +- `getFallbackReasons(df)` is empty. + +The check is a plan-wide yes or no. A plan with partial fallback, where some +operators run in Comet and others fall back to Spark, is not fully accelerated. +Use `getFallbackReasons(df)` to see what fell back. Codegen dispatch is not a +fallback, so a plan whose expressions use it can still be fully accelerated. + +Like `getFallbackReasons(df)`, it reads the executed plan and does not run the +query. With AQE, run the query first so the check covers the final plan: + +```scala +val df = spark.sql("SELECT ...") +df.collect() +assert(new ExtendedExplainInfo().isFullyAccelerated(df), "query fell back to Spark") +``` + ## Comet Operator Reference The following sections describe the Comet nodes you will see in plans, grouped diff --git a/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala b/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala index 06157d99207..f35cbc462af 100644 --- a/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala +++ b/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala @@ -75,13 +75,14 @@ class ExtendedExplainInfo extends ExtendedExplainGenerator { } /** - * Whether Comet is enabled for the session of `df` and reports no fallback reasons for its - * executed plan. Enabled means the check the Comet planner rules make, so this is false when - * Comet disabled itself, for example because Spark is not in off-heap mode or the Comet shuffle - * manager is not set, even if `spark.comet.enabled` is true. Like `getFallbackReasons(df)`, - * this does not run the query. + * Whether the executed plan of `df` is fully accelerated: Comet is enabled for its session and + * reports no fallback reasons for the plan. A plan with partial fallback, where some operators + * run in Comet and others fall back to Spark, is not fully accelerated. Enabled means the check + * the Comet planner rules make, so this is false when Comet disabled itself, for example + * because Spark is not in off-heap mode or the Comet shuffle manager is not set, even if + * `spark.comet.enabled` is true. Like `getFallbackReasons(df)`, this does not run the query. */ - def planAccelerated(df: DataFrame): Boolean = { + def isFullyAccelerated(df: DataFrame): Boolean = { val conf = df.queryExecution.sparkSession.sessionState.conf CometSparkSessionExtensions.isCometLoaded(conf) && getFallbackReasons(df).isEmpty } diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index 44caf7e5409..d19e44e03bf 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -239,7 +239,7 @@ class CometExecSuite extends CometTestBase { } } - test("getFallbackReasons and planAccelerated for a DataFrame") { + test("getFallbackReasons and isFullyAccelerated for a DataFrame") { withParquetTable((0 until 1000).map(i => (i, i + 1)), "tbl") { val info = new ExtendedExplainInfo() @@ -249,17 +249,17 @@ class CometExecSuite extends CometTestBase { val reasons = info.getFallbackReasons(fallback) assert(reasons.exists(_.contains("Sampling with replacement is not supported")), reasons) assert(reasons == info.getFallbackReasons(fallback.queryExecution.executedPlan)) - assert(!info.planAccelerated(fallback)) + assert(!info.isFullyAccelerated(fallback)) val native = sql("SELECT _1 + 1 FROM tbl WHERE _2 > 10") checkSparkAnswerAndOperator(native) assert(info.getFallbackReasons(native).isEmpty, info.getFallbackReasons(native)) - assert(info.planAccelerated(native)) + assert(info.isFullyAccelerated(native)) withSQLConf(CometConf.COMET_ENABLED.key -> "false") { val sparkOnly = sql("SELECT _1 + 1 FROM tbl WHERE _2 > 10") sparkOnly.collect() - assert(!info.planAccelerated(sparkOnly)) + assert(!info.isFullyAccelerated(sparkOnly)) } } }