diff --git a/docs/source/user-guide/latest/understanding-comet-plans.md b/docs/source/user-guide/latest/understanding-comet-plans.md index 9e7597597fe..a52b77ec48d 100644 --- a/docs/source/user-guide/latest/understanding-comet-plans.md +++ b/docs/source/user-guide/latest/understanding-comet-plans.md @@ -249,6 +249,11 @@ val info = new ExtendedExplainInfo() // Sorted, deduplicated list of fallback reasons across the whole plan. val reasons: Seq[String] = info.getFallbackReasons(plan) +// The same for a DataFrame's executed plan. It does not run the query, so for +// an adaptive query, run it first to get the reasons for the final plan. +val dfReasons: Seq[String] = info.getFallbackReasons(df) +// True when Comet is enabled for the session and the plan has no fallback reasons. +val fullyAccelerated: Boolean = info.isFullyAccelerated(df) // Sorted, deduplicated names of the expressions Comet accelerated, split by how // they run. Structural nodes (attribute references, literals, aliases) are not @@ -286,6 +291,30 @@ Project [COMET: from_unixtime(eventTime#5L, yyyy-MM-dd HH:mm:ss, Some(GMT)) is n Comet accelerated 1 out of 2 eligible operators (50%). Final plan contains 1 transitions between Spark and Comet. Accelerated expressions: 0 native, 0 codegen dispatch. ``` +### Checking That a Query Is Fully Accelerated + +`isFullyAccelerated(df)` returns `true` when both of the following hold: + +- Comet is enabled for the DataFrame's session. This is the same check Comet's + planner rules make, so it is `false` when Comet disables itself, for example + because Spark is not in off-heap mode or the Comet shuffle manager is not set, + even if `spark.comet.enabled` is `true`. +- `getFallbackReasons(df)` is empty. + +The check is a plan-wide yes or no. A plan with partial fallback, where some +operators run in Comet and others fall back to Spark, is not fully accelerated. +Use `getFallbackReasons(df)` to see what fell back. Codegen dispatch is not a +fallback, so a plan whose expressions use it can still be fully accelerated. + +Like `getFallbackReasons(df)`, it reads the executed plan and does not run the +query. With AQE, run the query first so the check covers the final plan: + +```scala +val df = spark.sql("SELECT ...") +df.collect() +assert(new ExtendedExplainInfo().isFullyAccelerated(df), "query fell back to Spark") +``` + ## Comet Operator Reference The following sections describe the Comet nodes you will see in plans, grouped diff --git a/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala b/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala index 2d10c851720..f35cbc462af 100644 --- a/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala +++ b/spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala @@ -23,7 +23,7 @@ import java.util.{Collections, IdentityHashMap, Locale} import scala.collection.mutable -import org.apache.spark.sql.ExtendedExplainGenerator +import org.apache.spark.sql.{DataFrame, ExtendedExplainGenerator} import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference, Expression, Literal, ScalaUDF} import org.apache.spark.sql.catalyst.trees.{TreeNode, TreeNodeTag} import org.apache.spark.sql.comet.{CometColumnarToRowExec, CometEmptyRelationExec, CometInMemoryTableScanExec, CometPlan, CometSparkToColumnarExec} @@ -64,6 +64,29 @@ class ExtendedExplainInfo extends ExtendedExplainGenerator { fallbackReasons(plan).toSeq.sorted } + /** + * The fallback reasons in the executed plan of `df`, as returned by `getFallbackReasons(plan)`. + * This does not run the query. For an adaptive query that has not run yet, the executed plan is + * the initial plan, so run the query first (for example with `collect()`) to get the reasons + * for the final plan and its query stages. + */ + def getFallbackReasons(df: DataFrame): Seq[String] = { + getFallbackReasons(df.queryExecution.executedPlan) + } + + /** + * Whether the executed plan of `df` is fully accelerated: Comet is enabled for its session and + * reports no fallback reasons for the plan. A plan with partial fallback, where some operators + * run in Comet and others fall back to Spark, is not fully accelerated. Enabled means the check + * the Comet planner rules make, so this is false when Comet disabled itself, for example + * because Spark is not in off-heap mode or the Comet shuffle manager is not set, even if + * `spark.comet.enabled` is true. Like `getFallbackReasons(df)`, this does not run the query. + */ + def isFullyAccelerated(df: DataFrame): Boolean = { + val conf = df.queryExecution.sparkSession.sessionState.conf + CometSparkSessionExtensions.isCometLoaded(conf) && getFallbackReasons(df).isEmpty + } + /** * Names of the expressions in `plan` that Comet lowered to native DataFusion expressions, * sorted alphabetically. Names are the expression's `prettyName` (the UDF name for a diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index 8e78885e6ce..d19e44e03bf 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -239,6 +239,31 @@ class CometExecSuite extends CometTestBase { } } + test("getFallbackReasons and isFullyAccelerated for a DataFrame") { + withParquetTable((0 until 1000).map(i => (i, i + 1)), "tbl") { + val info = new ExtendedExplainInfo() + + val fallback = + sql("SELECT * FROM tbl").sample(withReplacement = true, fraction = 0.3, seed = 42) + fallback.collect() + val reasons = info.getFallbackReasons(fallback) + assert(reasons.exists(_.contains("Sampling with replacement is not supported")), reasons) + assert(reasons == info.getFallbackReasons(fallback.queryExecution.executedPlan)) + assert(!info.isFullyAccelerated(fallback)) + + val native = sql("SELECT _1 + 1 FROM tbl WHERE _2 > 10") + checkSparkAnswerAndOperator(native) + assert(info.getFallbackReasons(native).isEmpty, info.getFallbackReasons(native)) + assert(info.isFullyAccelerated(native)) + + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + val sparkOnly = sql("SELECT _1 + 1 FROM tbl WHERE _2 > 10") + sparkOnly.collect() + assert(!info.isFullyAccelerated(sparkOnly)) + } + } + } + test("sample falls back to Spark when disabled") { withSQLConf(CometConf.COMET_EXEC_SAMPLE_ENABLED.key -> "false") { withParquetTable((0 until 1000).map(i => (i, i + 1)), "tbl") {