From 0a06dc748889d413d846146ad4bc92d9fba2d43c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 26 Oct 2026 07:06:32 -0600 Subject: [PATCH 1/3] docs: say that Spark 3.4 and 3.5 build a cache through rows under AQE --- .../user-guide/latest/in-memory-cache.md | 11 ++++- .../comet/exec/CometInMemoryCacheSuite.scala | 41 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/docs/source/user-guide/latest/in-memory-cache.md b/docs/source/user-guide/latest/in-memory-cache.md index 9a15ee39c3a..ff7eaa97416 100644 --- a/docs/source/user-guide/latest/in-memory-cache.md +++ b/docs/source/user-guide/latest/in-memory-cache.md @@ -224,7 +224,8 @@ above it. Building the cache is measured the same way, from the same source, with its rows produced either by Comet operators or by Spark operators. Comet's format is written straight from the Arrow batches of Comet operators, while Spark's is always built from rows, so above Comet operators it first converts -their batches. Measured on an Apple M3 Max (JDK 17, Spark 4.1, release build): +their batches. On Spark 3.4 and 3.5, Comet's format is written from Arrow batches only with AQE off; +see [Limitations](#limitations). Measured on an Apple M3 Max (JDK 17, Spark 4.1, release build): | Rows produced by | Spark's cache format | Comet's cache format, `zstd` | Comet's cache format, `none` | | ---------------- | -------------------: | ---------------------------: | ---------------------------: | @@ -301,6 +302,14 @@ operator reads a relation cached in Comet's format depends on the scan below it: Whenever Spark's scan reads Comet's format, which also happens for a relation whose cached plan records `Dataset.observe` metrics, Comet records a fallback reason on the scan. +On **Spark 3.4 and 3.5**, a relation cached under AQE (`spark.sql.adaptive.enabled`, on by default) is +built from rows. Spark builds a cache from the batches of its plan only when the plan supports +columnar output, and a cached `AdaptiveSparkPlanExec` does not until Spark 4.0. So the cached plan +runs natively, then converts its batches to rows, which are written into Arrow again before they +are compressed, where Spark 4.0 and later, or any version without AQE, write the Arrow batches as +they are. Comet cannot change this without a hook in Spark. The relation is read the same way once +it is built, but the build timings above are from Spark 4.1 and do not apply to this case. + Measured by the same benchmark over the same 5M-row relation, with native execution off so that Spark operators consume the cached data, Comet disabled for the row reader and enabled for the fused reader (Apple M4, JDK 17, Spark 4.1; the average of two runs): diff --git a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala index b0edda42338..849d7803be1 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala @@ -2758,6 +2758,47 @@ class CometInMemoryCacheSuite extends CometTestBase { assert(zstdSize == plainSize, s"the relation size should not depend on the codec: $sizes") } + test("Comet in-memory cache is built from rows under AQE before Spark 4.0") { + // Spark builds a cache from the batches of its plan only when the plan reports that it + // supports columnar output. A cached AdaptiveSparkPlanExec does not before Spark 4.0, so there + // a relation cached under AQE goes native plan, columnar-to-row, UnsafeRow and back to Arrow + // before it is compressed. Without AQE, or from 4.0 on, it is written from the Arrow batches. + // The user guide's build timings are from Spark 4.1 and say so. + Seq(true, false).foreach { adaptive => + withConversions( + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> adaptive.toString, + SQLConf.SHUFFLE_PARTITIONS.key -> "3", + CometConf.COMET_SHUFFLE_MODE.key -> "jvm", + CometConf.COMET_EXEC_IN_MEMORY_CACHE_ENABLED.key -> "true") { + spark.catalog.clearCache() + try { + spark + .range(0, 100, 1, 2) + .selectExpr("id % 7 AS k", "id AS v") + .groupBy("k") + .agg(sum("v").as("s")) + .createOrReplaceTempView("rows_build_cache") + spark.catalog.cacheTable("rows_build_cache") + assert(spark.table("rows_build_cache").count() == 7) + val cachedPlan = spark.sharedState.cacheManager + .lookupCachedData(spark.table("rows_build_cache")) + .get + .cachedRepresentation + .cacheBuilder + .cachedPlan + assert( + cachedPlan.isInstanceOf[AdaptiveSparkPlanExec] == adaptive, + s"AQE is $adaptive:\n$cachedPlan") + assert( + cachedPlan.supportsColumnar == (!adaptive || isSpark40Plus), + s"AQE is $adaptive:\n$cachedPlan") + } finally { + spark.catalog.clearCache() + } + } + } + } + test("Comet in-memory cache scans no columns for a row-count-only query") { // SELECT count(*) selects no columns, and the scan must keep it that way. Widening it -- to // the whole cache schema, or to a single placeholder column -- makes the emitted batches From f94fe706010f868d9f426825a554213279a7b273 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 06:22:10 -0600 Subject: [PATCH 2/3] docs: limit the AQE cache row-build note to Spark 3.5 and 3.4 with canChangeCachedPlanOutputPartitioning Spark 3.4 turns AQE off for a cached plan unless spark.sql.optimizer.canChangeCachedPlanOutputPartitioning is set, and that conf defaults to true only from 3.5. Say so in the guide, and set the conf in the test so its AQE case gets an adaptive cached plan on 3.4 as well. --- .../user-guide/latest/in-memory-cache.md | 22 +++++++++++-------- .../comet/exec/CometInMemoryCacheSuite.scala | 5 ++++- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/source/user-guide/latest/in-memory-cache.md b/docs/source/user-guide/latest/in-memory-cache.md index 9ac7e0611e1..04c8d6c43ef 100644 --- a/docs/source/user-guide/latest/in-memory-cache.md +++ b/docs/source/user-guide/latest/in-memory-cache.md @@ -224,8 +224,10 @@ above it. Building the cache is measured the same way, from the same source, with its rows produced either by Comet operators or by Spark operators. Comet's format is written straight from the Arrow batches of Comet operators, while Spark's is always built from rows, so above Comet operators it first converts -their batches. On Spark 3.4 and 3.5, Comet's format is written from Arrow batches only with AQE off; -see [Limitations](#limitations). Measured on an Apple M3 Max (JDK 17, Spark 4.1, release build): +their batches. On Spark 3.5, and on Spark 3.4 with +`spark.sql.optimizer.canChangeCachedPlanOutputPartitioning` set, Comet's format is written from +Arrow batches only with AQE off; see [Limitations](#limitations). Measured on an Apple M3 Max +(JDK 17, Spark 4.1, release build): | Rows produced by | Spark's cache format | Comet's cache format, `zstd` | Comet's cache format, `none` | | ---------------- | -------------------: | ---------------------------: | ---------------------------: | @@ -302,13 +304,15 @@ operator reads a relation cached in Comet's format depends on the scan below it: Whenever Spark's scan reads Comet's format, which also happens for a relation whose cached plan records `Dataset.observe` metrics, Comet records a fallback reason on the scan. -On **Spark 3.4 and 3.5**, a relation cached under AQE (`spark.sql.adaptive.enabled`, on by default) is -built from rows. Spark builds a cache from the batches of its plan only when the plan supports -columnar output, and a cached `AdaptiveSparkPlanExec` does not until Spark 4.0. So the cached plan -runs natively, then converts its batches to rows, which are written into Arrow again before they -are compressed, where Spark 4.0 and later, or any version without AQE, write the Arrow batches as -they are. Comet cannot change this without a hook in Spark. The relation is read the same way once -it is built, but the build timings above are from Spark 4.1 and do not apply to this case. +On **Spark 3.5**, a relation cached under AQE (`spark.sql.adaptive.enabled`, on by default) is built +from rows. Spark builds a cache from the batches of its plan only when the plan supports columnar +output, and a cached `AdaptiveSparkPlanExec` does not until Spark 4.0. So the cached plan runs +natively, then converts its batches to rows, which are written into Arrow again before they are +compressed, where Spark 4.0 and later, or any version without AQE, write the Arrow batches as they +are. Comet cannot change this without a hook in Spark. The relation is read the same way once it is +built, but the build timings above are from Spark 4.1 and do not apply to this case. Spark 3.4 does +the same only with `spark.sql.optimizer.canChangeCachedPlanOutputPartitioning` set to `true`. By +default it turns AQE off for the plan it caches, which is then written from its Arrow batches. Measured by the same benchmark over the same 5M-row relation, with native execution off so that Spark operators consume the cached data, Comet disabled for the row reader and enabled for the fused diff --git a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala index 7e7e6bd3fc2..bc1c2b6af47 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala @@ -2998,10 +2998,13 @@ class CometInMemoryCacheSuite extends CometTestBase { // supports columnar output. A cached AdaptiveSparkPlanExec does not before Spark 4.0, so there // a relation cached under AQE goes native plan, columnar-to-row, UnsafeRow and back to Arrow // before it is compressed. Without AQE, or from 4.0 on, it is written from the Arrow batches. - // The user guide's build timings are from Spark 4.1 and say so. + // Spark 3.4 turns AQE off for the cached plan unless canChangeCachedPlanOutputPartitioning, + // which defaults to true only from 3.5, is set. The user guide's build timings are from Spark + // 4.1 and say so. Seq(true, false).foreach { adaptive => withConversions( SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> adaptive.toString, + SQLConf.CAN_CHANGE_CACHED_PLAN_OUTPUT_PARTITIONING.key -> "true", SQLConf.SHUFFLE_PARTITIONS.key -> "3", CometConf.COMET_SHUFFLE_MODE.key -> "jvm", CometConf.COMET_EXEC_IN_MEMORY_CACHE_ENABLED.key -> "true") { From 4816d9247635f9856b40f3f615710aaf4ea3d213 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 16 Nov 2026 10:30:01 -0700 Subject: [PATCH 3/3] docs: drop the Spark 3.4 case from the AQE cache row-build note Spark 3.4 is no longer supported, so the note and its test cover Spark 3.5 only. The test no longer sets canChangeCachedPlanOutputPartitioning, which defaults to true on 3.5. --- docs/source/user-guide/latest/in-memory-cache.md | 10 +++------- .../apache/comet/exec/CometInMemoryCacheSuite.scala | 5 +---- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/docs/source/user-guide/latest/in-memory-cache.md b/docs/source/user-guide/latest/in-memory-cache.md index a0cb0f802ce..c522ac38510 100644 --- a/docs/source/user-guide/latest/in-memory-cache.md +++ b/docs/source/user-guide/latest/in-memory-cache.md @@ -220,10 +220,8 @@ above it. Building the cache is measured the same way, from the same source, with its rows produced either by Comet operators or by Spark operators. Comet's format is written straight from the Arrow batches of Comet operators, while Spark's is always built from rows, so above Comet operators it first converts -their batches. On Spark 3.5, and on Spark 3.4 with -`spark.sql.optimizer.canChangeCachedPlanOutputPartitioning` set, Comet's format is written from -Arrow batches only with AQE off; see [Limitations](#limitations). Measured on an Apple M3 Max -(JDK 17, Spark 4.1, release build): +their batches. On Spark 3.5, Comet's format is written from Arrow batches only with AQE off; see +[Limitations](#limitations). Measured on an Apple M3 Max (JDK 17, Spark 4.1, release build): | Rows produced by | Spark's cache format | Comet's cache format, `zstd` | Comet's cache format, `none` | | ---------------- | -------------------: | ---------------------------: | ---------------------------: | @@ -306,9 +304,7 @@ output, and a cached `AdaptiveSparkPlanExec` does not until Spark 4.0. So the ca natively, then converts its batches to rows, which are written into Arrow again before they are compressed, where Spark 4.0 and later, or any version without AQE, write the Arrow batches as they are. Comet cannot change this without a hook in Spark. The relation is read the same way once it is -built, but the build timings above are from Spark 4.1 and do not apply to this case. Spark 3.4 does -the same only with `spark.sql.optimizer.canChangeCachedPlanOutputPartitioning` set to `true`. By -default it turns AQE off for the plan it caches, which is then written from its Arrow batches. +built, but the build timings above are from Spark 4.1 and do not apply to this case. Measured by the same benchmark over the same 5M-row relation, with native execution off so that Spark operators consume the cached data, Comet disabled for the row reader and enabled for the fused diff --git a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala index 0386e331ad6..eee0d1ccc22 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala @@ -2996,13 +2996,10 @@ class CometInMemoryCacheSuite extends CometTestBase { // supports columnar output. A cached AdaptiveSparkPlanExec does not before Spark 4.0, so there // a relation cached under AQE goes native plan, columnar-to-row, UnsafeRow and back to Arrow // before it is compressed. Without AQE, or from 4.0 on, it is written from the Arrow batches. - // Spark 3.4 turns AQE off for the cached plan unless canChangeCachedPlanOutputPartitioning, - // which defaults to true only from 3.5, is set. The user guide's build timings are from Spark - // 4.1 and say so. + // The user guide's build timings are from Spark 4.1 and say so. Seq(true, false).foreach { adaptive => withConversions( SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> adaptive.toString, - SQLConf.CAN_CHANGE_CACHED_PLAN_OUTPUT_PARTITIONING.key -> "true", SQLConf.SHUFFLE_PARTITIONS.key -> "3", CometConf.COMET_SHUFFLE_MODE.key -> "jvm", CometConf.COMET_EXEC_IN_MEMORY_CACHE_ENABLED.key -> "true") {