From 572c51b2bab703c0b2f23a6db3c8e5234d696900 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 4 Nov 2026 08:41:33 -0700 Subject: [PATCH 1/4] test: tag each Comet suite with its CI test group, and add run-comet-tests.sh CI runs the Comet suites in four groups, listed by name in pr_build_linux.yml and pr_build_macos.yml. Only GitHub Actions can read those lists, so nothing else can run a group. Each concrete suite now also carries a tag for its group (ScansGroup, ShuffleGroup, ExecGroup or ExpressionsGroup), and dev/ci/run-comet-tests.sh runs a group by tag. check-suites.py requires exactly one tag per concrete suite and that both workflow files list the suite under its tag's group, so the tags and the lists cannot drift apart. The workflows are unchanged; a follow-up can drop the lists and run the script. --- .ai/skills/review-comet-pr/SKILL.md | 6 +- dev/ci/check-suites.py | 221 +++++++++++------- dev/ci/run-comet-tests.sh | 82 +++++++ docs/source/contributor-guide/development.md | 66 +++--- .../java/org/apache/comet/tags/ExecGroup.java | 33 +++ .../apache/comet/tags/ExpressionsGroup.java | 33 +++ .../org/apache/comet/tags/ScansGroup.java | 33 +++ .../org/apache/comet/tags/ShuffleGroup.java | 33 +++ .../comet/CometArrayExpressionSuite.scala | 2 + .../comet/CometBitwiseExpressionSuite.scala | 2 + .../comet/CometBroadcastBuildSideSuite.scala | 2 + .../apache/comet/CometCodegenFuzzSuite.scala | 2 + .../apache/comet/CometCodegenHOFSuite.scala | 3 + .../comet/CometCodegenSourceSuite.scala | 2 + .../org/apache/comet/CometCodegenSuite.scala | 2 + .../org/apache/comet/CometConfSuite.scala | 3 + .../comet/CometCoverageStatsSuite.scala | 3 + .../comet/CometCsvExpressionSuite.scala | 2 + .../comet/CometDateTimeUtilsSuite.scala | 3 + .../apache/comet/CometExpressionSuite.scala | 2 + .../comet/CometFallbackInvarianceSuite.scala | 3 + .../comet/CometFloatSemanticsSuite.scala | 3 + .../comet/CometFuzzAggregateSuite.scala | 2 + .../apache/comet/CometFuzzIcebergSuite.scala | 2 + .../org/apache/comet/CometFuzzMathSuite.scala | 3 + .../org/apache/comet/CometFuzzTestSuite.scala | 2 + .../comet/CometHashExpressionSuite.scala | 2 + .../comet/CometIcebergNativeSuite.scala | 2 + .../CometIcebergResidualPushdownSuite.scala | 3 + .../CometIcebergRewriteActionSuite.scala | 2 + ...IcebergSystemFunctionExtensionsSuite.scala | 3 + .../CometIcebergSystemFunctionSuite.scala | 2 + .../comet/CometIcebergWriteActionSuite.scala | 2 + .../CometIcebergWriteDetectionSuite.scala | 2 + .../comet/CometJsonExpressionSuite.scala | 2 + .../org/apache/comet/CometJsonJvmSuite.scala | 3 + .../comet/CometMapExpressionSuite.scala | 2 + .../comet/CometMathExpressionSuite.scala | 2 + .../apache/comet/CometNativeCastSuite.scala | 2 + .../org/apache/comet/CometNativeSuite.scala | 3 + .../apache/comet/CometNativeUdfSuite.scala | 2 + .../apache/comet/CometPublicApiSuite.scala | 2 + .../apache/comet/CometRegExpJvmSuite.scala | 2 + .../apache/comet/CometRegexParitySuite.scala | 2 + .../comet/CometScalaUDFClassLoaderSuite.scala | 3 + .../comet/CometSetOpWithGroupBySuite.scala | 3 + .../CometSparkSessionExtensionsSuite.scala | 2 + .../apache/comet/CometSqlFileTestSuite.scala | 3 + .../comet/CometStringExpressionSuite.scala | 2 + .../comet/CometTemporalExpressionSuite.scala | 2 + .../comet/CometUuidExpressionSuite.scala | 3 + .../comet/CometVariantProjectionSuite.scala | 2 + .../apache/comet/CometVariantTypeSuite.scala | 2 + .../org/apache/comet/DataGeneratorSuite.scala | 2 + .../DirectColumnarToRowConverterSuite.scala | 2 + .../org/apache/comet/GenerateDocsSuite.scala | 2 + .../comet/QueryContextInternerSuite.scala | 2 + .../comet/SparkErrorConverterSuite.scala | 2 + .../apache/comet/SqlFileTestParserSuite.scala | 3 + ...CometSpecializedGettersDispatchSuite.scala | 2 + .../comet/csv/CometCsvNativeReadSuite.scala | 2 + .../comet/exec/CometAggregateSuite.scala | 2 + .../exec/CometColumnarShuffleSuite.scala | 5 + .../exec/CometDirectColumnarToRowSuite.scala | 2 + .../comet/exec/CometExec3_4PlusSuite.scala | 2 + .../apache/comet/exec/CometExecSuite.scala | 2 + .../comet/exec/CometGenerateExecSuite.scala | 2 + .../exec/CometInMemoryCacheKryoSuite.scala | 4 + .../exec/CometInMemoryCachePruningSuite.scala | 2 + .../comet/exec/CometInMemoryCacheSuite.scala | 2 + .../apache/comet/exec/CometJoinSuite.scala | 2 + .../exec/CometNativeColumnarToRowSuite.scala | 2 + .../comet/exec/CometNativeReaderSuite.scala | 2 + .../comet/exec/CometNativeShuffleSuite.scala | 2 + .../comet/exec/CometRangeExecSuite.scala | 2 + .../CometShuffleInputConversionSuite.scala | 2 + .../apache/comet/exec/CometTopKSuite.scala | 2 + .../comet/exec/CometTypedDatasetSuite.scala | 2 + .../comet/exec/CometWindowExecSuite.scala | 2 + .../comet/expressions/CometRegexSuite.scala | 2 + .../conditional/CometCaseWhenSuite.scala | 3 + .../conditional/CometCoalesceSuite.scala | 3 + .../conditional/CometIfSuite.scala | 3 + .../iceberg/IcebergReflectionSuite.scala | 3 + .../objectstore/AbfsAuthParitySuite.scala | 2 + .../objectstore/AbfsAuthResolverSuite.scala | 2 + .../objectstore/AbfsScanFallbackSuite.scala | 2 + .../comet/objectstore/NativeConfigSuite.scala | 2 + .../CometParquetWriterRetrySuite.scala | 2 + .../parquet/CometParquetWriterSuite.scala | 2 + .../comet/parquet/ParquetReadSuite.scala | 2 + .../ParquetTimestampLtzAsNtzSuite.scala | 2 + .../comet/rules/CometExecRuleSuite.scala | 2 + .../rules/CometMultiStoreScanSuite.scala | 2 + .../comet/rules/CometScanContribSuite.scala | 2 + .../comet/rules/CometScanRuleSuite.scala | 2 + .../rules/CometScanSchemeFallbackSuite.scala | 2 + .../EliminateRedundantTransitionsSuite.scala | 2 + ...tNativeForTransitionHeavyStagesSuite.scala | 2 + .../comet/serde/CometLiteralSuite.scala | 2 + .../serde/CometScalarFunctionSuite.scala | 2 + .../comet/serde/QueryPlanSerdeSuite.scala | 2 + .../comet/serde/SerdeRegistrationSuite.scala | 3 + .../apache/comet/serde/SparkFixesSuite.scala | 3 + .../CometIcebergNativeScanSuite.scala | 2 + .../serde/operator/CometNativeScanSuite.scala | 2 + .../IcebergWriteProtoTranslationSuite.scala | 2 + .../CelebornShufflePartitionPusherSuite.scala | 3 + .../comet/udf/CometUdfBridgeSuite.scala | 2 + .../comet/vector/CometVectorUtilsSuite.scala | 3 + .../apache/comet/vector/NativeUtilSuite.scala | 2 + .../CometExecIteratorLifecycleSuite.scala | 2 + .../org/apache/spark/CometPluginsSuite.scala | 9 + .../spark/CometRuntimeShutdownSuite.scala | 3 + .../spark/CometTaskMemoryManagerSuite.scala | 3 + ...UnboundedShuffleMemoryAllocatorSuite.scala | 3 + ...etUnifiedShuffleMemoryAllocatorSuite.scala | 3 + .../spark/shuffle/sort/SpillSorterSuite.scala | 2 + .../spark/sql/CometTPCDSQuerySuite.scala | 2 + .../spark/sql/CometTPCDSQueryTestSuite.scala | 2 + .../spark/sql/CometTPCHQuerySuite.scala | 2 + .../sql/comet/CometBroadcastValueSuite.scala | 2 + .../sql/comet/CometCollectMetricsSuite.scala | 2 + .../comet/CometDecimalPromotionSuite.scala | 2 + .../CometDppFallbackRepro3949Suite.scala | 2 + .../sql/comet/CometNativeWriteSuite.scala | 2 + .../sql/comet/CometPlanEqualitySuite.scala | 2 + .../sql/comet/CometPlanStabilitySuite.scala | 3 + .../comet/CometScanWithPlanDataSuite.scala | 2 + .../comet/CometSessionExtensionsSuite.scala | 2 + .../CometShuffleFallbackStickinessSuite.scala | 2 + .../sql/comet/CometTaskMetricsSuite.scala | 2 + .../comet/CometUnsafeProjectionSuite.scala | 3 + .../sql/comet/DecimalPrecisionSuite.scala | 3 + .../sql/comet/ParquetEncryptionITCase.scala | 2 + ...lanDataInjectorShuffleLifecycleSuite.scala | 2 + .../sql/comet/PlanDataInjectorSuite.scala | 2 + .../arrow/CachedBatchRowIteratorSuite.scala | 2 + .../arrow/CometArrowStreamSuite.scala | 2 + .../arrow/CometArrowWriterSuite.scala | 2 + .../arrow/CometStringWriterSuite.scala | 3 + .../CometBlockStoreShuffleReaderSuite.scala | 3 + ...lebornConcurrentMaterializationSuite.scala | 2 + .../CometCelebornLocalFetchFailureSuite.scala | 2 + ...ometCelebornNativeShuffleWriterSuite.scala | 2 + .../CometCelebornShuffleFallbackSuite.scala | 2 + .../CometCelebornShuffleManagerSuite.scala | 2 + .../CometCelebornShufflePlanningSuite.scala | 2 + .../CometCelebornShuffleReaderSuite.scala | 2 + .../CometCelebornShuffleStatisticsSuite.scala | 2 + .../shuffle/CometDiskBlockWriterSuite.scala | 3 + ...CometNativePositionalRoundRobinSuite.scala | 2 + .../CometNativeShuffleInputRDDSuite.scala | 2 + .../CometNativeShuffleWriterSuite.scala | 2 + .../NativeBatchDecoderIteratorSuite.scala | 2 + .../spark/sql/comet/util/UtilsSuite.scala | 2 + .../comet/exec/CometMergeRowsSuite.scala | 2 + .../apache/comet/CometWidthBucketSuite.scala | 3 + .../comet/exec/CometMergeRowsSuite.scala | 3 + .../CometSparkInternalFunctionsSuite.scala | 3 + .../spark/sql/CometToPrettyStringSuite.scala | 2 + .../apache/comet/CometStringDecodeSuite.scala | 2 + .../comet/exec/CometMergeRowsSuite.scala | 3 + .../spark/sql/CometCollationSuite.scala | 2 + .../comet/exec/CometMergeRowsSuite.scala | 2 + .../spark/sql/CometCollationSuite.scala | 2 + .../sql/comet/CometArrowPythonUdfSuite.scala | 2 + .../CometDecimalArithmeticViewSuite.scala | 2 + .../comet/CometIcebergEncryptionSuite.scala | 2 + ...tInMemoryCacheLastAttemptMetricSuite.scala | 2 + .../comet/CometCastCollatedStringSuite.scala | 2 + .../apache/comet/CometWidthBucketSuite.scala | 3 + .../exec/CometEmptyRelationExecSuite.scala | 2 + .../comet/exec/CometShuffle4_0Suite.scala | 3 + ...CometEmptyRelationParquetWriterSuite.scala | 2 + .../CometEmptyRelationExecRuleSuite.scala | 2 + .../spark/sql/CometSortCollationSuite.scala | 2 + .../CometSparkInternalFunctionsSuite.scala | 3 + .../spark/sql/CometToPrettyStringSuite.scala | 2 + .../sql/CometVariantShreddingSuite.scala | 2 + .../sql/comet/CometMapInBatchSuite.scala | 2 + .../python/CometArrowPythonRunnerSuite.scala | 2 + 182 files changed, 801 insertions(+), 107 deletions(-) create mode 100755 dev/ci/run-comet-tests.sh create mode 100644 spark/src/test/java/org/apache/comet/tags/ExecGroup.java create mode 100644 spark/src/test/java/org/apache/comet/tags/ExpressionsGroup.java create mode 100644 spark/src/test/java/org/apache/comet/tags/ScansGroup.java create mode 100644 spark/src/test/java/org/apache/comet/tags/ShuffleGroup.java diff --git a/.ai/skills/review-comet-pr/SKILL.md b/.ai/skills/review-comet-pr/SKILL.md index ea1f57a9d1f..eac35a1005b 100644 --- a/.ai/skills/review-comet-pr/SKILL.md +++ b/.ai/skills/review-comet-pr/SKILL.md @@ -243,8 +243,10 @@ config, not a testing one, to turn it off before merge (#6466). - Do the tests compare against Spark? A comparison with Comet's own accumulator, or with a second implementation such as iceberg-rust instead of the Iceberg Java that Spark runs, can agree on the wrong answer (#6423, #6426). -- New suites must be registered in both `.github/workflows/pr_build_linux.yml` and - `pr_build_macos.yml`. +- Each new concrete suite needs exactly one test-group tag (`@ScansGroup`, `@ShuffleGroup`, + `@ExecGroup` or `@ExpressionsGroup`), and a listing under that group in both + `.github/workflows/pr_build_linux.yml` and `pr_build_macos.yml`. `dev/ci/check-suites.py` + enforces both; the group should match the area the suite covers. For a change on a default path, look for tests with the inputs that broke earlier changes: diff --git a/dev/ci/check-suites.py b/dev/ci/check-suites.py index 68bf972c5cf..24b6b3a8847 100644 --- a/dev/ci/check-suites.py +++ b/dev/ci/check-suites.py @@ -14,90 +14,153 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Check that every Comet test suite belongs to exactly one CI test group. + +CI runs the Comet test suites in groups, one job per group. Each concrete suite +carries the ScalaTest tag of its group (see +spark/src/test/java/org/apache/comet/tags/*Group.java), and the CI workflows +list it in the group of the same name. This fails on a suite with no group tag, +which would run in no job, on one with two, and on a suite whose workflow +listing does not match its tag. Suites deliberately left out of CI are listed +in IGNORE_LIST and must carry no group tag, so that no group picks them up. +""" import re import sys from pathlib import Path -def file_to_class_name(path: Path) -> str | None: - parts = path.parts - if "org" not in parts or "apache" not in parts: - return None - org_index = parts.index("org") - package_parts = parts[org_index:] - class_name = ".".join(package_parts) - class_name = class_name.replace(".scala", "") - return class_name +REPO_ROOT = Path(__file__).resolve().parents[2] -if __name__ == "__main__": +# One ScalaTest tag annotation per group, such as ExecGroup for the `exec` group. +TAGS_DIR = REPO_ROOT / "spark/src/test/java/org/apache/comet/tags" +GROUP_TAGS = sorted(path.stem for path in TAGS_DIR.glob("*Group.java")) - # ignore traits, abstract classes, and intentionally skipped test suites - ignore_list = [ - "org.apache.comet.parquet.ParquetReadSuite", # abstract - "org.apache.comet.parquet.ParquetReadFromS3Suite", # manual test suite - "org.apache.comet.parquet.ParquetWriteWithMagicCommitterSuite", # manual test suite - "org.apache.comet.csv.CsvReadFromS3Suite", # manual test suite - "org.apache.comet.parquet.ParquetReadFromFakeHadoopFsSuite", # manual test suite (loads libhdfs, see #5023) - "org.apache.comet.IcebergReadFromS3Suite", # manual test suite - "org.apache.comet.CometIcebergSchedulerFailureSuite", # manual, multiple hosts + SIGKILL - "org.apache.comet.cloud.s3.CometS3CredentialBridgeSuite", # manual test suite - "org.apache.comet.cloud.s3.HadoopS3ACredentialProviderAdapterBridgeSuite", # manual test suite - "org.apache.comet.shuffle.CelebornReflectionCompatibilitySuite", # dedicated version matrix - "org.apache.spark.sql.comet.CometPlanStabilitySuite", # abstract - "org.apache.spark.sql.comet.ParquetDatetimeRebaseSuite", # abstract - "org.apache.comet.exec.CometColumnarShuffleSuite" # abstract - ] - - for workflow_filename in [".github/workflows/pr_build_linux.yml", ".github/workflows/pr_build_macos.yml"]: - workflow = open(workflow_filename, encoding="utf-8").read() - - root = Path(".") - for path in root.rglob("*Suite.scala"): - # contrib suites run via their own module-level workflows - # (e.g. delta_contrib_test.yml), not the main PR build matrix - if path.parts[0] == "contrib": - continue - class_name = file_to_class_name(path) - if class_name: - if "Shim" in class_name: - continue - if class_name in ignore_list: - continue - if class_name not in workflow: - print(f"Suite not found in workflow {workflow_filename}: {class_name}") - sys.exit(-1) - print(f"Found {class_name} in {workflow_filename}") - - # Forward check: every suite named in a workflow must be declared in a source file. - # Filename-based discovery is not enough here, because several suites are declared - # inside a file named after a different class (e.g. CometShuffleSuite is declared in - # CometColumnarShuffleSuite.scala), and version-specific suites such as - # CometStringDecodeSuite live only in the spark-3.x sourceset. - declared = set() - suite_declaration = re.compile( - r"^\s*(?:(?:final|sealed|abstract|private|case)\s+)*(?:class|trait|object)\s+(\w*Suite)\b" - ) - for path in Path(".").rglob("*.scala"): - if "target" in path.parts: +# Suites that CI does not run, each with the reason. +IGNORE_LIST = { + "org.apache.comet.parquet.ParquetReadFromS3Suite": "manual test suite", + "org.apache.comet.parquet.ParquetWriteWithMagicCommitterSuite": "manual test suite", + "org.apache.comet.csv.CsvReadFromS3Suite": "manual test suite", + "org.apache.comet.parquet.ParquetReadFromFakeHadoopFsSuite": "manual test suite (loads libhdfs, see #5023)", + "org.apache.comet.IcebergReadFromS3Suite": "manual test suite", + "org.apache.comet.CometIcebergSchedulerFailureSuite": "manual, multiple hosts + SIGKILL", + "org.apache.comet.cloud.s3.CometS3CredentialBridgeSuite": "manual test suite", + "org.apache.comet.cloud.s3.HadoopS3ACredentialProviderAdapterBridgeSuite": "manual test suite", + "org.apache.comet.shuffle.CelebornReflectionCompatibilitySuite": "dedicated version matrix", +} + +CLASS_DECL = re.compile( + r"^(?P(?:(?:final|sealed|abstract|private(?:\[\w+\])?|case)\s+)*)" + r"(?Pclass|trait|object)\s+(?P\w+)" +) +ANNOTATION = re.compile(r"^@(\w+)") + + +def test_sources(): + """Every Scala test source outside contrib, in all source sets (scala, spark-3.4, spark-4.x, ...). + + The contrib modules run their suites from their own workflows, such as delta_contrib_test.yml. + """ + for path in sorted(REPO_ROOT.rglob("*.scala")): + parts = path.relative_to(REPO_ROOT).parts + if parts[0] != "contrib" and "target" not in parts and "test" in parts: + yield path + + +def declarations(text): + """Yield (name, concrete, annotations, line) for each top-level type in a source file. + + `concrete` is true for a class that is neither abstract nor a trait or object. + """ + lines = text.splitlines() + for number, line in enumerate(lines): + match = CLASS_DECL.match(line) + if not match: continue - class_name = file_to_class_name(path) - if class_name: - declared.add(class_name.rsplit(".", 1)[-1]) - for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): - match = suite_declaration.match(line) - if match: - declared.add(match.group(1)) - - listed_suite = re.compile(r"\borg\.apache\.[A-Za-z0-9_.]*Suite\b") - undeclared = [] - for workflow_filename in [".github/workflows/pr_build_linux.yml", ".github/workflows/pr_build_macos.yml"]: - workflow = open(workflow_filename, encoding="utf-8").read() - for name in sorted(set(listed_suite.findall(workflow))): - if name.rsplit(".", 1)[-1] not in declared: - undeclared.append((workflow_filename, name)) - - if undeclared: - for workflow_filename, name in undeclared: - print(f"Workflow lists an undeclared suite {name} in {workflow_filename}") - sys.exit(-1) - print("All workflow-listed suites are declared") + annotations = [] + above = number - 1 + while above >= 0 and (annotation := ANNOTATION.match(lines[above])): + annotations.append(annotation.group(1)) + above -= 1 + concrete = match.group("kind") == "class" and "abstract" not in match.group("mods") + yield match.group("name"), concrete, annotations, number + 1 + + +# The linux-test and macOS test matrices list each group's suites by name. +WORKFLOWS = [".github/workflows/pr_build_linux.yml", ".github/workflows/pr_build_macos.yml"] +MATRIX_GROUP = re.compile(r'^\s+- name: "(\w+)"\n\s+value: \|\n((?:\s+[\w.]+\n)+)', re.M) + + +def workflow_groups(workflow): + """Map each suite a workflow's test matrix lists to the group it is listed in.""" + text = (REPO_ROOT / workflow).read_text(encoding="utf-8") + return { + suite: group + for group, suites in MATRIX_GROUP.findall(text) + for suite in suites.split() + } + + +def group_name(tag): + """ExecGroup -> exec""" + return tag[: -len("Group")].lower() + + +def main(): + failures = [] + members = {tag: set() for tag in GROUP_TAGS} + seen_ignored = set() + for path in test_sources(): + text = path.read_text(encoding="utf-8", errors="ignore") + package = re.search(r"^package\s+([\w.]+)", text, re.M) + prefix = package.group(1) + "." if package else "" + for name, concrete, annotations, line in declarations(text): + fqn = prefix + name + where = f"{path.relative_to(REPO_ROOT)}:{line}" + tags = [a for a in annotations if a in members] + if not (concrete and re.search(r"(Suite|ITCase)$", name)): + if tags: + failures.append( + f"{where}: {fqn} is not a concrete suite, so its @{tags[0]} does not reach " + "the suites that extend it. Tag each concrete suite instead." + ) + continue + if fqn in IGNORE_LIST: + seen_ignored.add(fqn) + if tags: + failures.append( + f"{where}: {fqn} is in IGNORE_LIST ({IGNORE_LIST[fqn]}) but carries @{tags[0]}" + ) + continue + if len(tags) != 1: + found = ", ".join("@" + t for t in tags) if tags else "none" + failures.append( + f"{where}: {fqn} needs exactly one group tag ({', '.join('@' + t for t in GROUP_TAGS)}); " + f"found {found}" + ) + continue + members[tags[0]].add(fqn) + + for fqn in sorted(set(IGNORE_LIST) - seen_ignored): + failures.append(f"IGNORE_LIST names {fqn}, which no longer exists") + + tagged = {fqn: group_name(tag) for tag, suites in members.items() for fqn in suites} + for workflow in WORKFLOWS: + listed = workflow_groups(workflow) + for fqn, group in sorted(tagged.items()): + if fqn not in listed: + failures.append(f"{workflow} does not list {fqn}, which its tag puts in {group}") + elif listed[fqn] != group: + failures.append(f"{workflow} lists {fqn} in {listed[fqn]}, but its tag puts it in {group}") + for fqn in sorted(set(listed) - set(tagged)): + failures.append(f"{workflow} lists {fqn}, which is not a tagged concrete suite") + + if failures: + for failure in failures: + print(failure) + sys.exit(1) + summary = ", ".join(f"{tag} {len(suites)}" for tag, suites in members.items()) + print(f"Every suite has one group tag: {summary}") + + +if __name__ == "__main__": + main() diff --git a/dev/ci/run-comet-tests.sh b/dev/ci/run-comet-tests.sh new file mode 100755 index 00000000000..ea7b9ae86f9 --- /dev/null +++ b/dev/ci/run-comet-tests.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Run one group of the Comet JVM test suites. Each concrete suite carries one +# group tag, such as org.apache.comet.tags.ExecGroup, and CI runs each group as +# its own job. +# Arguments after the group go to Maven: +# +# dev/ci/run-comet-tests.sh exec -Pspark-4.0 +# +# Build the native library first: `make core`, or `make release` with -Prelease. +# Written for bash 3.2. + +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# Each group is a tag annotation here, such as ExecGroup.java for `exec`. +TAGS_DIR="$REPO/spark/src/test/java/org/apache/comet/tags" + +usage() { + groups="" + for tag in "$TAGS_DIR"/*Group.java; do + tag="$(basename "$tag" Group.java)" + groups="$groups $(printf '%s' "$tag" | tr '[:upper:]' '[:lower:]')" + done + cat >&2 <] [maven args...] + + groups $groups + --goal the Maven goal to run (default: test). install also runs the + spark-integration checks. +USAGE + exit 2 +} + +GOAL="test" +while [ $# -gt 0 ]; do + case "$1" in + --goal) + [ $# -ge 2 ] || usage + GOAL="$2" + shift 2 + ;; + -h | --help) usage ;; + *) break ;; + esac +done +[ $# -ge 1 ] || usage + +# exec -> ExecGroup +TAG="$(printf '%s' "${1:0:1}" | tr '[:lower:]' '[:upper:]')${1:1}Group" +if [ ! -f "$TAGS_DIR/$TAG.java" ]; then + echo "Unknown test group: $1" >&2 + usage +fi +shift + +cd "$REPO" + +export SPARK_HOME="${SPARK_HOME:-$REPO}" +export SPARK_LOCAL_HOSTNAME="${SPARK_LOCAL_HOSTNAME:-localhost}" +export SPARK_LOCAL_IP="${SPARK_LOCAL_IP:-127.0.0.1}" +export MAVEN_OPTS="${MAVEN_OPTS:--Xmx4G -Xms2G -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log}" + +set -x +exec ./mvnw -B "$GOAL" "-DtagsToInclude=org.apache.comet.tags.$TAG" "$@" diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md index 8efec3ea7c3..bf94f9cda12 100644 --- a/docs/source/contributor-guide/development.md +++ b/docs/source/contributor-guide/development.md @@ -710,19 +710,28 @@ make test-jvm ### 6. Register New Test Suites in CI -Comet's CI does not automatically discover test suites. Instead, test suites are explicitly listed -in the GitHub Actions workflow files so they can be grouped by category and run as separate parallel -jobs. This reduces overall CI time. - -If you add a new Comet Scala Test suite, you must add it to the `suite` matrix in **both** workflow files: +CI runs Comet's Scala test suites in four groups, each as its own parallel job. A new suite joins a +group in two places: its class carries the group's ScalaTest tag, and its fully qualified class name +is listed under the group in the `suite` matrix of **both** workflow files: - `.github/workflows/pr_build_linux.yml` - `.github/workflows/pr_build_macos.yml` -Each file contains a `suite` matrix with named groups such as `fuzz`, `shuffle`, `parquet`, `csv`, -`exec`, `expressions`, and `sql`. Add your new suite's fully qualified class name to the -appropriate group. For example, if you add a new expression test suite, add it to the -`expressions` group: +| Tag | Group | Covers | +| ------------------- | ------------- | -------------------------------------------------------------- | +| `@ScansGroup` | `scans` | Parquet, CSV and Iceberg reads and writes, and the fuzz suites | +| `@ShuffleGroup` | `shuffle` | Native and columnar shuffle, and the Celeborn integration | +| `@ExecGroup` | `exec` | Operators, plan rules, plugins, memory, and TPC queries | +| `@ExpressionsGroup` | `expressions` | Expression evaluation, casts, codegen, and the SQL file tests | + +For example, a new expression suite goes in the `expressions` group: + +```scala +import org.apache.comet.tags.ExpressionsGroup + +@ExpressionsGroup +class YourNewExpressionSuite extends CometTestBase { +``` ```yaml - name: "expressions" @@ -732,27 +741,32 @@ appropriate group. For example, if you add a new expression test suite, add it t org.apache.comet.YourNewExpressionSuite # <-- add here ``` -Choose the group that best matches the area your test covers: +Tag each concrete suite class. A tag on an abstract base class or a trait does not reach the suites +that extend it. A suite that has a copy in more than one Spark version source set, such as +`spark-3.5` and `spark-4.0`, needs the tag in each copy. -| Group | Covers | -| ------------- | ---------------------------------------------------------- | -| `fuzz` | Fuzz testing and data generation | -| `shuffle` | Shuffle operators and related exchange behavior | -| `parquet` | Parquet read/write and native reader tests | -| `csv` | CSV native read tests | -| `exec` | Execution operators, joins, aggregates, plan rules, TPC-\* | -| `expressions` | Expression evaluation, casts, and Comet SQL Tests | -| `sql` | SQL-level behavior tests | +The `preflight` job in `.github/workflows/ci.yml` runs `dev/ci/check-suites.py` on every pull +request. It fails when a concrete suite has no group tag or more than one, or when either workflow +file does not list it under the group its tag names. + +#### Test Groups + +`dev/ci/run-comet-tests.sh` runs one group locally: the suites that carry its tag, which +`dev/ci/check-suites.py` keeps the same as the group's CI job. + +```sh +make +dev/ci/run-comet-tests.sh expressions # the default Spark profile +dev/ci/run-comet-tests.sh exec -Pspark-4.0 # arguments after the group go to Maven +``` -**Important:** The suite lists in both workflow files must stay in sync. The `preflight` job in -`.github/workflows/ci.yml` runs `dev/ci/check-suites.py` on every pull request. It scans for all -`*Suite.scala` files in the repository and verifies that each one appears in both workflow files. -If any suite is missing, this check will fail and block the PR. +The script runs `mvnw test` by default. Pass `--goal install` to also run the `spark-integration` +checks, and `-Prelease` to use the library that `make release` builds. A small number of suites are deliberately **not** run in CI, because they need infrastructure CI -does not have or because running them there is not worth the cost. These are listed in the -`ignore_list` in `dev/ci/check-suites.py`, and each one documents in its own scaladoc why it is -excluded and how to run it. Run a manual suite with: +does not have or because running them there is not worth the cost. These are listed in +`IGNORE_LIST` in `dev/ci/check-suites.py` and carry no group tag. Each one documents in its own +scaladoc why it is excluded and how to run it. Run a manual suite with: ```sh ./mvnw test -Dtest=none -Dsuites="org.apache.comet.parquet.ParquetReadFromFakeHadoopFsSuite" diff --git a/spark/src/test/java/org/apache/comet/tags/ExecGroup.java b/spark/src/test/java/org/apache/comet/tags/ExecGroup.java new file mode 100644 index 00000000000..698e3ec44ba --- /dev/null +++ b/spark/src/test/java/org/apache/comet/tags/ExecGroup.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.tags; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.scalatest.TagAnnotation; + +/** Puts a suite in the `exec` CI test group. See "Test Groups" in the development guide. */ +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface ExecGroup {} diff --git a/spark/src/test/java/org/apache/comet/tags/ExpressionsGroup.java b/spark/src/test/java/org/apache/comet/tags/ExpressionsGroup.java new file mode 100644 index 00000000000..c7f37f5d107 --- /dev/null +++ b/spark/src/test/java/org/apache/comet/tags/ExpressionsGroup.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.tags; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.scalatest.TagAnnotation; + +/** Puts a suite in the `expressions` CI test group. See "Test Groups" in the development guide. */ +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface ExpressionsGroup {} diff --git a/spark/src/test/java/org/apache/comet/tags/ScansGroup.java b/spark/src/test/java/org/apache/comet/tags/ScansGroup.java new file mode 100644 index 00000000000..3ffd50c6846 --- /dev/null +++ b/spark/src/test/java/org/apache/comet/tags/ScansGroup.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.tags; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.scalatest.TagAnnotation; + +/** Puts a suite in the `scans` CI test group. See "Test Groups" in the development guide. */ +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface ScansGroup {} diff --git a/spark/src/test/java/org/apache/comet/tags/ShuffleGroup.java b/spark/src/test/java/org/apache/comet/tags/ShuffleGroup.java new file mode 100644 index 00000000000..63cc93c76df --- /dev/null +++ b/spark/src/test/java/org/apache/comet/tags/ShuffleGroup.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.tags; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.scalatest.TagAnnotation; + +/** Puts a suite in the `shuffle` CI test group. See "Test Groups" in the development guide. */ +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface ShuffleGroup {} diff --git a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala index 761bc954df7..fead26f9c9d 100644 --- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala @@ -35,8 +35,10 @@ import org.apache.spark.sql.types.{ArrayType, DataType, DoubleType, FloatType, I import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus} import org.apache.comet.DataTypeSupport.isComplexType import org.apache.comet.serde.{ArrayElementEqualitySupport, ArraySetSupport, CometArrayDistinct, CometArrayExcept, CometArrayJoin, CometArrayRemove, CometArrayReverse, CometArraysOverlap, CometArrayUnion, CometFlatten, CometSequence, Compatible, ExprOuterClass, Incompatible, SparkFixes} +import org.apache.comet.tags.ExpressionsGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} +@ExpressionsGroup class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("array set signed-zero Spark patch versions") { diff --git a/spark/src/test/scala/org/apache/comet/CometBitwiseExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometBitwiseExpressionSuite.scala index 36540bc4997..46b2cc7d57d 100644 --- a/spark/src/test/scala/org/apache/comet/CometBitwiseExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometBitwiseExpressionSuite.scala @@ -25,8 +25,10 @@ import org.apache.hadoop.fs.Path import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.comet.tags.ExpressionsGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} +@ExpressionsGroup class CometBitwiseExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("bitwise expressions") { diff --git a/spark/src/test/scala/org/apache/comet/CometBroadcastBuildSideSuite.scala b/spark/src/test/scala/org/apache/comet/CometBroadcastBuildSideSuite.scala index a1cb9ffdec2..6c4a213255c 100644 --- a/spark/src/test/scala/org/apache/comet/CometBroadcastBuildSideSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometBroadcastBuildSideSuite.scala @@ -33,11 +33,13 @@ import org.apache.spark.sql.execution.joins.BroadcastNestedLoopJoinExec import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometSparkSessionExtensions.isSpark35Plus +import org.apache.comet.tags.ScansGroup // Tests for the broadcast build-side bridge, spark.comet.convert.broadcastBuildSide.enabled (issue // #6008). A build-side scan that Comet cannot read natively, such as ORC or an Iceberg table // the native reader declines, is converted to Arrow so that the broadcast join runs natively. Each // query is compared with Spark, and the tests assert where the bridge must and must not appear. +@ScansGroup class CometBroadcastBuildSideSuite extends CometTestBase with CometIcebergTestBase { // Most tests use an ORC build side. Text, Parquet, JSON and CSV scans are converted only by their diff --git a/spark/src/test/scala/org/apache/comet/CometCodegenFuzzSuite.scala b/spark/src/test/scala/org/apache/comet/CometCodegenFuzzSuite.scala index 9c0940ad099..0b5ec27b8d1 100644 --- a/spark/src/test/scala/org/apache/comet/CometCodegenFuzzSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCodegenFuzzSuite.scala @@ -37,6 +37,7 @@ import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.DataTypeSupport.isComplexType import org.apache.comet.codegen.CometBatchKernelCodegen +import org.apache.comet.tags.ExpressionsGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, ParquetGenerator, SchemaGenOptions} import org.apache.comet.vector.CometVector @@ -47,6 +48,7 @@ import org.apache.comet.vector.CometVector * (not [[CometFuzzTestBase]]) because the base's `shuffle` x `nativeC2R` cross-product is * irrelevant for projection-only queries. */ +@ExpressionsGroup class CometCodegenFuzzSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometCodegenHOFSuite.scala b/spark/src/test/scala/org/apache/comet/CometCodegenHOFSuite.scala index 49fadd16e0e..e1aca516a0a 100644 --- a/spark/src/test/scala/org/apache/comet/CometCodegenHOFSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCodegenHOFSuite.scala @@ -23,6 +23,8 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.comet.tags.ExpressionsGroup + /** * Higher-order function regression coverage for the codegen dispatcher. * @@ -41,6 +43,7 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper * the per-task instance, not the companion, so concurrent partitions cannot race on a shared * `NamedLambdaVariable.value`. The two-collects test below regresses this. */ +@ExpressionsGroup class CometCodegenHOFSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometCodegenSourceSuite.scala b/spark/src/test/scala/org/apache/comet/CometCodegenSourceSuite.scala index 54b59dec9dd..019551dca00 100644 --- a/spark/src/test/scala/org/apache/comet/CometCodegenSourceSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCodegenSourceSuite.scala @@ -31,6 +31,7 @@ import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.codegen.CometBatchKernelCodegen import org.apache.comet.codegen.CometBatchKernelCodegen.{ArrayColumnSpec, ArrowColumnSpec, MapColumnSpec, ScalarColumnSpec, StructColumnSpec, StructFieldSpec} +import org.apache.comet.tags.ExpressionsGroup import org.apache.comet.udf.codegen.CometScalaUDFCodegen // Resolve Arrow vector classes through the codegen object so tests see the same `Class` objects @@ -53,6 +54,7 @@ import org.apache.comet.udf.codegen.CometScalaUDFCodegen * These are the smallest durable tests that the claimed optimizations actually reach the * generated Java, and they document the shapes future contributors should preserve. */ +@ExpressionsGroup class CometCodegenSourceSuite extends AnyFunSuite { private val varCharVectorClass = diff --git a/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala b/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala index 63343c23c7c..144a800944f 100644 --- a/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala @@ -45,6 +45,7 @@ import org.apache.comet.codegen.{CometBatchKernelCodegen, DispatchOccurrence} import org.apache.comet.codegen.CometBatchKernelCodegen.ArrowColumnSpec import org.apache.comet.serde.{CometInvokeTargets, CometScalaUDF, QueryPlanSerde} import org.apache.comet.serde.ExprOuterClass.Expr.ExprStructCase +import org.apache.comet.tags.ExpressionsGroup import org.apache.comet.udf.CometUdfBridge import org.apache.comet.udf.codegen.CometScalaUDFCodegen import org.apache.comet.vector.CometVector @@ -58,6 +59,7 @@ import org.apache.comet.vector.CometVector * rather than `checkSparkAnswerAndOperator` because ScalaUDF has no Comet-native path. Under * fallback the project runs on the JVM Spark path. */ +@ExpressionsGroup class CometCodegenSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometConfSuite.scala b/spark/src/test/scala/org/apache/comet/CometConfSuite.scala index 5c269589310..39e29df298a 100644 --- a/spark/src/test/scala/org/apache/comet/CometConfSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometConfSuite.scala @@ -23,6 +23,9 @@ import org.scalatest.funsuite.AnyFunSuite import org.apache.spark.sql.internal.SQLConf +import org.apache.comet.tags.ExecGroup + +@ExecGroup class CometConfSuite extends AnyFunSuite { test("primary key wins over alternative when both are set") { diff --git a/spark/src/test/scala/org/apache/comet/CometCoverageStatsSuite.scala b/spark/src/test/scala/org/apache/comet/CometCoverageStatsSuite.scala index b3a9b020775..a3c4ba8eae6 100644 --- a/spark/src/test/scala/org/apache/comet/CometCoverageStatsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCoverageStatsSuite.scala @@ -26,10 +26,13 @@ import org.apache.spark.sql.execution.{ReusedSubqueryExec, SparkPlan, SubqueryEx import org.apache.spark.sql.execution.exchange.ReusedExchangeExec import org.apache.spark.sql.internal.SQLConf +import org.apache.comet.tags.ExpressionsGroup + /** * Tests for how [[CometCoverageStats]] categorizes plan nodes into accelerated operators, * un-accelerated Spark operators, and transitions. */ +@ExpressionsGroup class CometCoverageStatsSuite extends CometTestBase { private def somePlan: SparkPlan = diff --git a/spark/src/test/scala/org/apache/comet/CometCsvExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometCsvExpressionSuite.scala index 6e50e7cb9ae..b2e22e52745 100644 --- a/spark/src/test/scala/org/apache/comet/CometCsvExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCsvExpressionSuite.scala @@ -29,8 +29,10 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StringType +import org.apache.comet.tags.ExpressionsGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} +@ExpressionsGroup class CometCsvExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("to_csv - default options") { diff --git a/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala b/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala index 072009d7ab6..aa464212337 100644 --- a/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala @@ -26,11 +26,14 @@ import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.DataTypes +import org.apache.comet.tags.ExpressionsGroup + /** * Tests for string-to-timestamp (and string-to-date) cast correctness, ported from Spark's * DateTimeUtilsSuite. That suite lives in sql/catalyst and does not extend CometTestBase, so it * never runs in Comet CI. */ +@ExpressionsGroup class CometDateTimeUtilsSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 3a4f8140506..618dc01d7bb 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -36,8 +36,10 @@ import org.apache.spark.sql.types._ import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, isSpark41Plus, isSpark42Plus} import org.apache.comet.serde.{CometAdd, CometDivide, CometIntegralDivide, CometMultiply, CometRemainder, CometSubtract, Compatible, Unsupported} +import org.apache.comet.tags.ExpressionsGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} +@ExpressionsGroup class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/CometFallbackInvarianceSuite.scala b/spark/src/test/scala/org/apache/comet/CometFallbackInvarianceSuite.scala index eb56881db81..14c0d76e171 100644 --- a/spark/src/test/scala/org/apache/comet/CometFallbackInvarianceSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFallbackInvarianceSuite.scala @@ -26,6 +26,8 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} import org.apache.spark.sql.internal.SQLConf +import org.apache.comet.tags.ExpressionsGroup + /** * Checks that forcing an expression from Comet's native path back to Spark does not change the * outcome of a query. @@ -50,6 +52,7 @@ import org.apache.spark.sql.internal.SQLConf * everything and the gate above would be inoperative. Row order is canonicalised in the * comparator instead. */ +@ExpressionsGroup class CometFallbackInvarianceSuite extends CometFuzzTestBase { /** diff --git a/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala b/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala index 442cb7c41a7..e28a1291d76 100644 --- a/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala @@ -23,6 +23,8 @@ import org.scalatest.exceptions.TestFailedException import org.apache.spark.sql.CometTestBase +import org.apache.comet.tags.ExpressionsGroup + /** * Sweeps Spark's `-0.0` and NaN semantics across operator contexts and the expressions that * accept `FLOAT` or `DOUBLE` input (#6385). @@ -36,6 +38,7 @@ import org.apache.spark.sql.CometTestBase * answer from Spark. The case fails once it matches, so that a fix, or a DataFusion upgrade that * changes float behavior, removes its entry from the list instead of leaving it stale. */ +@ExpressionsGroup class CometFloatSemanticsSuite extends CometTestBase { import CometFloatSemanticsSuite._ diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzAggregateSuite.scala b/spark/src/test/scala/org/apache/comet/CometFuzzAggregateSuite.scala index 3674887f80f..0fd06e33c26 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzAggregateSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzAggregateSuite.scala @@ -20,7 +20,9 @@ package org.apache.comet import org.apache.comet.DataTypeSupport.isComplexType +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometFuzzAggregateSuite extends CometFuzzTestBase { test("count distinct - simple columns") { diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzIcebergSuite.scala b/spark/src/test/scala/org/apache/comet/CometFuzzIcebergSuite.scala index ad9bd99b599..2b1838b6800 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzIcebergSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzIcebergSuite.scala @@ -28,7 +28,9 @@ import org.apache.spark.sql.types._ import org.apache.comet.DataTypeSupport.isComplexType import org.apache.comet.serde.OperatorOuterClass +import org.apache.comet.tags.ScansGroup +@ScansGroup class CometFuzzIcebergSuite extends CometFuzzIcebergBase { test("select *") { diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzMathSuite.scala b/spark/src/test/scala/org/apache/comet/CometFuzzMathSuite.scala index 50a4425f973..3bc6b0ded5c 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzMathSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzMathSuite.scala @@ -23,6 +23,9 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DecimalType, IntegerType, LongType} +import org.apache.comet.tags.ExpressionsGroup + +@ExpressionsGroup class CometFuzzMathSuite extends CometFuzzTestBase { // The integer-math fuzz tests intentionally cover overflowing inputs; under Spark 4's diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala index f8d8f307934..abb440210db 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala @@ -28,9 +28,11 @@ import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType import org.apache.spark.sql.types._ import org.apache.comet.DataTypeSupport.isComplexType +import org.apache.comet.tags.ScansGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator} import org.apache.comet.testing.FuzzDataGenerator.{doubleNaNLiteral, floatNaNLiteral} +@ScansGroup class CometFuzzTestSuite extends CometFuzzTestBase { test("select *") { diff --git a/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala index 7e823fb347c..9c54e5d3d00 100644 --- a/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala @@ -24,6 +24,7 @@ import scala.util.Random import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.comet.tags.ExpressionsGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, ParquetGenerator, SchemaGenOptions} import org.apache.comet.udf.codegen.CometScalaUDFCodegen @@ -33,6 +34,7 @@ import org.apache.comet.udf.codegen.CometScalaUDFCodegen * These tests verify that Comet's native implementations produce identical results to Spark for * all supported data types. */ +@ExpressionsGroup class CometHashExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { private def checkNativeHashes(query: String): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala index 4153498516f..944fe9fd914 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala @@ -49,6 +49,7 @@ import org.apache.comet.cloud.s3.MinioLocationScopedCredentialProvider import org.apache.comet.iceberg.{IcebergReflection, RESTCatalogHelper} import org.apache.comet.rules.CometScanRule import org.apache.comet.serde.{OperatorOuterClass, QueryPlanSerde} +import org.apache.comet.tags.ScansGroup import org.apache.comet.testing.{FuzzDataGenerator, SchemaGenOptions} /** @@ -56,6 +57,7 @@ import org.apache.comet.testing.{FuzzDataGenerator, SchemaGenOptions} * * Note: Requires Iceberg dependencies to be added to pom.xml */ +@ScansGroup class CometIcebergNativeSuite extends CometTestBase with RESTCatalogHelper diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergResidualPushdownSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergResidualPushdownSuite.scala index cdeaaa9cd40..b16cd56b630 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergResidualPushdownSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergResidualPushdownSuite.scala @@ -24,6 +24,8 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.comet.CometIcebergNativeScanExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.comet.tags.ScansGroup + /** * Residual pushdown for predicates that only reach the Iceberg scan filter when Iceberg's SQL * extensions are installed. @@ -33,6 +35,7 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper * so Iceberg can push the comparison, which puts an `UnboundTransform` term into each file's * residual. CometIcebergNativeSuite runs without the extensions, so it never sees one. */ +@ScansGroup class CometIcebergResidualPushdownSuite extends CometTestBase with CometIcebergTestBase diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergRewriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergRewriteActionSuite.scala index bb554ca095e..73ccb969eda 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergRewriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergRewriteActionSuite.scala @@ -34,6 +34,7 @@ import org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart import org.apache.comet.CometSparkSessionExtensions.isSpark35Plus import org.apache.comet.iceberg.IcebergReflection +import org.apache.comet.tags.ScansGroup /** * Verifies that Comet engages on the per-group reads issued by Iceberg's @@ -41,6 +42,7 @@ import org.apache.comet.iceberg.IcebergReflection * via ScanTaskSetManager and then reads them back through SparkStagedScan, which Comet's * CometScanRule recognises alongside SparkBatchQueryScan. */ +@ScansGroup class CometIcebergRewriteActionSuite extends CometTestBase with CometIcebergTestBase { private val catalog = "test_cat" diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionExtensionsSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionExtensionsSuite.scala index e71acf4e5c9..d09244f4fd4 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionExtensionsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionExtensionsSuite.scala @@ -24,6 +24,8 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression import org.apache.spark.sql.catalyst.plans.logical.Filter +import org.apache.comet.tags.ScansGroup + /** * Iceberg's system functions once Iceberg's SQL extensions have rewritten them. * @@ -33,6 +35,7 @@ import org.apache.spark.sql.catalyst.plans.logical.Filter * it with the same native kernels as the `StaticInvoke` that CometIcebergSystemFunctionSuite * covers. `spark.sql.extensions` is static, so this path needs a suite of its own. */ +@ScansGroup class CometIcebergSystemFunctionExtensionsSuite extends CometTestBase with CometIcebergTestBase { override protected def sparkConf: SparkConf = diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala index 613f726767d..7dcd368da2c 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala @@ -43,6 +43,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.comet.serde.{CometApplyFunctionExpression, CometExpressionSerde, CometIcebergBucket, CometIcebergSystemFunctions, CometIcebergTruncate, CometStaticInvoke, Compatible, SupportLevel, Unsupported} +import org.apache.comet.tags.ScansGroup /** * Native support for Iceberg's system functions (`bucket`, `truncate`, `years`, `months`, `days`, @@ -55,6 +56,7 @@ import org.apache.comet.serde.{CometApplyFunctionExpression, CometExpressionSerd * out-of-order partitions); in a filter or projection it would be a silently wrong answer, which * is why the coverage is per type rather than a few hand-picked rows. */ +@ScansGroup class CometIcebergSystemFunctionSuite extends CometTestBase with CometIcebergTestBase diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index 6f752267a63..32f980fd14d 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -60,6 +60,7 @@ import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark41Plu import org.apache.comet.iceberg.{IcebergReflection, IcebergWriteReportListener, PositionDeltaWrite} import org.apache.comet.serde.Unsupported import org.apache.comet.serde.operator.CometIcebergNativeWrite +import org.apache.comet.tags.ScansGroup private case class WriteSnapshot(snapshotDelta: Long, plans: Seq[SparkPlan]) @@ -69,6 +70,7 @@ private case class ReportedWrite( reasons: Seq[String], failed: Boolean) +@ScansGroup class CometIcebergWriteActionSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index f2fcc3010fe..5cc8e5a9720 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -48,7 +48,9 @@ import org.apache.comet.rules.EliminateRedundantTransitions import org.apache.comet.serde.{Compatible, SupportLevel, Unsupported} import org.apache.comet.serde.OperatorOuterClass.Operator import org.apache.comet.serde.operator.CometIcebergNativeWrite +import org.apache.comet.tags.ScansGroup +@ScansGroup class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTestBase { override protected def sparkConf: SparkConf = { diff --git a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala index 2356a985ef7..8dc108259a2 100644 --- a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala @@ -31,8 +31,10 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions._ import org.apache.comet.serde.CometStructsToJson +import org.apache.comet.tags.ExpressionsGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} +@ExpressionsGroup class CometJsonExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit diff --git a/spark/src/test/scala/org/apache/comet/CometJsonJvmSuite.scala b/spark/src/test/scala/org/apache/comet/CometJsonJvmSuite.scala index 4a4df2e9b1a..ce12e86b16e 100644 --- a/spark/src/test/scala/org/apache/comet/CometJsonJvmSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometJsonJvmSuite.scala @@ -23,6 +23,9 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.comet.tags.ExpressionsGroup + +@ExpressionsGroup class CometJsonJvmSuite extends CometTestBase with AdaptiveSparkPlanHelper { // No per-expression `allowIncompatible` is set, so all JSON expressions run through the diff --git a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala index 6b8202d240c..ab6443fa20c 100644 --- a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala @@ -29,8 +29,10 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.BinaryType import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.tags.ExpressionsGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} +@ExpressionsGroup class CometMapExpressionSuite extends CometTestBase { test("read map[int, int] from parquet") { diff --git a/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala index fdad9e3d3df..0e1f32dda0a 100644 --- a/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala @@ -27,8 +27,10 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataTypes, StructField, StructType} import org.apache.comet.CometSparkSessionExtensions.isSpark35Plus +import org.apache.comet.tags.ExpressionsGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} +@ExpressionsGroup class CometMathExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("abs") { diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 29ddba35457..1b6a2a4a5a6 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -40,6 +40,7 @@ import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, import org.apache.comet.expressions.{CometCast, CometEvalMode} import org.apache.comet.rules.CometScanTypeChecker import org.apache.comet.serde.{Compatible, Incompatible, QueryPlanSerde, SparkFixes, Unsupported} +import org.apache.comet.tags.ExpressionsGroup /** * Spark-parity coverage for Comet's **native** `Cast` implementation, and for the @@ -62,6 +63,7 @@ import org.apache.comet.serde.{Compatible, Incompatible, QueryPlanSerde, SparkFi * through the dispatcher instead. Adding a native cast implementation therefore means moving a * pair out of the `Unsupported` assertions and into the parity matrix below. */ +@ExpressionsGroup class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeSuite.scala index 08e0a81ccae..fa62424afe1 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeSuite.scala @@ -30,6 +30,9 @@ import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.{LongType, StructField, StructType} import org.apache.spark.sql.vectorized.ColumnarBatch +import org.apache.comet.tags.ExecGroup + +@ExecGroup class CometNativeSuite extends CometTestBase { test("test handling NPE thrown by JVM") { val rdd = spark.range(0, 1).rdd.map { value => diff --git a/spark/src/test/scala/org/apache/comet/CometNativeUdfSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeUdfSuite.scala index 765476df231..344ed527a7a 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeUdfSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeUdfSuite.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.comet.CometProjectExec import org.apache.spark.sql.functions.{col, expr, struct} import org.apache.spark.sql.types._ +import org.apache.comet.tags.ExpressionsGroup import org.apache.comet.udf.{CometNativeUDF, CometNativeUdfAbiException, CometNativeUdfLoadException, CometNativeUdfNotAllowedException} /** @@ -46,6 +47,7 @@ import org.apache.comet.udf.{CometNativeUDF, CometNativeUdfAbiException, CometNa * ./mvnw test -Dsuites="org.apache.comet.CometNativeUdfSuite" -Dtest=none * }}} */ +@ExpressionsGroup class CometNativeUdfSuite extends CometTestBase { private lazy val libPath: String = { diff --git a/spark/src/test/scala/org/apache/comet/CometPublicApiSuite.scala b/spark/src/test/scala/org/apache/comet/CometPublicApiSuite.scala index f6e15cfa8e6..fbdac66ed4c 100644 --- a/spark/src/test/scala/org/apache/comet/CometPublicApiSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometPublicApiSuite.scala @@ -27,6 +27,7 @@ import scala.jdk.CollectionConverters._ import org.scalatest.funsuite.AnyFunSuite import org.apache.comet.annotation.Public +import org.apache.comet.tags.ExecGroup /** * Guards the enumerated public API described in `docs/source/about/versioning_policy.md`. @@ -36,6 +37,7 @@ import org.apache.comet.annotation.Public * accident, or the policy is genuinely changing and both the doc and this list need updating in * the same PR. */ +@ExecGroup class CometPublicApiSuite extends AnyFunSuite { private val expected: Set[String] = Set( diff --git a/spark/src/test/scala/org/apache/comet/CometRegExpJvmSuite.scala b/spark/src/test/scala/org/apache/comet/CometRegExpJvmSuite.scala index 46caadc31f8..8bd60e410c0 100644 --- a/spark/src/test/scala/org/apache/comet/CometRegExpJvmSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometRegExpJvmSuite.scala @@ -23,9 +23,11 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.functions.col import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus} +import org.apache.comet.tags.ExpressionsGroup // Regex expressions other than in-subset `rlike` run through the codegen dispatcher by default // (Spark's own code, enabled by default) rather than the native rust path. +@ExpressionsGroup class CometRegExpJvmSuite extends CometTestBase { // A Java-only pattern used to check the opt-in behavior when Rust rejects it. diff --git a/spark/src/test/scala/org/apache/comet/CometRegexParitySuite.scala b/spark/src/test/scala/org/apache/comet/CometRegexParitySuite.scala index 60e0369ee63..da51027d046 100644 --- a/spark/src/test/scala/org/apache/comet/CometRegexParitySuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometRegexParitySuite.scala @@ -26,12 +26,14 @@ import org.apache.spark.sql.types.{StringType, StructField, StructType} import org.apache.comet.expressions.{CometRegex, RegexFlavor} import org.apache.comet.serde.Compatible +import org.apache.comet.tags.ExpressionsGroup /** * Differential corpus: every pattern the plan-time analyzer admits is evaluated on Spark (Java * `Pattern`) and on Comet's native Rust path, and the results must match. Routing is asserted via * extended explain so a dispatcher fallback cannot silently satisfy the comparison. */ +@ExpressionsGroup class CometRegexParitySuite extends CometTestBase with AdaptiveSparkPlanHelper { // One pattern per whitelist production, plus the lexer-boundary cases. diff --git a/spark/src/test/scala/org/apache/comet/CometScalaUDFClassLoaderSuite.scala b/spark/src/test/scala/org/apache/comet/CometScalaUDFClassLoaderSuite.scala index 17fd0fea095..bfd8cfc5c02 100644 --- a/spark/src/test/scala/org/apache/comet/CometScalaUDFClassLoaderSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometScalaUDFClassLoaderSuite.scala @@ -29,6 +29,8 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.{CometTestBase, Encoders} import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.comet.tags.ExpressionsGroup + /** * Regression coverage for UDF closures whose capturing class lives in a user jar, which used to * fail with: @@ -50,6 +52,7 @@ import org.apache.spark.sql.catalyst.expressions.Expression * `MutableURLClassLoader`, which becomes the task thread's context ClassLoader while staying * invisible to the ClassLoader that loaded Comet. */ +@ExpressionsGroup class CometScalaUDFClassLoaderSuite extends CometTestBase { import CometScalaUDFClassLoaderSuite._ diff --git a/spark/src/test/scala/org/apache/comet/CometSetOpWithGroupBySuite.scala b/spark/src/test/scala/org/apache/comet/CometSetOpWithGroupBySuite.scala index c6595ba42e1..b82d8250c57 100644 --- a/spark/src/test/scala/org/apache/comet/CometSetOpWithGroupBySuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSetOpWithGroupBySuite.scala @@ -24,6 +24,8 @@ import org.apache.spark.sql.comet.CometUnionExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.internal.SQLConf +import org.apache.comet.tags.ExecGroup + /** * Regression test for issue #4122: on Spark 4.1 (SPARK-52921), EXCEPT ALL / INTERSECT ALL whose * sides are themselves GROUP BY aggregates are lowered to a plan where the union inherits a hash @@ -31,6 +33,7 @@ import org.apache.spark.sql.internal.SQLConf * If Comet's columnar Union concatenates partitions it breaks that partitioning invariant and the * resulting sums/counts collapse two sides into the wrong partitions. */ +@ExecGroup class CometSetOpWithGroupBySuite extends CometTestBase with AdaptiveSparkPlanHelper { test("issue #4122: EXCEPT ALL with GROUP BY under both sides") { diff --git a/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala b/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala index 92bac5dba86..9cf5f12fd81 100644 --- a/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala @@ -30,7 +30,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.LongType import org.apache.comet.serde.OperatorOuterClass +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometSparkSessionExtensionsSuite extends CometTestBase { import CometSparkSessionExtensions._ diff --git a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala index ba6e804d664..ca947954c55 100644 --- a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala @@ -31,6 +31,9 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.ArrayType +import org.apache.comet.tags.ExpressionsGroup + +@ExpressionsGroup class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { /** Check if the current Spark version meets a minimum version requirement. */ diff --git a/spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala index f921c7bda79..84105eb8cbf 100644 --- a/spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala @@ -28,9 +28,11 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, DataTypes, StructField, StructType} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.tags.ExpressionsGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} import org.apache.comet.udf.codegen.CometScalaUDFCodegen +@ExpressionsGroup class CometStringExpressionSuite extends CometTestBase with CometCodegenAssertions { // scalastyle:off private val edgeCases = Seq( diff --git a/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala index c9e97bbfddd..729a64cf266 100644 --- a/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala @@ -36,8 +36,10 @@ import org.apache.spark.sql.types.{DataTypes, StructField, StructType} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.serde.{CometDateFormat, CometTimeZone, CometTruncDate, CometTruncTimestamp} +import org.apache.comet.tags.ExpressionsGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} +@ExpressionsGroup class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { /** Timezones used to verify that TimestampNTZ operations are timezone-independent. */ diff --git a/spark/src/test/scala/org/apache/comet/CometUuidExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometUuidExpressionSuite.scala index 56029cba282..22fbd383c22 100644 --- a/spark/src/test/scala/org/apache/comet/CometUuidExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometUuidExpressionSuite.scala @@ -24,6 +24,8 @@ import org.apache.spark.sql.catalyst.expressions.Uuid import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions.col +import org.apache.comet.tags.ExpressionsGroup + /** * Bit-for-bit comparisons of seeded `uuid` against Spark. * @@ -38,6 +40,7 @@ import org.apache.spark.sql.functions.col * form is not, so building the expression directly gives all profiles a real cross-engine * assertion and removes that dependence. */ +@ExpressionsGroup class CometUuidExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("seeded uuid matches Spark bit for bit") { diff --git a/spark/src/test/scala/org/apache/comet/CometVariantProjectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometVariantProjectionSuite.scala index 8bb9f62a72c..58e4d8af55f 100644 --- a/spark/src/test/scala/org/apache/comet/CometVariantProjectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometVariantProjectionSuite.scala @@ -35,7 +35,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{IntegerType, StructField, StructType} import org.apache.comet.serde.operator.CometNativeScan +import org.apache.comet.tags.ScansGroup +@ScansGroup class CometVariantProjectionSuite extends CometTestBase { override protected def sparkConf: SparkConf = super.sparkConf .set(SQLConf.USE_V1_SOURCE_LIST.key, "parquet") diff --git a/spark/src/test/scala/org/apache/comet/CometVariantTypeSuite.scala b/spark/src/test/scala/org/apache/comet/CometVariantTypeSuite.scala index 92c2ec76f13..172e5346f60 100644 --- a/spark/src/test/scala/org/apache/comet/CometVariantTypeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometVariantTypeSuite.scala @@ -34,7 +34,9 @@ import org.apache.spark.sql.types.{ArrayType, BinaryType, StructField, StructTyp import org.apache.comet.rules.CometScanTypeChecker import org.apache.comet.serde.{CometAttributeReference, QueryPlanSerde, Unsupported} +import org.apache.comet.tags.ExpressionsGroup +@ExpressionsGroup class CometVariantTypeSuite extends AnyFunSuite { private val storageType = StructType( Seq( diff --git a/spark/src/test/scala/org/apache/comet/DataGeneratorSuite.scala b/spark/src/test/scala/org/apache/comet/DataGeneratorSuite.scala index d1a321099c6..0d629c4df34 100644 --- a/spark/src/test/scala/org/apache/comet/DataGeneratorSuite.scala +++ b/spark/src/test/scala/org/apache/comet/DataGeneratorSuite.scala @@ -24,8 +24,10 @@ import scala.util.Random import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, MapType, StructField, StructType} +import org.apache.comet.tags.ScansGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, SchemaGenOptions} +@ScansGroup class DataGeneratorSuite extends CometTestBase { test("allowNull produces nulls in every type that honours it") { diff --git a/spark/src/test/scala/org/apache/comet/DirectColumnarToRowConverterSuite.scala b/spark/src/test/scala/org/apache/comet/DirectColumnarToRowConverterSuite.scala index 887ef5cb646..059111fabcc 100644 --- a/spark/src/test/scala/org/apache/comet/DirectColumnarToRowConverterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/DirectColumnarToRowConverterSuite.scala @@ -35,12 +35,14 @@ import org.apache.spark.sql.types._ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.unsafe.types.UTF8String +import org.apache.comet.tags.ExecGroup import org.apache.comet.vector.{CometDictionary, CometDictionaryVector, CometPlainVector} /** * Verifies that [[DirectColumnarToRowConverter]] produces rows byte-identical to * `UnsafeProjection`, which matters because `UnsafeRow` equality and hashing are byte-wise. */ +@ExecGroup class DirectColumnarToRowConverterSuite extends AnyFunSuite { private def assertMatchesUnsafeProjection( diff --git a/spark/src/test/scala/org/apache/comet/GenerateDocsSuite.scala b/spark/src/test/scala/org/apache/comet/GenerateDocsSuite.scala index 3d00d764900..aa755990490 100644 --- a/spark/src/test/scala/org/apache/comet/GenerateDocsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/GenerateDocsSuite.scala @@ -22,7 +22,9 @@ package org.apache.comet import org.scalatest.funsuite.AnyFunSuite import org.apache.comet.serde.{CometMakeTimestamp, CometRLike, CometToUnixTimestamp, NativeOptInAvailable} +import org.apache.comet.tags.ExpressionsGroup +@ExpressionsGroup class GenerateDocsSuite extends AnyFunSuite { private val configKey = "spark.comet.expression.RLike.allowIncompatible" diff --git a/spark/src/test/scala/org/apache/comet/QueryContextInternerSuite.scala b/spark/src/test/scala/org/apache/comet/QueryContextInternerSuite.scala index fd33f249947..fb08537e445 100644 --- a/spark/src/test/scala/org/apache/comet/QueryContextInternerSuite.scala +++ b/spark/src/test/scala/org/apache/comet/QueryContextInternerSuite.scala @@ -25,7 +25,9 @@ import org.apache.spark.sql.{CometTestBase, DataFrame} import org.apache.spark.sql.comet.CometNativeExec import org.apache.comet.serde.{ExprOuterClass, OperatorOuterClass} +import org.apache.comet.tags.ExecGroup +@ExecGroup class QueryContextInternerSuite extends CometTestBase { private val sqlText = diff --git a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala index c36d5d69f33..0e5869f4820 100644 --- a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala @@ -24,7 +24,9 @@ import org.scalatest.funsuite.AnyFunSuite import org.apache.spark.{SparkException, SparkThrowable} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.tags.ExpressionsGroup +@ExpressionsGroup class SparkErrorConverterSuite extends AnyFunSuite { test("ArithmeticOverflow preserves the suggested function") { diff --git a/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala b/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala index 61d2877ee0c..a2822e7111c 100644 --- a/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala +++ b/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala @@ -25,11 +25,14 @@ import java.nio.file.Files import org.scalatest.funsuite.AnyFunSuite +import org.apache.comet.tags.ExpressionsGroup + /** * Unit tests for [[SqlFileTestParser]]. Pure text parsing, so no Spark session is needed. The * end-to-end behaviour of each query mode is covered by `CometSqlFileTestSuite` running the * fixtures themselves. */ +@ExpressionsGroup class SqlFileTestParserSuite extends AnyFunSuite { private def parseQueries(lines: String*): Seq[SqlQuery] = diff --git a/spark/src/test/scala/org/apache/comet/codegen/CometSpecializedGettersDispatchSuite.scala b/spark/src/test/scala/org/apache/comet/codegen/CometSpecializedGettersDispatchSuite.scala index 7ad93fd9fc1..538ac4541e3 100644 --- a/spark/src/test/scala/org/apache/comet/codegen/CometSpecializedGettersDispatchSuite.scala +++ b/spark/src/test/scala/org/apache/comet/codegen/CometSpecializedGettersDispatchSuite.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.comet.util.Utils import org.apache.spark.sql.types.{DataType, DateType, LongType} import org.apache.comet.CometSparkSessionExtensions.isSpark41Plus +import org.apache.comet.tags.ExpressionsGroup /** * Unit coverage for the generic `SpecializedGetters.get(ordinal, dataType)` dispatch the kernel @@ -35,6 +36,7 @@ import org.apache.comet.CometSparkSessionExtensions.isSpark41Plus * has to match what `CometBatchKernelCodegen.isSupportedDataType` admits at plan time. A type * accepted there but missing here throws at execute time, when there is no longer a fallback. */ +@ExpressionsGroup class CometSpecializedGettersDispatchSuite extends AnyFunSuite { /** Minimal `CometInternalRow` over one fixed long, to drive the dispatch through `get`. */ diff --git a/spark/src/test/scala/org/apache/comet/csv/CometCsvNativeReadSuite.scala b/spark/src/test/scala/org/apache/comet/csv/CometCsvNativeReadSuite.scala index eeb809764be..75e40bf78fe 100644 --- a/spark/src/test/scala/org/apache/comet/csv/CometCsvNativeReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/csv/CometCsvNativeReadSuite.scala @@ -29,7 +29,9 @@ import org.apache.spark.sql.types.{IntegerType, StringType, StructType, Timestam import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.CometSparkSessionExtensions.isSpark41Plus +import org.apache.comet.tags.ScansGroup +@ScansGroup class CometCsvNativeReadSuite extends CometTestBase { private val TEST_CSV_PATH_NO_HEADER = "src/test/resources/test-data/csv-test-1.csv" private val TEST_CSV_PATH_HAS_HEADER = "src/test/resources/test-data/csv-test-2.csv" diff --git a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala index de829dd6094..259cd237971 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala @@ -47,11 +47,13 @@ import org.apache.comet.CometConf.COMET_EXEC_STRICT_FLOATING_POINT import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, isSpark41Plus} import org.apache.comet.rules.CometExecRule import org.apache.comet.serde.{CometCorr, CometRegrR2, ExprOuterClass, RegrSparkVersions} +import org.apache.comet.tags.ExecGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, ParquetGenerator, SchemaGenOptions} /** * Test suite dedicated to Comet native aggregate operator */ +@ExecGroup class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala index 01a5e31daa5..3f6475c5656 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala @@ -40,6 +40,7 @@ import org.apache.spark.sql.types._ import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.tags.ShuffleGroup abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper { protected val adaptiveExecutionEnabled: Boolean @@ -935,6 +936,7 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar } } +@ShuffleGroup class CometShuffleSuite extends CometColumnarShuffleSuite { protected val adaptiveExecutionEnabled: Boolean = true @@ -986,6 +988,7 @@ class CometShuffleSuite extends CometColumnarShuffleSuite { } } +@ShuffleGroup class DisableAQECometShuffleSuite extends CometColumnarShuffleSuite { protected val adaptiveExecutionEnabled: Boolean = false @@ -1032,6 +1035,7 @@ class DisableAQECometShuffleSuite extends CometColumnarShuffleSuite { } } +@ShuffleGroup class CometShuffleEncryptionSuite extends CometTestBase { import testImplicits._ @@ -1067,6 +1071,7 @@ class CometShuffleEncryptionSuite extends CometTestBase { } } +@ShuffleGroup class CometShuffleManagerSuite extends CometTestBase { test("should not bypass merge sort if executor cores are too high") { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometDirectColumnarToRowSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometDirectColumnarToRowSuite.scala index 1c5ff110f94..2c0a151e9b8 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometDirectColumnarToRowSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometDirectColumnarToRowSuite.scala @@ -24,12 +24,14 @@ import org.apache.spark.sql.comet.CometColumnarToRowExec import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf +import org.apache.comet.tags.ShuffleGroup /** * End-to-end tests for `spark.comet.exec.columnarToRow.direct.enabled`, which routes * `CometColumnarToRowExec`'s non-codegen conversion through `DirectColumnarToRowConverter`. * Whole-stage codegen is disabled so the operator's `doExecute` conversion path runs. */ +@ShuffleGroup class CometDirectColumnarToRowSuite extends CometTestBase { private def withDirectConverter(minBatchSize: Int = 0)(f: => Unit): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala index 516ece40be0..7c200358049 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala @@ -35,10 +35,12 @@ import org.apache.spark.util.sketch.BloomFilter import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus +import org.apache.comet.tags.ExecGroup /** * This test suite contains tests for only Spark 3.4+. */ +@ExecGroup class CometExec3_4PlusSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index 1ba17ea2dae..17ab091a08c 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -56,8 +56,10 @@ import org.apache.comet.{CometConf, CometExecIterator, CometNativeException, Ext import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, isSpark41Plus, isSpark42Plus} import org.apache.comet.rules.CometCoalesceShufflePartitions import org.apache.comet.serde.Config.ConfigMap +import org.apache.comet.tags.ExecGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} +@ExecGroup class CometExecSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala index 1ccfacdffe5..f88f5b3865b 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala @@ -27,7 +27,9 @@ import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometGenerateExecSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheKryoSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheKryoSuite.scala index 4455fad88ef..ee4a6db92d3 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheKryoSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheKryoSuite.scala @@ -32,6 +32,7 @@ import org.apache.spark.unsafe.types.UTF8String import com.esotericsoftware.kryo.Kryo import org.apache.comet.{CometConf, CometKryoRegistrator} +import org.apache.comet.tags.ExecGroup /** * Covers Comet's cached batch format under `spark.kryo.registrationRequired=true`. @@ -47,6 +48,7 @@ import org.apache.comet.{CometConf, CometKryoRegistrator} * This needs its own suite because `spark.serializer` and `spark.kryo.registrator` are read when * `SparkEnv` builds the serializer, so they cannot be changed per test. */ +@ExecGroup class CometInMemoryCacheKryoSuite extends CometTestBase { override protected def beforeAll(): Unit = { @@ -270,6 +272,7 @@ abstract class CometInMemoryCacheKryoRegistrationSuite(expectedBatch: String) } /** Registers Spark's cached batch and nothing of Comet's, so the plugin keeps Spark's format. */ +@ExecGroup class CometInMemoryCacheKryoUnregisteredSuite extends CometInMemoryCacheKryoRegistrationSuite(classOf[DefaultCachedBatch].getName) { override protected def registrations: Seq[(String, String)] = @@ -281,6 +284,7 @@ class CometInMemoryCacheKryoUnregisteredSuite * [[CometKryoRegistrator]], and not Spark's cached batch, so before Spark 4.1 only Comet's format * can be stored, and the plugin has to install it. */ +@ExecGroup class CometInMemoryCacheKryoClassesToRegisterSuite extends CometInMemoryCacheKryoRegistrationSuite( ArrowCachedBatchSerializer.cachedBatchClass.getName) { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCachePruningSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCachePruningSuite.scala index 0f9464b89b7..2468f839dba 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCachePruningSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCachePruningSuite.scala @@ -31,7 +31,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.comet.CometConf +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometInMemoryCachePruningSuite extends CometTestBase { override protected def beforeAll(): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala index 59cede8d87e..c2729e805a8 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala @@ -58,10 +58,12 @@ import org.apache.spark.storage.StorageLevel import org.apache.comet.{CometArrowAllocator, CometConf, CometKryoRegistrator, ExtendedExplainInfo} import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus} import org.apache.comet.rules.{CometCacheBuildRule, CometCacheColumnarRule} +import org.apache.comet.tags.ExecGroup import org.apache.comet.vector.{CometPlainVector, CometVector} import net.jpountz.lz4.LZ4FrameInputStream +@ExecGroup class CometInMemoryCacheSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala index b76f52330ac..365e2b8c068 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala @@ -53,7 +53,9 @@ import org.apache.comet.{CometConf, CometExecIterator, CometExplainInfo, Extende import org.apache.comet.CometSparkSessionExtensions.{hasFallbackReason, isSpark35Plus} import org.apache.comet.rules.RewriteJoin import org.apache.comet.serde.OperatorOuterClass +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometJoinSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala index 2dc6fcfd325..771a2d87873 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeColumnarToRowSuite.scala @@ -33,6 +33,7 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, NativeColumnarToRowConverter} +import org.apache.comet.tags.ShuffleGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, SchemaGenOptions} /** @@ -41,6 +42,7 @@ import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, SchemaGenOpt * These tests verify that CometNativeColumnarToRowExec produces correct results for all supported * data types. */ +@ShuffleGroup class CometNativeColumnarToRowSuite extends CometTestBase with AdaptiveSparkPlanHelper { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala index 7e35d22002d..7b2561078d4 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala @@ -38,7 +38,9 @@ import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, CometNativeException, ExtendedExplainInfo} import org.apache.comet.CometSparkSessionExtensions.isSpark41Plus +import org.apache.comet.tags.ScansGroup +@ScansGroup class CometNativeReaderSuite extends CometTestBase with AdaptiveSparkPlanHelper { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala index bb06a09fbf3..e9c5a62ae31 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala @@ -59,7 +59,9 @@ import org.apache.comet.{CometConf, CometExecIterator, CometExplainInfo, CometSh import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, withFallbackReason} import org.apache.comet.serde.{OperatorOuterClass, PartitioningOuterClass} import org.apache.comet.shuffle.ShufflePartitionPusher +import org.apache.comet.tags.ShuffleGroup +@ShuffleGroup class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometRangeExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometRangeExecSuite.scala index 15d13ced8e4..5e57010805a 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometRangeExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometRangeExecSuite.scala @@ -29,7 +29,9 @@ import org.apache.spark.sql.functions.{col, count, sum} import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometRangeExecSuite extends CometTestBase { // CometTestBase enables the Spark-to-Arrow conversions, including the one for Range. Use their diff --git a/spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala index e626410aae9..2b8c1bb1469 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala @@ -33,11 +33,13 @@ import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, import org.apache.spark.unsafe.types.CalendarInterval import org.apache.comet.{CometConf, ExtendedExplainInfo} +import org.apache.comet.tags.ShuffleGroup // Top-level, so the encoder needs no outer pointer. case class ShuffleInputRec(a: Int, b: String) /** Tests for [[CometConf.COMET_CONVERT_FROM_SHUFFLE_INPUT_ENABLED]]. */ +@ShuffleGroup class CometShuffleInputConversionSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala index 4bd78cf7aee..a92bb2efcc1 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala @@ -35,7 +35,9 @@ import org.apache.spark.sql.types.{ByteType, IntegerType, LongType, ShortType} import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometTopKSuite extends CometTestBase { private def localTopK(plan: SparkPlan): CometLocalTopKExec = { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometTypedDatasetSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometTypedDatasetSuite.scala index f2222b37f5d..5698038bf81 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometTypedDatasetSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometTypedDatasetSuite.scala @@ -30,6 +30,7 @@ import org.apache.spark.sql.functions.{broadcast, col, size, sum} import org.apache.spark.sql.internal.SQLConf import org.apache.comet.{CometConf, ExtendedExplainInfo} +import org.apache.comet.tags.ExecGroup // Top-level, so the encoders need no outer pointer, which is the ordinary user shape. case class TypedDsRec(a: Int, b: String) @@ -50,6 +51,7 @@ object TypedDsCounter { } /** Tests for [[CometConf.COMET_CONVERT_FROM_TYPED_DATASET_ENABLED]]. */ +@ExecGroup class CometTypedDatasetSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala index 642a73e337a..72d75801a3b 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala @@ -37,7 +37,9 @@ import org.apache.spark.sql.types.DecimalType import org.apache.comet.{CometConf, CometExplainInfo, ExtendedExplainInfo} import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus} +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometWindowExecSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/expressions/CometRegexSuite.scala b/spark/src/test/scala/org/apache/comet/expressions/CometRegexSuite.scala index 4ad8eb63b18..fd452001bd4 100644 --- a/spark/src/test/scala/org/apache/comet/expressions/CometRegexSuite.scala +++ b/spark/src/test/scala/org/apache/comet/expressions/CometRegexSuite.scala @@ -22,7 +22,9 @@ package org.apache.comet.expressions import org.scalatest.funsuite.AnyFunSuite import org.apache.comet.serde.{Compatible, Incompatible} +import org.apache.comet.tags.ExpressionsGroup +@ExpressionsGroup class CometRegexSuite extends AnyFunSuite { private def assertCompatible(pattern: String): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCaseWhenSuite.scala b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCaseWhenSuite.scala index 8e825462c4e..9f7d26d6136 100644 --- a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCaseWhenSuite.scala +++ b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCaseWhenSuite.scala @@ -22,6 +22,9 @@ package org.apache.comet.expressions.conditional import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.comet.tags.ExpressionsGroup + +@ExpressionsGroup class CometCaseWhenSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("case_when") { diff --git a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCoalesceSuite.scala b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCoalesceSuite.scala index b4e4ceb6d9c..4d856afbef5 100644 --- a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCoalesceSuite.scala +++ b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCoalesceSuite.scala @@ -24,6 +24,9 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.internal.SQLConf +import org.apache.comet.tags.ExpressionsGroup + +@ExpressionsGroup class CometCoalesceSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("coalesce should return correct datatype") { diff --git a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometIfSuite.scala b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometIfSuite.scala index 788667aad2e..070f6183fae 100644 --- a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometIfSuite.scala +++ b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometIfSuite.scala @@ -22,6 +22,9 @@ package org.apache.comet.expressions.conditional import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.comet.tags.ExpressionsGroup + +@ExpressionsGroup class CometIfSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("if expression") { diff --git a/spark/src/test/scala/org/apache/comet/iceberg/IcebergReflectionSuite.scala b/spark/src/test/scala/org/apache/comet/iceberg/IcebergReflectionSuite.scala index 02ffb737a7d..b9aef7c3271 100644 --- a/spark/src/test/scala/org/apache/comet/iceberg/IcebergReflectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/iceberg/IcebergReflectionSuite.scala @@ -34,6 +34,9 @@ import org.apache.iceberg.TableMetadata import org.apache.iceberg.io.FileIO import org.apache.iceberg.types.Types +import org.apache.comet.tags.ScansGroup + +@ScansGroup class IcebergReflectionSuite extends AnyFunSuite { /** Mimics HiveTableOperations/GlueTableOperations which inherit current(). */ diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala index eeca4d0fb99..8d8fbe0267e 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala @@ -44,6 +44,7 @@ import org.apache.comet.objectstore.AbfsAuthParityCases._ import org.apache.comet.objectstore.AbfsAuthResolver._ import org.apache.comet.objectstore.AbfsReflection.Handles import org.apache.comet.tags.AzureTest +import org.apache.comet.tags.ExecGroup /** * Pins the resolver's behaviour per hadoop-azure version and cross-checks it against the real @@ -68,6 +69,7 @@ import org.apache.comet.tags.AzureTest * assert while recording, so a recorded file never contradicts the hadoop-azure source. The file * is written only when every case ran, so a filtered run cannot leave a partial file behind. */ +@ExecGroup class AbfsAuthParitySuite extends AnyFunSuite with Matchers with BeforeAndAfterAll { import AbfsAuthParitySuite._ diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala index 4d69c2e01a3..5216bb2cc00 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala @@ -35,12 +35,14 @@ import org.apache.hadoop.fs.azurebfs.services.KeyProvider import org.apache.comet.objectstore.AbfsAuthResolver._ import org.apache.comet.objectstore.AbfsReflection.Handles +import org.apache.comet.tags.ExecGroup /** * Runs the resolver against the hadoop-azure on the test classpath (3.3.4 on the spark-3.x * profiles, 3.4.1, 3.4.2 and 3.5.0 on spark-4.0, 4.1 and 4.2). Cases that depend on a feature one * line lacks are gated on the capability probes rather than on a version number. */ +@ExecGroup class AbfsAuthResolverSuite extends AnyFunSuite with Matchers { private val account = "myacct.dfs.core.windows.net" diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala index 5386697de1b..23274f6d4b6 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala @@ -31,12 +31,14 @@ import org.apache.spark.sql.types.{IntegerType, StructType} import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.hadoop.fs.FakeAbfsFileSystem +import org.apache.comet.tags.ExecGroup /** * Plans scans over `abfss://` paths served by [[FakeAbfsFileSystem]], with authentication * resolved offline by the hadoop-azure on the test classpath. A scan Comet declines runs on * Spark's own scan against the local files. */ +@ExecGroup class AbfsScanFallbackSuite extends CometTestBase { private val multipleAuthoritiesReason = "more than one container or account" diff --git a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala index 1246fbac154..bec786d5a93 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala @@ -28,7 +28,9 @@ import org.apache.hadoop.conf.Configuration import org.apache.comet.CometConf.{COMET_LIBHDFS_SCHEMES_KEY, COMET_S3_COMPLIANT_SCHEMES_KEY} import org.apache.comet.objectstore.AbfsAuthResolver.{Keys, Markers, Providers} +import org.apache.comet.tags.ExecGroup +@ExecGroup class NativeConfigSuite extends AnyFunSuite with Matchers { /** diff --git a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterRetrySuite.scala b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterRetrySuite.scala index 40cd10e6ebd..1e98e720524 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterRetrySuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterRetrySuite.scala @@ -26,12 +26,14 @@ import scala.jdk.CollectionConverters._ import org.apache.spark.SparkConf import org.apache.comet.hadoop.fs.RecordingFileSystem +import org.apache.comet.tags.ScansGroup /** * Native Parquet writes whose tasks Spark retries. A suite of its own because a task gets more * than one attempt only under a `local[N,M]` master, and the failure tests elsewhere expect a * failed task to fail its job. */ +@ScansGroup class CometParquetWriterRetrySuite extends CometParquetWriterTestBase { override protected def sparkConf: SparkConf = diff --git a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala index 29a19307768..b31216c441d 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala @@ -58,8 +58,10 @@ import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plu import org.apache.comet.hadoop.fs.RecordingFileSystem import org.apache.comet.rules.RevertNativeForTransitionHeavyStages import org.apache.comet.serde.operator.NativeWriteUtils +import org.apache.comet.tags.ScansGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, SchemaGenOptions} +@ScansGroup class CometParquetWriterSuite extends CometParquetWriterTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index 56743367a4a..b3446ecf0c3 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -54,6 +54,7 @@ import com.google.common.primitives.UnsignedLong import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, isSpark41Plus} +import org.apache.comet.tags.ScansGroup import org.apache.comet.vector.CometVector abstract class ParquetReadSuite extends CometTestBase { @@ -3111,6 +3112,7 @@ abstract class ParquetReadSuite extends CometTestBase { } } +@ScansGroup class ParquetReadV1Suite extends ParquetReadSuite with AdaptiveSparkPlanHelper { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetTimestampLtzAsNtzSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetTimestampLtzAsNtzSuite.scala index 4305ff5d872..3d8f1962b00 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetTimestampLtzAsNtzSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetTimestampLtzAsNtzSuite.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.tags.ScansGroup /** * Tests for reading Parquet TimestampLTZ columns as TimestampNTZ. @@ -37,6 +38,7 @@ import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus * * See https://github.com/apache/datafusion-comet/issues/4219 */ +@ScansGroup class ParquetTimestampLtzAsNtzSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala index ffbe1d0e1b0..456c74593b5 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala @@ -54,6 +54,7 @@ import org.apache.comet.{CometConf, CometCoverageStats, CometExplainInfo, CometS import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, isSpark42Plus, withFallbackReason} import org.apache.comet.serde.{CometAggregateExpressionSerde, Compatible, ExprOuterClass, OperatorOuterClass, QueryPlanSerde, Unsupported} import org.apache.comet.shims.ShimCometWindowGroupLimit +import org.apache.comet.tags.ExecGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} /** @@ -61,6 +62,7 @@ import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} * transform Spark operators to Comet operators, fallback mechanisms, configuration handling, and * edge cases. */ +@ExecGroup class CometExecRuleSuite extends CometTestBase { // The observers are active only during the DPP lifecycle regression below. AQE can prepare diff --git a/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala index 3db9f2fced3..ebb8f165177 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala @@ -44,12 +44,14 @@ import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.CometConf.{COMET_LIBHDFS_SCHEMES_KEY, COMET_S3_COMPLIANT_SCHEMES_KEY} import org.apache.comet.hadoop.fs.{FakeAbfsFileSystem, FakeHdfsAuthorityFileSystem} import org.apache.comet.objectstore.NativeConfig +import org.apache.comet.tags.ExecGroup /** * Native scans over files in more than one object store, without a cloud store: `hdfs://nn1` and * `hdfs://nn2` are two native stores backed by the local disk. Native execution cannot read them, * so claimed scans are checked on their plans, and declined scans are run by Spark. */ +@ExecGroup class CometMultiStoreScanSuite extends CometTestBase with AdaptiveSparkPlanHelper { private var rootDir: File = _ diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala index 4e41a9b147a..15e08dd7952 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala @@ -44,6 +44,7 @@ import org.apache.spark.sql.execution.datasources.HadoopFsRelation import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.comet.ContribServices +import org.apache.comet.tags.ExecGroup import org.apache.comet.util.ClassLoaders /** @@ -62,6 +63,7 @@ import org.apache.comet.util.ClassLoaders * Sibling to `CometScanWithPlanDataSuite`, which covers the other half of the contrib SPI (plan * data). Everything here runs on the default build. */ +@ExecGroup class CometScanContribSuite extends AnyFunSuite { /** diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala index 52c6159d11e..f0c43dc21e7 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala @@ -30,11 +30,13 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, DataTypeSupport} +import org.apache.comet.tags.ExecGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} /** * Test suite specifically for CometScanRule transformation logic. */ +@ExecGroup class CometScanRuleSuite extends CometTestBase { /** Helper method to apply CometExecRule and return the transformed plan */ diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala index 8cb1765fa39..345779a4d3c 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala @@ -41,6 +41,7 @@ import org.apache.comet.{CometConf, CometIcebergTestBase, ExtendedExplainInfo, N import org.apache.comet.hadoop.fs.{FakeHdfsAuthorityFileSystem, FakeHDFSFileSystem, FakeHdfsSchemeFileSystem, FakeWasbSchemeFileSystem} import org.apache.comet.iceberg.IcebergStorageSchemes import org.apache.comet.objectstore.NativeConfig +import org.apache.comet.tags.ExecGroup /** * Comet's native readers go through object_store, which understands a fixed set of URL schemes. A @@ -52,6 +53,7 @@ import org.apache.comet.objectstore.NativeConfig * native probe is cached per probe URL so an authorityless URL can't poison the authority-bearing * form of the same scheme. */ +@ExecGroup class CometScanSchemeFallbackSuite extends CometTestBase with CometIcebergTestBase { private var fakeRootDir: File = _ diff --git a/spark/src/test/scala/org/apache/comet/rules/EliminateRedundantTransitionsSuite.scala b/spark/src/test/scala/org/apache/comet/rules/EliminateRedundantTransitionsSuite.scala index 33293c3d342..3e91b4f8ca1 100644 --- a/spark/src/test/scala/org/apache/comet/rules/EliminateRedundantTransitionsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/EliminateRedundantTransitionsSuite.scala @@ -24,7 +24,9 @@ import org.apache.spark.sql.comet.{CometColumnarToRowExec, CometNativeColumnarTo import org.apache.spark.sql.execution.{ColumnarToRowExec, SparkPlan, UnionExec} import org.apache.comet.CometConf +import org.apache.comet.tags.ExecGroup +@ExecGroup class EliminateRedundantTransitionsSuite extends CometTestBase { private def sparkPlanFor(sql: String): SparkPlan = { diff --git a/spark/src/test/scala/org/apache/comet/rules/RevertNativeForTransitionHeavyStagesSuite.scala b/spark/src/test/scala/org/apache/comet/rules/RevertNativeForTransitionHeavyStagesSuite.scala index 6eb7c65a43c..0b811200307 100644 --- a/spark/src/test/scala/org/apache/comet/rules/RevertNativeForTransitionHeavyStagesSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/RevertNativeForTransitionHeavyStagesSuite.scala @@ -43,6 +43,7 @@ import org.apache.spark.sql.util.QueryExecutionListener import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus} import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.tags.ExecGroup private case class AliasingFallbackCometExec( override val originalPlan: SparkPlan, @@ -67,6 +68,7 @@ private case class BothFormatsExec(child: SparkPlan) extends UnaryExecNode { copy(child = newChild) } +@ExecGroup class RevertNativeForTransitionHeavyStagesSuite extends CometTestBase { // Spark 4.0+ builds a cache through columnar output only when the cache serializer accepts diff --git a/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala b/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala index 44c8691ed47..0a52f560de3 100644 --- a/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala @@ -32,6 +32,7 @@ import org.apache.spark.sql.types._ import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus} import org.apache.comet.serde.literals.CometLiteral import org.apache.comet.shims.CometTypeShim +import org.apache.comet.tags.ExpressionsGroup /** * Pins `CometLiteral.listLiteralElementSupported` against the arms of @@ -40,6 +41,7 @@ import org.apache.comet.shims.CometTypeShim * raises a `MatchError` in the middle of planning, which Spark surfaces as `[INTERNAL_ERROR] The * Spark SQL phase planning failed with an internal error`. */ +@ExpressionsGroup class CometLiteralSuite extends CometTestBase with CometTypeShim { test("a null literal with duplicate struct field names is unsupported") { diff --git a/spark/src/test/scala/org/apache/comet/serde/CometScalarFunctionSuite.scala b/spark/src/test/scala/org/apache/comet/serde/CometScalarFunctionSuite.scala index 8cf88ec5593..ff83eec7b26 100644 --- a/spark/src/test/scala/org/apache/comet/serde/CometScalarFunctionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/CometScalarFunctionSuite.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.expressions.{Abs, Cos, Expression, Literal, import org.apache.spark.sql.types.{DataType, DoubleType, IntegerType} import org.apache.comet.{CometExplainInfo, CometSparkSessionExtensions} +import org.apache.comet.tags.ExpressionsGroup /** * Synthetic expression whose constructor declares `evalMode`, used to prove class-level detection @@ -85,6 +86,7 @@ case class TestEvalContextExpression(child: Expression, evalContext: Any) copy(child = newChildren.head) } +@ExpressionsGroup class CometScalarFunctionSuite extends CometTestBase { private def fallbackReasons(expr: Expression): Set[String] = { diff --git a/spark/src/test/scala/org/apache/comet/serde/QueryPlanSerdeSuite.scala b/spark/src/test/scala/org/apache/comet/serde/QueryPlanSerdeSuite.scala index d7e595d4a83..6c0ba08274e 100644 --- a/spark/src/test/scala/org/apache/comet/serde/QueryPlanSerdeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/QueryPlanSerdeSuite.scala @@ -33,7 +33,9 @@ import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus} import org.apache.comet.serde.QueryPlanSerde.{canShortCircuitNulls, canSkipEvaluation, supportedDataType, withSharedSubexpressions} +import org.apache.comet.tags.ExpressionsGroup +@ExpressionsGroup class QueryPlanSerdeSuite extends AnyFunSuite { test("make_date IN evaluates the probe but can skip list expressions") { diff --git a/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala b/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala index 300d19081f1..cc444c16e45 100644 --- a/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala @@ -21,6 +21,9 @@ package org.apache.comet.serde import org.scalatest.funsuite.AnyFunSuite +import org.apache.comet.tags.ExpressionsGroup + +@ExpressionsGroup class SerdeRegistrationSuite extends AnyFunSuite { // A version shim only adds serdes for classes the shared map cannot name on every Spark diff --git a/spark/src/test/scala/org/apache/comet/serde/SparkFixesSuite.scala b/spark/src/test/scala/org/apache/comet/serde/SparkFixesSuite.scala index 646b6c3f06e..c2908ad9339 100644 --- a/spark/src/test/scala/org/apache/comet/serde/SparkFixesSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/SparkFixesSuite.scala @@ -21,6 +21,9 @@ package org.apache.comet.serde import org.scalatest.funsuite.AnyFunSuite +import org.apache.comet.tags.ExpressionsGroup + +@ExpressionsGroup class SparkFixesSuite extends AnyFunSuite { test("fixes by Spark version") { diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala index 6ff5daf27fe..ec1ac28ab37 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala @@ -30,8 +30,10 @@ import org.apache.spark.sql.catalyst.expressions.AttributeReference import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, StructType} import org.apache.comet.iceberg.IcebergReflection +import org.apache.comet.tags.ExecGroup /** Unit tests for Iceberg native-scan serde helpers that do not require a Spark session. */ +@ExecGroup class CometIcebergNativeScanSuite extends AnyFunSuite with Matchers { test("complex type null residuals are not serialized") { diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala index a2bcde5d3c8..b7ca53a5278 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala @@ -30,7 +30,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{IntegerType, LongType, MetadataBuilder, StringType, StructField, StructType} import org.apache.comet.parquet.CometParquetUtils +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometNativeScanSuite extends AnyFunSuite { import CometNativeScan.{constantMetadataFieldPrefix, uniqueConstantMetadataFields} diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala index f1dd09ccf7e..9654f52c0a0 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala @@ -22,11 +22,13 @@ package org.apache.comet.serde.operator import org.scalatest.funsuite.AnyFunSuite import org.apache.comet.serde.OperatorOuterClass.{CompressionCodec, IcebergWriterMode} +import org.apache.comet.tags.ScansGroup /** * Unit tests for the protobuf translation. Iceberg must be on the classpath because the property * keys / numeric defaults are looked up reflectively from `org.apache.iceberg.TableProperties`. */ +@ScansGroup class IcebergWriteProtoTranslationSuite extends AnyFunSuite { import IcebergWriteProtoTranslation._ diff --git a/spark/src/test/scala/org/apache/comet/shuffle/CelebornShufflePartitionPusherSuite.scala b/spark/src/test/scala/org/apache/comet/shuffle/CelebornShufflePartitionPusherSuite.scala index d2da70a4e3b..0bf5a21ba6e 100644 --- a/spark/src/test/scala/org/apache/comet/shuffle/CelebornShufflePartitionPusherSuite.scala +++ b/spark/src/test/scala/org/apache/comet/shuffle/CelebornShufflePartitionPusherSuite.scala @@ -32,6 +32,9 @@ import org.scalatest.funsuite.AnyFunSuite import org.apache.spark.SparkConf +import org.apache.comet.tags.ShuffleGroup + +@ShuffleGroup class CelebornShufflePartitionPusherSuite extends AnyFunSuite { private val encodedAttemptId = (4 << 16) | 7 diff --git a/spark/src/test/scala/org/apache/comet/udf/CometUdfBridgeSuite.scala b/spark/src/test/scala/org/apache/comet/udf/CometUdfBridgeSuite.scala index 71ec1f856e5..3412a2ada1e 100644 --- a/spark/src/test/scala/org/apache/comet/udf/CometUdfBridgeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/udf/CometUdfBridgeSuite.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.types.IntegerType import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.comet.{CometArrowAllocator, CometArrowImportAllocator} +import org.apache.comet.tags.ExecGroup import org.apache.comet.vector.NativeUtil /** @@ -81,6 +82,7 @@ class IdentityUdf extends CometUDF { override def evaluate(inputs: Array[ValueVector], numRows: Int): ValueVector = inputs.head } +@ExecGroup class CometUdfBridgeSuite extends AnyFunSuite { /** Exports a single-column batch and invokes the bridge against it. */ diff --git a/spark/src/test/scala/org/apache/comet/vector/CometVectorUtilsSuite.scala b/spark/src/test/scala/org/apache/comet/vector/CometVectorUtilsSuite.scala index 8e61b3e0bcd..25388126bf5 100644 --- a/spark/src/test/scala/org/apache/comet/vector/CometVectorUtilsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/vector/CometVectorUtilsSuite.scala @@ -33,7 +33,10 @@ import org.apache.arrow.vector.dictionary.Dictionary import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider import org.apache.arrow.vector.types.pojo.{ArrowType, DictionaryEncoding, FieldType} +import org.apache.comet.tags.ExecGroup + /** Check failure cleanup without transferring ownership of the helper's source vectors. */ +@ExecGroup class CometVectorUtilsSuite extends AnyFunSuite with Matchers { for (missingDictionary <- Seq(true, false)) { diff --git a/spark/src/test/scala/org/apache/comet/vector/NativeUtilSuite.scala b/spark/src/test/scala/org/apache/comet/vector/NativeUtilSuite.scala index 50eb2959cf1..7643a9bab2c 100644 --- a/spark/src/test/scala/org/apache/comet/vector/NativeUtilSuite.scala +++ b/spark/src/test/scala/org/apache/comet/vector/NativeUtilSuite.scala @@ -42,7 +42,9 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.comet.{CometArrowAllocator, CometArrowImportAllocator, CometConf, CometRuntimeException} import org.apache.comet.serde.{OperatorOuterClass, QueryPlanSerde} +import org.apache.comet.tags.ExecGroup +@ExecGroup class NativeUtilSuite extends CometTestBase { private def withIsolatedStructAllocator( diff --git a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala index 1caaec887fa..f9fd49936f4 100644 --- a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala @@ -47,6 +47,7 @@ import org.apache.spark.util.JsonProtocol import org.apache.comet.{CometArrowAllocator, CometConf, CometExecIterator, CometExecutorMemoryUsage, CometNativeException, CometShuffleBlockIterator, Native} import org.apache.comet.serde.Config.ConfigMap import org.apache.comet.serde.OperatorOuterClass +import org.apache.comet.tags.ExecGroup import org.apache.comet.vector.CometPlainVector /** @@ -54,6 +55,7 @@ import org.apache.comet.vector.CometPlainVector * one release of the native execution context and of its task-shared memory pool reference, even * when a step of the lifecycle fails partway through. See issue #5212 (positions 2, 3 and 8). */ +@ExecGroup class CometExecIteratorLifecycleSuite extends CometTestBase { private def withTaskContext[T](taskAttemptId: Long)(f: => T): T = { diff --git a/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala b/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala index f85045a57ec..82c1eddda70 100644 --- a/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala @@ -34,7 +34,9 @@ import org.apache.spark.sql.internal.StaticSQLConf import org.apache.spark.util.{JsonProtocol, ManualClock, Utils} import org.apache.comet.{COMET_VERSION, CometConf, CometExecIterator, CometExecutorMemoryUsage} +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometPluginsSuite extends CometTestBase { override protected def sparkConf: SparkConf = { val conf = new SparkConf() @@ -175,6 +177,7 @@ class CometPluginsSuite extends CometTestBase { } } +@ExecGroup class CometPluginsDefaultSuite extends CometTestBase { override protected def sparkConf: SparkConf = { val conf = new SparkConf() @@ -193,6 +196,7 @@ class CometPluginsDefaultSuite extends CometTestBase { } } +@ExecGroup class CometPluginsMemoryOverheadWarningSuite extends CometTestBase { private val warning = @@ -276,6 +280,7 @@ class CometPluginsMemoryOverheadWarningSuite extends CometTestBase { } } +@ExecGroup class CometPluginsMemoryPoolFractionWarningSuite extends CometTestBase { private val warning = "spark.comet.exec.memoryPool.fraction=0.8 is deprecated" @@ -300,6 +305,7 @@ class CometPluginsMemoryPoolFractionWarningSuite extends CometTestBase { } } +@ExecGroup class CometPluginsUnifiedModeSuite extends CometTestBase { override protected def sparkConf: SparkConf = { val conf = new SparkConf() @@ -328,6 +334,7 @@ class CometPluginsUnifiedModeSuite extends CometTestBase { } } +@ExecGroup class CometPluginsExtensionOnlySuite extends CometTestBase { // No plugin and no off-heap memory. CometTestBase registers CometSparkSessionExtensions // directly, as an application can with spark.sql.extensions. @@ -389,6 +396,7 @@ class CometPluginsExtensionOnlySuite extends CometTestBase { } } +@ExecGroup class CometPluginsSparkShuffleManagerSuite extends CometTestBase { // The application runs Spark's own shuffle manager. override protected def sparkConf: SparkConf = { @@ -430,6 +438,7 @@ class CometPluginsSparkShuffleManagerSuite extends CometTestBase { } } +@ExecGroup class CometPluginsEventLogSuite extends SparkFunSuite { import CometExecIterator.{memoryUsageEvent, JvmArrowMemory} diff --git a/spark/src/test/scala/org/apache/spark/CometRuntimeShutdownSuite.scala b/spark/src/test/scala/org/apache/spark/CometRuntimeShutdownSuite.scala index 69425c7eae4..dc9aa8cebc8 100644 --- a/spark/src/test/scala/org/apache/spark/CometRuntimeShutdownSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometRuntimeShutdownSuite.scala @@ -33,6 +33,8 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.functions.{expr, sum} import org.apache.spark.util.Utils +import org.apache.comet.tags.ExecGroup + /** * Entry point run in a child JVM by [[CometRuntimeShutdownSuite]]. Executes a native Comet query * and then returns from `main` without calling `SparkContext.stop()`. @@ -78,6 +80,7 @@ object CometRuntimeShutdownMain { * Runs Comet in a child JVM because the failure mode under test, the JVM being unable to exit, * cannot be observed from inside the affected JVM. */ +@ExecGroup class CometRuntimeShutdownSuite extends AnyFunSuite { test("JVM exits when main returns without SparkContext.stop()") { diff --git a/spark/src/test/scala/org/apache/spark/CometTaskMemoryManagerSuite.scala b/spark/src/test/scala/org/apache/spark/CometTaskMemoryManagerSuite.scala index 82c5731da7a..af3316b4e13 100644 --- a/spark/src/test/scala/org/apache/spark/CometTaskMemoryManagerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometTaskMemoryManagerSuite.scala @@ -26,6 +26,9 @@ import org.apache.logging.log4j.Level import org.apache.logging.log4j.core.LogEvent import org.apache.spark.memory.{MemoryConsumer, TaskMemoryManager, TestMemoryManager} +import org.apache.comet.tags.ExecGroup + +@ExecGroup class CometTaskMemoryManagerSuite extends TaskMemoryTestUtils { test("native memory usage is visible to Spark's memory consumer") { diff --git a/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnboundedShuffleMemoryAllocatorSuite.scala b/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnboundedShuffleMemoryAllocatorSuite.scala index 7b2be4fbdff..902e063f0a5 100644 --- a/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnboundedShuffleMemoryAllocatorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnboundedShuffleMemoryAllocatorSuite.scala @@ -27,6 +27,9 @@ import org.apache.spark.SparkConf import org.apache.spark.memory.{SparkOutOfMemoryError, TaskMemoryManager, TestMemoryManager} import org.apache.spark.unsafe.memory.MemoryBlock +import org.apache.comet.tags.ShuffleGroup + +@ShuffleGroup class CometUnboundedShuffleMemoryAllocatorSuite extends AnyFunSuite { private val pageSize = 4096L diff --git a/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnifiedShuffleMemoryAllocatorSuite.scala b/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnifiedShuffleMemoryAllocatorSuite.scala index ecf2e47be21..385ccc4b885 100644 --- a/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnifiedShuffleMemoryAllocatorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnifiedShuffleMemoryAllocatorSuite.scala @@ -24,6 +24,9 @@ import org.apache.logging.log4j.core.LogEvent import org.apache.spark.{CometTaskMemoryManager, TaskContext, TaskContextImpl, TaskMemoryTestUtils} import org.apache.spark.memory.SparkOutOfMemoryError +import org.apache.comet.tags.ShuffleGroup + +@ShuffleGroup class CometUnifiedShuffleMemoryAllocatorSuite extends TaskMemoryTestUtils { private val PageSize = 20L diff --git a/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala b/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala index bf691530ccd..ee9a79f23af 100644 --- a/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala @@ -44,6 +44,7 @@ import org.apache.spark.unsafe.Platform import org.apache.spark.unsafe.UnsafeAlignedOffset import org.apache.comet.Native +import org.apache.comet.tags.ShuffleGroup import org.apache.comet.vector.NativeUtil /** @@ -52,6 +53,7 @@ import org.apache.comet.vector.NativeUtil * These tests verify SpillSorter behavior using Spark's test memory management infrastructure, * without needing a full SparkContext. */ +@ShuffleGroup class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { private val INITIAL_SIZE = 1024 diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala index bf80f12e9bb..27442e08c84 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala @@ -24,7 +24,9 @@ import org.apache.spark.internal.config.{MEMORY_OFFHEAP_ENABLED, MEMORY_OFFHEAP_ import org.apache.spark.sql.comet.shims.ShimCometTPCDSQuerySuite import org.apache.comet.CometConf +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometTPCDSQuerySuite extends { val tpcdsAllQueries: Seq[String] = Seq( diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala index 0e2fd64683e..f544a5dd2e1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala @@ -31,11 +31,13 @@ import org.apache.spark.sql.test.TestSparkSession import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.tags.ExecGroup /** * Because we need to modify some methods of Spark `TPCDSQueryTestSuite` but they are private, we * copy Spark `TPCDSQueryTestSuite`. */ +@ExecGroup class CometTPCDSQueryTestSuite extends QueryTest with TPCDSBase with CometSQLQueryTestHelper { val tpcdsExtendedQueries: Seq[String] = Seq("q72") diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala index 0f942927091..d295f7dc7a9 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala @@ -33,6 +33,7 @@ import org.apache.spark.sql.test.TestSparkSession import org.apache.comet.CometConf import org.apache.comet.shims.ShimCometTPCHQuerySuite +import org.apache.comet.tags.ExecGroup /** * End-to-end tests to check TPCH query results. @@ -49,6 +50,7 @@ import org.apache.comet.shims.ShimCometTPCHQuerySuite * ./mvnw -Dsuites=org.apache.spark.sql.CometTPCHQuerySuite test * }}} */ +@ExecGroup class CometTPCHQuerySuite extends QueryTest with TPCBase with ShimCometTPCHQuerySuite { private val tpchDataPath = sys.env.get("SPARK_TPCH_DATA") diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala index 15b7c706490..758e058777d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala @@ -42,8 +42,10 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.util.io.ChunkedByteBuffer import org.apache.comet.{CometArrowAllocator, CometKryoRegistrator} +import org.apache.comet.tags.ExecGroup import org.apache.comet.vector.CometVector +@ExecGroup class CometBroadcastValueSuite extends CometTestBase { private val Timeout = 30.seconds diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometCollectMetricsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometCollectMetricsSuite.scala index ad5aef6241d..695013e447e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometCollectMetricsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometCollectMetricsSuite.scala @@ -37,7 +37,9 @@ import org.apache.spark.sql.util.QueryExecutionListener import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark35Plus import org.apache.comet.rules.CometExecRule +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometCollectMetricsSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometDecimalPromotionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometDecimalPromotionSuite.scala index 68f88e18255..a3977ccb1b6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometDecimalPromotionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometDecimalPromotionSuite.scala @@ -27,7 +27,9 @@ import org.apache.spark.sql.types.{DecimalType, IntegerType} import org.apache.comet.{CometConf, CometExplainInfo} import org.apache.comet.serde.{CometDivide, ExprOuterClass, QueryPlanSerde, Unsupported} +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometDecimalPromotionSuite extends CometTestBase { private case class Operation( diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometDppFallbackRepro3949Suite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometDppFallbackRepro3949Suite.scala index f672ebc082f..6d0bbc6fab0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometDppFallbackRepro3949Suite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometDppFallbackRepro3949Suite.scala @@ -32,6 +32,7 @@ import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec import org.apache.spark.sql.internal.SQLConf import org.apache.comet.{CometConf, CometExplainInfo} +import org.apache.comet.tags.ExecGroup /** * Attempts an end-to-end reproduction of issue #3949. @@ -61,6 +62,7 @@ import org.apache.comet.{CometConf, CometExplainInfo} * shuffle whose child subtree (descending through `QueryStageExec.plan`) still contains a DPP * scan -- i.e. an inconsistency that the bug would produce. */ +@ExecGroup class CometDppFallbackRepro3949Suite extends CometTestBase { // ---------------------------------------------------------------------- diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometNativeWriteSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometNativeWriteSuite.scala index 098b1c16bc5..9a1dbd6b94c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometNativeWriteSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometNativeWriteSuite.scala @@ -49,8 +49,10 @@ import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.serde.{Incompatible, Unsupported} import org.apache.comet.serde.OperatorOuterClass.Operator import org.apache.comet.serde.operator.{schema2Proto, CometDataWritingCommand, NativeWriteUtils} +import org.apache.comet.tags.ScansGroup /** Exercises the retained Spark 3.x writer, including both terminal execution entry points. */ +@ScansGroup class CometNativeWriteSuite extends CometTestBase { import NativeWriteCommitProtocol._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala index 52a5e1fe953..a6a0117bcfd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala @@ -32,8 +32,10 @@ import org.apache.xbean.asm9.tree.{ClassNode, FieldInsnNode, MethodInsnNode} import com.google.common.reflect.ClassPath import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus +import org.apache.comet.tags.ExecGroup import org.apache.comet.util.ClassLoaders +@ExecGroup class CometPlanEqualitySuite extends AnyFunSuite { // Native serialization state and the source plan are deliberately not the identity of a diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala index b63db32128b..3430af55b4c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala @@ -36,6 +36,7 @@ import org.apache.spark.sql.test.TestSparkSession import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, isSpark41Plus, isSpark42Plus} +import org.apache.comet.tags.ExecGroup /** * Similar to [[org.apache.spark.sql.PlanStabilitySuite]], checks that TPC-DS Comet plans don't @@ -282,6 +283,7 @@ private object CometPlanStabilitySuite { } } +@ExecGroup class CometTPCDSV1_4_PlanStabilitySuite extends CometPlanStabilitySuite { private val planNames: Seq[String] = CometPlanStabilitySuite.planNameChain("v1_4") override val goldenFilePath: String = @@ -296,6 +298,7 @@ class CometTPCDSV1_4_PlanStabilitySuite extends CometPlanStabilitySuite { } } +@ExecGroup class CometTPCDSV2_7_PlanStabilitySuite extends CometPlanStabilitySuite { private val planNames: Seq[String] = CometPlanStabilitySuite.planNameChain("v2_7") override val goldenFilePath: String = diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometScanWithPlanDataSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometScanWithPlanDataSuite.scala index 9bf49125d7d..631758c1c64 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometScanWithPlanDataSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometScanWithPlanDataSuite.scala @@ -36,6 +36,7 @@ import org.apache.spark.sql.execution.{BinaryExecNode, SparkPlan} import org.apache.comet.CometRuntimeException import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.tags.ExecGroup /** * Unit coverage for the core SPI that lets out-of-tree contrib leaf scans (e.g. the Delta @@ -50,6 +51,7 @@ import org.apache.comet.serde.OperatorOuterClass.Operator * Deliberately does not exercise the per-op injector registry mechanics; that surface is owned by * `PlanDataInjectorSuite`. */ +@ExecGroup class CometScanWithPlanDataSuite extends AnyFunSuite { /** diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala index 49a2f83b6e7..4fc88f82b5c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala @@ -23,7 +23,9 @@ import org.apache.spark.sql.{CometTestBase, SparkSessionExtensions} import org.apache.comet.CometSparkSessionExtensions import org.apache.comet.rules.CometRule +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometSessionExtensionsSuite extends CometTestBase { private def countCometRules(extensions: SparkSessionExtensions): (Int, Int) = { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometShuffleFallbackStickinessSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometShuffleFallbackStickinessSuite.scala index 23f5ad5d413..42ed1cc0962 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometShuffleFallbackStickinessSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometShuffleFallbackStickinessSuite.scala @@ -31,6 +31,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{hasFallbackReason, withFallbackReason} +import org.apache.comet.tags.ExecGroup /** * Pins the sticky-fallback invariant for Comet shuffle decisions: `shuffleSupported` must return @@ -46,6 +47,7 @@ import org.apache.comet.CometSparkSessionExtensions.{hasFallbackReason, withFall * The coordinator tags the node with `withFallbackReasons` only on total fallback and * short-circuits via `hasFallbackReason` on subsequent passes. */ +@ExecGroup class CometShuffleFallbackStickinessSuite extends CometTestBase { test("shuffleSupported returns None when the shuffle already carries explain info") { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala index 4266f7f1605..182ae73a183 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala @@ -47,7 +47,9 @@ import org.apache.spark.unsafe.Platform import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus} import org.apache.comet.serde.{Metric, OperatorOuterClass} +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper { override protected def sparkConf: SparkConf = { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometUnsafeProjectionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometUnsafeProjectionSuite.scala index c592d5b0a35..28d30a2927f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometUnsafeProjectionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometUnsafeProjectionSuite.scala @@ -30,6 +30,9 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} import org.apache.spark.sql.types._ +import org.apache.comet.tags.ExecGroup + +@ExecGroup class CometUnsafeProjectionSuite extends CometTestBase with AdaptiveSparkPlanHelper { /** A schema with every writer GenerateUnsafeProjection nests: structs, arrays and maps. */ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/DecimalPrecisionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/DecimalPrecisionSuite.scala index be4c7db5de8..281ddd81170 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/DecimalPrecisionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/DecimalPrecisionSuite.scala @@ -23,6 +23,9 @@ import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.expressions.{Add, Alias, AttributeReference, CaseWhen, Cast, CheckOverflow, Coalesce, Divide, Expression, GreaterThan, IsNotNull, Literal, Multiply, Remainder, Subtract} import org.apache.spark.sql.types.{DecimalType, IntegerType, LongType, StringType} +import org.apache.comet.tags.ExecGroup + +@ExecGroup class DecimalPrecisionSuite extends SparkFunSuite { private val decA = AttributeReference("a", DecimalType(10, 2))() diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala b/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala index 1423981b1c4..7a5e43ba424 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala @@ -37,12 +37,14 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SQLTestUtils import org.apache.comet.{CometConf, IntegrationTestSuite} +import org.apache.comet.tags.ScansGroup /** * A integration test suite that tests parquet modular encryption usage. */ @RunWith(classOf[JUnitRunner]) @IntegrationTestSuite +@ScansGroup class ParquetEncryptionITCase extends CometTestBase with SQLTestUtils { private val encoder = Base64.getEncoder private val footerKey = diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorShuffleLifecycleSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorShuffleLifecycleSuite.scala index ae7db078cb6..51abe82c14d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorShuffleLifecycleSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorShuffleLifecycleSuite.scala @@ -26,6 +26,7 @@ import org.apache.spark.sql.functions.col import org.apache.spark.util.Utils import org.apache.comet.CometConf +import org.apache.comet.tags.ExecGroup /** * End-to-end lifecycle of the executor-side plan data held by [[PlanDataInjector]]. A map-only @@ -37,6 +38,7 @@ import org.apache.comet.CometConf * The recreated-context test stops the suite's SparkContext, so it runs last and nothing else may * follow it. */ +@ExecGroup class PlanDataInjectorShuffleLifecycleSuite extends CometTestBase { private def withNativeShuffle[T](session: SparkSession)(f: => T): T = { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorSuite.scala index bb165f58346..a5fcd8de0ef 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorSuite.scala @@ -26,7 +26,9 @@ import org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager import org.apache.comet.serde.OperatorOuterClass import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.tags.ExecGroup +@ExecGroup class PlanDataInjectorSuite extends AnyFunSuite { /** Builds an un-injected IcebergScan operator: hasCommon, zero file_scan_tasks. */ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchRowIteratorSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchRowIteratorSuite.scala index 108694a7b5a..00353f01032 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchRowIteratorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchRowIteratorSuite.scala @@ -32,8 +32,10 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{IntegerType, LongType, StringType} import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} +import org.apache.comet.tags.ExecGroup import org.apache.comet.vector.CometPlainVector +@ExecGroup class CachedBatchRowIteratorSuite extends AnyFunSuite { Seq("CODEGEN_ONLY", "NO_CODEGEN").foreach { mode => def withMode(settings: (String, String)*)(f: => Unit): Unit = { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala index 34bb32d908f..a9f24b0b1f1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala @@ -44,6 +44,7 @@ import org.apache.spark.sql.types.{ArrayType, BooleanType, ByteType, CalendarInt import org.apache.spark.sql.vectorized.{ColumnarArray, ColumnarBatch, ColumnarMap, ColumnVector} import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} +import org.apache.comet.tags.ExecGroup import org.apache.comet.vector.{CometPlainVector, CometVector, NativeUtil} /** @@ -52,6 +53,7 @@ import org.apache.comet.vector.{CometPlainVector, CometVector, NativeUtil} * `CometMathExpressionSuite`, but that test only catches *one* function-level type drift. This * suite covers the boundary contract independently of any specific function. */ +@ExecGroup class CometArrowStreamSuite extends AnyFunSuite with Matchers { private def expectedSchema(types: (String, ArrowType)*): Schema = { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowWriterSuite.scala index 400d9aad797..0ba0f8ac7de 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowWriterSuite.scala @@ -43,12 +43,14 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.CometSparkSessionExtensions.isSpark41Plus +import org.apache.comet.tags.ExecGroup /** * The columnar paths of [[ArrowWriter]] copy Spark's vectors in bulk where they can. Whatever * they take, they must produce what the row path writes for the same rows, which reads every * value through Spark's getters one at a time. */ +@ExecGroup class CometArrowWriterSuite extends AnyFunSuite with Matchers { private val numRows = 300 diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometStringWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometStringWriterSuite.scala index 3f23851d76f..50d9e11272b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometStringWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometStringWriterSuite.scala @@ -39,6 +39,9 @@ import org.apache.spark.sql.comet.util.Utils import org.apache.spark.sql.types.{ArrayType, StringType, StructField, StructType} import org.apache.spark.unsafe.types.UTF8String +import org.apache.comet.tags.ExecGroup + +@ExecGroup class CometStringWriterSuite extends AnyFunSuite with Matchers { private val sourceOffset = 7 private val schema = StructType(Seq(StructField("text", StringType))) diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReaderSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReaderSuite.scala index 6eda56cdb76..3221ce055dc 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReaderSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReaderSuite.scala @@ -24,6 +24,9 @@ import java.io.InputStream import org.scalatest.funsuite.AnyFunSuite +import org.apache.comet.tags.ShuffleGroup + +@ShuffleGroup class CometBlockStoreShuffleReaderSuite extends AnyFunSuite { test("closing a partially consumed raw stream does not fetch remaining blocks") { class TrackingStream extends ByteArrayInputStream(Array[Byte](1, 2, 3)) { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala index 7603c22d784..dd598eb11d4 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala @@ -34,7 +34,9 @@ import org.apache.spark.sql.comet.shims.ShimCometShuffleMaterialization import org.apache.spark.util.ThreadUtils import org.apache.comet.CometConf +import org.apache.comet.tags.ShuffleGroup +@ShuffleGroup class CometCelebornConcurrentMaterializationSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala index a2f8f1e41e5..92f35c9d650 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala @@ -30,7 +30,9 @@ import org.apache.spark.shuffle.{FetchFailedException, ShuffleHandle, ShuffleRea import org.apache.spark.sql.CometTestBase import org.apache.comet.CometConf +import org.apache.comet.tags.ShuffleGroup +@ShuffleGroup class CometCelebornLocalFetchFailureSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornNativeShuffleWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornNativeShuffleWriterSuite.scala index 048b4c09a6e..dff55f73601 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornNativeShuffleWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornNativeShuffleWriterSuite.scala @@ -30,8 +30,10 @@ import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.{CometConf, CometShuffleSizeLimitException} import org.apache.comet.shuffle.{CelebornShufflePartitionPusher, CelebornShufflePusherFactory, RecordingCelebornPushClient} +import org.apache.comet.tags.ShuffleGroup /** Exercises real Spark map tasks, native RSS planning, and the Celeborn map lifecycle. */ +@ShuffleGroup class CometCelebornNativeShuffleWriterSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala index de3381febfe..05c0221425b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala @@ -38,10 +38,12 @@ import org.apache.spark.sql.CometTestBase import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.shuffle.{CelebornShufflePusherFactory, RecordingCelebornPushClient, ResolvedCelebornShufflePusher} +import org.apache.comet.tags.ShuffleGroup /** * Real Spark stage recovery and local Comet I/O; only the optional Celeborn client is replaced. */ +@ShuffleGroup class CometCelebornShuffleFallbackSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManagerSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManagerSuite.scala index de3cc5de317..43daa9f16fb 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManagerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManagerSuite.scala @@ -31,7 +31,9 @@ import org.apache.spark.shuffle.{BaseShuffleHandle, ShuffleBlockResolver, Shuffl import org.apache.spark.sql.comet.{NativeScanPlanDataInjector, PlanDataInjector} import org.apache.comet.serde.OperatorOuterClass +import org.apache.comet.tags.ShuffleGroup +@ShuffleGroup class CometCelebornShuffleManagerSuite extends AnyFunSuite { // Match the optional CelebornConf API, including its enum-valued policy and Long threshold. diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShufflePlanningSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShufflePlanningSuite.scala index 625013aa9e9..98de0ca99d8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShufflePlanningSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShufflePlanningSuite.scala @@ -42,12 +42,14 @@ import org.apache.comet.{CometConf, CometExplainInfo} import org.apache.comet.CometSparkSessionExtensions.{isCometShuffleEnabled, isCometShuffleManagerEnabled} import org.apache.comet.rules.{CometExecRule, RevertNativeForTransitionHeavyStages} import org.apache.comet.serde.{Compatible, OperatorOuterClass} +import org.apache.comet.tags.ShuffleGroup /** * Plans against the actual composite manager without requiring the optional Celeborn client. Only * Spark-fallback queries execute: the local test backend must never receive a native dependency. * Native plan assertions below are not transport or live-cluster integration tests. */ +@ShuffleGroup class CometCelebornShufflePlanningSuite extends CometTestBase { override protected val shuffleManager: String = diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReaderSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReaderSuite.scala index d315bddfe02..5b3a6ce1df5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReaderSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReaderSuite.scala @@ -53,7 +53,9 @@ import com.google.flatbuffers.Table import org.apache.comet.{CometConf, CometExecIterator, CometShuffleBlockIterator, Native} import org.apache.comet.serde.{OperatorOuterClass, QueryPlanSerde} +import org.apache.comet.tags.ShuffleGroup +@ShuffleGroup class CometCelebornShuffleReaderSuite extends CometTestBase { import NativeShuffleTestFrames._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleStatisticsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleStatisticsSuite.scala index f4be2680366..c9dcfdf6916 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleStatisticsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleStatisticsSuite.scala @@ -27,7 +27,9 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.metric.SQLShuffleWriteMetricsReporter.{SHUFFLE_BYTES_WRITTEN, SHUFFLE_RECORDS_WRITTEN} import org.apache.comet.CometConf +import org.apache.comet.tags.ShuffleGroup +@ShuffleGroup class CometCelebornShuffleStatisticsSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index f18e50658c7..758af60d7e1 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -43,6 +43,9 @@ import org.apache.spark.sql.types.{BinaryType, IntegerType, MetadataBuilder, Str import org.apache.spark.unsafe.UnsafeAlignedOffset import org.apache.spark.util.Utils +import org.apache.comet.tags.ShuffleGroup + +@ShuffleGroup class CometDiskBlockWriterSuite extends AnyFunSuite { private val schema = StructType(Seq(StructField("a", BinaryType))) diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala index bfab3727cab..c9a09c5b98f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.functions.{col, lit, rand, udf} import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf +import org.apache.comet.tags.ShuffleGroup /** * Positional round robin places rows by their ordinal within the map task rather than by hashing @@ -39,6 +40,7 @@ import org.apache.comet.CometConf * `CometShuffleExchangeExec.usesPositionalRoundRobin` directly rather than inferring the decision * from output. */ +@ShuffleGroup class CometNativePositionalRoundRobinSuite extends CometTestBase with AdaptiveSparkPlanHelper { private val numPartitions = 8 diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleInputRDDSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleInputRDDSuite.scala index 6ca87258b92..1e026d4425e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleInputRDDSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleInputRDDSuite.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.tags.ShuffleGroup /** * Ensure the serialized shuffle-map-stage task binary does not grow with the number of @@ -43,6 +44,7 @@ import org.apache.comet.serde.OperatorOuterClass.Operator * Lives in the `execution.shuffle` package so it can construct the `private[shuffle]` * [[CometNativeShuffleInputRDD]] and the `private[comet]` [[NativeExecContext]] directly. */ +@ShuffleGroup class CometNativeShuffleInputRDDSuite extends CometTestBase { test("native shuffle input preserves its parents' determinism") { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala index 8e72e37384a..6e5af62d57a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala @@ -24,11 +24,13 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.CometConf +import org.apache.comet.tags.ShuffleGroup /** * Checks the plan [[CometNativeShuffleWriter]] hands to native code. Lives in the * `execution.shuffle` package so it can call the `private[shuffle]` `buildUnifiedPlan`. */ +@ShuffleGroup class CometNativeShuffleWriterSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIteratorSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIteratorSuite.scala index 865d1a34cb8..9fb5768799f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIteratorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIteratorSuite.scala @@ -28,8 +28,10 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.metric.SQLMetrics import org.apache.comet.Native +import org.apache.comet.tags.ShuffleGroup import org.apache.comet.vector.NativeUtil +@ShuffleGroup class NativeBatchDecoderIteratorSuite extends CometTestBase { import NativeShuffleTestFrames.{intFrame, TrackingInputStream} diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala index 86a4fe7a899..e8f060300bd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala @@ -39,8 +39,10 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.util.io.ChunkedByteBuffer import org.apache.comet.CometArrowAllocator +import org.apache.comet.tags.ExecGroup import org.apache.comet.vector.CometVector +@ExecGroup class UtilsSuite extends CometTestBase { test("serializeBatches round trips schema metadata larger than the output chunk estimate") { diff --git a/spark/src/test/spark-3.4/org/apache/comet/exec/CometMergeRowsSuite.scala b/spark/src/test/spark-3.4/org/apache/comet/exec/CometMergeRowsSuite.scala index b97c0e5bb5a..54bb6d981cd 100644 --- a/spark/src/test/spark-3.4/org/apache/comet/exec/CometMergeRowsSuite.scala +++ b/spark/src/test/spark-3.4/org/apache/comet/exec/CometMergeRowsSuite.scala @@ -22,8 +22,10 @@ package org.apache.comet.exec import org.apache.spark.sql.CometTestBase import org.apache.comet.shims.ShimCometMergeRows +import org.apache.comet.tags.ExecGroup /** Spark 3.4 compatibility coverage for the version-specific MergeRows registration shim. */ +@ExecGroup class CometMergeRowsSuite extends CometTestBase { test("Spark 3.4 leaves native MergeRows execution unregistered") { assert(ShimCometMergeRows.nativeExecs.isEmpty) diff --git a/spark/src/test/spark-3.5/org/apache/comet/CometWidthBucketSuite.scala b/spark/src/test/spark-3.5/org/apache/comet/CometWidthBucketSuite.scala index c2a0034a01b..e8c958b4e2a 100644 --- a/spark/src/test/spark-3.5/org/apache/comet/CometWidthBucketSuite.scala +++ b/spark/src/test/spark-3.5/org/apache/comet/CometWidthBucketSuite.scala @@ -22,6 +22,9 @@ package org.apache.comet import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} +import org.apache.comet.tags.ExpressionsGroup + +@ExpressionsGroup class CometWidthBucketSuite extends CometTestBase { private def countSparkProjectExec(plan: SparkPlan): Int = diff --git a/spark/src/test/spark-3.5/org/apache/comet/exec/CometMergeRowsSuite.scala b/spark/src/test/spark-3.5/org/apache/comet/exec/CometMergeRowsSuite.scala index a621b5a9f52..9af8721741e 100644 --- a/spark/src/test/spark-3.5/org/apache/comet/exec/CometMergeRowsSuite.scala +++ b/spark/src/test/spark-3.5/org/apache/comet/exec/CometMergeRowsSuite.scala @@ -19,5 +19,8 @@ package org.apache.comet.exec +import org.apache.comet.tags.ExecGroup + /** Spark 3.5.x entry point for the shared native MergeRows contract suite. */ +@ExecGroup class CometMergeRowsSuite extends CometMergeRowsNativeSuiteBase diff --git a/spark/src/test/spark-3.5/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala b/spark/src/test/spark-3.5/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala index 6f645747394..24b189f9143 100644 --- a/spark/src/test/spark-3.5/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala +++ b/spark/src/test/spark-3.5/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala @@ -22,6 +22,9 @@ package org.apache.spark.sql import org.apache.spark.sql.catalyst.expressions.Empty2Null import org.apache.spark.sql.functions.col +import org.apache.comet.tags.ExecGroup + +@ExecGroup class CometSparkInternalFunctionsSuite extends CometTestBase { test("empty2null is offloaded to Comet") { diff --git a/spark/src/test/spark-3.5/org/apache/spark/sql/CometToPrettyStringSuite.scala b/spark/src/test/spark-3.5/org/apache/spark/sql/CometToPrettyStringSuite.scala index beb37211029..85d895823ca 100644 --- a/spark/src/test/spark-3.5/org/apache/spark/sql/CometToPrettyStringSuite.scala +++ b/spark/src/test/spark-3.5/org/apache/spark/sql/CometToPrettyStringSuite.scala @@ -30,7 +30,9 @@ import org.apache.comet.CometConf import org.apache.comet.CometFuzzTestBase import org.apache.comet.expressions.{CometCast, CometEvalMode} import org.apache.comet.serde.Compatible +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometToPrettyStringSuite extends CometFuzzTestBase { test("ToPrettyString") { diff --git a/spark/src/test/spark-3.x/org/apache/comet/CometStringDecodeSuite.scala b/spark/src/test/spark-3.x/org/apache/comet/CometStringDecodeSuite.scala index 40c238e6566..daff1180701 100644 --- a/spark/src/test/spark-3.x/org/apache/comet/CometStringDecodeSuite.scala +++ b/spark/src/test/spark-3.x/org/apache/comet/CometStringDecodeSuite.scala @@ -23,7 +23,9 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.tags.ExpressionsGroup +@ExpressionsGroup class CometStringDecodeSuite extends CometTestBase { private def countSparkProjectExec(plan: SparkPlan): Int = diff --git a/spark/src/test/spark-4.0/org/apache/comet/exec/CometMergeRowsSuite.scala b/spark/src/test/spark-4.0/org/apache/comet/exec/CometMergeRowsSuite.scala index f0da046a99d..b3a15368773 100644 --- a/spark/src/test/spark-4.0/org/apache/comet/exec/CometMergeRowsSuite.scala +++ b/spark/src/test/spark-4.0/org/apache/comet/exec/CometMergeRowsSuite.scala @@ -19,5 +19,8 @@ package org.apache.comet.exec +import org.apache.comet.tags.ExecGroup + /** Spark 4.0 entry point for the shared native MergeRows contract suite. */ +@ExecGroup class CometMergeRowsSuite extends CometMergeRowsNativeSuiteBase diff --git a/spark/src/test/spark-4.0/org/apache/spark/sql/CometCollationSuite.scala b/spark/src/test/spark-4.0/org/apache/spark/sql/CometCollationSuite.scala index 8502eb44c0b..e73b06c1e5e 100644 --- a/spark/src/test/spark-4.0/org/apache/spark/sql/CometCollationSuite.scala +++ b/spark/src/test/spark-4.0/org/apache/spark/sql/CometCollationSuite.scala @@ -30,7 +30,9 @@ import org.apache.spark.sql.types.StringType import org.apache.comet.{CometConf, CometExplainInfo, ExtendedExplainInfo} import org.apache.comet.serde.OperatorOuterClass +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometCollationSuite extends CometTestBase { // Queries that group, sort, or shuffle on a non-default collated string must fall back to diff --git a/spark/src/test/spark-4.1+/org/apache/comet/exec/CometMergeRowsSuite.scala b/spark/src/test/spark-4.1+/org/apache/comet/exec/CometMergeRowsSuite.scala index 49d2f2d4c10..b9d75a7ec28 100644 --- a/spark/src/test/spark-4.1+/org/apache/comet/exec/CometMergeRowsSuite.scala +++ b/spark/src/test/spark-4.1+/org/apache/comet/exec/CometMergeRowsSuite.scala @@ -35,11 +35,13 @@ import org.apache.spark.sql.util.QueryExecutionListener import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus import org.apache.comet.shims.{MergeRowsMetricsShim, ShimCometMergeRows} +import org.apache.comet.tags.ExecGroup /** * Spark 4.1+ stock V2 writers build MergeSummary by locating the concrete Spark MergeRowsExec. * Native MergeRows is therefore kept on Spark when the enclosing writer remains on Spark. */ +@ExecGroup class CometMergeRowsSuite extends CometTestBase with AdaptiveSparkPlanHelper { private val catalog = "generic_rowlevel" diff --git a/spark/src/test/spark-4.1+/org/apache/spark/sql/CometCollationSuite.scala b/spark/src/test/spark-4.1+/org/apache/spark/sql/CometCollationSuite.scala index 9ebd3002905..7cc235fc415 100644 --- a/spark/src/test/spark-4.1+/org/apache/spark/sql/CometCollationSuite.scala +++ b/spark/src/test/spark-4.1+/org/apache/spark/sql/CometCollationSuite.scala @@ -20,7 +20,9 @@ package org.apache.spark.sql import org.apache.comet.{CometConf, ExtendedExplainInfo} +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometCollationSuite extends CometTestBase { // Queries that group, sort, or shuffle on a non-default collated string must fall back to diff --git a/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometArrowPythonUdfSuite.scala b/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometArrowPythonUdfSuite.scala index 1e742873a5a..76384556e51 100644 --- a/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometArrowPythonUdfSuite.scala +++ b/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometArrowPythonUdfSuite.scala @@ -36,7 +36,9 @@ import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, import org.apache.comet.{CometConf, CometExplainInfo, NativeBase, SparkErrorConverter} import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus import org.apache.comet.exceptions.CometQueryExecutionException +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometArrowPythonUdfSuite extends CometTestBase { private def python: String = sys.env.getOrElse("PYSPARK_PYTHON", "python3") diff --git a/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometDecimalArithmeticViewSuite.scala b/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometDecimalArithmeticViewSuite.scala index 3e68ffb80b0..0c8b56da1c3 100644 --- a/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometDecimalArithmeticViewSuite.scala +++ b/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometDecimalArithmeticViewSuite.scala @@ -25,7 +25,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.DecimalType import org.apache.comet.serde.{CometScalarFunction, ExprOuterClass, QueryPlanSerde} +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometDecimalArithmeticViewSuite extends CometTestBase { // Spark 4.1+ (SPARK-53968) stores `spark.sql.decimalOperations.allowPrecisionLoss` per diff --git a/spark/src/test/spark-4.1/org/apache/comet/CometIcebergEncryptionSuite.scala b/spark/src/test/spark-4.1/org/apache/comet/CometIcebergEncryptionSuite.scala index 1b221bf454d..1cfe8c438a4 100644 --- a/spark/src/test/spark-4.1/org/apache/comet/CometIcebergEncryptionSuite.scala +++ b/spark/src/test/spark-4.1/org/apache/comet/CometIcebergEncryptionSuite.scala @@ -32,12 +32,14 @@ import org.apache.spark.sql.comet.CometIcebergNativeScanExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.comet.iceberg.CometTestKMS +import org.apache.comet.tags.ScansGroup /** * Encryption-specific Iceberg tests. Lives in the spark-4.1 source set because Iceberg table * encryption is a V3 feature and the encryption APIs (KeyManagementClient, StandardEncryption * manager) are only public from Iceberg 1.11, which Comet pairs with Spark 4.1. */ +@ScansGroup class CometIcebergEncryptionSuite extends CometTestBase with CometIcebergTestBase diff --git a/spark/src/test/spark-4.2/org/apache/comet/exec/CometInMemoryCacheLastAttemptMetricSuite.scala b/spark/src/test/spark-4.2/org/apache/comet/exec/CometInMemoryCacheLastAttemptMetricSuite.scala index 3f30e44aa06..5577b0e87e9 100644 --- a/spark/src/test/spark-4.2/org/apache/comet/exec/CometInMemoryCacheLastAttemptMetricSuite.scala +++ b/spark/src/test/spark-4.2/org/apache/comet/exec/CometInMemoryCacheLastAttemptMetricSuite.scala @@ -27,8 +27,10 @@ import org.apache.spark.sql.execution.columnar.CometInMemoryRelationHelper import org.apache.spark.sql.execution.metric.SQLLastAttemptMetrics import org.apache.comet.CometConf +import org.apache.comet.tags.ExecGroup /** Spark 4.2's last-attempt metrics over a relation cached in Comet's format. */ +@ExecGroup class CometInMemoryCacheLastAttemptMetricSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/spark-4.x/org/apache/comet/CometCastCollatedStringSuite.scala b/spark/src/test/spark-4.x/org/apache/comet/CometCastCollatedStringSuite.scala index f547836a7b7..211fbe00a9c 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/CometCastCollatedStringSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/CometCastCollatedStringSuite.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, IntegerType, import org.apache.comet.expressions.{CometCast, CometEvalMode} import org.apache.comet.serde.{Compatible, Unsupported} +import org.apache.comet.tags.ExpressionsGroup /** * https://github.com/apache/datafusion-comet/issues/4489 @@ -62,6 +63,7 @@ import org.apache.comet.serde.{Compatible, Unsupported} * This lives under `spark-4.x`, shared by every 4.x profile, rather than `spark-4.1+`, because * collation is a Spark 4.0 feature and `StringType(collationName)` already resolves there. */ +@ExpressionsGroup class CometCastCollatedStringSuite extends CometTestBase { private val lcase = StringType("UTF8_LCASE") diff --git a/spark/src/test/spark-4.x/org/apache/comet/CometWidthBucketSuite.scala b/spark/src/test/spark-4.x/org/apache/comet/CometWidthBucketSuite.scala index c2a0034a01b..e8c958b4e2a 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/CometWidthBucketSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/CometWidthBucketSuite.scala @@ -22,6 +22,9 @@ package org.apache.comet import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} +import org.apache.comet.tags.ExpressionsGroup + +@ExpressionsGroup class CometWidthBucketSuite extends CometTestBase { private def countSparkProjectExec(plan: SparkPlan): Int = diff --git a/spark/src/test/spark-4.x/org/apache/comet/exec/CometEmptyRelationExecSuite.scala b/spark/src/test/spark-4.x/org/apache/comet/exec/CometEmptyRelationExecSuite.scala index 76b378e4181..45efe1da972 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/exec/CometEmptyRelationExecSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/exec/CometEmptyRelationExecSuite.scala @@ -28,7 +28,9 @@ import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometEmptyRelationExecSuite extends CometTestBase { // CometTestBase enables the Spark-to-Arrow conversions; use their production defaults here. diff --git a/spark/src/test/spark-4.x/org/apache/comet/exec/CometShuffle4_0Suite.scala b/spark/src/test/spark-4.x/org/apache/comet/exec/CometShuffle4_0Suite.scala index 8629d1a78c1..e753a5085a6 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/exec/CometShuffle4_0Suite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/exec/CometShuffle4_0Suite.scala @@ -28,6 +28,9 @@ import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{FloatType, LongType, StringType, TimestampType} +import org.apache.comet.tags.ShuffleGroup + +@ShuffleGroup class CometShuffle4_0Suite extends CometColumnarShuffleSuite { protected val adaptiveExecutionEnabled: Boolean = true diff --git a/spark/src/test/spark-4.x/org/apache/comet/parquet/CometEmptyRelationParquetWriterSuite.scala b/spark/src/test/spark-4.x/org/apache/comet/parquet/CometEmptyRelationParquetWriterSuite.scala index c739e420315..35a772a83c6 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/parquet/CometEmptyRelationParquetWriterSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/parquet/CometEmptyRelationParquetWriterSuite.scala @@ -27,7 +27,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.comet.CometConf +import org.apache.comet.tags.ScansGroup +@ScansGroup class CometEmptyRelationParquetWriterSuite extends CometParquetWriterTestBase { test("EmptyRelationExec keeps AQE empty parquet output readable") { diff --git a/spark/src/test/spark-4.x/org/apache/comet/rules/CometEmptyRelationExecRuleSuite.scala b/spark/src/test/spark-4.x/org/apache/comet/rules/CometEmptyRelationExecRuleSuite.scala index 77811d9e4f4..4c37b4ca524 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/rules/CometEmptyRelationExecRuleSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/rules/CometEmptyRelationExecRuleSuite.scala @@ -34,7 +34,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, BooleanType, DecimalType, IntegerType, StructType} import org.apache.comet.{CometConf, CometCoverageStats, CometExplainInfo, ExtendedExplainInfo} +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometEmptyRelationExecRuleSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/CometSortCollationSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/CometSortCollationSuite.scala index 6916ae72b3c..78a0380ef45 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/CometSortCollationSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/CometSortCollationSuite.scala @@ -28,7 +28,9 @@ import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, CometExplainInfo} import org.apache.comet.serde.QueryPlanSerde +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometSortCollationSuite extends CometTestBase { private val sortCollationReason = "Sort does not support non-default string collation" diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala index b8b107f2e8d..4621be87958 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala @@ -23,6 +23,9 @@ import org.apache.spark.sql.catalyst.expressions.Empty2Null import org.apache.spark.sql.classic.ExpressionUtils import org.apache.spark.sql.functions._ +import org.apache.comet.tags.ExecGroup + +@ExecGroup class CometSparkInternalFunctionsSuite extends CometTestBase { test("empty2null is offloaded to Comet") { diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/CometToPrettyStringSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/CometToPrettyStringSuite.scala index 80063d08e54..70dadc7f6d1 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/CometToPrettyStringSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/CometToPrettyStringSuite.scala @@ -33,7 +33,9 @@ import org.apache.comet.CometConf import org.apache.comet.CometFuzzTestBase import org.apache.comet.expressions.{CometCast, CometEvalMode} import org.apache.comet.serde.Compatible +import org.apache.comet.tags.ExecGroup +@ExecGroup class CometToPrettyStringSuite extends CometFuzzTestBase { test("ToPrettyString") { diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/CometVariantShreddingSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/CometVariantShreddingSuite.scala index c5359c3512c..2180581633e 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/CometVariantShreddingSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/CometVariantShreddingSuite.scala @@ -27,8 +27,10 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.TestSparkSession import org.apache.comet.{CometConf, CometSparkSessionExtensions} +import org.apache.comet.tags.ScansGroup /** Run Spark's unchanged reconstruction assertions in Comet's regular Spark 4 CI jobs. */ +@ScansGroup class CometVariantShreddingSuite extends VariantShreddingSuite { override protected def sparkConf: SparkConf = super.sparkConf .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false") diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/comet/CometMapInBatchSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/comet/CometMapInBatchSuite.scala index 2a3a0f9920e..e4c1b6c1f5f 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/comet/CometMapInBatchSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/comet/CometMapInBatchSuite.scala @@ -32,6 +32,7 @@ import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.rules.EliminateRedundantTransitions +import org.apache.comet.tags.ExecGroup /** Minimal CometPlan leaf used to anchor the rule's transform without triggering execution. */ private case class StubCometLeaf(override val output: Seq[Attribute]) @@ -53,6 +54,7 @@ private case class StubCometLeaf(override val output: Seq[Attribute]) * `PythonFunction` / `PythonAccumulatorV2` / `PythonBroadcast` classes when fabricating a stub * `PythonUDF` for `MapInArrowExec` to wrap. */ +@ExecGroup class CometMapInBatchSuite extends CometTestBase { private def stubPythonUDF: PythonUDF = { diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/execution/python/CometArrowPythonRunnerSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/execution/python/CometArrowPythonRunnerSuite.scala index cda8fb3834a..daf889297c7 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/execution/python/CometArrowPythonRunnerSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/execution/python/CometArrowPythonRunnerSuite.scala @@ -52,8 +52,10 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.util.DirectByteBufferOutputStream import org.apache.comet.CometArrowAllocator +import org.apache.comet.tags.ExecGroup import org.apache.comet.vector.{CometDecodedVector, CometDictionary, CometDictionaryVector, CometPlainVector, CometStructVector} +@ExecGroup class CometArrowPythonRunnerSuite extends AnyFunSuite with Matchers { /** From 432e0c798368db39a45165321e44f28faa204853 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 4 Nov 2026 15:53:42 -0700 Subject: [PATCH 2/4] docs: say the group tag does not select integration suites, and update ci.md --- dev/ci/run-comet-tests.sh | 8 +++++++- docs/source/contributor-guide/ci.md | 2 +- docs/source/contributor-guide/development.md | 7 +++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/dev/ci/run-comet-tests.sh b/dev/ci/run-comet-tests.sh index ea7b9ae86f9..4f0742ca232 100755 --- a/dev/ci/run-comet-tests.sh +++ b/dev/ci/run-comet-tests.sh @@ -24,6 +24,11 @@ # # dev/ci/run-comet-tests.sh exec -Pspark-4.0 # +# The group tag does not select integration suites (@IntegrationTestSuite, such as +# ParquetEncryptionITCase). The ScalaTest plugin excludes them, and they run under +# maven-failsafe-plugin, which ignores the tag. So the default `test` goal never runs +# them, and `--goal install` runs all of them whichever group is named. +# # Build the native library first: `make core`, or `make release` with -Prelease. # Written for bash 3.2. @@ -44,7 +49,8 @@ Usage: dev/ci/run-comet-tests.sh [--goal ] [maven args...] groups $groups --goal the Maven goal to run (default: test). install also runs the - spark-integration checks. + spark-integration checks, which are not selected by the group: + every *ITCase suite runs, whichever group is named. USAGE exit 2 } diff --git a/docs/source/contributor-guide/ci.md b/docs/source/contributor-guide/ci.md index 9088ddbcf22..574cc3b195d 100644 --- a/docs/source/contributor-guide/ci.md +++ b/docs/source/contributor-guide/ci.md @@ -524,7 +524,7 @@ python3 dev/ci/check-ci-config.py actionlint --shellcheck=off ``` -A new test suite has to be registered in the workflow files by hand; see +A new test suite needs a group tag and a listing in the workflow files; see [Register New Test Suites in CI](development.md#6-register-new-test-suites-in-ci). A new job in `ci.yml` needs an entry in both `FILTERS` and `POLICY` in `dev/ci/compute-changes.py`, a case in `dev/ci/check-ci-config.py`, and a line in `required_checks.needs`. The check will tell you which diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md index bf94f9cda12..0cf2e367cd5 100644 --- a/docs/source/contributor-guide/development.md +++ b/docs/source/contributor-guide/development.md @@ -763,6 +763,13 @@ dev/ci/run-comet-tests.sh exec -Pspark-4.0 # arguments after the group go to The script runs `mvnw test` by default. Pass `--goal install` to also run the `spark-integration` checks, and `-Prelease` to use the library that `make release` builds. +The group tag does not select an integration suite, one marked `@IntegrationTestSuite` such as +`ParquetEncryptionITCase`. The ScalaTest plugin excludes those, and Maven runs them under +`maven-failsafe-plugin` by their `*ITCase` name, which ignores the tag. The default `test` goal +therefore never runs them, and `--goal install` runs every one of them whichever group you name. +Such a suite still carries a tag and a workflow listing so that the checks above hold, but until CI +runs them for one group only, the listing does not decide where they run. + A small number of suites are deliberately **not** run in CI, because they need infrastructure CI does not have or because running them there is not worth the cost. These are listed in `IGNORE_LIST` in `dev/ci/check-suites.py` and carry no group tag. Each one documents in its own From 53b705aadc726e482d834fa38d59dd00282146b1 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 4 Nov 2026 15:58:10 -0700 Subject: [PATCH 3/4] docs: say CI still selects suites by the workflow lists --- docs/source/contributor-guide/development.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md index 0cf2e367cd5..cfe288ffb3f 100644 --- a/docs/source/contributor-guide/development.md +++ b/docs/source/contributor-guide/development.md @@ -767,8 +767,10 @@ The group tag does not select an integration suite, one marked `@IntegrationTest `ParquetEncryptionITCase`. The ScalaTest plugin excludes those, and Maven runs them under `maven-failsafe-plugin` by their `*ITCase` name, which ignores the tag. The default `test` goal therefore never runs them, and `--goal install` runs every one of them whichever group you name. -Such a suite still carries a tag and a workflow listing so that the checks above hold, but until CI -runs them for one group only, the listing does not decide where they run. + +CI still selects suites by the lists in the workflow files, not by tag, and runs `install`. An +integration suite is excluded from that selection too, so it runs in every group's job whichever +group the list names it under. A small number of suites are deliberately **not** run in CI, because they need infrastructure CI does not have or because running them there is not worth the cost. These are listed in From 16e25826a15e56490f4b329ab4dd6d30159012f7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 12 Nov 2026 12:09:57 -0700 Subject: [PATCH 4/4] test: rename the CI test group tags to *TestGroup ExecGroup, ExpressionsGroup, ScansGroup and ShuffleGroup become ExecTestGroup, ExpressionsTestGroup, ScansTestGroup and ShuffleTestGroup, so the name says the tag selects a CI test group. check-suites.py, run-comet-tests.sh and the docs follow the new names. --- .ai/skills/review-comet-pr/SKILL.md | 4 ++-- dev/ci/check-suites.py | 10 +++++----- dev/ci/run-comet-tests.sh | 12 ++++++------ docs/source/contributor-guide/development.md | 16 ++++++++-------- .../{ExecGroup.java => ExecTestGroup.java} | 2 +- ...onsGroup.java => ExpressionsTestGroup.java} | 2 +- .../{ScansGroup.java => ScansTestGroup.java} | 2 +- ...ShuffleGroup.java => ShuffleTestGroup.java} | 2 +- .../comet/CometArrayExpressionSuite.scala | 4 ++-- .../comet/CometBitwiseExpressionSuite.scala | 4 ++-- .../comet/CometBroadcastBuildSideSuite.scala | 4 ++-- .../apache/comet/CometCodegenFuzzSuite.scala | 4 ++-- .../apache/comet/CometCodegenHOFSuite.scala | 4 ++-- .../apache/comet/CometCodegenSourceSuite.scala | 4 ++-- .../org/apache/comet/CometCodegenSuite.scala | 4 ++-- .../org/apache/comet/CometConfSuite.scala | 4 ++-- .../apache/comet/CometCoverageStatsSuite.scala | 4 ++-- .../apache/comet/CometCsvExpressionSuite.scala | 4 ++-- .../apache/comet/CometDateTimeUtilsSuite.scala | 4 ++-- .../apache/comet/CometExpressionSuite.scala | 4 ++-- .../comet/CometFallbackInvarianceSuite.scala | 4 ++-- .../comet/CometFloatSemanticsSuite.scala | 4 ++-- .../apache/comet/CometFuzzAggregateSuite.scala | 4 ++-- .../apache/comet/CometFuzzIcebergSuite.scala | 4 ++-- .../org/apache/comet/CometFuzzMathSuite.scala | 4 ++-- .../org/apache/comet/CometFuzzTestSuite.scala | 4 ++-- .../comet/CometHashExpressionSuite.scala | 4 ++-- .../apache/comet/CometIcebergHdfsSuite.scala | 4 ++-- .../apache/comet/CometIcebergNativeSuite.scala | 4 ++-- .../CometIcebergResidualPushdownSuite.scala | 4 ++-- .../comet/CometIcebergRewriteActionSuite.scala | 4 ++-- ...tIcebergSystemFunctionExtensionsSuite.scala | 4 ++-- .../CometIcebergSystemFunctionSuite.scala | 4 ++-- .../comet/CometIcebergWriteActionSuite.scala | 4 ++-- .../CometIcebergWriteDetectionSuite.scala | 4 ++-- .../comet/CometJsonExpressionSuite.scala | 4 ++-- .../org/apache/comet/CometJsonJvmSuite.scala | 4 ++-- .../apache/comet/CometMapExpressionSuite.scala | 4 ++-- .../comet/CometMathExpressionSuite.scala | 4 ++-- .../apache/comet/CometNativeCastSuite.scala | 4 ++-- .../org/apache/comet/CometNativeSuite.scala | 4 ++-- .../org/apache/comet/CometNativeUdfSuite.scala | 4 ++-- .../org/apache/comet/CometPublicApiSuite.scala | 4 ++-- .../org/apache/comet/CometRegExpJvmSuite.scala | 4 ++-- .../apache/comet/CometRegexParitySuite.scala | 4 ++-- .../comet/CometScalaUDFClassLoaderSuite.scala | 4 ++-- .../comet/CometSetOpWithGroupBySuite.scala | 4 ++-- .../CometSparkSessionExtensionsSuite.scala | 4 ++-- .../apache/comet/CometSqlFileTestSuite.scala | 4 ++-- .../comet/CometStringExpressionSuite.scala | 4 ++-- .../comet/CometTemporalExpressionSuite.scala | 4 ++-- .../comet/CometUuidExpressionSuite.scala | 4 ++-- .../comet/CometVariantProjectionSuite.scala | 4 ++-- .../apache/comet/CometVariantTypeSuite.scala | 4 ++-- .../org/apache/comet/DataGeneratorSuite.scala | 4 ++-- .../DirectColumnarToRowConverterSuite.scala | 4 ++-- .../org/apache/comet/GenerateDocsSuite.scala | 4 ++-- .../comet/QueryContextInternerSuite.scala | 4 ++-- .../comet/SparkErrorConverterSuite.scala | 4 ++-- .../apache/comet/SqlFileTestParserSuite.scala | 4 ++-- .../CometSpecializedGettersDispatchSuite.scala | 4 ++-- .../comet/csv/CometCsvNativeReadSuite.scala | 4 ++-- .../comet/exec/CometAggregateSuite.scala | 4 ++-- .../comet/exec/CometColumnarShuffleSuite.scala | 10 +++++----- .../exec/CometDirectColumnarToRowSuite.scala | 4 ++-- .../comet/exec/CometExec3_4PlusSuite.scala | 4 ++-- .../org/apache/comet/exec/CometExecSuite.scala | 4 ++-- .../comet/exec/CometGenerateExecSuite.scala | 4 ++-- .../exec/CometInMemoryCacheKryoSuite.scala | 8 ++++---- .../exec/CometInMemoryCachePruningSuite.scala | 4 ++-- .../comet/exec/CometInMemoryCacheSuite.scala | 4 ++-- .../org/apache/comet/exec/CometJoinSuite.scala | 4 ++-- .../comet/exec/CometNativeReaderSuite.scala | 4 ++-- .../comet/exec/CometNativeShuffleSuite.scala | 4 ++-- .../comet/exec/CometRangeExecSuite.scala | 4 ++-- .../CometShuffleInputConversionSuite.scala | 4 ++-- .../org/apache/comet/exec/CometTopKSuite.scala | 4 ++-- .../comet/exec/CometTypedDatasetSuite.scala | 4 ++-- .../comet/exec/CometWindowExecSuite.scala | 4 ++-- .../comet/expressions/CometRegexSuite.scala | 4 ++-- .../conditional/CometCaseWhenSuite.scala | 4 ++-- .../conditional/CometCoalesceSuite.scala | 4 ++-- .../expressions/conditional/CometIfSuite.scala | 4 ++-- .../comet/iceberg/IcebergReflectionSuite.scala | 4 ++-- .../objectstore/AbfsAuthParitySuite.scala | 4 ++-- .../objectstore/AbfsAuthResolverSuite.scala | 4 ++-- .../objectstore/AbfsScanFallbackSuite.scala | 4 ++-- .../comet/objectstore/NativeConfigSuite.scala | 4 ++-- .../parquet/CometParquetWriterRetrySuite.scala | 4 ++-- .../parquet/CometParquetWriterSuite.scala | 4 ++-- .../comet/parquet/ParquetReadSuite.scala | 4 ++-- .../ParquetTimestampLtzAsNtzSuite.scala | 4 ++-- .../comet/rules/CometExecRuleSuite.scala | 4 ++-- .../comet/rules/CometMultiStoreScanSuite.scala | 4 ++-- .../comet/rules/CometScanContribSuite.scala | 4 ++-- .../comet/rules/CometScanRuleSuite.scala | 4 ++-- .../rules/CometScanSchemeFallbackSuite.scala | 4 ++-- .../EliminateRedundantTransitionsSuite.scala | 4 ++-- ...rtNativeForTransitionHeavyStagesSuite.scala | 4 ++-- .../apache/comet/serde/CometLiteralSuite.scala | 4 ++-- .../comet/serde/CometScalarFunctionSuite.scala | 4 ++-- .../comet/serde/QueryPlanSerdeSuite.scala | 4 ++-- .../comet/serde/SerdeRegistrationSuite.scala | 4 ++-- .../apache/comet/serde/SparkFixesSuite.scala | 4 ++-- .../operator/CometIcebergNativeScanSuite.scala | 4 ++-- .../serde/operator/CometNativeScanSuite.scala | 4 ++-- .../IcebergWriteProtoTranslationSuite.scala | 4 ++-- .../CelebornShufflePartitionPusherSuite.scala | 4 ++-- .../apache/comet/udf/CometUdfBridgeSuite.scala | 4 ++-- .../comet/vector/CometVectorUtilsSuite.scala | 4 ++-- .../apache/comet/vector/NativeUtilSuite.scala | 4 ++-- .../CometExecIteratorLifecycleSuite.scala | 4 ++-- .../org/apache/spark/CometPluginsSuite.scala | 18 +++++++++--------- .../spark/CometRuntimeShutdownSuite.scala | 4 ++-- .../spark/CometTaskMemoryManagerSuite.scala | 4 ++-- ...tUnboundedShuffleMemoryAllocatorSuite.scala | 4 ++-- ...metUnifiedShuffleMemoryAllocatorSuite.scala | 4 ++-- .../spark/shuffle/sort/SpillSorterSuite.scala | 4 ++-- .../spark/sql/CometTPCDSQuerySuite.scala | 4 ++-- .../spark/sql/CometTPCDSQueryTestSuite.scala | 4 ++-- .../apache/spark/sql/CometTPCHQuerySuite.scala | 4 ++-- .../sql/comet/CometBroadcastValueSuite.scala | 4 ++-- .../sql/comet/CometCollectMetricsSuite.scala | 4 ++-- .../sql/comet/CometDecimalPromotionSuite.scala | 4 ++-- .../comet/CometDppFallbackRepro3949Suite.scala | 4 ++-- .../sql/comet/CometNativeWriteSuite.scala | 4 ++-- .../sql/comet/CometPlanEqualitySuite.scala | 4 ++-- .../sql/comet/CometPlanStabilitySuite.scala | 6 +++--- .../sql/comet/CometScanWithPlanDataSuite.scala | 4 ++-- .../comet/CometSessionExtensionsSuite.scala | 4 ++-- .../CometShuffleFallbackStickinessSuite.scala | 4 ++-- .../sql/comet/CometTaskMetricsSuite.scala | 4 ++-- .../sql/comet/CometUnsafeProjectionSuite.scala | 4 ++-- .../sql/comet/DecimalPrecisionSuite.scala | 4 ++-- .../sql/comet/ParquetEncryptionITCase.scala | 4 ++-- ...PlanDataInjectorShuffleLifecycleSuite.scala | 4 ++-- .../sql/comet/PlanDataInjectorSuite.scala | 4 ++-- .../arrow/CachedBatchRowIteratorSuite.scala | 4 ++-- .../arrow/CometArrowStreamSuite.scala | 4 ++-- .../arrow/CometArrowWriterSuite.scala | 4 ++-- .../arrow/CometStringWriterSuite.scala | 4 ++-- .../arrow/CometZstdCompressionCodecSuite.scala | 4 ++-- .../CometBlockStoreShuffleReaderSuite.scala | 4 ++-- ...elebornConcurrentMaterializationSuite.scala | 4 ++-- .../CometCelebornLocalFetchFailureSuite.scala | 4 ++-- ...CometCelebornNativeShuffleWriterSuite.scala | 4 ++-- .../CometCelebornShuffleFallbackSuite.scala | 4 ++-- .../CometCelebornShuffleManagerSuite.scala | 4 ++-- .../CometCelebornShufflePlanningSuite.scala | 4 ++-- .../CometCelebornShuffleReaderSuite.scala | 4 ++-- .../CometCelebornShuffleStatisticsSuite.scala | 4 ++-- .../shuffle/CometDiskBlockWriterSuite.scala | 4 ++-- .../CometNativePositionalRoundRobinSuite.scala | 4 ++-- .../CometNativeShuffleInputRDDSuite.scala | 4 ++-- .../CometNativeShuffleWriterSuite.scala | 4 ++-- .../NativeBatchDecoderIteratorSuite.scala | 4 ++-- .../spark/sql/comet/util/UtilsSuite.scala | 4 ++-- .../apache/comet/CometWidthBucketSuite.scala | 4 ++-- .../comet/exec/CometMergeRowsSuite.scala | 4 ++-- .../sql/CometSparkInternalFunctionsSuite.scala | 4 ++-- .../spark/sql/CometToPrettyStringSuite.scala | 4 ++-- .../apache/comet/CometStringDecodeSuite.scala | 4 ++-- .../comet/exec/CometMergeRowsSuite.scala | 4 ++-- .../apache/spark/sql/CometCollationSuite.scala | 4 ++-- .../comet/exec/CometMergeRowsSuite.scala | 4 ++-- .../apache/spark/sql/CometCollationSuite.scala | 4 ++-- .../sql/comet/CometArrowPythonUdfSuite.scala | 4 ++-- .../CometDecimalArithmeticViewSuite.scala | 4 ++-- .../comet/CometIcebergEncryptionSuite.scala | 4 ++-- ...etInMemoryCacheLastAttemptMetricSuite.scala | 4 ++-- .../comet/CometCastCollatedStringSuite.scala | 4 ++-- .../apache/comet/CometWidthBucketSuite.scala | 4 ++-- .../exec/CometEmptyRelationExecSuite.scala | 4 ++-- .../comet/exec/CometShuffle4_0Suite.scala | 4 ++-- .../CometEmptyRelationParquetWriterSuite.scala | 4 ++-- .../CometEmptyRelationExecRuleSuite.scala | 4 ++-- .../spark/sql/CometSortCollationSuite.scala | 4 ++-- .../sql/CometSparkInternalFunctionsSuite.scala | 4 ++-- .../spark/sql/CometToPrettyStringSuite.scala | 4 ++-- .../spark/sql/CometVariantShreddingSuite.scala | 4 ++-- .../spark/sql/comet/CometMapInBatchSuite.scala | 4 ++-- .../python/CometArrowPythonRunnerSuite.scala | 4 ++-- 182 files changed, 386 insertions(+), 386 deletions(-) rename spark/src/test/java/org/apache/comet/tags/{ExecGroup.java => ExecTestGroup.java} (97%) rename spark/src/test/java/org/apache/comet/tags/{ExpressionsGroup.java => ExpressionsTestGroup.java} (96%) rename spark/src/test/java/org/apache/comet/tags/{ScansGroup.java => ScansTestGroup.java} (97%) rename spark/src/test/java/org/apache/comet/tags/{ShuffleGroup.java => ShuffleTestGroup.java} (96%) diff --git a/.ai/skills/review-comet-pr/SKILL.md b/.ai/skills/review-comet-pr/SKILL.md index f7a15470c76..39a3eb4fee2 100644 --- a/.ai/skills/review-comet-pr/SKILL.md +++ b/.ai/skills/review-comet-pr/SKILL.md @@ -244,8 +244,8 @@ config, not a testing one, to turn it off before merge (#6466). - Do the tests compare against Spark? A comparison with Comet's own accumulator, or with a second implementation such as iceberg-rust instead of the Iceberg Java that Spark runs, can agree on the wrong answer (#6423, #6426). -- Each new concrete suite needs exactly one test-group tag (`@ScansGroup`, `@ShuffleGroup`, - `@ExecGroup` or `@ExpressionsGroup`), and a listing under that group in both +- Each new concrete suite needs exactly one test-group tag (`@ScansTestGroup`, `@ShuffleTestGroup`, + `@ExecTestGroup` or `@ExpressionsTestGroup`), and a listing under that group in both `.github/workflows/pr_build_linux.yml` and `pr_build_macos.yml`. `dev/ci/check-suites.py` enforces both; the group should match the area the suite covers. diff --git a/dev/ci/check-suites.py b/dev/ci/check-suites.py index 404d249d42d..97d6a51c460 100644 --- a/dev/ci/check-suites.py +++ b/dev/ci/check-suites.py @@ -18,7 +18,7 @@ CI runs the Comet test suites in groups, one job per group. Each concrete suite carries the ScalaTest tag of its group (see -spark/src/test/java/org/apache/comet/tags/*Group.java), and the CI workflows +spark/src/test/java/org/apache/comet/tags/*TestGroup.java), and the CI workflows list it in the group of the same name. This fails on a suite with no group tag, which would run in no job, on one with two, and on a suite whose workflow listing does not match its tag. Suites deliberately left out of CI are listed @@ -31,9 +31,9 @@ REPO_ROOT = Path(__file__).resolve().parents[2] -# One ScalaTest tag annotation per group, such as ExecGroup for the `exec` group. +# One ScalaTest tag annotation per group, such as ExecTestGroup for the `exec` group. TAGS_DIR = REPO_ROOT / "spark/src/test/java/org/apache/comet/tags" -GROUP_TAGS = sorted(path.stem for path in TAGS_DIR.glob("*Group.java")) +GROUP_TAGS = sorted(path.stem for path in TAGS_DIR.glob("*TestGroup.java")) # Suites that CI does not run, each with the reason. IGNORE_LIST = { @@ -101,8 +101,8 @@ def workflow_groups(workflow): def group_name(tag): - """ExecGroup -> exec""" - return tag[: -len("Group")].lower() + """ExecTestGroup -> exec""" + return tag[: -len("TestGroup")].lower() def main(): diff --git a/dev/ci/run-comet-tests.sh b/dev/ci/run-comet-tests.sh index 4f0742ca232..50249fa04c2 100755 --- a/dev/ci/run-comet-tests.sh +++ b/dev/ci/run-comet-tests.sh @@ -18,7 +18,7 @@ # under the License. # # Run one group of the Comet JVM test suites. Each concrete suite carries one -# group tag, such as org.apache.comet.tags.ExecGroup, and CI runs each group as +# group tag, such as org.apache.comet.tags.ExecTestGroup, and CI runs each group as # its own job. # Arguments after the group go to Maven: # @@ -35,13 +35,13 @@ set -euo pipefail REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -# Each group is a tag annotation here, such as ExecGroup.java for `exec`. +# Each group is a tag annotation here, such as ExecTestGroup.java for `exec`. TAGS_DIR="$REPO/spark/src/test/java/org/apache/comet/tags" usage() { groups="" - for tag in "$TAGS_DIR"/*Group.java; do - tag="$(basename "$tag" Group.java)" + for tag in "$TAGS_DIR"/*TestGroup.java; do + tag="$(basename "$tag" TestGroup.java)" groups="$groups $(printf '%s' "$tag" | tr '[:upper:]' '[:lower:]')" done cat >&2 < ExecGroup -TAG="$(printf '%s' "${1:0:1}" | tr '[:lower:]' '[:upper:]')${1:1}Group" +# exec -> ExecTestGroup +TAG="$(printf '%s' "${1:0:1}" | tr '[:lower:]' '[:upper:]')${1:1}TestGroup" if [ ! -f "$TAGS_DIR/$TAG.java" ]; then echo "Unknown test group: $1" >&2 usage diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md index 3a3b0c1c911..34d344ca0c8 100644 --- a/docs/source/contributor-guide/development.md +++ b/docs/source/contributor-guide/development.md @@ -713,19 +713,19 @@ is listed under the group in the `suite` matrix of **both** workflow files: - `.github/workflows/pr_build_linux.yml` - `.github/workflows/pr_build_macos.yml` -| Tag | Group | Covers | -| ------------------- | ------------- | -------------------------------------------------------------- | -| `@ScansGroup` | `scans` | Parquet, CSV and Iceberg reads and writes, and the fuzz suites | -| `@ShuffleGroup` | `shuffle` | Native and columnar shuffle, and the Celeborn integration | -| `@ExecGroup` | `exec` | Operators, plan rules, plugins, memory, and TPC queries | -| `@ExpressionsGroup` | `expressions` | Expression evaluation, casts, codegen, and the SQL file tests | +| Tag | Group | Covers | +| ----------------------- | ------------- | -------------------------------------------------------------- | +| `@ScansTestGroup` | `scans` | Parquet, CSV and Iceberg reads and writes, and the fuzz suites | +| `@ShuffleTestGroup` | `shuffle` | Native and columnar shuffle, and the Celeborn integration | +| `@ExecTestGroup` | `exec` | Operators, plan rules, plugins, memory, and TPC queries | +| `@ExpressionsTestGroup` | `expressions` | Expression evaluation, casts, codegen, and the SQL file tests | For example, a new expression suite goes in the `expressions` group: ```scala -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class YourNewExpressionSuite extends CometTestBase { ``` diff --git a/spark/src/test/java/org/apache/comet/tags/ExecGroup.java b/spark/src/test/java/org/apache/comet/tags/ExecTestGroup.java similarity index 97% rename from spark/src/test/java/org/apache/comet/tags/ExecGroup.java rename to spark/src/test/java/org/apache/comet/tags/ExecTestGroup.java index 698e3ec44ba..16f946ff088 100644 --- a/spark/src/test/java/org/apache/comet/tags/ExecGroup.java +++ b/spark/src/test/java/org/apache/comet/tags/ExecTestGroup.java @@ -30,4 +30,4 @@ @TagAnnotation @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.TYPE}) -public @interface ExecGroup {} +public @interface ExecTestGroup {} diff --git a/spark/src/test/java/org/apache/comet/tags/ExpressionsGroup.java b/spark/src/test/java/org/apache/comet/tags/ExpressionsTestGroup.java similarity index 96% rename from spark/src/test/java/org/apache/comet/tags/ExpressionsGroup.java rename to spark/src/test/java/org/apache/comet/tags/ExpressionsTestGroup.java index c7f37f5d107..b9fac52b36c 100644 --- a/spark/src/test/java/org/apache/comet/tags/ExpressionsGroup.java +++ b/spark/src/test/java/org/apache/comet/tags/ExpressionsTestGroup.java @@ -30,4 +30,4 @@ @TagAnnotation @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.TYPE}) -public @interface ExpressionsGroup {} +public @interface ExpressionsTestGroup {} diff --git a/spark/src/test/java/org/apache/comet/tags/ScansGroup.java b/spark/src/test/java/org/apache/comet/tags/ScansTestGroup.java similarity index 97% rename from spark/src/test/java/org/apache/comet/tags/ScansGroup.java rename to spark/src/test/java/org/apache/comet/tags/ScansTestGroup.java index 3ffd50c6846..efe21288d24 100644 --- a/spark/src/test/java/org/apache/comet/tags/ScansGroup.java +++ b/spark/src/test/java/org/apache/comet/tags/ScansTestGroup.java @@ -30,4 +30,4 @@ @TagAnnotation @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.TYPE}) -public @interface ScansGroup {} +public @interface ScansTestGroup {} diff --git a/spark/src/test/java/org/apache/comet/tags/ShuffleGroup.java b/spark/src/test/java/org/apache/comet/tags/ShuffleTestGroup.java similarity index 96% rename from spark/src/test/java/org/apache/comet/tags/ShuffleGroup.java rename to spark/src/test/java/org/apache/comet/tags/ShuffleTestGroup.java index 63cc93c76df..310f531b0a9 100644 --- a/spark/src/test/java/org/apache/comet/tags/ShuffleGroup.java +++ b/spark/src/test/java/org/apache/comet/tags/ShuffleTestGroup.java @@ -30,4 +30,4 @@ @TagAnnotation @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.TYPE}) -public @interface ShuffleGroup {} +public @interface ShuffleTestGroup {} diff --git a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala index d98e6e34edc..7d608fc4475 100644 --- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala @@ -37,7 +37,7 @@ import org.apache.spark.sql.types.{ArrayType, DataType, DoubleType, FloatType, I import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus} import org.apache.comet.DataTypeSupport.isComplexType import org.apache.comet.serde.{ArrayElementEqualitySupport, ArraySetSupport, CometArrayDistinct, CometArrayExcept, CometArrayJoin, CometArrayRemove, CometArrayReverse, CometArraysOverlap, CometArrayUnion, CometFlatten, CometSequence, Compatible, ExprOuterClass, Incompatible, SparkFixes} -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} /** Counts calls to a test UDF. Comet tests run in local mode, so tasks see this object. */ @@ -45,7 +45,7 @@ object SizeChildCounter { val calls = new AtomicLong(0) } -@ExpressionsGroup +@ExpressionsTestGroup class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("array set signed-zero support levels") { diff --git a/spark/src/test/scala/org/apache/comet/CometBitwiseExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometBitwiseExpressionSuite.scala index 46b2cc7d57d..d51a9b1e03a 100644 --- a/spark/src/test/scala/org/apache/comet/CometBitwiseExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometBitwiseExpressionSuite.scala @@ -25,10 +25,10 @@ import org.apache.hadoop.fs.Path import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} -@ExpressionsGroup +@ExpressionsTestGroup class CometBitwiseExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("bitwise expressions") { diff --git a/spark/src/test/scala/org/apache/comet/CometBroadcastBuildSideSuite.scala b/spark/src/test/scala/org/apache/comet/CometBroadcastBuildSideSuite.scala index 6c4a213255c..5d43195d5d7 100644 --- a/spark/src/test/scala/org/apache/comet/CometBroadcastBuildSideSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometBroadcastBuildSideSuite.scala @@ -33,13 +33,13 @@ import org.apache.spark.sql.execution.joins.BroadcastNestedLoopJoinExec import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometSparkSessionExtensions.isSpark35Plus -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup // Tests for the broadcast build-side bridge, spark.comet.convert.broadcastBuildSide.enabled (issue // #6008). A build-side scan that Comet cannot read natively, such as ORC or an Iceberg table // the native reader declines, is converted to Arrow so that the broadcast join runs natively. Each // query is compared with Spark, and the tests assert where the bridge must and must not appear. -@ScansGroup +@ScansTestGroup class CometBroadcastBuildSideSuite extends CometTestBase with CometIcebergTestBase { // Most tests use an ORC build side. Text, Parquet, JSON and CSV scans are converted only by their diff --git a/spark/src/test/scala/org/apache/comet/CometCodegenFuzzSuite.scala b/spark/src/test/scala/org/apache/comet/CometCodegenFuzzSuite.scala index bc8b308346d..10d4eb00de0 100644 --- a/spark/src/test/scala/org/apache/comet/CometCodegenFuzzSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCodegenFuzzSuite.scala @@ -37,7 +37,7 @@ import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.DataTypeSupport.isComplexType import org.apache.comet.codegen.CometBatchKernelCodegen -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, ParquetGenerator, SchemaGenOptions} import org.apache.comet.vector.CometVector @@ -48,7 +48,7 @@ import org.apache.comet.vector.CometVector * (not [[CometFuzzTestBase]]) because the base's per-shuffle-mode runs are irrelevant for * projection-only queries. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometCodegenFuzzSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometCodegenHOFSuite.scala b/spark/src/test/scala/org/apache/comet/CometCodegenHOFSuite.scala index e1aca516a0a..fd29d7b3b1f 100644 --- a/spark/src/test/scala/org/apache/comet/CometCodegenHOFSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCodegenHOFSuite.scala @@ -23,7 +23,7 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup /** * Higher-order function regression coverage for the codegen dispatcher. @@ -43,7 +43,7 @@ import org.apache.comet.tags.ExpressionsGroup * the per-task instance, not the companion, so concurrent partitions cannot race on a shared * `NamedLambdaVariable.value`. The two-collects test below regresses this. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometCodegenHOFSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometCodegenSourceSuite.scala b/spark/src/test/scala/org/apache/comet/CometCodegenSourceSuite.scala index 019551dca00..230bf253ada 100644 --- a/spark/src/test/scala/org/apache/comet/CometCodegenSourceSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCodegenSourceSuite.scala @@ -31,7 +31,7 @@ import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.codegen.CometBatchKernelCodegen import org.apache.comet.codegen.CometBatchKernelCodegen.{ArrayColumnSpec, ArrowColumnSpec, MapColumnSpec, ScalarColumnSpec, StructColumnSpec, StructFieldSpec} -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.udf.codegen.CometScalaUDFCodegen // Resolve Arrow vector classes through the codegen object so tests see the same `Class` objects @@ -54,7 +54,7 @@ import org.apache.comet.udf.codegen.CometScalaUDFCodegen * These are the smallest durable tests that the claimed optimizations actually reach the * generated Java, and they document the shapes future contributors should preserve. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometCodegenSourceSuite extends AnyFunSuite { private val varCharVectorClass = diff --git a/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala b/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala index 144a800944f..05090207f43 100644 --- a/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala @@ -45,7 +45,7 @@ import org.apache.comet.codegen.{CometBatchKernelCodegen, DispatchOccurrence} import org.apache.comet.codegen.CometBatchKernelCodegen.ArrowColumnSpec import org.apache.comet.serde.{CometInvokeTargets, CometScalaUDF, QueryPlanSerde} import org.apache.comet.serde.ExprOuterClass.Expr.ExprStructCase -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.udf.CometUdfBridge import org.apache.comet.udf.codegen.CometScalaUDFCodegen import org.apache.comet.vector.CometVector @@ -59,7 +59,7 @@ import org.apache.comet.vector.CometVector * rather than `checkSparkAnswerAndOperator` because ScalaUDF has no Comet-native path. Under * fallback the project runs on the JVM Spark path. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometCodegenSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometConfSuite.scala b/spark/src/test/scala/org/apache/comet/CometConfSuite.scala index 39e29df298a..a44da43d5e4 100644 --- a/spark/src/test/scala/org/apache/comet/CometConfSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometConfSuite.scala @@ -23,9 +23,9 @@ import org.scalatest.funsuite.AnyFunSuite import org.apache.spark.sql.internal.SQLConf -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometConfSuite extends AnyFunSuite { test("primary key wins over alternative when both are set") { diff --git a/spark/src/test/scala/org/apache/comet/CometCoverageStatsSuite.scala b/spark/src/test/scala/org/apache/comet/CometCoverageStatsSuite.scala index a3c4ba8eae6..2de20f93a98 100644 --- a/spark/src/test/scala/org/apache/comet/CometCoverageStatsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCoverageStatsSuite.scala @@ -26,13 +26,13 @@ import org.apache.spark.sql.execution.{ReusedSubqueryExec, SparkPlan, SubqueryEx import org.apache.spark.sql.execution.exchange.ReusedExchangeExec import org.apache.spark.sql.internal.SQLConf -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup /** * Tests for how [[CometCoverageStats]] categorizes plan nodes into accelerated operators, * un-accelerated Spark operators, and transitions. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometCoverageStatsSuite extends CometTestBase { private def somePlan: SparkPlan = diff --git a/spark/src/test/scala/org/apache/comet/CometCsvExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometCsvExpressionSuite.scala index b2e22e52745..217f0833e88 100644 --- a/spark/src/test/scala/org/apache/comet/CometCsvExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCsvExpressionSuite.scala @@ -29,10 +29,10 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StringType -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} -@ExpressionsGroup +@ExpressionsTestGroup class CometCsvExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("to_csv - default options") { diff --git a/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala b/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala index aa464212337..3a7569ea31f 100644 --- a/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala @@ -26,14 +26,14 @@ import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.DataTypes -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup /** * Tests for string-to-timestamp (and string-to-date) cast correctness, ported from Spark's * DateTimeUtilsSuite. That suite lives in sql/catalyst and does not extend CometTestBase, so it * never runs in Comet CI. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometDateTimeUtilsSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 618dc01d7bb..63b4b6a3176 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -36,10 +36,10 @@ import org.apache.spark.sql.types._ import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, isSpark41Plus, isSpark42Plus} import org.apache.comet.serde.{CometAdd, CometDivide, CometIntegralDivide, CometMultiply, CometRemainder, CometSubtract, Compatible, Unsupported} -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} -@ExpressionsGroup +@ExpressionsTestGroup class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/CometFallbackInvarianceSuite.scala b/spark/src/test/scala/org/apache/comet/CometFallbackInvarianceSuite.scala index 14c0d76e171..a047911ab34 100644 --- a/spark/src/test/scala/org/apache/comet/CometFallbackInvarianceSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFallbackInvarianceSuite.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} import org.apache.spark.sql.internal.SQLConf -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup /** * Checks that forcing an expression from Comet's native path back to Spark does not change the @@ -52,7 +52,7 @@ import org.apache.comet.tags.ExpressionsGroup * everything and the gate above would be inoperative. Row order is canonicalised in the * comparator instead. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometFallbackInvarianceSuite extends CometFuzzTestBase { /** diff --git a/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala b/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala index e28a1291d76..eb04f30ca51 100644 --- a/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala @@ -23,7 +23,7 @@ import org.scalatest.exceptions.TestFailedException import org.apache.spark.sql.CometTestBase -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup /** * Sweeps Spark's `-0.0` and NaN semantics across operator contexts and the expressions that @@ -38,7 +38,7 @@ import org.apache.comet.tags.ExpressionsGroup * answer from Spark. The case fails once it matches, so that a fix, or a DataFusion upgrade that * changes float behavior, removes its entry from the list instead of leaving it stale. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometFloatSemanticsSuite extends CometTestBase { import CometFloatSemanticsSuite._ diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzAggregateSuite.scala b/spark/src/test/scala/org/apache/comet/CometFuzzAggregateSuite.scala index 0fd06e33c26..e532a6d12e9 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzAggregateSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzAggregateSuite.scala @@ -20,9 +20,9 @@ package org.apache.comet import org.apache.comet.DataTypeSupport.isComplexType -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometFuzzAggregateSuite extends CometFuzzTestBase { test("count distinct - simple columns") { diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzIcebergSuite.scala b/spark/src/test/scala/org/apache/comet/CometFuzzIcebergSuite.scala index 2b1838b6800..87a40503416 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzIcebergSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzIcebergSuite.scala @@ -28,9 +28,9 @@ import org.apache.spark.sql.types._ import org.apache.comet.DataTypeSupport.isComplexType import org.apache.comet.serde.OperatorOuterClass -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup -@ScansGroup +@ScansTestGroup class CometFuzzIcebergSuite extends CometFuzzIcebergBase { test("select *") { diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzMathSuite.scala b/spark/src/test/scala/org/apache/comet/CometFuzzMathSuite.scala index 3bc6b0ded5c..b849f1288aa 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzMathSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzMathSuite.scala @@ -23,9 +23,9 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DecimalType, IntegerType, LongType} -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class CometFuzzMathSuite extends CometFuzzTestBase { // The integer-math fuzz tests intentionally cover overflowing inputs; under Spark 4's diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala index abb440210db..508c4b737c0 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala @@ -28,11 +28,11 @@ import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType import org.apache.spark.sql.types._ import org.apache.comet.DataTypeSupport.isComplexType -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator} import org.apache.comet.testing.FuzzDataGenerator.{doubleNaNLiteral, floatNaNLiteral} -@ScansGroup +@ScansTestGroup class CometFuzzTestSuite extends CometFuzzTestBase { test("select *") { diff --git a/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala index 9c54e5d3d00..57303d7e069 100644 --- a/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala @@ -24,7 +24,7 @@ import scala.util.Random import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, ParquetGenerator, SchemaGenOptions} import org.apache.comet.udf.codegen.CometScalaUDFCodegen @@ -34,7 +34,7 @@ import org.apache.comet.udf.codegen.CometScalaUDFCodegen * These tests verify that Comet's native implementations produce identical results to Spark for * all supported data types. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometHashExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { private def checkNativeHashes(query: String): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergHdfsSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergHdfsSuite.scala index 03c41259c9c..eb64c8deeb7 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergHdfsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergHdfsSuite.scala @@ -45,7 +45,7 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus import org.apache.comet.iceberg.IcebergReflection -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup /** * End-to-end coverage of the native Iceberg scan against an `hdfs://` warehouse, backed by @@ -86,7 +86,7 @@ import org.apache.comet.tags.ScansGroup * the fix once an Iceberg runtime exists. Until then this suite skips there and starts * nothing. */ -@ScansGroup +@ScansTestGroup class CometIcebergHdfsSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala index f054a9d6f71..b58fd5bb970 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala @@ -53,7 +53,7 @@ import org.apache.comet.cloud.s3.MinioLocationScopedCredentialProvider import org.apache.comet.iceberg.{IcebergReflection, RESTCatalogHelper} import org.apache.comet.rules.CometScanRule import org.apache.comet.serde.{OperatorOuterClass, QueryPlanSerde} -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup import org.apache.comet.testing.{FuzzDataGenerator, SchemaGenOptions} /** @@ -61,7 +61,7 @@ import org.apache.comet.testing.{FuzzDataGenerator, SchemaGenOptions} * * Note: Requires Iceberg dependencies to be added to pom.xml */ -@ScansGroup +@ScansTestGroup class CometIcebergNativeSuite extends CometTestBase with RESTCatalogHelper diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergResidualPushdownSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergResidualPushdownSuite.scala index b16cd56b630..a0d3d0cbc08 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergResidualPushdownSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergResidualPushdownSuite.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.comet.CometIcebergNativeScanExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup /** * Residual pushdown for predicates that only reach the Iceberg scan filter when Iceberg's SQL @@ -35,7 +35,7 @@ import org.apache.comet.tags.ScansGroup * so Iceberg can push the comparison, which puts an `UnboundTransform` term into each file's * residual. CometIcebergNativeSuite runs without the extensions, so it never sees one. */ -@ScansGroup +@ScansTestGroup class CometIcebergResidualPushdownSuite extends CometTestBase with CometIcebergTestBase diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergRewriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergRewriteActionSuite.scala index 73ccb969eda..e61cd8bea14 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergRewriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergRewriteActionSuite.scala @@ -34,7 +34,7 @@ import org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart import org.apache.comet.CometSparkSessionExtensions.isSpark35Plus import org.apache.comet.iceberg.IcebergReflection -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup /** * Verifies that Comet engages on the per-group reads issued by Iceberg's @@ -42,7 +42,7 @@ import org.apache.comet.tags.ScansGroup * via ScanTaskSetManager and then reads them back through SparkStagedScan, which Comet's * CometScanRule recognises alongside SparkBatchQueryScan. */ -@ScansGroup +@ScansTestGroup class CometIcebergRewriteActionSuite extends CometTestBase with CometIcebergTestBase { private val catalog = "test_cat" diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionExtensionsSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionExtensionsSuite.scala index d09244f4fd4..214fd8bb533 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionExtensionsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionExtensionsSuite.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression import org.apache.spark.sql.catalyst.plans.logical.Filter -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup /** * Iceberg's system functions once Iceberg's SQL extensions have rewritten them. @@ -35,7 +35,7 @@ import org.apache.comet.tags.ScansGroup * it with the same native kernels as the `StaticInvoke` that CometIcebergSystemFunctionSuite * covers. `spark.sql.extensions` is static, so this path needs a suite of its own. */ -@ScansGroup +@ScansTestGroup class CometIcebergSystemFunctionExtensionsSuite extends CometTestBase with CometIcebergTestBase { override protected def sparkConf: SparkConf = diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala index 7dcd368da2c..47d4c1ed35f 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala @@ -43,7 +43,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.comet.serde.{CometApplyFunctionExpression, CometExpressionSerde, CometIcebergBucket, CometIcebergSystemFunctions, CometIcebergTruncate, CometStaticInvoke, Compatible, SupportLevel, Unsupported} -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup /** * Native support for Iceberg's system functions (`bucket`, `truncate`, `years`, `months`, `days`, @@ -56,7 +56,7 @@ import org.apache.comet.tags.ScansGroup * out-of-order partitions); in a filter or projection it would be a silently wrong answer, which * is why the coverage is per type rather than a few hand-picked rows. */ -@ScansGroup +@ScansTestGroup class CometIcebergSystemFunctionSuite extends CometTestBase with CometIcebergTestBase diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index 5c9fe90819d..0c12578c0b1 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -60,7 +60,7 @@ import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark41Plu import org.apache.comet.iceberg.{IcebergReflection, IcebergWriteReportListener, PositionDeltaWrite} import org.apache.comet.serde.Unsupported import org.apache.comet.serde.operator.CometIcebergNativeWrite -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup private case class WriteSnapshot(snapshotDelta: Long, plans: Seq[SparkPlan]) @@ -70,7 +70,7 @@ private case class ReportedWrite( reasons: Seq[String], failed: Boolean) -@ScansGroup +@ScansTestGroup class CometIcebergWriteActionSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index 9aee45d2e27..44845e48663 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -48,9 +48,9 @@ import org.apache.comet.rules.EliminateRedundantTransitions import org.apache.comet.serde.{Compatible, SupportLevel, Unsupported} import org.apache.comet.serde.OperatorOuterClass.Operator import org.apache.comet.serde.operator.CometIcebergNativeWrite -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup -@ScansGroup +@ScansTestGroup class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTestBase { override protected def sparkConf: SparkConf = { diff --git a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala index 8dc108259a2..11fa06e5929 100644 --- a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala @@ -31,10 +31,10 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions._ import org.apache.comet.serde.CometStructsToJson -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} -@ExpressionsGroup +@ExpressionsTestGroup class CometJsonExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit diff --git a/spark/src/test/scala/org/apache/comet/CometJsonJvmSuite.scala b/spark/src/test/scala/org/apache/comet/CometJsonJvmSuite.scala index ce12e86b16e..d37532b0f95 100644 --- a/spark/src/test/scala/org/apache/comet/CometJsonJvmSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometJsonJvmSuite.scala @@ -23,9 +23,9 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class CometJsonJvmSuite extends CometTestBase with AdaptiveSparkPlanHelper { // No per-expression `allowIncompatible` is set, so all JSON expressions run through the diff --git a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala index ab6443fa20c..cba9aa8f85d 100644 --- a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala @@ -29,10 +29,10 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.BinaryType import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} -@ExpressionsGroup +@ExpressionsTestGroup class CometMapExpressionSuite extends CometTestBase { test("read map[int, int] from parquet") { diff --git a/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala index 0e1f32dda0a..5c33ed20f2e 100644 --- a/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala @@ -27,10 +27,10 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataTypes, StructField, StructType} import org.apache.comet.CometSparkSessionExtensions.isSpark35Plus -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} -@ExpressionsGroup +@ExpressionsTestGroup class CometMathExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("abs") { diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index cb9925198a6..136fa9f58f1 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -40,7 +40,7 @@ import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, import org.apache.comet.expressions.{CometCast, CometEvalMode} import org.apache.comet.rules.CometScanTypeChecker import org.apache.comet.serde.{Compatible, Incompatible, QueryPlanSerde, SparkFixes, Unsupported} -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup /** * Spark-parity coverage for Comet's **native** `Cast` implementation, and for the @@ -63,7 +63,7 @@ import org.apache.comet.tags.ExpressionsGroup * through the dispatcher instead. Adding a native cast implementation therefore means moving a * pair out of the `Unsupported` assertions and into the parity matrix below. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeSuite.scala index fa62424afe1..01c6bbca5ea 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeSuite.scala @@ -30,9 +30,9 @@ import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.{LongType, StructField, StructType} import org.apache.spark.sql.vectorized.ColumnarBatch -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometNativeSuite extends CometTestBase { test("test handling NPE thrown by JVM") { val rdd = spark.range(0, 1).rdd.map { value => diff --git a/spark/src/test/scala/org/apache/comet/CometNativeUdfSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeUdfSuite.scala index 344ed527a7a..dee24fbe65f 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeUdfSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeUdfSuite.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.comet.CometProjectExec import org.apache.spark.sql.functions.{col, expr, struct} import org.apache.spark.sql.types._ -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.udf.{CometNativeUDF, CometNativeUdfAbiException, CometNativeUdfLoadException, CometNativeUdfNotAllowedException} /** @@ -47,7 +47,7 @@ import org.apache.comet.udf.{CometNativeUDF, CometNativeUdfAbiException, CometNa * ./mvnw test -Dsuites="org.apache.comet.CometNativeUdfSuite" -Dtest=none * }}} */ -@ExpressionsGroup +@ExpressionsTestGroup class CometNativeUdfSuite extends CometTestBase { private lazy val libPath: String = { diff --git a/spark/src/test/scala/org/apache/comet/CometPublicApiSuite.scala b/spark/src/test/scala/org/apache/comet/CometPublicApiSuite.scala index fbdac66ed4c..42a0bdb2379 100644 --- a/spark/src/test/scala/org/apache/comet/CometPublicApiSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometPublicApiSuite.scala @@ -27,7 +27,7 @@ import scala.jdk.CollectionConverters._ import org.scalatest.funsuite.AnyFunSuite import org.apache.comet.annotation.Public -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Guards the enumerated public API described in `docs/source/about/versioning_policy.md`. @@ -37,7 +37,7 @@ import org.apache.comet.tags.ExecGroup * accident, or the policy is genuinely changing and both the doc and this list need updating in * the same PR. */ -@ExecGroup +@ExecTestGroup class CometPublicApiSuite extends AnyFunSuite { private val expected: Set[String] = Set( diff --git a/spark/src/test/scala/org/apache/comet/CometRegExpJvmSuite.scala b/spark/src/test/scala/org/apache/comet/CometRegExpJvmSuite.scala index 8bd60e410c0..f99a54f67a5 100644 --- a/spark/src/test/scala/org/apache/comet/CometRegExpJvmSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometRegExpJvmSuite.scala @@ -23,11 +23,11 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.functions.col import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus} -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup // Regex expressions other than in-subset `rlike` run through the codegen dispatcher by default // (Spark's own code, enabled by default) rather than the native rust path. -@ExpressionsGroup +@ExpressionsTestGroup class CometRegExpJvmSuite extends CometTestBase { // A Java-only pattern used to check the opt-in behavior when Rust rejects it. diff --git a/spark/src/test/scala/org/apache/comet/CometRegexParitySuite.scala b/spark/src/test/scala/org/apache/comet/CometRegexParitySuite.scala index da51027d046..cb50ba13b7d 100644 --- a/spark/src/test/scala/org/apache/comet/CometRegexParitySuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometRegexParitySuite.scala @@ -26,14 +26,14 @@ import org.apache.spark.sql.types.{StringType, StructField, StructType} import org.apache.comet.expressions.{CometRegex, RegexFlavor} import org.apache.comet.serde.Compatible -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup /** * Differential corpus: every pattern the plan-time analyzer admits is evaluated on Spark (Java * `Pattern`) and on Comet's native Rust path, and the results must match. Routing is asserted via * extended explain so a dispatcher fallback cannot silently satisfy the comparison. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometRegexParitySuite extends CometTestBase with AdaptiveSparkPlanHelper { // One pattern per whitelist production, plus the lexer-boundary cases. diff --git a/spark/src/test/scala/org/apache/comet/CometScalaUDFClassLoaderSuite.scala b/spark/src/test/scala/org/apache/comet/CometScalaUDFClassLoaderSuite.scala index bfd8cfc5c02..f75d9a748e0 100644 --- a/spark/src/test/scala/org/apache/comet/CometScalaUDFClassLoaderSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometScalaUDFClassLoaderSuite.scala @@ -29,7 +29,7 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.{CometTestBase, Encoders} import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup /** * Regression coverage for UDF closures whose capturing class lives in a user jar, which used to @@ -52,7 +52,7 @@ import org.apache.comet.tags.ExpressionsGroup * `MutableURLClassLoader`, which becomes the task thread's context ClassLoader while staying * invisible to the ClassLoader that loaded Comet. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometScalaUDFClassLoaderSuite extends CometTestBase { import CometScalaUDFClassLoaderSuite._ diff --git a/spark/src/test/scala/org/apache/comet/CometSetOpWithGroupBySuite.scala b/spark/src/test/scala/org/apache/comet/CometSetOpWithGroupBySuite.scala index b82d8250c57..af35dd63b73 100644 --- a/spark/src/test/scala/org/apache/comet/CometSetOpWithGroupBySuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSetOpWithGroupBySuite.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.comet.CometUnionExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.internal.SQLConf -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Regression test for issue #4122: on Spark 4.1 (SPARK-52921), EXCEPT ALL / INTERSECT ALL whose @@ -33,7 +33,7 @@ import org.apache.comet.tags.ExecGroup * If Comet's columnar Union concatenates partitions it breaks that partitioning invariant and the * resulting sums/counts collapse two sides into the wrong partitions. */ -@ExecGroup +@ExecTestGroup class CometSetOpWithGroupBySuite extends CometTestBase with AdaptiveSparkPlanHelper { test("issue #4122: EXCEPT ALL with GROUP BY under both sides") { diff --git a/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala b/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala index 9cf5f12fd81..5fba5c6eecb 100644 --- a/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala @@ -30,9 +30,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.LongType import org.apache.comet.serde.OperatorOuterClass -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometSparkSessionExtensionsSuite extends CometTestBase { import CometSparkSessionExtensions._ diff --git a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala index ca947954c55..84ff0242dcb 100644 --- a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala @@ -31,9 +31,9 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.ArrayType -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { /** Check if the current Spark version meets a minimum version requirement. */ diff --git a/spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala index 84105eb8cbf..4dbf78f6a40 100644 --- a/spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala @@ -28,11 +28,11 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, DataTypes, StructField, StructType} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} import org.apache.comet.udf.codegen.CometScalaUDFCodegen -@ExpressionsGroup +@ExpressionsTestGroup class CometStringExpressionSuite extends CometTestBase with CometCodegenAssertions { // scalastyle:off private val edgeCases = Seq( diff --git a/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala index 729a64cf266..8322f3f030a 100644 --- a/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala @@ -36,10 +36,10 @@ import org.apache.spark.sql.types.{DataTypes, StructField, StructType} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.serde.{CometDateFormat, CometTimeZone, CometTruncDate, CometTruncTimestamp} -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} -@ExpressionsGroup +@ExpressionsTestGroup class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { /** Timezones used to verify that TimestampNTZ operations are timezone-independent. */ diff --git a/spark/src/test/scala/org/apache/comet/CometUuidExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometUuidExpressionSuite.scala index 22fbd383c22..f1c89637dde 100644 --- a/spark/src/test/scala/org/apache/comet/CometUuidExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometUuidExpressionSuite.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.expressions.Uuid import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions.col -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup /** * Bit-for-bit comparisons of seeded `uuid` against Spark. @@ -40,7 +40,7 @@ import org.apache.comet.tags.ExpressionsGroup * form is not, so building the expression directly gives all profiles a real cross-engine * assertion and removes that dependence. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometUuidExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("seeded uuid matches Spark bit for bit") { diff --git a/spark/src/test/scala/org/apache/comet/CometVariantProjectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometVariantProjectionSuite.scala index 39e5539327c..a8c53cf708a 100644 --- a/spark/src/test/scala/org/apache/comet/CometVariantProjectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometVariantProjectionSuite.scala @@ -34,9 +34,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{IntegerType, StructField, StructType} import org.apache.comet.serde.operator.CometNativeScan -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup -@ScansGroup +@ScansTestGroup class CometVariantProjectionSuite extends CometTestBase { override protected def sparkConf: SparkConf = super.sparkConf .set(SQLConf.USE_V1_SOURCE_LIST.key, "parquet") diff --git a/spark/src/test/scala/org/apache/comet/CometVariantTypeSuite.scala b/spark/src/test/scala/org/apache/comet/CometVariantTypeSuite.scala index 0641063da93..a23e0e07ca8 100644 --- a/spark/src/test/scala/org/apache/comet/CometVariantTypeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometVariantTypeSuite.scala @@ -33,9 +33,9 @@ import org.apache.spark.sql.types.{ArrayType, BinaryType, StructField, StructTyp import org.apache.comet.rules.CometScanTypeChecker import org.apache.comet.serde.{CometAttributeReference, QueryPlanSerde, Unsupported} -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class CometVariantTypeSuite extends AnyFunSuite { private val storageType = StructType( Seq( diff --git a/spark/src/test/scala/org/apache/comet/DataGeneratorSuite.scala b/spark/src/test/scala/org/apache/comet/DataGeneratorSuite.scala index 0d629c4df34..c04e865fda8 100644 --- a/spark/src/test/scala/org/apache/comet/DataGeneratorSuite.scala +++ b/spark/src/test/scala/org/apache/comet/DataGeneratorSuite.scala @@ -24,10 +24,10 @@ import scala.util.Random import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, MapType, StructField, StructType} -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, SchemaGenOptions} -@ScansGroup +@ScansTestGroup class DataGeneratorSuite extends CometTestBase { test("allowNull produces nulls in every type that honours it") { diff --git a/spark/src/test/scala/org/apache/comet/DirectColumnarToRowConverterSuite.scala b/spark/src/test/scala/org/apache/comet/DirectColumnarToRowConverterSuite.scala index 059111fabcc..1dea95c6f83 100644 --- a/spark/src/test/scala/org/apache/comet/DirectColumnarToRowConverterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/DirectColumnarToRowConverterSuite.scala @@ -35,14 +35,14 @@ import org.apache.spark.sql.types._ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.unsafe.types.UTF8String -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.{CometDictionary, CometDictionaryVector, CometPlainVector} /** * Verifies that [[DirectColumnarToRowConverter]] produces rows byte-identical to * `UnsafeProjection`, which matters because `UnsafeRow` equality and hashing are byte-wise. */ -@ExecGroup +@ExecTestGroup class DirectColumnarToRowConverterSuite extends AnyFunSuite { private def assertMatchesUnsafeProjection( diff --git a/spark/src/test/scala/org/apache/comet/GenerateDocsSuite.scala b/spark/src/test/scala/org/apache/comet/GenerateDocsSuite.scala index aa755990490..0c999d831d1 100644 --- a/spark/src/test/scala/org/apache/comet/GenerateDocsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/GenerateDocsSuite.scala @@ -22,9 +22,9 @@ package org.apache.comet import org.scalatest.funsuite.AnyFunSuite import org.apache.comet.serde.{CometMakeTimestamp, CometRLike, CometToUnixTimestamp, NativeOptInAvailable} -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class GenerateDocsSuite extends AnyFunSuite { private val configKey = "spark.comet.expression.RLike.allowIncompatible" diff --git a/spark/src/test/scala/org/apache/comet/QueryContextInternerSuite.scala b/spark/src/test/scala/org/apache/comet/QueryContextInternerSuite.scala index fb08537e445..10e11d331e6 100644 --- a/spark/src/test/scala/org/apache/comet/QueryContextInternerSuite.scala +++ b/spark/src/test/scala/org/apache/comet/QueryContextInternerSuite.scala @@ -25,9 +25,9 @@ import org.apache.spark.sql.{CometTestBase, DataFrame} import org.apache.spark.sql.comet.CometNativeExec import org.apache.comet.serde.{ExprOuterClass, OperatorOuterClass} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class QueryContextInternerSuite extends CometTestBase { private val sqlText = diff --git a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala index 0e5869f4820..dc5a95227ec 100644 --- a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala @@ -24,9 +24,9 @@ import org.scalatest.funsuite.AnyFunSuite import org.apache.spark.{SparkException, SparkThrowable} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class SparkErrorConverterSuite extends AnyFunSuite { test("ArithmeticOverflow preserves the suggested function") { diff --git a/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala b/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala index a2822e7111c..39a3a35dea8 100644 --- a/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala +++ b/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala @@ -25,14 +25,14 @@ import java.nio.file.Files import org.scalatest.funsuite.AnyFunSuite -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup /** * Unit tests for [[SqlFileTestParser]]. Pure text parsing, so no Spark session is needed. The * end-to-end behaviour of each query mode is covered by `CometSqlFileTestSuite` running the * fixtures themselves. */ -@ExpressionsGroup +@ExpressionsTestGroup class SqlFileTestParserSuite extends AnyFunSuite { private def parseQueries(lines: String*): Seq[SqlQuery] = diff --git a/spark/src/test/scala/org/apache/comet/codegen/CometSpecializedGettersDispatchSuite.scala b/spark/src/test/scala/org/apache/comet/codegen/CometSpecializedGettersDispatchSuite.scala index 538ac4541e3..ca65055b132 100644 --- a/spark/src/test/scala/org/apache/comet/codegen/CometSpecializedGettersDispatchSuite.scala +++ b/spark/src/test/scala/org/apache/comet/codegen/CometSpecializedGettersDispatchSuite.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.comet.util.Utils import org.apache.spark.sql.types.{DataType, DateType, LongType} import org.apache.comet.CometSparkSessionExtensions.isSpark41Plus -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup /** * Unit coverage for the generic `SpecializedGetters.get(ordinal, dataType)` dispatch the kernel @@ -36,7 +36,7 @@ import org.apache.comet.tags.ExpressionsGroup * has to match what `CometBatchKernelCodegen.isSupportedDataType` admits at plan time. A type * accepted there but missing here throws at execute time, when there is no longer a fallback. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometSpecializedGettersDispatchSuite extends AnyFunSuite { /** Minimal `CometInternalRow` over one fixed long, to drive the dispatch through `get`. */ diff --git a/spark/src/test/scala/org/apache/comet/csv/CometCsvNativeReadSuite.scala b/spark/src/test/scala/org/apache/comet/csv/CometCsvNativeReadSuite.scala index 75e40bf78fe..70c9f30c863 100644 --- a/spark/src/test/scala/org/apache/comet/csv/CometCsvNativeReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/csv/CometCsvNativeReadSuite.scala @@ -29,9 +29,9 @@ import org.apache.spark.sql.types.{IntegerType, StringType, StructType, Timestam import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.CometSparkSessionExtensions.isSpark41Plus -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup -@ScansGroup +@ScansTestGroup class CometCsvNativeReadSuite extends CometTestBase { private val TEST_CSV_PATH_NO_HEADER = "src/test/resources/test-data/csv-test-1.csv" private val TEST_CSV_PATH_HAS_HEADER = "src/test/resources/test-data/csv-test-2.csv" diff --git a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala index d7f7b879e6d..51687cc2ecd 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala @@ -47,13 +47,13 @@ import org.apache.comet.CometConf.COMET_EXEC_STRICT_FLOATING_POINT import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, isSpark41Plus, isSpark42Plus} import org.apache.comet.rules.CometExecRule import org.apache.comet.serde.{CometCorr, CometRegrR2, ExprOuterClass, SparkFixes} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, ParquetGenerator, SchemaGenOptions} /** * Test suite dedicated to Comet native aggregate operator */ -@ExecGroup +@ExecTestGroup class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala index 3f6475c5656..8eba9dffa10 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala @@ -40,7 +40,7 @@ import org.apache.spark.sql.types._ import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper { protected val adaptiveExecutionEnabled: Boolean @@ -936,7 +936,7 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar } } -@ShuffleGroup +@ShuffleTestGroup class CometShuffleSuite extends CometColumnarShuffleSuite { protected val adaptiveExecutionEnabled: Boolean = true @@ -988,7 +988,7 @@ class CometShuffleSuite extends CometColumnarShuffleSuite { } } -@ShuffleGroup +@ShuffleTestGroup class DisableAQECometShuffleSuite extends CometColumnarShuffleSuite { protected val adaptiveExecutionEnabled: Boolean = false @@ -1035,7 +1035,7 @@ class DisableAQECometShuffleSuite extends CometColumnarShuffleSuite { } } -@ShuffleGroup +@ShuffleTestGroup class CometShuffleEncryptionSuite extends CometTestBase { import testImplicits._ @@ -1071,7 +1071,7 @@ class CometShuffleEncryptionSuite extends CometTestBase { } } -@ShuffleGroup +@ShuffleTestGroup class CometShuffleManagerSuite extends CometTestBase { test("should not bypass merge sort if executor cores are too high") { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometDirectColumnarToRowSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometDirectColumnarToRowSuite.scala index bc56e4eab59..9982c5b0b19 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometDirectColumnarToRowSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometDirectColumnarToRowSuite.scala @@ -24,14 +24,14 @@ import org.apache.spark.sql.comet.CometColumnarToRowExec import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup /** * End-to-end tests for `spark.comet.exec.columnarToRow.direct.enabled`, which routes * `CometColumnarToRowExec`'s non-codegen conversion through `DirectColumnarToRowConverter`. * Whole-stage codegen is disabled so the operator's `doExecute` conversion path runs. */ -@ShuffleGroup +@ShuffleTestGroup class CometDirectColumnarToRowSuite extends CometTestBase { private def withDirectConverter(minBatchSize: Int = 0)(f: => Unit): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala index 7c200358049..0beb16520e2 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExec3_4PlusSuite.scala @@ -35,12 +35,12 @@ import org.apache.spark.util.sketch.BloomFilter import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * This test suite contains tests for only Spark 3.4+. */ -@ExecGroup +@ExecTestGroup class CometExec3_4PlusSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index eda8bdb98f1..165c42c0be6 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -56,10 +56,10 @@ import org.apache.comet.{CometConf, CometExecIterator, CometNativeException, Ext import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, isSpark41Plus, isSpark42Plus} import org.apache.comet.rules.CometCoalesceShufflePartitions import org.apache.comet.serde.Config.ConfigMap -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} -@ExecGroup +@ExecTestGroup class CometExecSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala index f88f5b3865b..1e1c3891615 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala @@ -27,9 +27,9 @@ import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometGenerateExecSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheKryoSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheKryoSuite.scala index ee4a6db92d3..cd86c258682 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheKryoSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheKryoSuite.scala @@ -32,7 +32,7 @@ import org.apache.spark.unsafe.types.UTF8String import com.esotericsoftware.kryo.Kryo import org.apache.comet.{CometConf, CometKryoRegistrator} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Covers Comet's cached batch format under `spark.kryo.registrationRequired=true`. @@ -48,7 +48,7 @@ import org.apache.comet.tags.ExecGroup * This needs its own suite because `spark.serializer` and `spark.kryo.registrator` are read when * `SparkEnv` builds the serializer, so they cannot be changed per test. */ -@ExecGroup +@ExecTestGroup class CometInMemoryCacheKryoSuite extends CometTestBase { override protected def beforeAll(): Unit = { @@ -272,7 +272,7 @@ abstract class CometInMemoryCacheKryoRegistrationSuite(expectedBatch: String) } /** Registers Spark's cached batch and nothing of Comet's, so the plugin keeps Spark's format. */ -@ExecGroup +@ExecTestGroup class CometInMemoryCacheKryoUnregisteredSuite extends CometInMemoryCacheKryoRegistrationSuite(classOf[DefaultCachedBatch].getName) { override protected def registrations: Seq[(String, String)] = @@ -284,7 +284,7 @@ class CometInMemoryCacheKryoUnregisteredSuite * [[CometKryoRegistrator]], and not Spark's cached batch, so before Spark 4.1 only Comet's format * can be stored, and the plugin has to install it. */ -@ExecGroup +@ExecTestGroup class CometInMemoryCacheKryoClassesToRegisterSuite extends CometInMemoryCacheKryoRegistrationSuite( ArrowCachedBatchSerializer.cachedBatchClass.getName) { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCachePruningSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCachePruningSuite.scala index 2468f839dba..dcde6912b77 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCachePruningSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCachePruningSuite.scala @@ -31,9 +31,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.comet.CometConf -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometInMemoryCachePruningSuite extends CometTestBase { override protected def beforeAll(): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala index ee8cb455f57..4db5944799b 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala @@ -58,12 +58,12 @@ import org.apache.spark.storage.StorageLevel import org.apache.comet.{CometArrowAllocator, CometConf, CometKryoRegistrator, ExtendedExplainInfo} import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus} import org.apache.comet.rules.{CometCacheBuildRule, CometCacheColumnarRule} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.{CometPlainVector, CometVector} import net.jpountz.lz4.LZ4FrameInputStream -@ExecGroup +@ExecTestGroup class CometInMemoryCacheSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala index e82c87e4c88..db1869f1c8e 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala @@ -53,9 +53,9 @@ import org.apache.comet.{CometConf, CometExecIterator, CometExplainInfo, Extende import org.apache.comet.CometSparkSessionExtensions.{hasFallbackReason, isSpark35Plus} import org.apache.comet.rules.RewriteJoin import org.apache.comet.serde.OperatorOuterClass -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometJoinSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala index 7b2561078d4..0741cb3c677 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala @@ -38,9 +38,9 @@ import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, CometNativeException, ExtendedExplainInfo} import org.apache.comet.CometSparkSessionExtensions.isSpark41Plus -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup -@ScansGroup +@ScansTestGroup class CometNativeReaderSuite extends CometTestBase with AdaptiveSparkPlanHelper { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala index e9c5a62ae31..5f343e14add 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala @@ -59,9 +59,9 @@ import org.apache.comet.{CometConf, CometExecIterator, CometExplainInfo, CometSh import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, withFallbackReason} import org.apache.comet.serde.{OperatorOuterClass, PartitioningOuterClass} import org.apache.comet.shuffle.ShufflePartitionPusher -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup -@ShuffleGroup +@ShuffleTestGroup class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometRangeExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometRangeExecSuite.scala index 5e57010805a..982df0bf786 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometRangeExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometRangeExecSuite.scala @@ -29,9 +29,9 @@ import org.apache.spark.sql.functions.{col, count, sum} import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometRangeExecSuite extends CometTestBase { // CometTestBase enables the Spark-to-Arrow conversions, including the one for Range. Use their diff --git a/spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala index 2b8c1bb1469..6cdb841606f 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala @@ -33,13 +33,13 @@ import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, import org.apache.spark.unsafe.types.CalendarInterval import org.apache.comet.{CometConf, ExtendedExplainInfo} -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup // Top-level, so the encoder needs no outer pointer. case class ShuffleInputRec(a: Int, b: String) /** Tests for [[CometConf.COMET_CONVERT_FROM_SHUFFLE_INPUT_ENABLED]]. */ -@ShuffleGroup +@ShuffleTestGroup class CometShuffleInputConversionSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala index a92bb2efcc1..2593dbd18e4 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala @@ -35,9 +35,9 @@ import org.apache.spark.sql.types.{ByteType, IntegerType, LongType, ShortType} import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.serde.OperatorOuterClass.Operator -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometTopKSuite extends CometTestBase { private def localTopK(plan: SparkPlan): CometLocalTopKExec = { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometTypedDatasetSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometTypedDatasetSuite.scala index 5698038bf81..94972c81a53 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometTypedDatasetSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometTypedDatasetSuite.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.functions.{broadcast, col, size, sum} import org.apache.spark.sql.internal.SQLConf import org.apache.comet.{CometConf, ExtendedExplainInfo} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup // Top-level, so the encoders need no outer pointer, which is the ordinary user shape. case class TypedDsRec(a: Int, b: String) @@ -51,7 +51,7 @@ object TypedDsCounter { } /** Tests for [[CometConf.COMET_CONVERT_FROM_TYPED_DATASET_ENABLED]]. */ -@ExecGroup +@ExecTestGroup class CometTypedDatasetSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala index 13c3f332fb4..0a7cfe8a4c3 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala @@ -37,9 +37,9 @@ import org.apache.spark.sql.types.DecimalType import org.apache.comet.{CometConf, CometExplainInfo, ExtendedExplainInfo} import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometWindowExecSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/expressions/CometRegexSuite.scala b/spark/src/test/scala/org/apache/comet/expressions/CometRegexSuite.scala index fd452001bd4..cc1b8393730 100644 --- a/spark/src/test/scala/org/apache/comet/expressions/CometRegexSuite.scala +++ b/spark/src/test/scala/org/apache/comet/expressions/CometRegexSuite.scala @@ -22,9 +22,9 @@ package org.apache.comet.expressions import org.scalatest.funsuite.AnyFunSuite import org.apache.comet.serde.{Compatible, Incompatible} -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class CometRegexSuite extends AnyFunSuite { private def assertCompatible(pattern: String): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCaseWhenSuite.scala b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCaseWhenSuite.scala index 9f7d26d6136..9d9b0639920 100644 --- a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCaseWhenSuite.scala +++ b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCaseWhenSuite.scala @@ -22,9 +22,9 @@ package org.apache.comet.expressions.conditional import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class CometCaseWhenSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("case_when") { diff --git a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCoalesceSuite.scala b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCoalesceSuite.scala index 4d856afbef5..1e137772d71 100644 --- a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCoalesceSuite.scala +++ b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCoalesceSuite.scala @@ -24,9 +24,9 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.internal.SQLConf -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class CometCoalesceSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("coalesce should return correct datatype") { diff --git a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometIfSuite.scala b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometIfSuite.scala index 070f6183fae..aca4d7e2788 100644 --- a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometIfSuite.scala +++ b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometIfSuite.scala @@ -22,9 +22,9 @@ package org.apache.comet.expressions.conditional import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class CometIfSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("if expression") { diff --git a/spark/src/test/scala/org/apache/comet/iceberg/IcebergReflectionSuite.scala b/spark/src/test/scala/org/apache/comet/iceberg/IcebergReflectionSuite.scala index b9aef7c3271..09e4d4b54f1 100644 --- a/spark/src/test/scala/org/apache/comet/iceberg/IcebergReflectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/iceberg/IcebergReflectionSuite.scala @@ -34,9 +34,9 @@ import org.apache.iceberg.TableMetadata import org.apache.iceberg.io.FileIO import org.apache.iceberg.types.Types -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup -@ScansGroup +@ScansTestGroup class IcebergReflectionSuite extends AnyFunSuite { /** Mimics HiveTableOperations/GlueTableOperations which inherit current(). */ diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala index 8d8fbe0267e..94959cff7af 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala @@ -44,7 +44,7 @@ import org.apache.comet.objectstore.AbfsAuthParityCases._ import org.apache.comet.objectstore.AbfsAuthResolver._ import org.apache.comet.objectstore.AbfsReflection.Handles import org.apache.comet.tags.AzureTest -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Pins the resolver's behaviour per hadoop-azure version and cross-checks it against the real @@ -69,7 +69,7 @@ import org.apache.comet.tags.ExecGroup * assert while recording, so a recorded file never contradicts the hadoop-azure source. The file * is written only when every case ran, so a filtered run cannot leave a partial file behind. */ -@ExecGroup +@ExecTestGroup class AbfsAuthParitySuite extends AnyFunSuite with Matchers with BeforeAndAfterAll { import AbfsAuthParitySuite._ diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala index 7382a09e399..63abfee9761 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala @@ -44,14 +44,14 @@ import org.apache.logging.log4j.core.config.Property import org.apache.comet.objectstore.AbfsAuthResolver._ import org.apache.comet.objectstore.AbfsReflection.Handles -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Runs the resolver against the hadoop-azure on the test classpath (3.3.4 on the spark-3.x * profiles, 3.4.1, 3.4.2 and 3.5.0 on spark-4.0, 4.1 and 4.2). Cases that depend on a feature one * line lacks are gated on the capability probes rather than on a version number. */ -@ExecGroup +@ExecTestGroup class AbfsAuthResolverSuite extends AnyFunSuite with Matchers { private val account = "myacct.dfs.core.windows.net" diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala index 23274f6d4b6..ff51b83211a 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala @@ -31,14 +31,14 @@ import org.apache.spark.sql.types.{IntegerType, StructType} import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.hadoop.fs.FakeAbfsFileSystem -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Plans scans over `abfss://` paths served by [[FakeAbfsFileSystem]], with authentication * resolved offline by the hadoop-azure on the test classpath. A scan Comet declines runs on * Spark's own scan against the local files. */ -@ExecGroup +@ExecTestGroup class AbfsScanFallbackSuite extends CometTestBase { private val multipleAuthoritiesReason = "more than one container or account" diff --git a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala index e0000815ce4..72dc9738899 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala @@ -28,9 +28,9 @@ import org.apache.hadoop.conf.Configuration import org.apache.comet.CometConf.{COMET_LIBHDFS_SCHEMES_KEY, COMET_S3_COMPLIANT_SCHEMES_KEY} import org.apache.comet.objectstore.AbfsAuthResolver.{Keys, Markers, Providers} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class NativeConfigSuite extends AnyFunSuite with Matchers { /** diff --git a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterRetrySuite.scala b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterRetrySuite.scala index 1e98e720524..5fe13903aad 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterRetrySuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterRetrySuite.scala @@ -26,14 +26,14 @@ import scala.jdk.CollectionConverters._ import org.apache.spark.SparkConf import org.apache.comet.hadoop.fs.RecordingFileSystem -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup /** * Native Parquet writes whose tasks Spark retries. A suite of its own because a task gets more * than one attempt only under a `local[N,M]` master, and the failure tests elsewhere expect a * failed task to fail its job. */ -@ScansGroup +@ScansTestGroup class CometParquetWriterRetrySuite extends CometParquetWriterTestBase { override protected def sparkConf: SparkConf = diff --git a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala index 59e90bcbd19..891b9070fcf 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala @@ -58,10 +58,10 @@ import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plu import org.apache.comet.hadoop.fs.RecordingFileSystem import org.apache.comet.rules.RevertNativeForTransitionHeavyStages import org.apache.comet.serde.operator.NativeWriteUtils -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, SchemaGenOptions} -@ScansGroup +@ScansTestGroup class CometParquetWriterSuite extends CometParquetWriterTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index b3446ecf0c3..1bc3af72d45 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -54,7 +54,7 @@ import com.google.common.primitives.UnsignedLong import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, isSpark41Plus} -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup import org.apache.comet.vector.CometVector abstract class ParquetReadSuite extends CometTestBase { @@ -3112,7 +3112,7 @@ abstract class ParquetReadSuite extends CometTestBase { } } -@ScansGroup +@ScansTestGroup class ParquetReadV1Suite extends ParquetReadSuite with AdaptiveSparkPlanHelper { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetTimestampLtzAsNtzSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetTimestampLtzAsNtzSuite.scala index 3d8f1962b00..926aa13609a 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetTimestampLtzAsNtzSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetTimestampLtzAsNtzSuite.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup /** * Tests for reading Parquet TimestampLTZ columns as TimestampNTZ. @@ -38,7 +38,7 @@ import org.apache.comet.tags.ScansGroup * * See https://github.com/apache/datafusion-comet/issues/4219 */ -@ScansGroup +@ScansTestGroup class ParquetTimestampLtzAsNtzSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala index 456c74593b5..cf7612c753a 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala @@ -54,7 +54,7 @@ import org.apache.comet.{CometConf, CometCoverageStats, CometExplainInfo, CometS import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, isSpark42Plus, withFallbackReason} import org.apache.comet.serde.{CometAggregateExpressionSerde, Compatible, ExprOuterClass, OperatorOuterClass, QueryPlanSerde, Unsupported} import org.apache.comet.shims.ShimCometWindowGroupLimit -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} /** @@ -62,7 +62,7 @@ import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} * transform Spark operators to Comet operators, fallback mechanisms, configuration handling, and * edge cases. */ -@ExecGroup +@ExecTestGroup class CometExecRuleSuite extends CometTestBase { // The observers are active only during the DPP lifecycle regression below. AQE can prepare diff --git a/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala index ebb8f165177..0daf44809f5 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala @@ -44,14 +44,14 @@ import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.CometConf.{COMET_LIBHDFS_SCHEMES_KEY, COMET_S3_COMPLIANT_SCHEMES_KEY} import org.apache.comet.hadoop.fs.{FakeAbfsFileSystem, FakeHdfsAuthorityFileSystem} import org.apache.comet.objectstore.NativeConfig -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Native scans over files in more than one object store, without a cloud store: `hdfs://nn1` and * `hdfs://nn2` are two native stores backed by the local disk. Native execution cannot read them, * so claimed scans are checked on their plans, and declined scans are run by Spark. */ -@ExecGroup +@ExecTestGroup class CometMultiStoreScanSuite extends CometTestBase with AdaptiveSparkPlanHelper { private var rootDir: File = _ diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala index 15e08dd7952..9c6febb9015 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala @@ -44,7 +44,7 @@ import org.apache.spark.sql.execution.datasources.HadoopFsRelation import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.comet.ContribServices -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.util.ClassLoaders /** @@ -63,7 +63,7 @@ import org.apache.comet.util.ClassLoaders * Sibling to `CometScanWithPlanDataSuite`, which covers the other half of the contrib SPI (plan * data). Everything here runs on the default build. */ -@ExecGroup +@ExecTestGroup class CometScanContribSuite extends AnyFunSuite { /** diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala index f0c43dc21e7..96b0dfb1fb7 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala @@ -30,13 +30,13 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, DataTypeSupport} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} /** * Test suite specifically for CometScanRule transformation logic. */ -@ExecGroup +@ExecTestGroup class CometScanRuleSuite extends CometTestBase { /** Helper method to apply CometExecRule and return the transformed plan */ diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala index 6c60613dd6c..2ba03809a68 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala @@ -44,7 +44,7 @@ import org.apache.comet.{CometConf, CometExplainInfo, CometIcebergTestBase, Exte import org.apache.comet.hadoop.fs.{FakeHdfsAuthorityFileSystem, FakeHDFSFileSystem, FakeHdfsSchemeFileSystem, FakeWasbSchemeFileSystem} import org.apache.comet.iceberg.IcebergStorageSchemes import org.apache.comet.objectstore.NativeConfig -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Comet's native readers go through object_store, which understands a fixed set of URL schemes. A @@ -56,7 +56,7 @@ import org.apache.comet.tags.ExecGroup * native probe is cached per probe URL so an authorityless URL can't poison the authority-bearing * form of the same scheme. */ -@ExecGroup +@ExecTestGroup class CometScanSchemeFallbackSuite extends CometTestBase with CometIcebergTestBase { private var fakeRootDir: File = _ diff --git a/spark/src/test/scala/org/apache/comet/rules/EliminateRedundantTransitionsSuite.scala b/spark/src/test/scala/org/apache/comet/rules/EliminateRedundantTransitionsSuite.scala index 654c776433d..d349f6e1644 100644 --- a/spark/src/test/scala/org/apache/comet/rules/EliminateRedundantTransitionsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/EliminateRedundantTransitionsSuite.scala @@ -24,9 +24,9 @@ import org.apache.spark.sql.comet.{CometColumnarToRowExec, CometPlan} import org.apache.spark.sql.execution.{ColumnarToRowExec, SparkPlan, UnionExec} import org.apache.comet.CometConf -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class EliminateRedundantTransitionsSuite extends CometTestBase { private def sparkPlanFor(sql: String): SparkPlan = { diff --git a/spark/src/test/scala/org/apache/comet/rules/RevertNativeForTransitionHeavyStagesSuite.scala b/spark/src/test/scala/org/apache/comet/rules/RevertNativeForTransitionHeavyStagesSuite.scala index 45a32372ef9..b2574894396 100644 --- a/spark/src/test/scala/org/apache/comet/rules/RevertNativeForTransitionHeavyStagesSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/RevertNativeForTransitionHeavyStagesSuite.scala @@ -43,7 +43,7 @@ import org.apache.spark.sql.util.QueryExecutionListener import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus} import org.apache.comet.serde.OperatorOuterClass.Operator -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup private case class AliasingFallbackCometExec( override val originalPlan: SparkPlan, @@ -68,7 +68,7 @@ private case class BothFormatsExec(child: SparkPlan) extends UnaryExecNode { copy(child = newChild) } -@ExecGroup +@ExecTestGroup class RevertNativeForTransitionHeavyStagesSuite extends CometTestBase { // Spark 4.0+ builds a cache through columnar output only when the cache serializer accepts diff --git a/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala b/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala index 0a52f560de3..37e426173ef 100644 --- a/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala @@ -32,7 +32,7 @@ import org.apache.spark.sql.types._ import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus} import org.apache.comet.serde.literals.CometLiteral import org.apache.comet.shims.CometTypeShim -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup /** * Pins `CometLiteral.listLiteralElementSupported` against the arms of @@ -41,7 +41,7 @@ import org.apache.comet.tags.ExpressionsGroup * raises a `MatchError` in the middle of planning, which Spark surfaces as `[INTERNAL_ERROR] The * Spark SQL phase planning failed with an internal error`. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometLiteralSuite extends CometTestBase with CometTypeShim { test("a null literal with duplicate struct field names is unsupported") { diff --git a/spark/src/test/scala/org/apache/comet/serde/CometScalarFunctionSuite.scala b/spark/src/test/scala/org/apache/comet/serde/CometScalarFunctionSuite.scala index ff83eec7b26..44b723d241f 100644 --- a/spark/src/test/scala/org/apache/comet/serde/CometScalarFunctionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/CometScalarFunctionSuite.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.expressions.{Abs, Cos, Expression, Literal, import org.apache.spark.sql.types.{DataType, DoubleType, IntegerType} import org.apache.comet.{CometExplainInfo, CometSparkSessionExtensions} -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup /** * Synthetic expression whose constructor declares `evalMode`, used to prove class-level detection @@ -86,7 +86,7 @@ case class TestEvalContextExpression(child: Expression, evalContext: Any) copy(child = newChildren.head) } -@ExpressionsGroup +@ExpressionsTestGroup class CometScalarFunctionSuite extends CometTestBase { private def fallbackReasons(expr: Expression): Set[String] = { diff --git a/spark/src/test/scala/org/apache/comet/serde/QueryPlanSerdeSuite.scala b/spark/src/test/scala/org/apache/comet/serde/QueryPlanSerdeSuite.scala index 6c0ba08274e..b6d6ffb8baf 100644 --- a/spark/src/test/scala/org/apache/comet/serde/QueryPlanSerdeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/QueryPlanSerdeSuite.scala @@ -33,9 +33,9 @@ import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus} import org.apache.comet.serde.QueryPlanSerde.{canShortCircuitNulls, canSkipEvaluation, supportedDataType, withSharedSubexpressions} -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class QueryPlanSerdeSuite extends AnyFunSuite { test("make_date IN evaluates the probe but can skip list expressions") { diff --git a/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala b/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala index cc444c16e45..e71b5547bdc 100644 --- a/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala @@ -21,9 +21,9 @@ package org.apache.comet.serde import org.scalatest.funsuite.AnyFunSuite -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class SerdeRegistrationSuite extends AnyFunSuite { // A version shim only adds serdes for classes the shared map cannot name on every Spark diff --git a/spark/src/test/scala/org/apache/comet/serde/SparkFixesSuite.scala b/spark/src/test/scala/org/apache/comet/serde/SparkFixesSuite.scala index d80d21ac061..f4163a07539 100644 --- a/spark/src/test/scala/org/apache/comet/serde/SparkFixesSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/SparkFixesSuite.scala @@ -21,13 +21,13 @@ package org.apache.comet.serde import org.scalatest.funsuite.AnyFunSuite -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup /** * Pins the version table of every fix in `SparkFixes`, so that the boundaries are checked on all * Spark versions in one run and not only on the one a test suite runs against. */ -@ExpressionsGroup +@ExpressionsTestGroup class SparkFixesSuite extends AnyFunSuite { private def checkVersions(name: String, fixed: String => Boolean)( diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala index 4073c18d175..27a4d36e62e 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala @@ -36,10 +36,10 @@ import org.apache.spark.sql.catalyst.expressions.AttributeReference import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, StructType} import org.apache.comet.iceberg.IcebergReflection -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** Unit tests for Iceberg native-scan serde helpers that do not require a Spark session. */ -@ExecGroup +@ExecTestGroup class CometIcebergNativeScanSuite extends AnyFunSuite with Matchers { test("complex type null residuals are not serialized") { diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala index b7ca53a5278..4fd9466e76e 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala @@ -30,9 +30,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{IntegerType, LongType, MetadataBuilder, StringType, StructField, StructType} import org.apache.comet.parquet.CometParquetUtils -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometNativeScanSuite extends AnyFunSuite { import CometNativeScan.{constantMetadataFieldPrefix, uniqueConstantMetadataFields} diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala index 4fc9126055e..c670633cb9f 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala @@ -22,13 +22,13 @@ package org.apache.comet.serde.operator import org.scalatest.funsuite.AnyFunSuite import org.apache.comet.serde.OperatorOuterClass.{CompressionCodec, IcebergWriterMode} -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup /** * Unit tests for the protobuf translation. Iceberg must be on the classpath because the property * keys / numeric defaults are looked up reflectively from `org.apache.iceberg.TableProperties`. */ -@ScansGroup +@ScansTestGroup class IcebergWriteProtoTranslationSuite extends AnyFunSuite { import IcebergWriteProtoTranslation._ diff --git a/spark/src/test/scala/org/apache/comet/shuffle/CelebornShufflePartitionPusherSuite.scala b/spark/src/test/scala/org/apache/comet/shuffle/CelebornShufflePartitionPusherSuite.scala index 0bf5a21ba6e..5dda97e324f 100644 --- a/spark/src/test/scala/org/apache/comet/shuffle/CelebornShufflePartitionPusherSuite.scala +++ b/spark/src/test/scala/org/apache/comet/shuffle/CelebornShufflePartitionPusherSuite.scala @@ -32,9 +32,9 @@ import org.scalatest.funsuite.AnyFunSuite import org.apache.spark.SparkConf -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup -@ShuffleGroup +@ShuffleTestGroup class CelebornShufflePartitionPusherSuite extends AnyFunSuite { private val encodedAttemptId = (4 << 16) | 7 diff --git a/spark/src/test/scala/org/apache/comet/udf/CometUdfBridgeSuite.scala b/spark/src/test/scala/org/apache/comet/udf/CometUdfBridgeSuite.scala index 3412a2ada1e..88aec7e5fd1 100644 --- a/spark/src/test/scala/org/apache/comet/udf/CometUdfBridgeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/udf/CometUdfBridgeSuite.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.types.IntegerType import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.comet.{CometArrowAllocator, CometArrowImportAllocator} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.NativeUtil /** @@ -82,7 +82,7 @@ class IdentityUdf extends CometUDF { override def evaluate(inputs: Array[ValueVector], numRows: Int): ValueVector = inputs.head } -@ExecGroup +@ExecTestGroup class CometUdfBridgeSuite extends AnyFunSuite { /** Exports a single-column batch and invokes the bridge against it. */ diff --git a/spark/src/test/scala/org/apache/comet/vector/CometVectorUtilsSuite.scala b/spark/src/test/scala/org/apache/comet/vector/CometVectorUtilsSuite.scala index 25388126bf5..55521f60347 100644 --- a/spark/src/test/scala/org/apache/comet/vector/CometVectorUtilsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/vector/CometVectorUtilsSuite.scala @@ -33,10 +33,10 @@ import org.apache.arrow.vector.dictionary.Dictionary import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider import org.apache.arrow.vector.types.pojo.{ArrowType, DictionaryEncoding, FieldType} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** Check failure cleanup without transferring ownership of the helper's source vectors. */ -@ExecGroup +@ExecTestGroup class CometVectorUtilsSuite extends AnyFunSuite with Matchers { for (missingDictionary <- Seq(true, false)) { diff --git a/spark/src/test/scala/org/apache/comet/vector/NativeUtilSuite.scala b/spark/src/test/scala/org/apache/comet/vector/NativeUtilSuite.scala index 7643a9bab2c..822d005263d 100644 --- a/spark/src/test/scala/org/apache/comet/vector/NativeUtilSuite.scala +++ b/spark/src/test/scala/org/apache/comet/vector/NativeUtilSuite.scala @@ -42,9 +42,9 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.comet.{CometArrowAllocator, CometArrowImportAllocator, CometConf, CometRuntimeException} import org.apache.comet.serde.{OperatorOuterClass, QueryPlanSerde} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class NativeUtilSuite extends CometTestBase { private def withIsolatedStructAllocator( diff --git a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala index f9fd49936f4..8ab788bc66b 100644 --- a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala @@ -47,7 +47,7 @@ import org.apache.spark.util.JsonProtocol import org.apache.comet.{CometArrowAllocator, CometConf, CometExecIterator, CometExecutorMemoryUsage, CometNativeException, CometShuffleBlockIterator, Native} import org.apache.comet.serde.Config.ConfigMap import org.apache.comet.serde.OperatorOuterClass -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.CometPlainVector /** @@ -55,7 +55,7 @@ import org.apache.comet.vector.CometPlainVector * one release of the native execution context and of its task-shared memory pool reference, even * when a step of the lifecycle fails partway through. See issue #5212 (positions 2, 3 and 8). */ -@ExecGroup +@ExecTestGroup class CometExecIteratorLifecycleSuite extends CometTestBase { private def withTaskContext[T](taskAttemptId: Long)(f: => T): T = { diff --git a/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala b/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala index 29908da5a1b..fa2cfd710e4 100644 --- a/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala @@ -34,9 +34,9 @@ import org.apache.spark.sql.internal.StaticSQLConf import org.apache.spark.util.{JsonProtocol, ManualClock, Utils} import org.apache.comet.{COMET_VERSION, CometConf, CometExecIterator, CometExecutorMemoryUsage} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometPluginsSuite extends CometTestBase { override protected def sparkConf: SparkConf = { val conf = new SparkConf() @@ -177,7 +177,7 @@ class CometPluginsSuite extends CometTestBase { } } -@ExecGroup +@ExecTestGroup class CometPluginsDefaultSuite extends CometTestBase { override protected def sparkConf: SparkConf = { val conf = new SparkConf() @@ -196,7 +196,7 @@ class CometPluginsDefaultSuite extends CometTestBase { } } -@ExecGroup +@ExecTestGroup class CometPluginsMemoryOverheadWarningSuite extends CometTestBase { private val warning = @@ -280,7 +280,7 @@ class CometPluginsMemoryOverheadWarningSuite extends CometTestBase { } } -@ExecGroup +@ExecTestGroup class CometPluginsDeprecatedConfigWarningSuite extends CometTestBase { private def warningsFor(conf: SparkConf): Seq[String] = { @@ -307,7 +307,7 @@ class CometPluginsDeprecatedConfigWarningSuite extends CometTestBase { } } -@ExecGroup +@ExecTestGroup class CometPluginsUnifiedModeSuite extends CometTestBase { override protected def sparkConf: SparkConf = { val conf = new SparkConf() @@ -336,7 +336,7 @@ class CometPluginsUnifiedModeSuite extends CometTestBase { } } -@ExecGroup +@ExecTestGroup class CometPluginsExtensionOnlySuite extends CometTestBase { // No plugin and no off-heap memory. CometTestBase registers CometSparkSessionExtensions // directly, as an application can with spark.sql.extensions. @@ -398,7 +398,7 @@ class CometPluginsExtensionOnlySuite extends CometTestBase { } } -@ExecGroup +@ExecTestGroup class CometPluginsSparkShuffleManagerSuite extends CometTestBase { // The application runs Spark's own shuffle manager. override protected def sparkConf: SparkConf = { @@ -440,7 +440,7 @@ class CometPluginsSparkShuffleManagerSuite extends CometTestBase { } } -@ExecGroup +@ExecTestGroup class CometPluginsEventLogSuite extends SparkFunSuite { import CometExecIterator.{memoryUsageEvent, JvmArrowMemory} diff --git a/spark/src/test/scala/org/apache/spark/CometRuntimeShutdownSuite.scala b/spark/src/test/scala/org/apache/spark/CometRuntimeShutdownSuite.scala index dc9aa8cebc8..383b906957b 100644 --- a/spark/src/test/scala/org/apache/spark/CometRuntimeShutdownSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometRuntimeShutdownSuite.scala @@ -33,7 +33,7 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.functions.{expr, sum} import org.apache.spark.util.Utils -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Entry point run in a child JVM by [[CometRuntimeShutdownSuite]]. Executes a native Comet query @@ -80,7 +80,7 @@ object CometRuntimeShutdownMain { * Runs Comet in a child JVM because the failure mode under test, the JVM being unable to exit, * cannot be observed from inside the affected JVM. */ -@ExecGroup +@ExecTestGroup class CometRuntimeShutdownSuite extends AnyFunSuite { test("JVM exits when main returns without SparkContext.stop()") { diff --git a/spark/src/test/scala/org/apache/spark/CometTaskMemoryManagerSuite.scala b/spark/src/test/scala/org/apache/spark/CometTaskMemoryManagerSuite.scala index af3316b4e13..38ce8c7f922 100644 --- a/spark/src/test/scala/org/apache/spark/CometTaskMemoryManagerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometTaskMemoryManagerSuite.scala @@ -26,9 +26,9 @@ import org.apache.logging.log4j.Level import org.apache.logging.log4j.core.LogEvent import org.apache.spark.memory.{MemoryConsumer, TaskMemoryManager, TestMemoryManager} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometTaskMemoryManagerSuite extends TaskMemoryTestUtils { test("native memory usage is visible to Spark's memory consumer") { diff --git a/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnboundedShuffleMemoryAllocatorSuite.scala b/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnboundedShuffleMemoryAllocatorSuite.scala index 902e063f0a5..748dd91774a 100644 --- a/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnboundedShuffleMemoryAllocatorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnboundedShuffleMemoryAllocatorSuite.scala @@ -27,9 +27,9 @@ import org.apache.spark.SparkConf import org.apache.spark.memory.{SparkOutOfMemoryError, TaskMemoryManager, TestMemoryManager} import org.apache.spark.unsafe.memory.MemoryBlock -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup -@ShuffleGroup +@ShuffleTestGroup class CometUnboundedShuffleMemoryAllocatorSuite extends AnyFunSuite { private val pageSize = 4096L diff --git a/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnifiedShuffleMemoryAllocatorSuite.scala b/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnifiedShuffleMemoryAllocatorSuite.scala index 385ccc4b885..34ba0cc03cf 100644 --- a/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnifiedShuffleMemoryAllocatorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnifiedShuffleMemoryAllocatorSuite.scala @@ -24,9 +24,9 @@ import org.apache.logging.log4j.core.LogEvent import org.apache.spark.{CometTaskMemoryManager, TaskContext, TaskContextImpl, TaskMemoryTestUtils} import org.apache.spark.memory.SparkOutOfMemoryError -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup -@ShuffleGroup +@ShuffleTestGroup class CometUnifiedShuffleMemoryAllocatorSuite extends TaskMemoryTestUtils { private val PageSize = 20L diff --git a/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala b/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala index ee9a79f23af..aa6baa40ced 100644 --- a/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala @@ -44,7 +44,7 @@ import org.apache.spark.unsafe.Platform import org.apache.spark.unsafe.UnsafeAlignedOffset import org.apache.comet.Native -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup import org.apache.comet.vector.NativeUtil /** @@ -53,7 +53,7 @@ import org.apache.comet.vector.NativeUtil * These tests verify SpillSorter behavior using Spark's test memory management infrastructure, * without needing a full SparkContext. */ -@ShuffleGroup +@ShuffleTestGroup class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { private val INITIAL_SIZE = 1024 diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala index 27442e08c84..328bf8634f3 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala @@ -24,9 +24,9 @@ import org.apache.spark.internal.config.{MEMORY_OFFHEAP_ENABLED, MEMORY_OFFHEAP_ import org.apache.spark.sql.comet.shims.ShimCometTPCDSQuerySuite import org.apache.comet.CometConf -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometTPCDSQuerySuite extends { val tpcdsAllQueries: Seq[String] = Seq( diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala index f544a5dd2e1..a05308696ec 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala @@ -31,13 +31,13 @@ import org.apache.spark.sql.test.TestSparkSession import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Because we need to modify some methods of Spark `TPCDSQueryTestSuite` but they are private, we * copy Spark `TPCDSQueryTestSuite`. */ -@ExecGroup +@ExecTestGroup class CometTPCDSQueryTestSuite extends QueryTest with TPCDSBase with CometSQLQueryTestHelper { val tpcdsExtendedQueries: Seq[String] = Seq("q72") diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala index d295f7dc7a9..faa4b95205b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala @@ -33,7 +33,7 @@ import org.apache.spark.sql.test.TestSparkSession import org.apache.comet.CometConf import org.apache.comet.shims.ShimCometTPCHQuerySuite -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * End-to-end tests to check TPCH query results. @@ -50,7 +50,7 @@ import org.apache.comet.tags.ExecGroup * ./mvnw -Dsuites=org.apache.spark.sql.CometTPCHQuerySuite test * }}} */ -@ExecGroup +@ExecTestGroup class CometTPCHQuerySuite extends QueryTest with TPCBase with ShimCometTPCHQuerySuite { private val tpchDataPath = sys.env.get("SPARK_TPCH_DATA") diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala index 758e058777d..5a3f5e67afb 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala @@ -42,10 +42,10 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.util.io.ChunkedByteBuffer import org.apache.comet.{CometArrowAllocator, CometKryoRegistrator} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.CometVector -@ExecGroup +@ExecTestGroup class CometBroadcastValueSuite extends CometTestBase { private val Timeout = 30.seconds diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometCollectMetricsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometCollectMetricsSuite.scala index 695013e447e..f064a64928b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometCollectMetricsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometCollectMetricsSuite.scala @@ -37,9 +37,9 @@ import org.apache.spark.sql.util.QueryExecutionListener import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark35Plus import org.apache.comet.rules.CometExecRule -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometCollectMetricsSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometDecimalPromotionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometDecimalPromotionSuite.scala index a3977ccb1b6..e3623574c6e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometDecimalPromotionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometDecimalPromotionSuite.scala @@ -27,9 +27,9 @@ import org.apache.spark.sql.types.{DecimalType, IntegerType} import org.apache.comet.{CometConf, CometExplainInfo} import org.apache.comet.serde.{CometDivide, ExprOuterClass, QueryPlanSerde, Unsupported} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometDecimalPromotionSuite extends CometTestBase { private case class Operation( diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometDppFallbackRepro3949Suite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometDppFallbackRepro3949Suite.scala index 6d0bbc6fab0..75896b13476 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometDppFallbackRepro3949Suite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometDppFallbackRepro3949Suite.scala @@ -32,7 +32,7 @@ import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec import org.apache.spark.sql.internal.SQLConf import org.apache.comet.{CometConf, CometExplainInfo} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Attempts an end-to-end reproduction of issue #3949. @@ -62,7 +62,7 @@ import org.apache.comet.tags.ExecGroup * shuffle whose child subtree (descending through `QueryStageExec.plan`) still contains a DPP * scan -- i.e. an inconsistency that the bug would produce. */ -@ExecGroup +@ExecTestGroup class CometDppFallbackRepro3949Suite extends CometTestBase { // ---------------------------------------------------------------------- diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometNativeWriteSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometNativeWriteSuite.scala index 9a1dbd6b94c..ce8f188c705 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometNativeWriteSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometNativeWriteSuite.scala @@ -49,10 +49,10 @@ import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.serde.{Incompatible, Unsupported} import org.apache.comet.serde.OperatorOuterClass.Operator import org.apache.comet.serde.operator.{schema2Proto, CometDataWritingCommand, NativeWriteUtils} -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup /** Exercises the retained Spark 3.x writer, including both terminal execution entry points. */ -@ScansGroup +@ScansTestGroup class CometNativeWriteSuite extends CometTestBase { import NativeWriteCommitProtocol._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala index a6a0117bcfd..b104f40098d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala @@ -32,10 +32,10 @@ import org.apache.xbean.asm9.tree.{ClassNode, FieldInsnNode, MethodInsnNode} import com.google.common.reflect.ClassPath import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.util.ClassLoaders -@ExecGroup +@ExecTestGroup class CometPlanEqualitySuite extends AnyFunSuite { // Native serialization state and the source plan are deliberately not the identity of a diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala index 3430af55b4c..fa6cdacc649 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.test.TestSparkSession import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus, isSpark41Plus, isSpark42Plus} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Similar to [[org.apache.spark.sql.PlanStabilitySuite]], checks that TPC-DS Comet plans don't @@ -283,7 +283,7 @@ private object CometPlanStabilitySuite { } } -@ExecGroup +@ExecTestGroup class CometTPCDSV1_4_PlanStabilitySuite extends CometPlanStabilitySuite { private val planNames: Seq[String] = CometPlanStabilitySuite.planNameChain("v1_4") override val goldenFilePath: String = @@ -298,7 +298,7 @@ class CometTPCDSV1_4_PlanStabilitySuite extends CometPlanStabilitySuite { } } -@ExecGroup +@ExecTestGroup class CometTPCDSV2_7_PlanStabilitySuite extends CometPlanStabilitySuite { private val planNames: Seq[String] = CometPlanStabilitySuite.planNameChain("v2_7") override val goldenFilePath: String = diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometScanWithPlanDataSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometScanWithPlanDataSuite.scala index 631758c1c64..7c39aa585d8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometScanWithPlanDataSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometScanWithPlanDataSuite.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.execution.{BinaryExecNode, SparkPlan} import org.apache.comet.CometRuntimeException import org.apache.comet.serde.OperatorOuterClass.Operator -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Unit coverage for the core SPI that lets out-of-tree contrib leaf scans (e.g. the Delta @@ -51,7 +51,7 @@ import org.apache.comet.tags.ExecGroup * Deliberately does not exercise the per-op injector registry mechanics; that surface is owned by * `PlanDataInjectorSuite`. */ -@ExecGroup +@ExecTestGroup class CometScanWithPlanDataSuite extends AnyFunSuite { /** diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala index 6cb84e07717..ce8a491d901 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala @@ -25,9 +25,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometSparkSessionExtensions import org.apache.comet.rules.CometRule -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometSessionExtensionsSuite extends CometTestBase { test("captured configs do not override a SQL execution after a setting is unset") { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometShuffleFallbackStickinessSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometShuffleFallbackStickinessSuite.scala index 42ed1cc0962..cdd3ad84ad6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometShuffleFallbackStickinessSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometShuffleFallbackStickinessSuite.scala @@ -31,7 +31,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{hasFallbackReason, withFallbackReason} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Pins the sticky-fallback invariant for Comet shuffle decisions: `shuffleSupported` must return @@ -47,7 +47,7 @@ import org.apache.comet.tags.ExecGroup * The coordinator tags the node with `withFallbackReasons` only on total fallback and * short-circuits via `hasFallbackReason` on subsequent passes. */ -@ExecGroup +@ExecTestGroup class CometShuffleFallbackStickinessSuite extends CometTestBase { test("shuffleSupported returns None when the shuffle already carries explain info") { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala index 6ae2449d2e8..39ad7b80fc4 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala @@ -47,9 +47,9 @@ import org.apache.spark.unsafe.Platform import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus} import org.apache.comet.serde.{Metric, OperatorOuterClass} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper { override protected def sparkConf: SparkConf = { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometUnsafeProjectionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometUnsafeProjectionSuite.scala index 28d30a2927f..b738848ee81 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometUnsafeProjectionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometUnsafeProjectionSuite.scala @@ -30,9 +30,9 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} import org.apache.spark.sql.types._ -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometUnsafeProjectionSuite extends CometTestBase with AdaptiveSparkPlanHelper { /** A schema with every writer GenerateUnsafeProjection nests: structs, arrays and maps. */ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/DecimalPrecisionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/DecimalPrecisionSuite.scala index 281ddd81170..682d242ff6e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/DecimalPrecisionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/DecimalPrecisionSuite.scala @@ -23,9 +23,9 @@ import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.expressions.{Add, Alias, AttributeReference, CaseWhen, Cast, CheckOverflow, Coalesce, Divide, Expression, GreaterThan, IsNotNull, Literal, Multiply, Remainder, Subtract} import org.apache.spark.sql.types.{DecimalType, IntegerType, LongType, StringType} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class DecimalPrecisionSuite extends SparkFunSuite { private val decA = AttributeReference("a", DecimalType(10, 2))() diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala b/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala index 7a5e43ba424..20c0cfb1563 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala @@ -37,14 +37,14 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SQLTestUtils import org.apache.comet.{CometConf, IntegrationTestSuite} -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup /** * A integration test suite that tests parquet modular encryption usage. */ @RunWith(classOf[JUnitRunner]) @IntegrationTestSuite -@ScansGroup +@ScansTestGroup class ParquetEncryptionITCase extends CometTestBase with SQLTestUtils { private val encoder = Base64.getEncoder private val footerKey = diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorShuffleLifecycleSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorShuffleLifecycleSuite.scala index 51abe82c14d..9f1ba00febb 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorShuffleLifecycleSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorShuffleLifecycleSuite.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.functions.col import org.apache.spark.util.Utils import org.apache.comet.CometConf -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * End-to-end lifecycle of the executor-side plan data held by [[PlanDataInjector]]. A map-only @@ -38,7 +38,7 @@ import org.apache.comet.tags.ExecGroup * The recreated-context test stops the suite's SparkContext, so it runs last and nothing else may * follow it. */ -@ExecGroup +@ExecTestGroup class PlanDataInjectorShuffleLifecycleSuite extends CometTestBase { private def withNativeShuffle[T](session: SparkSession)(f: => T): T = { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorSuite.scala index a5fcd8de0ef..e4e6408e251 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorSuite.scala @@ -26,9 +26,9 @@ import org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager import org.apache.comet.serde.OperatorOuterClass import org.apache.comet.serde.OperatorOuterClass.Operator -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class PlanDataInjectorSuite extends AnyFunSuite { /** Builds an un-injected IcebergScan operator: hasCommon, zero file_scan_tasks. */ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchRowIteratorSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchRowIteratorSuite.scala index 00353f01032..c305e3a8494 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchRowIteratorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchRowIteratorSuite.scala @@ -32,10 +32,10 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{IntegerType, LongType, StringType} import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.CometPlainVector -@ExecGroup +@ExecTestGroup class CachedBatchRowIteratorSuite extends AnyFunSuite { Seq("CODEGEN_ONLY", "NO_CODEGEN").foreach { mode => def withMode(settings: (String, String)*)(f: => Unit): Unit = { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala index a9f24b0b1f1..81a15d0a8db 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala @@ -44,7 +44,7 @@ import org.apache.spark.sql.types.{ArrayType, BooleanType, ByteType, CalendarInt import org.apache.spark.sql.vectorized.{ColumnarArray, ColumnarBatch, ColumnarMap, ColumnVector} import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.{CometPlainVector, CometVector, NativeUtil} /** @@ -53,7 +53,7 @@ import org.apache.comet.vector.{CometPlainVector, CometVector, NativeUtil} * `CometMathExpressionSuite`, but that test only catches *one* function-level type drift. This * suite covers the boundary contract independently of any specific function. */ -@ExecGroup +@ExecTestGroup class CometArrowStreamSuite extends AnyFunSuite with Matchers { private def expectedSchema(types: (String, ArrowType)*): Schema = { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowWriterSuite.scala index 0ba0f8ac7de..e37fde2235a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowWriterSuite.scala @@ -43,14 +43,14 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.CometSparkSessionExtensions.isSpark41Plus -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * The columnar paths of [[ArrowWriter]] copy Spark's vectors in bulk where they can. Whatever * they take, they must produce what the row path writes for the same rows, which reads every * value through Spark's getters one at a time. */ -@ExecGroup +@ExecTestGroup class CometArrowWriterSuite extends AnyFunSuite with Matchers { private val numRows = 300 diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometStringWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometStringWriterSuite.scala index 50d9e11272b..21b70924c18 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometStringWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometStringWriterSuite.scala @@ -39,9 +39,9 @@ import org.apache.spark.sql.comet.util.Utils import org.apache.spark.sql.types.{ArrayType, StringType, StructField, StructType} import org.apache.spark.unsafe.types.UTF8String -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometStringWriterSuite extends AnyFunSuite with Matchers { private val sourceOffset = 7 private val schema = StructType(Seq(StructField("text", StringType))) diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodecSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodecSuite.scala index bc9b0a570f5..92f8bbb166d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodecSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodecSuite.scala @@ -35,7 +35,7 @@ import org.apache.spark.SparkException import com.github.luben.zstd.Zstd -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Checks that the zstd codec never writes past the buffer it allocates for its output. @@ -46,7 +46,7 @@ import org.apache.comet.tags.ExecGroup * The defect is explained on [[CometZstdCompressionCodec]]. The inputs are the ones that provoke * it: bytes that look structured enough for zstd to try entropy coding but do not compress. */ -@ExecGroup +@ExecTestGroup class CometZstdCompressionCodecSuite extends AnyFunSuite with Matchers { // zstd-jni unpacks its native library into java.io.tmpdir, which the test JVM points at diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReaderSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReaderSuite.scala index 3221ce055dc..d86cc470e7b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReaderSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReaderSuite.scala @@ -24,9 +24,9 @@ import java.io.InputStream import org.scalatest.funsuite.AnyFunSuite -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup -@ShuffleGroup +@ShuffleTestGroup class CometBlockStoreShuffleReaderSuite extends AnyFunSuite { test("closing a partially consumed raw stream does not fetch remaining blocks") { class TrackingStream extends ByteArrayInputStream(Array[Byte](1, 2, 3)) { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala index dd598eb11d4..56f5534758a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala @@ -34,9 +34,9 @@ import org.apache.spark.sql.comet.shims.ShimCometShuffleMaterialization import org.apache.spark.util.ThreadUtils import org.apache.comet.CometConf -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup -@ShuffleGroup +@ShuffleTestGroup class CometCelebornConcurrentMaterializationSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala index 92f35c9d650..36e2dbfb7cd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala @@ -30,9 +30,9 @@ import org.apache.spark.shuffle.{FetchFailedException, ShuffleHandle, ShuffleRea import org.apache.spark.sql.CometTestBase import org.apache.comet.CometConf -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup -@ShuffleGroup +@ShuffleTestGroup class CometCelebornLocalFetchFailureSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornNativeShuffleWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornNativeShuffleWriterSuite.scala index dff55f73601..a0e3271965d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornNativeShuffleWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornNativeShuffleWriterSuite.scala @@ -30,10 +30,10 @@ import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.{CometConf, CometShuffleSizeLimitException} import org.apache.comet.shuffle.{CelebornShufflePartitionPusher, CelebornShufflePusherFactory, RecordingCelebornPushClient} -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup /** Exercises real Spark map tasks, native RSS planning, and the Celeborn map lifecycle. */ -@ShuffleGroup +@ShuffleTestGroup class CometCelebornNativeShuffleWriterSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala index 05c0221425b..afce8b7f025 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala @@ -38,12 +38,12 @@ import org.apache.spark.sql.CometTestBase import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.shuffle.{CelebornShufflePusherFactory, RecordingCelebornPushClient, ResolvedCelebornShufflePusher} -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup /** * Real Spark stage recovery and local Comet I/O; only the optional Celeborn client is replaced. */ -@ShuffleGroup +@ShuffleTestGroup class CometCelebornShuffleFallbackSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManagerSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManagerSuite.scala index 43daa9f16fb..008c4e620df 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManagerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManagerSuite.scala @@ -31,9 +31,9 @@ import org.apache.spark.shuffle.{BaseShuffleHandle, ShuffleBlockResolver, Shuffl import org.apache.spark.sql.comet.{NativeScanPlanDataInjector, PlanDataInjector} import org.apache.comet.serde.OperatorOuterClass -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup -@ShuffleGroup +@ShuffleTestGroup class CometCelebornShuffleManagerSuite extends AnyFunSuite { // Match the optional CelebornConf API, including its enum-valued policy and Long threshold. diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShufflePlanningSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShufflePlanningSuite.scala index 98de0ca99d8..7e212ccd1c0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShufflePlanningSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShufflePlanningSuite.scala @@ -42,14 +42,14 @@ import org.apache.comet.{CometConf, CometExplainInfo} import org.apache.comet.CometSparkSessionExtensions.{isCometShuffleEnabled, isCometShuffleManagerEnabled} import org.apache.comet.rules.{CometExecRule, RevertNativeForTransitionHeavyStages} import org.apache.comet.serde.{Compatible, OperatorOuterClass} -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup /** * Plans against the actual composite manager without requiring the optional Celeborn client. Only * Spark-fallback queries execute: the local test backend must never receive a native dependency. * Native plan assertions below are not transport or live-cluster integration tests. */ -@ShuffleGroup +@ShuffleTestGroup class CometCelebornShufflePlanningSuite extends CometTestBase { override protected val shuffleManager: String = diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReaderSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReaderSuite.scala index 5b3a6ce1df5..5f83fc23b53 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReaderSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReaderSuite.scala @@ -53,9 +53,9 @@ import com.google.flatbuffers.Table import org.apache.comet.{CometConf, CometExecIterator, CometShuffleBlockIterator, Native} import org.apache.comet.serde.{OperatorOuterClass, QueryPlanSerde} -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup -@ShuffleGroup +@ShuffleTestGroup class CometCelebornShuffleReaderSuite extends CometTestBase { import NativeShuffleTestFrames._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleStatisticsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleStatisticsSuite.scala index c9dcfdf6916..a941533e782 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleStatisticsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleStatisticsSuite.scala @@ -27,9 +27,9 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.metric.SQLShuffleWriteMetricsReporter.{SHUFFLE_BYTES_WRITTEN, SHUFFLE_RECORDS_WRITTEN} import org.apache.comet.CometConf -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup -@ShuffleGroup +@ShuffleTestGroup class CometCelebornShuffleStatisticsSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index 758af60d7e1..15a6fc1c595 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -43,9 +43,9 @@ import org.apache.spark.sql.types.{BinaryType, IntegerType, MetadataBuilder, Str import org.apache.spark.unsafe.UnsafeAlignedOffset import org.apache.spark.util.Utils -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup -@ShuffleGroup +@ShuffleTestGroup class CometDiskBlockWriterSuite extends AnyFunSuite { private val schema = StructType(Seq(StructField("a", BinaryType))) diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala index c9a09c5b98f..80198308b8f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.functions.{col, lit, rand, udf} import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup /** * Positional round robin places rows by their ordinal within the map task rather than by hashing @@ -40,7 +40,7 @@ import org.apache.comet.tags.ShuffleGroup * `CometShuffleExchangeExec.usesPositionalRoundRobin` directly rather than inferring the decision * from output. */ -@ShuffleGroup +@ShuffleTestGroup class CometNativePositionalRoundRobinSuite extends CometTestBase with AdaptiveSparkPlanHelper { private val numPartitions = 8 diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleInputRDDSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleInputRDDSuite.scala index 1e026d4425e..4fc6db8166e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleInputRDDSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleInputRDDSuite.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.serde.OperatorOuterClass.Operator -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup /** * Ensure the serialized shuffle-map-stage task binary does not grow with the number of @@ -44,7 +44,7 @@ import org.apache.comet.tags.ShuffleGroup * Lives in the `execution.shuffle` package so it can construct the `private[shuffle]` * [[CometNativeShuffleInputRDD]] and the `private[comet]` [[NativeExecContext]] directly. */ -@ShuffleGroup +@ShuffleTestGroup class CometNativeShuffleInputRDDSuite extends CometTestBase { test("native shuffle input preserves its parents' determinism") { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala index 6e5af62d57a..6ed184b6487 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala @@ -24,13 +24,13 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.CometConf -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup /** * Checks the plan [[CometNativeShuffleWriter]] hands to native code. Lives in the * `execution.shuffle` package so it can call the `private[shuffle]` `buildUnifiedPlan`. */ -@ShuffleGroup +@ShuffleTestGroup class CometNativeShuffleWriterSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIteratorSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIteratorSuite.scala index 9fb5768799f..26531a97649 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIteratorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIteratorSuite.scala @@ -28,10 +28,10 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.metric.SQLMetrics import org.apache.comet.Native -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup import org.apache.comet.vector.NativeUtil -@ShuffleGroup +@ShuffleTestGroup class NativeBatchDecoderIteratorSuite extends CometTestBase { import NativeShuffleTestFrames.{intFrame, TrackingInputStream} diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala index e8f060300bd..c7b1e7d4d06 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala @@ -39,10 +39,10 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.util.io.ChunkedByteBuffer import org.apache.comet.CometArrowAllocator -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.CometVector -@ExecGroup +@ExecTestGroup class UtilsSuite extends CometTestBase { test("serializeBatches round trips schema metadata larger than the output chunk estimate") { diff --git a/spark/src/test/spark-3.5/org/apache/comet/CometWidthBucketSuite.scala b/spark/src/test/spark-3.5/org/apache/comet/CometWidthBucketSuite.scala index e8c958b4e2a..34b6258acae 100644 --- a/spark/src/test/spark-3.5/org/apache/comet/CometWidthBucketSuite.scala +++ b/spark/src/test/spark-3.5/org/apache/comet/CometWidthBucketSuite.scala @@ -22,9 +22,9 @@ package org.apache.comet import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class CometWidthBucketSuite extends CometTestBase { private def countSparkProjectExec(plan: SparkPlan): Int = diff --git a/spark/src/test/spark-3.5/org/apache/comet/exec/CometMergeRowsSuite.scala b/spark/src/test/spark-3.5/org/apache/comet/exec/CometMergeRowsSuite.scala index 9af8721741e..986eb8c9462 100644 --- a/spark/src/test/spark-3.5/org/apache/comet/exec/CometMergeRowsSuite.scala +++ b/spark/src/test/spark-3.5/org/apache/comet/exec/CometMergeRowsSuite.scala @@ -19,8 +19,8 @@ package org.apache.comet.exec -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** Spark 3.5.x entry point for the shared native MergeRows contract suite. */ -@ExecGroup +@ExecTestGroup class CometMergeRowsSuite extends CometMergeRowsNativeSuiteBase diff --git a/spark/src/test/spark-3.5/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala b/spark/src/test/spark-3.5/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala index 24b189f9143..6e5b976b42c 100644 --- a/spark/src/test/spark-3.5/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala +++ b/spark/src/test/spark-3.5/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala @@ -22,9 +22,9 @@ package org.apache.spark.sql import org.apache.spark.sql.catalyst.expressions.Empty2Null import org.apache.spark.sql.functions.col -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometSparkInternalFunctionsSuite extends CometTestBase { test("empty2null is offloaded to Comet") { diff --git a/spark/src/test/spark-3.5/org/apache/spark/sql/CometToPrettyStringSuite.scala b/spark/src/test/spark-3.5/org/apache/spark/sql/CometToPrettyStringSuite.scala index 85d895823ca..28c7094d236 100644 --- a/spark/src/test/spark-3.5/org/apache/spark/sql/CometToPrettyStringSuite.scala +++ b/spark/src/test/spark-3.5/org/apache/spark/sql/CometToPrettyStringSuite.scala @@ -30,9 +30,9 @@ import org.apache.comet.CometConf import org.apache.comet.CometFuzzTestBase import org.apache.comet.expressions.{CometCast, CometEvalMode} import org.apache.comet.serde.Compatible -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometToPrettyStringSuite extends CometFuzzTestBase { test("ToPrettyString") { diff --git a/spark/src/test/spark-3.x/org/apache/comet/CometStringDecodeSuite.scala b/spark/src/test/spark-3.x/org/apache/comet/CometStringDecodeSuite.scala index daff1180701..d4d70d8f0c9 100644 --- a/spark/src/test/spark-3.x/org/apache/comet/CometStringDecodeSuite.scala +++ b/spark/src/test/spark-3.x/org/apache/comet/CometStringDecodeSuite.scala @@ -23,9 +23,9 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class CometStringDecodeSuite extends CometTestBase { private def countSparkProjectExec(plan: SparkPlan): Int = diff --git a/spark/src/test/spark-4.0/org/apache/comet/exec/CometMergeRowsSuite.scala b/spark/src/test/spark-4.0/org/apache/comet/exec/CometMergeRowsSuite.scala index b3a15368773..2aafd71e107 100644 --- a/spark/src/test/spark-4.0/org/apache/comet/exec/CometMergeRowsSuite.scala +++ b/spark/src/test/spark-4.0/org/apache/comet/exec/CometMergeRowsSuite.scala @@ -19,8 +19,8 @@ package org.apache.comet.exec -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** Spark 4.0 entry point for the shared native MergeRows contract suite. */ -@ExecGroup +@ExecTestGroup class CometMergeRowsSuite extends CometMergeRowsNativeSuiteBase diff --git a/spark/src/test/spark-4.0/org/apache/spark/sql/CometCollationSuite.scala b/spark/src/test/spark-4.0/org/apache/spark/sql/CometCollationSuite.scala index e73b06c1e5e..b69990c4333 100644 --- a/spark/src/test/spark-4.0/org/apache/spark/sql/CometCollationSuite.scala +++ b/spark/src/test/spark-4.0/org/apache/spark/sql/CometCollationSuite.scala @@ -30,9 +30,9 @@ import org.apache.spark.sql.types.StringType import org.apache.comet.{CometConf, CometExplainInfo, ExtendedExplainInfo} import org.apache.comet.serde.OperatorOuterClass -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometCollationSuite extends CometTestBase { // Queries that group, sort, or shuffle on a non-default collated string must fall back to diff --git a/spark/src/test/spark-4.1+/org/apache/comet/exec/CometMergeRowsSuite.scala b/spark/src/test/spark-4.1+/org/apache/comet/exec/CometMergeRowsSuite.scala index b9d75a7ec28..d57585a9a67 100644 --- a/spark/src/test/spark-4.1+/org/apache/comet/exec/CometMergeRowsSuite.scala +++ b/spark/src/test/spark-4.1+/org/apache/comet/exec/CometMergeRowsSuite.scala @@ -35,13 +35,13 @@ import org.apache.spark.sql.util.QueryExecutionListener import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus import org.apache.comet.shims.{MergeRowsMetricsShim, ShimCometMergeRows} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** * Spark 4.1+ stock V2 writers build MergeSummary by locating the concrete Spark MergeRowsExec. * Native MergeRows is therefore kept on Spark when the enclosing writer remains on Spark. */ -@ExecGroup +@ExecTestGroup class CometMergeRowsSuite extends CometTestBase with AdaptiveSparkPlanHelper { private val catalog = "generic_rowlevel" diff --git a/spark/src/test/spark-4.1+/org/apache/spark/sql/CometCollationSuite.scala b/spark/src/test/spark-4.1+/org/apache/spark/sql/CometCollationSuite.scala index 7cc235fc415..203577ffabf 100644 --- a/spark/src/test/spark-4.1+/org/apache/spark/sql/CometCollationSuite.scala +++ b/spark/src/test/spark-4.1+/org/apache/spark/sql/CometCollationSuite.scala @@ -20,9 +20,9 @@ package org.apache.spark.sql import org.apache.comet.{CometConf, ExtendedExplainInfo} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometCollationSuite extends CometTestBase { // Queries that group, sort, or shuffle on a non-default collated string must fall back to diff --git a/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometArrowPythonUdfSuite.scala b/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometArrowPythonUdfSuite.scala index 76384556e51..412c9abbe15 100644 --- a/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometArrowPythonUdfSuite.scala +++ b/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometArrowPythonUdfSuite.scala @@ -36,9 +36,9 @@ import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, import org.apache.comet.{CometConf, CometExplainInfo, NativeBase, SparkErrorConverter} import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus import org.apache.comet.exceptions.CometQueryExecutionException -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometArrowPythonUdfSuite extends CometTestBase { private def python: String = sys.env.getOrElse("PYSPARK_PYTHON", "python3") diff --git a/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometDecimalArithmeticViewSuite.scala b/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometDecimalArithmeticViewSuite.scala index 0c8b56da1c3..a869807a22c 100644 --- a/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometDecimalArithmeticViewSuite.scala +++ b/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometDecimalArithmeticViewSuite.scala @@ -25,9 +25,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.DecimalType import org.apache.comet.serde.{CometScalarFunction, ExprOuterClass, QueryPlanSerde} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometDecimalArithmeticViewSuite extends CometTestBase { // Spark 4.1+ (SPARK-53968) stores `spark.sql.decimalOperations.allowPrecisionLoss` per diff --git a/spark/src/test/spark-4.1/org/apache/comet/CometIcebergEncryptionSuite.scala b/spark/src/test/spark-4.1/org/apache/comet/CometIcebergEncryptionSuite.scala index 1cfe8c438a4..662b9e04bd5 100644 --- a/spark/src/test/spark-4.1/org/apache/comet/CometIcebergEncryptionSuite.scala +++ b/spark/src/test/spark-4.1/org/apache/comet/CometIcebergEncryptionSuite.scala @@ -32,14 +32,14 @@ import org.apache.spark.sql.comet.CometIcebergNativeScanExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.comet.iceberg.CometTestKMS -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup /** * Encryption-specific Iceberg tests. Lives in the spark-4.1 source set because Iceberg table * encryption is a V3 feature and the encryption APIs (KeyManagementClient, StandardEncryption * manager) are only public from Iceberg 1.11, which Comet pairs with Spark 4.1. */ -@ScansGroup +@ScansTestGroup class CometIcebergEncryptionSuite extends CometTestBase with CometIcebergTestBase diff --git a/spark/src/test/spark-4.2/org/apache/comet/exec/CometInMemoryCacheLastAttemptMetricSuite.scala b/spark/src/test/spark-4.2/org/apache/comet/exec/CometInMemoryCacheLastAttemptMetricSuite.scala index 5577b0e87e9..1b751de08b1 100644 --- a/spark/src/test/spark-4.2/org/apache/comet/exec/CometInMemoryCacheLastAttemptMetricSuite.scala +++ b/spark/src/test/spark-4.2/org/apache/comet/exec/CometInMemoryCacheLastAttemptMetricSuite.scala @@ -27,10 +27,10 @@ import org.apache.spark.sql.execution.columnar.CometInMemoryRelationHelper import org.apache.spark.sql.execution.metric.SQLLastAttemptMetrics import org.apache.comet.CometConf -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** Spark 4.2's last-attempt metrics over a relation cached in Comet's format. */ -@ExecGroup +@ExecTestGroup class CometInMemoryCacheLastAttemptMetricSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/spark-4.x/org/apache/comet/CometCastCollatedStringSuite.scala b/spark/src/test/spark-4.x/org/apache/comet/CometCastCollatedStringSuite.scala index 211fbe00a9c..0b140b3e47d 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/CometCastCollatedStringSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/CometCastCollatedStringSuite.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, IntegerType, import org.apache.comet.expressions.{CometCast, CometEvalMode} import org.apache.comet.serde.{Compatible, Unsupported} -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup /** * https://github.com/apache/datafusion-comet/issues/4489 @@ -63,7 +63,7 @@ import org.apache.comet.tags.ExpressionsGroup * This lives under `spark-4.x`, shared by every 4.x profile, rather than `spark-4.1+`, because * collation is a Spark 4.0 feature and `StringType(collationName)` already resolves there. */ -@ExpressionsGroup +@ExpressionsTestGroup class CometCastCollatedStringSuite extends CometTestBase { private val lcase = StringType("UTF8_LCASE") diff --git a/spark/src/test/spark-4.x/org/apache/comet/CometWidthBucketSuite.scala b/spark/src/test/spark-4.x/org/apache/comet/CometWidthBucketSuite.scala index e8c958b4e2a..34b6258acae 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/CometWidthBucketSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/CometWidthBucketSuite.scala @@ -22,9 +22,9 @@ package org.apache.comet import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} -import org.apache.comet.tags.ExpressionsGroup +import org.apache.comet.tags.ExpressionsTestGroup -@ExpressionsGroup +@ExpressionsTestGroup class CometWidthBucketSuite extends CometTestBase { private def countSparkProjectExec(plan: SparkPlan): Int = diff --git a/spark/src/test/spark-4.x/org/apache/comet/exec/CometEmptyRelationExecSuite.scala b/spark/src/test/spark-4.x/org/apache/comet/exec/CometEmptyRelationExecSuite.scala index 45efe1da972..359ef10a8a8 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/exec/CometEmptyRelationExecSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/exec/CometEmptyRelationExecSuite.scala @@ -28,9 +28,9 @@ import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometEmptyRelationExecSuite extends CometTestBase { // CometTestBase enables the Spark-to-Arrow conversions; use their production defaults here. diff --git a/spark/src/test/spark-4.x/org/apache/comet/exec/CometShuffle4_0Suite.scala b/spark/src/test/spark-4.x/org/apache/comet/exec/CometShuffle4_0Suite.scala index e753a5085a6..46c268cb5ba 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/exec/CometShuffle4_0Suite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/exec/CometShuffle4_0Suite.scala @@ -28,9 +28,9 @@ import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{FloatType, LongType, StringType, TimestampType} -import org.apache.comet.tags.ShuffleGroup +import org.apache.comet.tags.ShuffleTestGroup -@ShuffleGroup +@ShuffleTestGroup class CometShuffle4_0Suite extends CometColumnarShuffleSuite { protected val adaptiveExecutionEnabled: Boolean = true diff --git a/spark/src/test/spark-4.x/org/apache/comet/parquet/CometEmptyRelationParquetWriterSuite.scala b/spark/src/test/spark-4.x/org/apache/comet/parquet/CometEmptyRelationParquetWriterSuite.scala index 35a772a83c6..c61ceda7649 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/parquet/CometEmptyRelationParquetWriterSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/parquet/CometEmptyRelationParquetWriterSuite.scala @@ -27,9 +27,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.comet.CometConf -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup -@ScansGroup +@ScansTestGroup class CometEmptyRelationParquetWriterSuite extends CometParquetWriterTestBase { test("EmptyRelationExec keeps AQE empty parquet output readable") { diff --git a/spark/src/test/spark-4.x/org/apache/comet/rules/CometEmptyRelationExecRuleSuite.scala b/spark/src/test/spark-4.x/org/apache/comet/rules/CometEmptyRelationExecRuleSuite.scala index 4c37b4ca524..e30e6770a51 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/rules/CometEmptyRelationExecRuleSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/rules/CometEmptyRelationExecRuleSuite.scala @@ -34,9 +34,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, BooleanType, DecimalType, IntegerType, StructType} import org.apache.comet.{CometConf, CometCoverageStats, CometExplainInfo, ExtendedExplainInfo} -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometEmptyRelationExecRuleSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/CometSortCollationSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/CometSortCollationSuite.scala index 78a0380ef45..d521db80f06 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/CometSortCollationSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/CometSortCollationSuite.scala @@ -28,9 +28,9 @@ import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, CometExplainInfo} import org.apache.comet.serde.QueryPlanSerde -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometSortCollationSuite extends CometTestBase { private val sortCollationReason = "Sort does not support non-default string collation" diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala index 4621be87958..5c522e3b5ca 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala @@ -23,9 +23,9 @@ import org.apache.spark.sql.catalyst.expressions.Empty2Null import org.apache.spark.sql.classic.ExpressionUtils import org.apache.spark.sql.functions._ -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometSparkInternalFunctionsSuite extends CometTestBase { test("empty2null is offloaded to Comet") { diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/CometToPrettyStringSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/CometToPrettyStringSuite.scala index 70dadc7f6d1..f86cc680458 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/CometToPrettyStringSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/CometToPrettyStringSuite.scala @@ -33,9 +33,9 @@ import org.apache.comet.CometConf import org.apache.comet.CometFuzzTestBase import org.apache.comet.expressions.{CometCast, CometEvalMode} import org.apache.comet.serde.Compatible -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup -@ExecGroup +@ExecTestGroup class CometToPrettyStringSuite extends CometFuzzTestBase { test("ToPrettyString") { diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/CometVariantShreddingSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/CometVariantShreddingSuite.scala index 2180581633e..6ed56ca72bd 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/CometVariantShreddingSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/CometVariantShreddingSuite.scala @@ -27,10 +27,10 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.TestSparkSession import org.apache.comet.{CometConf, CometSparkSessionExtensions} -import org.apache.comet.tags.ScansGroup +import org.apache.comet.tags.ScansTestGroup /** Run Spark's unchanged reconstruction assertions in Comet's regular Spark 4 CI jobs. */ -@ScansGroup +@ScansTestGroup class CometVariantShreddingSuite extends VariantShreddingSuite { override protected def sparkConf: SparkConf = super.sparkConf .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false") diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/comet/CometMapInBatchSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/comet/CometMapInBatchSuite.scala index e4c1b6c1f5f..c467f6c3144 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/comet/CometMapInBatchSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/comet/CometMapInBatchSuite.scala @@ -32,7 +32,7 @@ import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.rules.EliminateRedundantTransitions -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup /** Minimal CometPlan leaf used to anchor the rule's transform without triggering execution. */ private case class StubCometLeaf(override val output: Seq[Attribute]) @@ -54,7 +54,7 @@ private case class StubCometLeaf(override val output: Seq[Attribute]) * `PythonFunction` / `PythonAccumulatorV2` / `PythonBroadcast` classes when fabricating a stub * `PythonUDF` for `MapInArrowExec` to wrap. */ -@ExecGroup +@ExecTestGroup class CometMapInBatchSuite extends CometTestBase { private def stubPythonUDF: PythonUDF = { diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/execution/python/CometArrowPythonRunnerSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/execution/python/CometArrowPythonRunnerSuite.scala index daf889297c7..8cb569fce41 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/execution/python/CometArrowPythonRunnerSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/execution/python/CometArrowPythonRunnerSuite.scala @@ -52,10 +52,10 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.util.DirectByteBufferOutputStream import org.apache.comet.CometArrowAllocator -import org.apache.comet.tags.ExecGroup +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.{CometDecodedVector, CometDictionary, CometDictionaryVector, CometPlainVector, CometStructVector} -@ExecGroup +@ExecTestGroup class CometArrowPythonRunnerSuite extends AnyFunSuite with Matchers { /**