From fb6260914e170276ec0bd87d105bbafdbfcc572e Mon Sep 17 00:00:00 2001 From: Erik Bogado Date: Mon, 21 Sep 2026 22:36:19 -0300 Subject: [PATCH 1/5] ci: key TPC dataset caches on pinned generators The TPC-H and TPC-DS dataset caches were keyed on the hash of pr_build_linux.yml, so every workflow edit regenerated both datasets and left useless entries that evict main's. The generators were also unpinned: GenTPCHData cloned tpch-dbgen HEAD and the tpcds-kit checkout had no ref, so a generator change could not rotate a dataset key. Pin both generators and key the datasets on their inputs instead: - TPC-H key is the hash of GenTPCHData.scala; the generator now checks out a fixed tpch-dbgen commit before applying the stdout patch. - TPC-DS key embeds the pinned tpcds-kit commit, which the checkout now names explicitly. Save both from main only, matching the cache-write policy for other large caches. check_tpc_dataset_caches() in dev/ci/check-ci-config.py asserts all of the above so the old shape cannot come back. --- .github/workflows/pr_build_linux.yml | 35 ++++++-- dev/ci/check-ci-config.py | 84 +++++++++++++++++++ .../org/apache/spark/sql/GenTPCHData.scala | 10 ++- 3 files changed, 120 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index a9c184a20b0..1d305b9a8df 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -699,12 +699,12 @@ jobs: restore-keys: | ${{ runner.os }}-java-maven- - - name: Cache TPC-H data + - name: Restore TPC-H data id: cache-tpch - uses: actions/cache@v6 + uses: actions/cache/restore@v6 with: path: ./tpch - key: tpch-${{ hashFiles('.github/workflows/pr_build_linux.yml') }} + key: tpch-sf1-${{ hashFiles('spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala') }} - name: Build project run: | @@ -720,6 +720,17 @@ jobs: run: | SPARK_HOME=`pwd` SPARK_TPCH_DATA=`pwd`/tpch/sf1_parquet ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCHQuerySuite test + # The dataset changes only when the generator does, so main caches it like + # any other large entry. Saved only on main: an entry written from a pull + # request or a `gh-readonly-queue/*` branch can never be restored by a + # later run and evicts main's from the shared budget. + - name: Save TPC-H data + if: ${{ github.ref == 'refs/heads/main' && steps.cache-tpch.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v6 + with: + path: ./tpch + key: tpch-sf1-${{ hashFiles('spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala') }} + # Saved only on main: an entry written from a pull request or a # `gh-readonly-queue/*` branch cannot be restored by any later run, and # it evicts main's from the shared budget. See "Large caches are written @@ -773,12 +784,12 @@ jobs: restore-keys: | ${{ runner.os }}-java-maven- - - name: Cache TPC-DS data + - name: Restore TPC-DS data id: cache-tpcds - uses: actions/cache@v6 + uses: actions/cache/restore@v6 with: path: ./tpcds-sf-1 - key: tpcds-${{ hashFiles('.github/workflows/pr_build_linux.yml') }} + key: tpcds-sf1-1b7fb7529eda - name: Build project run: | @@ -789,6 +800,7 @@ jobs: uses: actions/checkout@v7 with: repository: databricks/tpcds-kit + ref: 1b7fb7529edae091684201fab142d956d6afd881 path: ./tpcds-kit - name: Build tpcds-kit @@ -829,6 +841,17 @@ jobs: spark.sql.autoBroadcastJoinThreshold=-1 spark.sql.join.forceApplyShuffledHashJoin=true + # The dataset changes only when the generator does, so main caches it like + # any other large entry. Saved only on main: an entry written from a pull + # request or a `gh-readonly-queue/*` branch can never be restored by a + # later run and evicts main's from the shared budget. + - name: Save TPC-DS data + if: ${{ github.ref == 'refs/heads/main' && steps.cache-tpcds.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v6 + with: + path: ./tpcds-sf-1 + key: tpcds-sf1-1b7fb7529eda + # Saved only on main: an entry written from a pull request or a # `gh-readonly-queue/*` branch cannot be restored by any later run, and # it evicts main's from the shared budget. See "Large caches are written diff --git a/dev/ci/check-ci-config.py b/dev/ci/check-ci-config.py index 55edd5f5e27..92a02957624 100644 --- a/dev/ci/check-ci-config.py +++ b/dev/ci/check-ci-config.py @@ -1200,6 +1200,89 @@ def check_cache_save_scope(): return not failures +TPC_WORKFLOW = WORKFLOWS / "pr_build_linux.yml" +TPC_WORKFLOW_HASH = re.compile( + r"hashFiles\(\s*['\"]\.github/workflows/pr_build_linux\.yml['\"]\s*\)" +) +TPCDS_KIT_CHECKOUT = re.compile( + r"repository:[ \t]*databricks/tpcds-kit\n" + r"(?:[ \t]+[^\n]*\n)*?" + r"[ \t]+ref:[ \t]*([0-9a-f]{40})" +) +TPCH_CACHE_KEY = re.compile( + r"key:\s*tpch-sf1-\$\{\{\s*hashFiles\(" + r"'spark/src/test/scala/org/apache/spark/sql/GenTPCHData\.scala'\)\s*\}\}" +) +TPCDS_CACHE_KEY = re.compile(r"key:\s*tpcds-sf1-([0-9a-f]{12})\b") +GIT_COMMIT = re.compile(r"\b[0-9a-f]{40}\b") +GIT_CHECKOUT = re.compile(r"git checkout") + + +def check_tpc_dataset_caches(): + """The TPC datasets are keyed on a pinned generator, not on the workflow file. + + Both dataset caches used `hashFiles('.github/workflows/pr_build_linux.yml')` + as their key, and PR #5973 left them exempt from the main-only save rule + because `./tpch` and `./tpcds-sf-1` are not dependency trees. The workflow + file is one of the most edited files in the repository, so the key rotated + with every CI change, orphaning main's entry and regenerating a + multi-hundred-megabyte dataset -- and the dataset kept competing with the + ~4.1 GiB `Linux-cargo-debug` cache whose eviction costs a ~26 minute cold + build. Key them on the generator input and pin the generators themselves, so + a dataset is regenerated only when the generator changes. + """ + failures = [] + text = TPC_WORKFLOW.read_text(encoding="utf-8") + workflow_hash = TPC_WORKFLOW_HASH.search(text) + if workflow_hash: + failures.append( + f"{TPC_WORKFLOW}: `{workflow_hash.group(0)}` keys a TPC dataset on " + f"this workflow file, so every CI edit rotates the cache key and " + f"regenerates the SF=1 dataset. Key it on the pinned generator" + ) + if not TPCH_CACHE_KEY.search(text): + failures.append( + f"{TPC_WORKFLOW}: the TPC-H dataset cache key is not keyed on " + f"`GenTPCHData.scala`, so a generator change would not rotate it" + ) + kit = TPCDS_KIT_CHECKOUT.search(text) + if not kit: + failures.append( + f"{TPC_WORKFLOW}: the `databricks/tpcds-kit` checkout has no pinned " + f"`ref:`, so the TPC-DS dataset can change without the key rotating" + ) + else: + key = TPCDS_CACHE_KEY.search(text) + if not key or key.group(1) != kit.group(1)[:12]: + failures.append( + f"{TPC_WORKFLOW}: the TPC-DS dataset cache key does not embed the " + f"`tpcds-kit` revision `{kit.group(1)[:12]}`, so a generator bump " + f"would not rotate it" + ) + for line_no, uses, step in _cache_steps(text.splitlines()): + if not CACHE_RW_USES.match(uses): + continue + body = "\n".join(step) + if "tpch" not in body and "tpcds" not in body: + continue + failures.append( + f"{TPC_WORKFLOW}:{line_no}: a TPC dataset cache uses " + f"`actions/cache@vN`, whose save runs in an implicit post step that " + f"no `if:` can reach. Split it into `actions/cache/restore` plus a " + f"main-only `actions/cache/save`" + ) + generator = Path("spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala") + generator_text = generator.read_text(encoding="utf-8") + if not (GIT_COMMIT.search(generator_text) and GIT_CHECKOUT.search(generator_text)): + failures.append( + f"{generator}: the tpch-dbgen checkout is not pinned to a commit, so " + f"the TPC-H dataset can change without the cache key rotating" + ) + for failure in failures: + print(f"tpc dataset cache: {failure}") + return not failures + + if __name__ == "__main__": ok = check_change_filters() ok = check_event_policy() and ok @@ -1212,6 +1295,7 @@ def check_cache_save_scope(): ok = check_nightly_scope() and ok ok = check_nightly_base_fallback() and ok ok = check_cache_save_scope() and ok + ok = check_tpc_dataset_caches() and ok ok = check_local_ci_config() and ok if not ok: sys.exit(1) diff --git a/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala b/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala index e25d4e51e46..ada083a2e34 100644 --- a/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala +++ b/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala @@ -119,14 +119,18 @@ object GenTPCHData { } } - // Install tpch-dbgen (with the stdout patch) + // Install tpch-dbgen, pinned to a commit so the TPC-H dataset cache, which is + // keyed on this file, rotates only when the generator changes. def installDBGEN( baseDir: String, url: String = "https://github.com/databricks/tpch-dbgen.git", useStdout: Boolean = true)(i: java.lang.Long): Unit = { - // Check if we want the revision which makes dbgen output to stdout + val dbgenRevision = "6985da461c641fd0d255b214f2d693f1bf08bc33" + // The revision whose bm_utils.c makes dbgen output to stdout. + val stdoutPatchRevision = "0469309147b42abac8857fa61b4cf69a6d3128a8" val checkoutRevision: String = - if (useStdout) "git checkout 0469309147b42abac8857fa61b4cf69a6d3128a8 -- bm_utils.c" else "" + s"git checkout $dbgenRevision" + + (if (useStdout) s" && git checkout $stdoutPatchRevision -- bm_utils.c" else "") Seq("mkdir", "-p", baseDir).! val pw = new PrintWriter(s"${baseDir}/dbgen_$i.sh") From 3f2a119d356eafb3b50fddbb0f6d9e2087d39218 Mon Sep 17 00:00:00 2001 From: Erik Bogado Date: Mon, 21 Sep 2026 22:36:19 -0300 Subject: [PATCH 2/5] ci: key TPC dataset caches on pinned generators The TPC-H and TPC-DS dataset caches were keyed on the hash of pr_build_linux.yml, so every workflow edit regenerated both datasets and left useless entries that evict main's. The generators were also unpinned: GenTPCHData cloned tpch-dbgen HEAD and the tpcds-kit checkout had no ref, so a generator change could not rotate a dataset key. Pin both generators and key the datasets on their inputs instead: - TPC-H key is the hash of GenTPCHData.scala; the generator now checks out a fixed tpch-dbgen commit before applying the stdout patch. - TPC-DS key embeds the pinned tpcds-kit commit, which the checkout now names explicitly. Save both from main only, matching the cache-write policy for other large caches. check_tpc_dataset_caches() in dev/ci/check-ci-config.py asserts all of the above so the old shape cannot come back. --- .github/workflows/pr_build_linux.yml | 35 ++++++-- dev/ci/check-ci-config.py | 84 +++++++++++++++++++ .../org/apache/spark/sql/GenTPCHData.scala | 10 ++- 3 files changed, 120 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 8befa94b7ca..129ee093cb7 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -701,12 +701,12 @@ jobs: restore-keys: | ${{ runner.os }}-java-maven- - - name: Cache TPC-H data + - name: Restore TPC-H data id: cache-tpch - uses: actions/cache@v6 + uses: actions/cache/restore@v6 with: path: ./tpch - key: tpch-${{ hashFiles('.github/workflows/pr_build_linux.yml') }} + key: tpch-sf1-${{ hashFiles('spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala') }} - name: Build project run: | @@ -722,6 +722,17 @@ jobs: run: | SPARK_HOME=`pwd` SPARK_TPCH_DATA=`pwd`/tpch/sf1_parquet ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCHQuerySuite test + # The dataset changes only when the generator does, so main caches it like + # any other large entry. Saved only on main: an entry written from a pull + # request or a `gh-readonly-queue/*` branch can never be restored by a + # later run and evicts main's from the shared budget. + - name: Save TPC-H data + if: ${{ github.ref == 'refs/heads/main' && steps.cache-tpch.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v6 + with: + path: ./tpch + key: tpch-sf1-${{ hashFiles('spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala') }} + # Saved only on main: an entry written from a pull request or a # `gh-readonly-queue/*` branch cannot be restored by any later run, and # it evicts main's from the shared budget. See "Large caches are written @@ -775,12 +786,12 @@ jobs: restore-keys: | ${{ runner.os }}-java-maven- - - name: Cache TPC-DS data + - name: Restore TPC-DS data id: cache-tpcds - uses: actions/cache@v6 + uses: actions/cache/restore@v6 with: path: ./tpcds-sf-1 - key: tpcds-${{ hashFiles('.github/workflows/pr_build_linux.yml') }} + key: tpcds-sf1-1b7fb7529eda - name: Build project run: | @@ -791,6 +802,7 @@ jobs: uses: actions/checkout@v7 with: repository: databricks/tpcds-kit + ref: 1b7fb7529edae091684201fab142d956d6afd881 path: ./tpcds-kit - name: Build tpcds-kit @@ -831,6 +843,17 @@ jobs: spark.sql.autoBroadcastJoinThreshold=-1 spark.sql.join.forceApplyShuffledHashJoin=true + # The dataset changes only when the generator does, so main caches it like + # any other large entry. Saved only on main: an entry written from a pull + # request or a `gh-readonly-queue/*` branch can never be restored by a + # later run and evicts main's from the shared budget. + - name: Save TPC-DS data + if: ${{ github.ref == 'refs/heads/main' && steps.cache-tpcds.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v6 + with: + path: ./tpcds-sf-1 + key: tpcds-sf1-1b7fb7529eda + # Saved only on main: an entry written from a pull request or a # `gh-readonly-queue/*` branch cannot be restored by any later run, and # it evicts main's from the shared budget. See "Large caches are written diff --git a/dev/ci/check-ci-config.py b/dev/ci/check-ci-config.py index 3cf407f1520..ab930e9127d 100644 --- a/dev/ci/check-ci-config.py +++ b/dev/ci/check-ci-config.py @@ -1479,6 +1479,89 @@ def check_cache_save_scope(): return not failures +TPC_WORKFLOW = WORKFLOWS / "pr_build_linux.yml" +TPC_WORKFLOW_HASH = re.compile( + r"hashFiles\(\s*['\"]\.github/workflows/pr_build_linux\.yml['\"]\s*\)" +) +TPCDS_KIT_CHECKOUT = re.compile( + r"repository:[ \t]*databricks/tpcds-kit\n" + r"(?:[ \t]+[^\n]*\n)*?" + r"[ \t]+ref:[ \t]*([0-9a-f]{40})" +) +TPCH_CACHE_KEY = re.compile( + r"key:\s*tpch-sf1-\$\{\{\s*hashFiles\(" + r"'spark/src/test/scala/org/apache/spark/sql/GenTPCHData\.scala'\)\s*\}\}" +) +TPCDS_CACHE_KEY = re.compile(r"key:\s*tpcds-sf1-([0-9a-f]{12})\b") +GIT_COMMIT = re.compile(r"\b[0-9a-f]{40}\b") +GIT_CHECKOUT = re.compile(r"git checkout") + + +def check_tpc_dataset_caches(): + """The TPC datasets are keyed on a pinned generator, not on the workflow file. + + Both dataset caches used `hashFiles('.github/workflows/pr_build_linux.yml')` + as their key, and PR #5973 left them exempt from the main-only save rule + because `./tpch` and `./tpcds-sf-1` are not dependency trees. The workflow + file is one of the most edited files in the repository, so the key rotated + with every CI change, orphaning main's entry and regenerating a + multi-hundred-megabyte dataset -- and the dataset kept competing with the + ~4.1 GiB `Linux-cargo-debug` cache whose eviction costs a ~26 minute cold + build. Key them on the generator input and pin the generators themselves, so + a dataset is regenerated only when the generator changes. + """ + failures = [] + text = TPC_WORKFLOW.read_text(encoding="utf-8") + workflow_hash = TPC_WORKFLOW_HASH.search(text) + if workflow_hash: + failures.append( + f"{TPC_WORKFLOW}: `{workflow_hash.group(0)}` keys a TPC dataset on " + f"this workflow file, so every CI edit rotates the cache key and " + f"regenerates the SF=1 dataset. Key it on the pinned generator" + ) + if not TPCH_CACHE_KEY.search(text): + failures.append( + f"{TPC_WORKFLOW}: the TPC-H dataset cache key is not keyed on " + f"`GenTPCHData.scala`, so a generator change would not rotate it" + ) + kit = TPCDS_KIT_CHECKOUT.search(text) + if not kit: + failures.append( + f"{TPC_WORKFLOW}: the `databricks/tpcds-kit` checkout has no pinned " + f"`ref:`, so the TPC-DS dataset can change without the key rotating" + ) + else: + key = TPCDS_CACHE_KEY.search(text) + if not key or key.group(1) != kit.group(1)[:12]: + failures.append( + f"{TPC_WORKFLOW}: the TPC-DS dataset cache key does not embed the " + f"`tpcds-kit` revision `{kit.group(1)[:12]}`, so a generator bump " + f"would not rotate it" + ) + for line_no, uses, step in _cache_steps(text.splitlines()): + if not CACHE_RW_USES.match(uses): + continue + body = "\n".join(step) + if "tpch" not in body and "tpcds" not in body: + continue + failures.append( + f"{TPC_WORKFLOW}:{line_no}: a TPC dataset cache uses " + f"`actions/cache@vN`, whose save runs in an implicit post step that " + f"no `if:` can reach. Split it into `actions/cache/restore` plus a " + f"main-only `actions/cache/save`" + ) + generator = Path("spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala") + generator_text = generator.read_text(encoding="utf-8") + if not (GIT_COMMIT.search(generator_text) and GIT_CHECKOUT.search(generator_text)): + failures.append( + f"{generator}: the tpch-dbgen checkout is not pinned to a commit, so " + f"the TPC-H dataset can change without the cache key rotating" + ) + for failure in failures: + print(f"tpc dataset cache: {failure}") + return not failures + + if __name__ == "__main__": ok = check_change_filters() ok = check_event_policy() and ok @@ -1494,6 +1577,7 @@ def check_cache_save_scope(): ok = check_nightly_scope() and ok ok = check_nightly_base_fallback() and ok ok = check_cache_save_scope() and ok + ok = check_tpc_dataset_caches() and ok ok = check_local_ci_config() and ok if not ok: sys.exit(1) diff --git a/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala b/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala index 6e8ce177d62..af9ce62153e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala +++ b/spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala @@ -119,14 +119,18 @@ object GenTPCHData { } } - // Install tpch-dbgen (with the stdout patch) + // Install tpch-dbgen, pinned to a commit so the TPC-H dataset cache, which is + // keyed on this file, rotates only when the generator changes. def installDBGEN( baseDir: String, url: String = "https://github.com/databricks/tpch-dbgen.git", useStdout: Boolean = true)(i: java.lang.Long): Unit = { - // Check if we want the revision which makes dbgen output to stdout + val dbgenRevision = "6985da461c641fd0d255b214f2d693f1bf08bc33" + // The revision whose bm_utils.c makes dbgen output to stdout. + val stdoutPatchRevision = "0469309147b42abac8857fa61b4cf69a6d3128a8" val checkoutRevision: String = - if (useStdout) "git checkout 0469309147b42abac8857fa61b4cf69a6d3128a8 -- bm_utils.c" else "" + s"git checkout $dbgenRevision" + + (if (useStdout) s" && git checkout $stdoutPatchRevision -- bm_utils.c" else "") Seq("mkdir", "-p", baseDir).! val pw = new PrintWriter(s"${baseDir}/dbgen_$i.sh") From 7f68df22fe93635abce40c5ca66052c9879bcf23 Mon Sep 17 00:00:00 2001 From: Erik Bogado Date: Fri, 2 Oct 2026 03:43:47 -0300 Subject: [PATCH 3/5] ci: couple TPC cache keys to generation inputs --- .github/workflows/README.md | 10 ++++--- .github/workflows/pr_build_linux.yml | 40 +++++++++++++++------------- dev/ci/check-ci-config.py | 40 +++++++++++++++++++++++++--- 3 files changed, 65 insertions(+), 25 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 77dc9d08034..e1ec8837e79 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -429,10 +429,12 @@ any of the paths above, and rejects a `save` that is missing the `github.ref` guard. `publish_snapshot.yml` is exempt in `CACHE_SAVE_SCOPE_EXEMPT`, because it runs from main on a schedule already. -The TPC-H and TPC-DS dataset caches keep the read-write form and are out of -scope entirely: `./tpch` and `./tpcds-sf-1` are a few hundred MB, they are not -dependency trees, and they are keyed on this workflow file, so a pull request -that edits it would regenerate the data on every run rather than once. +The TPC-H and TPC-DS dataset caches also use separate restore and main-only +save steps. Their keys include the generator (`GenTPCHData.scala` for TPC-H, +the pinned `tpcds-kit` commit for TPC-DS), scale factor and partition count. +The generation arguments, keys and dataset paths use the same job environment +values, so changing an input invalidates the dataset without unrelated workflow +edits rotating its key. `dev/ci/check-ci-config.py` guards this coupling. A job that only ever runs on a pull request or in the queue keeps the guard anyway, and so never writes. That is deliberate — it restores from main's diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 129ee093cb7..a4d3f47db52 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -674,6 +674,8 @@ jobs: container: image: amd64/rust env: + TPC_SCALE_FACTOR: 1 + TPC_NUM_PARTITIONS: 1 JAVA_TOOL_OPTIONS: --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED steps: - uses: actions/checkout@v7 @@ -706,24 +708,24 @@ jobs: uses: actions/cache/restore@v6 with: path: ./tpch - key: tpch-sf1-${{ hashFiles('spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala') }} + key: tpch-sf${{ env.TPC_SCALE_FACTOR }}-p${{ env.TPC_NUM_PARTITIONS }}-${{ hashFiles('spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala') }} - name: Build project run: | ./mvnw -B -Prelease install -DskipTests - - name: Generate TPC-H data (SF=1) + - name: Generate TPC-H data if: steps.cache-tpch.outputs.cache-hit != 'true' run: | - cd spark && MAVEN_OPTS='-Xmx20g' ../mvnw -B -Prelease exec:java -Dexec.mainClass="org.apache.spark.sql.GenTPCHData" -Dexec.classpathScope="test" -Dexec.cleanupDaemonThreads="false" -Dexec.args="--location `pwd`/.. --scaleFactor 1 --numPartitions 1 --overwrite" + cd spark && MAVEN_OPTS='-Xmx20g' ../mvnw -B -Prelease exec:java -Dexec.mainClass="org.apache.spark.sql.GenTPCHData" -Dexec.classpathScope="test" -Dexec.cleanupDaemonThreads="false" -Dexec.args="--location `pwd`/.. --scaleFactor $TPC_SCALE_FACTOR --numPartitions $TPC_NUM_PARTITIONS --overwrite" - name: Run TPC-H queries if: ${{ !inputs.cache-refresh-only }} run: | - SPARK_HOME=`pwd` SPARK_TPCH_DATA=`pwd`/tpch/sf1_parquet ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCHQuerySuite test + SPARK_HOME=`pwd` SPARK_TPCH_DATA=`pwd`/tpch/sf${TPC_SCALE_FACTOR}_parquet ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCHQuerySuite test - # The dataset changes only when the generator does, so main caches it like - # any other large entry. Saved only on main: an entry written from a pull + # The dataset key includes the generator, scale factor and partition count. + # Main caches it like any other large entry. An entry written from a pull # request or a `gh-readonly-queue/*` branch can never be restored by a # later run and evicts main's from the shared budget. - name: Save TPC-H data @@ -731,7 +733,7 @@ jobs: uses: actions/cache/save@v6 with: path: ./tpch - key: tpch-sf1-${{ hashFiles('spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala') }} + key: tpch-sf${{ env.TPC_SCALE_FACTOR }}-p${{ env.TPC_NUM_PARTITIONS }}-${{ hashFiles('spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala') }} # Saved only on main: an entry written from a pull request or a # `gh-readonly-queue/*` branch cannot be restored by any later run, and @@ -759,6 +761,8 @@ jobs: container: image: amd64/rust env: + TPC_SCALE_FACTOR: 1 + TPC_NUM_PARTITIONS: 1 JAVA_TOOL_OPTIONS: --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED steps: - uses: actions/checkout@v7 @@ -790,8 +794,8 @@ jobs: id: cache-tpcds uses: actions/cache/restore@v6 with: - path: ./tpcds-sf-1 - key: tpcds-sf1-1b7fb7529eda + path: ./tpcds-sf-${{ env.TPC_SCALE_FACTOR }} + key: tpcds-sf${{ env.TPC_SCALE_FACTOR }}-p${{ env.TPC_NUM_PARTITIONS }}-1b7fb7529eda - name: Build project run: | @@ -812,15 +816,15 @@ jobs: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 120 --slave /usr/bin/g++ g++ /usr/bin/g++-12 cd tpcds-kit/tools && make OS=LINUX - - name: Generate TPC-DS data (SF=1) + - name: Generate TPC-DS data if: steps.cache-tpcds.outputs.cache-hit != 'true' run: | - cd spark && MAVEN_OPTS='-Xmx20g' ../mvnw -B -Prelease exec:java -Dexec.mainClass="org.apache.spark.sql.GenTPCDSData" -Dexec.classpathScope="test" -Dexec.cleanupDaemonThreads="false" -Dexec.args="--dsdgenDir `pwd`/../tpcds-kit/tools --location `pwd`/../tpcds-sf-1 --scaleFactor 1 --numPartitions 1" + cd spark && MAVEN_OPTS='-Xmx20g' ../mvnw -B -Prelease exec:java -Dexec.mainClass="org.apache.spark.sql.GenTPCDSData" -Dexec.classpathScope="test" -Dexec.cleanupDaemonThreads="false" -Dexec.args="--dsdgenDir `pwd`/../tpcds-kit/tools --location `pwd`/../tpcds-sf-$TPC_SCALE_FACTOR --scaleFactor $TPC_SCALE_FACTOR --numPartitions $TPC_NUM_PARTITIONS" - name: Run TPC-DS queries (Sort merge join) if: ${{ !inputs.cache-refresh-only }} run: | - SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test + SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-$TPC_SCALE_FACTOR ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test env: SPARK_TPCDS_JOIN_CONF: | spark.sql.autoBroadcastJoinThreshold=-1 @@ -829,7 +833,7 @@ jobs: - name: Run TPC-DS queries (Broadcast hash join) if: ${{ !inputs.cache-refresh-only }} run: | - SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test + SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-$TPC_SCALE_FACTOR ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test env: SPARK_TPCDS_JOIN_CONF: | spark.sql.autoBroadcastJoinThreshold=10485760 @@ -837,22 +841,22 @@ jobs: - name: Run TPC-DS queries (Shuffled hash join) if: ${{ !inputs.cache-refresh-only }} run: | - SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test + SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-$TPC_SCALE_FACTOR ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test env: SPARK_TPCDS_JOIN_CONF: | spark.sql.autoBroadcastJoinThreshold=-1 spark.sql.join.forceApplyShuffledHashJoin=true - # The dataset changes only when the generator does, so main caches it like - # any other large entry. Saved only on main: an entry written from a pull + # The dataset key includes the generator, scale factor and partition count. + # Main caches it like any other large entry. An entry written from a pull # request or a `gh-readonly-queue/*` branch can never be restored by a # later run and evicts main's from the shared budget. - name: Save TPC-DS data if: ${{ github.ref == 'refs/heads/main' && steps.cache-tpcds.outputs.cache-hit != 'true' }} uses: actions/cache/save@v6 with: - path: ./tpcds-sf-1 - key: tpcds-sf1-1b7fb7529eda + path: ./tpcds-sf-${{ env.TPC_SCALE_FACTOR }} + key: tpcds-sf${{ env.TPC_SCALE_FACTOR }}-p${{ env.TPC_NUM_PARTITIONS }}-1b7fb7529eda # Saved only on main: an entry written from a pull request or a # `gh-readonly-queue/*` branch cannot be restored by any later run, and diff --git a/dev/ci/check-ci-config.py b/dev/ci/check-ci-config.py index ab930e9127d..9c0512c78c4 100644 --- a/dev/ci/check-ci-config.py +++ b/dev/ci/check-ci-config.py @@ -1489,10 +1489,10 @@ def check_cache_save_scope(): r"[ \t]+ref:[ \t]*([0-9a-f]{40})" ) TPCH_CACHE_KEY = re.compile( - r"key:\s*tpch-sf1-\$\{\{\s*hashFiles\(" + r"key:\s*tpch-sf.*-\$\{\{\s*hashFiles\(" r"'spark/src/test/scala/org/apache/spark/sql/GenTPCHData\.scala'\)\s*\}\}" ) -TPCDS_CACHE_KEY = re.compile(r"key:\s*tpcds-sf1-([0-9a-f]{12})\b") +TPCDS_CACHE_KEY = re.compile(r"key:\s*tpcds-sf.*-([0-9a-f]{12})\b") GIT_COMMIT = re.compile(r"\b[0-9a-f]{40}\b") GIT_CHECKOUT = re.compile(r"git checkout") @@ -1508,7 +1508,7 @@ def check_tpc_dataset_caches(): multi-hundred-megabyte dataset -- and the dataset kept competing with the ~4.1 GiB `Linux-cargo-debug` cache whose eviction costs a ~26 minute cold build. Key them on the generator input and pin the generators themselves, so - a dataset is regenerated only when the generator changes. + a dataset is regenerated only when the generator or its arguments change. """ failures = [] text = TPC_WORKFLOW.read_text(encoding="utf-8") @@ -1550,6 +1550,40 @@ def check_tpc_dataset_caches(): f"no `if:` can reach. Split it into `actions/cache/restore` plus a " f"main-only `actions/cache/save`" ) + for dataset in ("tpch", "tpcds"): + job = f"verify-benchmark-results-{dataset}" + block = re.search(rf"(?ms)^ {job}:\n(.*?)(?=^ [A-Za-z0-9_-]+:|\Z)", text) + if not block: + failures.append(f"{TPC_WORKFLOW}: missing dataset job `{job}`") + continue + body = block.group(1) + for option, variable in ( + ("scaleFactor", "TPC_SCALE_FACTOR"), + ("numPartitions", "TPC_NUM_PARTITIONS"), + ): + if not re.search(rf"(?m)^ {variable}: [1-9][0-9]*$", body): + failures.append(f"{job}: define positive `{variable}` in job env") + if re.findall(rf"--{option}\s+([^\s\"]+)", body) != [f"${variable}"]: + failures.append(f"{job}: generator `--{option}` must use `${variable}`") + keys = re.findall(rf"(?m)^ key: ({dataset}-.*)$", body) + prefix = f"{dataset}-sf${{{{ env.TPC_SCALE_FACTOR }}}}-p${{{{ env.TPC_NUM_PARTITIONS }}}}-" + if len(keys) != 2 or keys[0] != keys[1] or not keys[0].startswith(prefix): + failures.append( + f"{job}: restore/save keys must match and include actual " + f"scale factor and partition count" + ) + restore = re.search( + r"(?ms)^ - name: Restore TPC-[A-Z-]+ data\n(.*?)(?=^ - |\Z)", body + ) + if restore and re.search(r"(?m)^ restore-keys:", restore.group(1)): + failures.append( + f"{job}: dataset restore must not fall back to different generation inputs" + ) + for _, uses, step in _cache_steps(body.splitlines()): + step_body = "\n".join(step) + if CACHE_SAVE_USES.match(uses) and f"key: {dataset}-" in step_body: + if not CACHE_MAIN_GUARD.search(step_body): + failures.append(f"{job}: dataset saves must be main-only") generator = Path("spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala") generator_text = generator.read_text(encoding="utf-8") if not (GIT_COMMIT.search(generator_text) and GIT_CHECKOUT.search(generator_text)): From fd0b0e68b0f44da495fa2ad598840ba85609fbac Mon Sep 17 00:00:00 2001 From: Erik Bogado Date: Mon, 26 Oct 2026 10:46:24 -0300 Subject: [PATCH 4/5] docs(ci): correct TPC cache key comments after the pinned-generator change --- .github/workflows/README.md | 11 +++++++---- dev/ci/check-ci-config.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index ecb71140356..f991c5c4cff 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -50,8 +50,10 @@ merge. Three jobs are still on `push`: `docs`, because it deploys to `pr_build_macos`, because of `actions/cache` scoping. A pull request can only restore caches saved on its own branch or on `main`, and the queue runs on a throwaway `gh-readonly-queue/*` branch whose caches are deleted with it. Without a push -run, a `Cargo.lock` or `pom.xml` change would leave the cargo-ci, Maven and -TPC-H/TPC-DS caches on `main` stale until the next unrelated change. +run, a `Cargo.lock` or `pom.xml` change would leave the cargo-ci and Maven +caches on `main` stale until the next unrelated change. The TPC-H and TPC-DS +datasets do not depend on those files; the push run re-warms them only after +eviction. Warming those caches is the only thing the push run is for, so on `push` the Linux build runs in **cache-refresh-only** mode: `build-native` and the two @@ -445,8 +447,9 @@ edits rotating its key. `dev/ci/check-ci-config.py` guards this coupling. A job that only ever runs on a pull request or in the queue keeps the guard anyway, and so never writes. That is deliberate — it restores from main's -entry through `restore-keys` and downloads whatever else it needs, which is -what a cold pull request already did. See the push-tier discussion above for +entry through `restore-keys` (the TPC dataset restores require an exact key +instead) and downloads whatever else it needs, which is what a cold pull +request already did. See the push-tier discussion above for which jobs do run on main and therefore do write. ## Retrying flaky network operations diff --git a/dev/ci/check-ci-config.py b/dev/ci/check-ci-config.py index cf701d183df..1b8b495e2bd 100644 --- a/dev/ci/check-ci-config.py +++ b/dev/ci/check-ci-config.py @@ -1583,7 +1583,7 @@ def check_tpc_dataset_caches(): Both dataset caches used `hashFiles('.github/workflows/pr_build_linux.yml')` as their key, and PR #5973 left them exempt from the main-only save rule - because `./tpch` and `./tpcds-sf-1` are not dependency trees. The workflow + because `./tpch` and `./tpcds-sf-` are not dependency trees. The workflow file is one of the most edited files in the repository, so the key rotated with every CI change, orphaning main's entry and regenerating a multi-hundred-megabyte dataset -- and the dataset kept competing with the From 67e2b73e29ea174b06b1b043139ae9658341361f Mon Sep 17 00:00:00 2001 From: Erik Bogado Date: Sun, 15 Nov 2026 18:46:07 -0300 Subject: [PATCH 5/5] fix(ci): include TPC generator inputs in cache keys --- .github/workflows/README.md | 7 +++++-- .github/workflows/pr_build_linux.yml | 10 ++++++---- dev/ci/check-ci-config.py | 13 ++++++++----- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 80445730785..3a965e4af5b 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -514,8 +514,11 @@ guard. `publish_snapshot.yml` is exempt in `CACHE_SAVE_SCOPE_EXEMPT`, because it runs from main on a schedule already. The TPC-H and TPC-DS dataset caches also use separate restore and main-only -save steps. Their keys include the generator (`GenTPCHData.scala` for TPC-H, -the pinned `tpcds-kit` commit for TPC-DS), scale factor and partition count. +save steps. Their keys include the generator (`GenTPCHData.scala` and `TPCH.scala` +for TPC-H, the pinned `tpcds-kit` commit and root `pom.xml` hash for TPC-DS), +scale factor and partition count. The root POM selects the Spark tests jar +providing `GenTPCDSData` and `TPCDSTables`; hashing it conservatively invalidates +TPC-DS on unrelated POM edits too. The generation arguments, keys and dataset paths use the same job environment values, so changing an input invalidates the dataset without unrelated workflow edits rotating its key. `dev/ci/check-ci-config.py` guards this coupling. diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index e093b6e869a..4ed59c6dd29 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -802,7 +802,7 @@ jobs: uses: actions/cache/restore@v6 with: path: ./tpch - key: tpch-sf${{ env.TPC_SCALE_FACTOR }}-p${{ env.TPC_NUM_PARTITIONS }}-${{ hashFiles('spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala') }} + key: tpch-sf${{ env.TPC_SCALE_FACTOR }}-p${{ env.TPC_NUM_PARTITIONS }}-${{ hashFiles('spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala', 'spark/src/test/scala/org/apache/spark/sql/TPCH.scala') }} - name: Build project run: | @@ -827,7 +827,7 @@ jobs: uses: actions/cache/save@v6 with: path: ./tpch - key: tpch-sf${{ env.TPC_SCALE_FACTOR }}-p${{ env.TPC_NUM_PARTITIONS }}-${{ hashFiles('spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala') }} + key: tpch-sf${{ env.TPC_SCALE_FACTOR }}-p${{ env.TPC_NUM_PARTITIONS }}-${{ hashFiles('spark/src/test/scala/org/apache/spark/sql/GenTPCHData.scala', 'spark/src/test/scala/org/apache/spark/sql/TPCH.scala') }} # Saved only on main: an entry written from a pull request or a # `gh-readonly-queue/*` branch cannot be restored by any later run, and @@ -884,12 +884,14 @@ jobs: restore-keys: | ${{ runner.os }}-java-maven- + # The root POM selects Spark's GenTPCDSData/TPCDSTables tests jar. + # Hashing it also invalidates the dataset on unrelated POM edits. - name: Restore TPC-DS data id: cache-tpcds uses: actions/cache/restore@v6 with: path: ./tpcds-sf-${{ env.TPC_SCALE_FACTOR }} - key: tpcds-sf${{ env.TPC_SCALE_FACTOR }}-p${{ env.TPC_NUM_PARTITIONS }}-1b7fb7529eda + key: tpcds-sf${{ env.TPC_SCALE_FACTOR }}-p${{ env.TPC_NUM_PARTITIONS }}-1b7fb7529eda-${{ hashFiles('pom.xml') }} - name: Build project run: | @@ -950,7 +952,7 @@ jobs: uses: actions/cache/save@v6 with: path: ./tpcds-sf-${{ env.TPC_SCALE_FACTOR }} - key: tpcds-sf${{ env.TPC_SCALE_FACTOR }}-p${{ env.TPC_NUM_PARTITIONS }}-1b7fb7529eda + key: tpcds-sf${{ env.TPC_SCALE_FACTOR }}-p${{ env.TPC_NUM_PARTITIONS }}-1b7fb7529eda-${{ hashFiles('pom.xml') }} # Saved only on main: an entry written from a pull request or a # `gh-readonly-queue/*` branch cannot be restored by any later run, and diff --git a/dev/ci/check-ci-config.py b/dev/ci/check-ci-config.py index b5a4b9031fd..774a9e971ae 100644 --- a/dev/ci/check-ci-config.py +++ b/dev/ci/check-ci-config.py @@ -1606,9 +1606,12 @@ def check_cache_save_scope(): ) TPCH_CACHE_KEY = re.compile( r"key:\s*tpch-sf.*-\$\{\{\s*hashFiles\(" - r"'spark/src/test/scala/org/apache/spark/sql/GenTPCHData\.scala'\)\s*\}\}" + r"'spark/src/test/scala/org/apache/spark/sql/GenTPCHData\.scala',\s*" + r"'spark/src/test/scala/org/apache/spark/sql/TPCH\.scala'\)\s*\}\}" +) +TPCDS_CACHE_KEY = re.compile( + r"key:\s*tpcds-sf.*-([0-9a-f]{12})-\$\{\{\s*hashFiles\('pom\.xml'\)\s*\}\}" ) -TPCDS_CACHE_KEY = re.compile(r"key:\s*tpcds-sf.*-([0-9a-f]{12})\b") GIT_COMMIT = re.compile(r"\b[0-9a-f]{40}\b") GIT_CHECKOUT = re.compile(r"git checkout") @@ -1638,7 +1641,7 @@ def check_tpc_dataset_caches(): if not TPCH_CACHE_KEY.search(text): failures.append( f"{TPC_WORKFLOW}: the TPC-H dataset cache key is not keyed on " - f"`GenTPCHData.scala`, so a generator change would not rotate it" + f"`GenTPCHData.scala` and `TPCH.scala`, so a generator change would not rotate it" ) kit = TPCDS_KIT_CHECKOUT.search(text) if not kit: @@ -1651,8 +1654,8 @@ def check_tpc_dataset_caches(): if not key or key.group(1) != kit.group(1)[:12]: failures.append( f"{TPC_WORKFLOW}: the TPC-DS dataset cache key does not embed the " - f"`tpcds-kit` revision `{kit.group(1)[:12]}`, so a generator bump " - f"would not rotate it" + f"`tpcds-kit` revision `{kit.group(1)[:12]}` and the root `pom.xml` " + f"hash, so a native or Spark generator bump would not rotate it" ) for line_no, uses, step in _cache_steps(text.splitlines()): if not CACHE_RW_USES.match(uses):