diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 6addfd923b9..d54eca33845 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -603,6 +603,26 @@ the `java-test`, `rust-test` and `setup-spark-builder` callers. A new job that runs `./mvnw` without going through a setup action needs the step before its first Maven use. +## Job timeouts + +Every job that runs on a runner sets `timeout-minutes`. Without it GitHub's +default is six hours, so a test suite that hangs holds a runner, and the +pipeline it belongs to, for six hours before anyone sees a failure. Each limit +is about two to three times the slowest successful run of that job in November +2026, rounded up: 180 minutes for the macOS test matrix, 120 for the Spark SQL, +Iceberg and Delta suites, 90 for native builds and the Linux test matrix, and +10 to 60 for lint and housekeeping jobs. A job that is legitimately getting +slower should have its limit raised in the same pull request that makes it +slower, not removed. A job that hits its limit fails with "The job has exceeded +the maximum execution time", which is a hang until shown otherwise, not a +flaky runner. + +A reusable workflow's caller cannot set `timeout-minutes`, so the limits live +on the jobs inside `pr_build_linux.yml`, `spark_sql_test_reusable.yml` and the +other reusable workflows. The merge queue's +`check_response_timeout_minutes: 300` (see below) is well above every job +limit, so a timed-out job fails the entry rather than leaving it waiting. + ## Merge queue `.asf.yaml` declares a `Merge Queue` ruleset for the default branch, so `main` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0780a9b87f..04e43fcfa4c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,6 +161,7 @@ jobs: && (github.event_name != 'push' || github.repository == 'apache/datafusion-comet' || github.workflow == 'Comet CI (fork run)') runs-on: ubuntu-slim + timeout-minutes: 30 steps: - uses: actions/checkout@v7 @@ -287,6 +288,7 @@ jobs: name: Detect changes needs: preflight runs-on: ubuntu-slim + timeout-minutes: 15 # `actions: read` is for dev/ci/nightly-base.py, which lists previous # scheduled runs to pick the nightly's diff base. permissions: @@ -691,6 +693,7 @@ jobs: - iceberg_1_10 - iceberg_1_11 runs-on: ubuntu-slim + timeout-minutes: 15 steps: - name: Summarize upstream results env: @@ -732,6 +735,7 @@ jobs: actions: read issues: write runs-on: ubuntu-slim + timeout-minutes: 15 steps: - uses: actions/github-script@v9 with: diff --git a/.github/workflows/ci_fork.yml b/.github/workflows/ci_fork.yml index a7b67b5b48f..50b4c0cfa63 100644 --- a/.github/workflows/ci_fork.yml +++ b/.github/workflows/ci_fork.yml @@ -66,6 +66,7 @@ jobs: # test it twice. if: github.repository != 'apache/datafusion-comet' runs-on: ubuntu-slim + timeout-minutes: 15 # Failing to cancel only leaves an older run to finish, so it never blocks the run. continue-on-error: true permissions: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index fdf0d6db746..79a59294ee3 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -40,6 +40,7 @@ jobs: # still get one. if: github.event_name != 'schedule' || github.repository == 'apache/datafusion-comet' runs-on: ubuntu-slim + timeout-minutes: 30 permissions: contents: read security-events: write diff --git a/.github/workflows/delta_build_gate.yml b/.github/workflows/delta_build_gate.yml index caf8a37ae3f..9d3d2b1765f 100644 --- a/.github/workflows/delta_build_gate.yml +++ b/.github/workflows/delta_build_gate.yml @@ -51,6 +51,7 @@ jobs: delta-build-gate: name: Build-gate verification (default builds exclude Delta) runs-on: ubuntu-24.04 + timeout-minutes: 60 container: image: amd64/rust env: diff --git a/.github/workflows/delta_contrib_test.yml b/.github/workflows/delta_contrib_test.yml index 10e6b9f1029..400b54ce39c 100644 --- a/.github/workflows/delta_contrib_test.yml +++ b/.github/workflows/delta_contrib_test.yml @@ -50,6 +50,7 @@ jobs: contrib-delta: name: Delta contrib (Spark ${{ inputs.spark }}) runs-on: ubuntu-24.04 + timeout-minutes: 120 container: image: amd64/rust steps: @@ -139,6 +140,7 @@ jobs: # Pinned to Spark 3.5, so only that caller runs it: once per ci.yml run. if: inputs.spark == '3.5' runs-on: ubuntu-24.04 + timeout-minutes: 90 steps: - uses: actions/checkout@v7 @@ -228,6 +230,7 @@ jobs: # Spark-independent, so only the 3.5 caller runs it: once per ci.yml run. if: inputs.spark == '3.5' runs-on: ubuntu-24.04 + timeout-minutes: 60 container: image: amd64/rust steps: @@ -265,6 +268,7 @@ jobs: # Spark-independent, so only the 3.5 caller runs it: once per ci.yml run. if: inputs.spark == '3.5' runs-on: ubuntu-24.04 + timeout-minutes: 15 steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 87968a44c99..eddf3da5313 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -27,6 +27,7 @@ jobs: name: Build docs if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ubuntu-24.04 + timeout-minutes: 60 permissions: contents: read steps: @@ -112,6 +113,7 @@ jobs: name: Deploy docs needs: build-docs runs-on: ubuntu-24.04 + timeout-minutes: 15 permissions: contents: write steps: diff --git a/.github/workflows/iceberg_spark_test_reusable.yml b/.github/workflows/iceberg_spark_test_reusable.yml index faa7ae610e5..4328b0fadb1 100644 --- a/.github/workflows/iceberg_spark_test_reusable.yml +++ b/.github/workflows/iceberg_spark_test_reusable.yml @@ -63,6 +63,7 @@ jobs: build-native: name: Build Native Library runs-on: ubuntu-24.04 + timeout-minutes: 90 outputs: shard-matrix: ${{ steps.shards.outputs.matrix }} shard-count: ${{ steps.shards.outputs.count }} @@ -126,6 +127,7 @@ jobs: matrix: ${{ fromJSON(needs.build-native.outputs.shard-matrix) }} name: iceberg-spark/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }}/shard-${{ matrix.shard }} runs-on: ubuntu-24.04 + timeout-minutes: 120 container: image: amd64/rust env: @@ -185,6 +187,7 @@ jobs: if: ${{ !cancelled() }} name: iceberg-spark-shard-coverage/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }} runs-on: ubuntu-slim + timeout-minutes: 15 steps: - uses: actions/checkout@v7 - name: Download Iceberg shard inventories @@ -208,6 +211,7 @@ jobs: needs: build-native name: iceberg-spark-extensions/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }} runs-on: ubuntu-24.04 + timeout-minutes: 120 container: image: amd64/rust env: @@ -257,6 +261,7 @@ jobs: needs: build-native name: iceberg-spark-runtime/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }} runs-on: ubuntu-24.04 + timeout-minutes: 60 container: image: amd64/rust env: diff --git a/.github/workflows/label_new_issues.yml b/.github/workflows/label_new_issues.yml index bbb4fb150d3..cd9879e8fe3 100644 --- a/.github/workflows/label_new_issues.yml +++ b/.github/workflows/label_new_issues.yml @@ -27,6 +27,7 @@ permissions: jobs: add-triage-label: runs-on: ubuntu-slim + timeout-minutes: 10 steps: - uses: actions/github-script@v9 with: diff --git a/.github/workflows/miri.yml b/.github/workflows/miri.yml index 3d8cf9b1bc7..28ecad90ce7 100644 --- a/.github/workflows/miri.yml +++ b/.github/workflows/miri.yml @@ -38,6 +38,7 @@ jobs: # The nightly runs upstream only. A fork can still run it by dispatch. if: github.event_name != 'schedule' || github.repository == 'apache/datafusion-comet' runs-on: ubuntu-24.04 + timeout-minutes: 90 strategy: fail-fast: false # Keep Miri focused on Comet's memory-access code. Whole-workspace tests @@ -86,6 +87,7 @@ jobs: permissions: issues: write runs-on: ubuntu-slim + timeout-minutes: 15 steps: - uses: actions/github-script@v9 with: diff --git a/.github/workflows/pr_benchmark_check.yml b/.github/workflows/pr_benchmark_check.yml index ff297cd3326..18785e050ac 100644 --- a/.github/workflows/pr_benchmark_check.yml +++ b/.github/workflows/pr_benchmark_check.yml @@ -33,6 +33,7 @@ jobs: benchmark-check: name: Benchmark Compile & Lint Check runs-on: ubuntu-24.04 + timeout-minutes: 30 container: image: amd64/rust steps: diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 3c48f07ae74..70f86501c2b 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -88,6 +88,7 @@ jobs: lint: name: Lint runs-on: ubuntu-24.04 + timeout-minutes: 30 container: image: amd64/rust outputs: @@ -116,6 +117,7 @@ jobs: name: Lint Scala (syntactic) if: ${{ !inputs.cache-refresh-only && inputs.profiles != 'nightly' }} runs-on: ubuntu-24.04 + timeout-minutes: 30 steps: - uses: actions/checkout@v7 @@ -157,6 +159,7 @@ jobs: name: Lint Java (${{ matrix.profile.name }}) if: ${{ !inputs.cache-refresh-only && inputs.profiles != 'nightly' }} runs-on: ubuntu-24.04 + timeout-minutes: 30 container: image: amd64/rust env: @@ -286,6 +289,7 @@ jobs: name: Strict Scala warnings (Spark 3.5, JDK 17) if: ${{ !inputs.cache-refresh-only && inputs.profiles != 'nightly' }} runs-on: ubuntu-24.04 + timeout-minutes: 30 container: image: amd64/rust steps: @@ -329,6 +333,7 @@ jobs: name: Celeborn ${{ matrix.celeborn_version }} reflection compatibility if: ${{ !inputs.cache-refresh-only && inputs.profiles != 'nightly' }} runs-on: ubuntu-24.04 + timeout-minutes: 30 container: image: amd64/rust env: @@ -394,6 +399,7 @@ jobs: needs: lint name: Build Native Library runs-on: ubuntu-24.04 + timeout-minutes: 90 container: image: amd64/rust steps: @@ -460,6 +466,7 @@ jobs: name: ubuntu-latest/rust-test if: ${{ !inputs.cache-refresh-only && inputs.profiles != 'nightly' }} runs-on: ubuntu-24.04 + timeout-minutes: 60 container: image: amd64/rust steps: @@ -485,6 +492,7 @@ jobs: name: ubuntu-latest/native-executors if: ${{ !inputs.cache-refresh-only && inputs.profiles != 'nightly' }} runs-on: ubuntu-24.04 + timeout-minutes: 60 container: image: amd64/rust steps: @@ -717,6 +725,7 @@ jobs: fail-fast: false name: ${{ matrix.profile.name }} [${{ matrix.suite.name }}] runs-on: ubuntu-24.04 + timeout-minutes: 90 container: image: amd64/rust env: @@ -768,6 +777,7 @@ jobs: name: Verify TPC-H Results if: ${{ inputs.profiles != 'nightly' }} runs-on: ubuntu-24.04 + timeout-minutes: 60 container: image: amd64/rust env: @@ -842,6 +852,7 @@ jobs: name: Verify TPC-DS Results if: ${{ inputs.profiles != 'nightly' }} runs-on: ubuntu-24.04 + timeout-minutes: 60 container: image: amd64/rust env: diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index eae6aef64ad..f0ed36bc776 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -50,6 +50,7 @@ jobs: lint: name: Lint runs-on: ubuntu-24.04 + timeout-minutes: 30 container: image: amd64/rust steps: @@ -69,6 +70,7 @@ jobs: needs: lint name: Build Native Library (macOS) runs-on: macos-15 + timeout-minutes: 90 steps: - uses: actions/checkout@v7 @@ -356,6 +358,7 @@ jobs: fail-fast: false name: ${{ matrix.os }}/${{ matrix.profile.name }} [${{ matrix.suite.name }}] runs-on: ${{ matrix.os }} + timeout-minutes: 180 steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/pr_title_check.yml b/.github/workflows/pr_title_check.yml index 65c27465234..c501b04a82d 100644 --- a/.github/workflows/pr_title_check.yml +++ b/.github/workflows/pr_title_check.yml @@ -28,6 +28,7 @@ on: jobs: check-pr-title: runs-on: ubuntu-slim + timeout-minutes: 10 steps: - uses: actions/checkout@v7 - name: Check PR title diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml index 667a344870b..085f600ab8a 100644 --- a/.github/workflows/publish_snapshot.yml +++ b/.github/workflows/publish_snapshot.yml @@ -64,6 +64,7 @@ jobs: # anywhere so the workflow can be exercised before it lands. if: github.repository == 'apache/datafusion-comet' || inputs.dry_run runs-on: ubuntu-slim + timeout-minutes: 15 outputs: publish: ${{ steps.check.outputs.publish }} steps: diff --git a/.github/workflows/pyarrow_udf_test.yml b/.github/workflows/pyarrow_udf_test.yml index f29caa052ff..2bb311f6877 100644 --- a/.github/workflows/pyarrow_udf_test.yml +++ b/.github/workflows/pyarrow_udf_test.yml @@ -35,6 +35,7 @@ jobs: pyarrow-udf: name: PyArrow UDF (${{ matrix.name }}, JDK 17, Python 3.11) runs-on: ubuntu-24.04 + timeout-minutes: 60 strategy: fail-fast: false matrix: diff --git a/.github/workflows/spark_sql_test_reusable.yml b/.github/workflows/spark_sql_test_reusable.yml index 3a775d3eedf..8274eb46560 100644 --- a/.github/workflows/spark_sql_test_reusable.yml +++ b/.github/workflows/spark_sql_test_reusable.yml @@ -80,6 +80,7 @@ jobs: build: name: Build Native + JVM Test Classes runs-on: ubuntu-24.04 + timeout-minutes: 90 container: image: amd64/rust outputs: @@ -225,6 +226,7 @@ jobs: fail-fast: false name: spark-sql-${{ matrix.module.name }}/spark-${{ inputs.spark-full }}-jdk${{ inputs.java }} runs-on: ubuntu-24.04 + timeout-minutes: 120 container: image: amd64/rust steps: @@ -349,6 +351,7 @@ jobs: name: merge-fallback-logs needs: [spark-sql-test] runs-on: ubuntu-24.04 + timeout-minutes: 15 steps: - name: Download fallback log artifacts # Not wrapped in download-artifact-retry: that local action needs a diff --git a/.github/workflows/spark_sql_writer_tests.yml b/.github/workflows/spark_sql_writer_tests.yml index 6b729cc488b..76b561141cf 100644 --- a/.github/workflows/spark_sql_writer_tests.yml +++ b/.github/workflows/spark_sql_writer_tests.yml @@ -57,6 +57,7 @@ jobs: writer-test: name: spark-sql-writer/spark-${{ inputs.spark-version }} runs-on: ubuntu-24.04 + timeout-minutes: 120 container: image: amd64/rust steps: diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index e6d50fb82aa..860f346fd9d 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -25,6 +25,7 @@ jobs: # Upstream only: in a fork this would close the fork owner's own stale PRs. if: github.repository == 'apache/datafusion-comet' runs-on: ubuntu-slim + timeout-minutes: 15 permissions: issues: write pull-requests: write diff --git a/.github/workflows/take.yml b/.github/workflows/take.yml index 9e84da8da68..b9da6984c3d 100644 --- a/.github/workflows/take.yml +++ b/.github/workflows/take.yml @@ -26,6 +26,7 @@ permissions: jobs: issue_assign: runs-on: ubuntu-slim + timeout-minutes: 10 if: (!github.event.issue.pull_request) && (github.event.comment.body == 'take' || github.event.comment.body == 'untake') concurrency: group: ${{ github.actor }}-issue-assign