diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 1976ca2632a..08996a9bcc9 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -528,6 +528,26 @@ the `java-test`, `rust-test` and `setup-spark-builder` callers. A new job that runs `./mvnw` without going through a setup action needs the step before its first Maven use. +## Job timeouts + +Every job that runs on a runner sets `timeout-minutes`. Without it GitHub's +default is six hours, so a test suite that hangs holds a runner, and the +pipeline it belongs to, for six hours before anyone sees a failure. Each limit +is about two to three times the slowest successful run of that job in November +2026, rounded up: 180 minutes for the macOS test matrix, 120 for the Spark SQL, +Iceberg and Delta suites, 90 for native builds and the Linux test matrix, and +10 to 60 for lint and housekeeping jobs. A job that is legitimately getting +slower should have its limit raised in the same pull request that makes it +slower, not removed. A job that hits its limit fails with "The job has exceeded +the maximum execution time", which is a hang until shown otherwise, not a +flaky runner. + +A reusable workflow's caller cannot set `timeout-minutes`, so the limits live +on the jobs inside `pr_build_linux.yml`, `spark_sql_test_reusable.yml` and the +other reusable workflows. The merge queue's +`check_response_timeout_minutes: 300` (see below) is well above every job +limit, so a timed-out job fails the entry rather than leaving it waiting. + ## Merge queue `.asf.yaml` declares a `Merge Queue` ruleset for the default branch, so `main` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 936b804272d..d182388c82e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,6 +132,7 @@ jobs: preflight: name: Preflight runs-on: ubuntu-slim + timeout-minutes: 30 steps: - uses: actions/checkout@v7 @@ -238,6 +239,7 @@ jobs: name: Detect changes needs: preflight runs-on: ubuntu-slim + timeout-minutes: 15 # `actions: read` is for dev/ci/nightly-base.py, which lists previous # scheduled runs to pick the nightly's diff base. permissions: @@ -667,6 +669,7 @@ jobs: - iceberg_1_10 - iceberg_1_11 runs-on: ubuntu-slim + timeout-minutes: 15 steps: - name: Summarize upstream results env: @@ -704,6 +707,7 @@ jobs: actions: read issues: write runs-on: ubuntu-slim + timeout-minutes: 15 steps: - uses: actions/github-script@v9 with: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c834da33407..580b13bcfda 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -37,6 +37,7 @@ jobs: analyze: name: Analyze Actions runs-on: ubuntu-slim + timeout-minutes: 30 permissions: contents: read security-events: write diff --git a/.github/workflows/delta_build_gate.yml b/.github/workflows/delta_build_gate.yml index caf8a37ae3f..9d3d2b1765f 100644 --- a/.github/workflows/delta_build_gate.yml +++ b/.github/workflows/delta_build_gate.yml @@ -51,6 +51,7 @@ jobs: delta-build-gate: name: Build-gate verification (default builds exclude Delta) runs-on: ubuntu-24.04 + timeout-minutes: 60 container: image: amd64/rust env: diff --git a/.github/workflows/delta_contrib_test.yml b/.github/workflows/delta_contrib_test.yml index 10e6b9f1029..400b54ce39c 100644 --- a/.github/workflows/delta_contrib_test.yml +++ b/.github/workflows/delta_contrib_test.yml @@ -50,6 +50,7 @@ jobs: contrib-delta: name: Delta contrib (Spark ${{ inputs.spark }}) runs-on: ubuntu-24.04 + timeout-minutes: 120 container: image: amd64/rust steps: @@ -139,6 +140,7 @@ jobs: # Pinned to Spark 3.5, so only that caller runs it: once per ci.yml run. if: inputs.spark == '3.5' runs-on: ubuntu-24.04 + timeout-minutes: 90 steps: - uses: actions/checkout@v7 @@ -228,6 +230,7 @@ jobs: # Spark-independent, so only the 3.5 caller runs it: once per ci.yml run. if: inputs.spark == '3.5' runs-on: ubuntu-24.04 + timeout-minutes: 60 container: image: amd64/rust steps: @@ -265,6 +268,7 @@ jobs: # Spark-independent, so only the 3.5 caller runs it: once per ci.yml run. if: inputs.spark == '3.5' runs-on: ubuntu-24.04 + timeout-minutes: 15 steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 1222dbef58b..f2e4be5f630 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -27,6 +27,7 @@ jobs: name: Build docs if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ubuntu-24.04 + timeout-minutes: 60 steps: - name: Checkout docs sources uses: actions/checkout@v7 diff --git a/.github/workflows/iceberg_spark_test_reusable.yml b/.github/workflows/iceberg_spark_test_reusable.yml index faa7ae610e5..4328b0fadb1 100644 --- a/.github/workflows/iceberg_spark_test_reusable.yml +++ b/.github/workflows/iceberg_spark_test_reusable.yml @@ -63,6 +63,7 @@ jobs: build-native: name: Build Native Library runs-on: ubuntu-24.04 + timeout-minutes: 90 outputs: shard-matrix: ${{ steps.shards.outputs.matrix }} shard-count: ${{ steps.shards.outputs.count }} @@ -126,6 +127,7 @@ jobs: matrix: ${{ fromJSON(needs.build-native.outputs.shard-matrix) }} name: iceberg-spark/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }}/shard-${{ matrix.shard }} runs-on: ubuntu-24.04 + timeout-minutes: 120 container: image: amd64/rust env: @@ -185,6 +187,7 @@ jobs: if: ${{ !cancelled() }} name: iceberg-spark-shard-coverage/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }} runs-on: ubuntu-slim + timeout-minutes: 15 steps: - uses: actions/checkout@v7 - name: Download Iceberg shard inventories @@ -208,6 +211,7 @@ jobs: needs: build-native name: iceberg-spark-extensions/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }} runs-on: ubuntu-24.04 + timeout-minutes: 120 container: image: amd64/rust env: @@ -257,6 +261,7 @@ jobs: needs: build-native name: iceberg-spark-runtime/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }} runs-on: ubuntu-24.04 + timeout-minutes: 60 container: image: amd64/rust env: diff --git a/.github/workflows/label_new_issues.yml b/.github/workflows/label_new_issues.yml index bbb4fb150d3..cd9879e8fe3 100644 --- a/.github/workflows/label_new_issues.yml +++ b/.github/workflows/label_new_issues.yml @@ -27,6 +27,7 @@ permissions: jobs: add-triage-label: runs-on: ubuntu-slim + timeout-minutes: 10 steps: - uses: actions/github-script@v9 with: diff --git a/.github/workflows/label_prs.yml b/.github/workflows/label_prs.yml index 7eaff2baf8f..f94478a20e4 100644 --- a/.github/workflows/label_prs.yml +++ b/.github/workflows/label_prs.yml @@ -39,6 +39,7 @@ concurrency: jobs: label: runs-on: ubuntu-slim + timeout-minutes: 10 steps: # Checks out the base branch, never the pull request's code: under pull_request_target the # default ref is the base commit, and only dev/ci/pr-type-label.mjs is read from it. diff --git a/.github/workflows/miri.yml b/.github/workflows/miri.yml index 5310af2ee40..4e50c7fb220 100644 --- a/.github/workflows/miri.yml +++ b/.github/workflows/miri.yml @@ -36,6 +36,7 @@ jobs: miri: name: Miri (${{ matrix.suite }}) runs-on: ubuntu-24.04 + timeout-minutes: 90 strategy: fail-fast: false # Keep Miri focused on Comet's memory-access code. Whole-workspace tests @@ -84,6 +85,7 @@ jobs: permissions: issues: write runs-on: ubuntu-slim + timeout-minutes: 15 steps: - uses: actions/github-script@v9 with: diff --git a/.github/workflows/pr_benchmark_check.yml b/.github/workflows/pr_benchmark_check.yml index ff297cd3326..18785e050ac 100644 --- a/.github/workflows/pr_benchmark_check.yml +++ b/.github/workflows/pr_benchmark_check.yml @@ -33,6 +33,7 @@ jobs: benchmark-check: name: Benchmark Compile & Lint Check runs-on: ubuntu-24.04 + timeout-minutes: 30 container: image: amd64/rust steps: diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 995267bc60e..b7d7e5e9a65 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -88,6 +88,7 @@ jobs: lint: name: Lint runs-on: ubuntu-24.04 + timeout-minutes: 30 container: image: amd64/rust outputs: @@ -115,6 +116,7 @@ jobs: name: Lint Scala (syntactic) if: ${{ !inputs.cache-refresh-only && inputs.profiles != 'nightly' }} runs-on: ubuntu-24.04 + timeout-minutes: 30 steps: - uses: actions/checkout@v7 @@ -156,6 +158,7 @@ jobs: name: Lint Java (${{ matrix.profile.name }}) if: ${{ !inputs.cache-refresh-only && inputs.profiles != 'nightly' }} runs-on: ubuntu-24.04 + timeout-minutes: 30 container: image: amd64/rust env: @@ -288,6 +291,7 @@ jobs: name: Strict Scala warnings (Spark 3.5, JDK 17) if: ${{ !inputs.cache-refresh-only && inputs.profiles != 'nightly' }} runs-on: ubuntu-24.04 + timeout-minutes: 30 container: image: amd64/rust steps: @@ -331,6 +335,7 @@ jobs: name: Celeborn ${{ matrix.celeborn_version }} reflection compatibility if: ${{ !inputs.cache-refresh-only && inputs.profiles != 'nightly' }} runs-on: ubuntu-24.04 + timeout-minutes: 30 container: image: amd64/rust env: @@ -396,6 +401,7 @@ jobs: needs: lint name: Build Native Library runs-on: ubuntu-24.04 + timeout-minutes: 90 container: image: amd64/rust steps: @@ -462,6 +468,7 @@ jobs: name: ubuntu-latest/rust-test if: ${{ !inputs.cache-refresh-only && inputs.profiles != 'nightly' }} runs-on: ubuntu-24.04 + timeout-minutes: 60 container: image: amd64/rust steps: @@ -683,6 +690,7 @@ jobs: fail-fast: false name: ${{ matrix.profile.name }} [${{ matrix.suite.name }}] runs-on: ubuntu-24.04 + timeout-minutes: 90 container: image: amd64/rust env: @@ -734,6 +742,7 @@ jobs: name: Verify TPC-H Results if: ${{ inputs.profiles != 'nightly' }} runs-on: ubuntu-24.04 + timeout-minutes: 60 container: image: amd64/rust env: @@ -808,6 +817,7 @@ jobs: name: Verify TPC-DS Results if: ${{ inputs.profiles != 'nightly' }} runs-on: ubuntu-24.04 + timeout-minutes: 60 container: image: amd64/rust env: diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index 0e1404083ca..98fce56adb0 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -50,6 +50,7 @@ jobs: lint: name: Lint runs-on: ubuntu-24.04 + timeout-minutes: 30 container: image: amd64/rust steps: @@ -69,6 +70,7 @@ jobs: needs: lint name: Build Native Library (macOS) runs-on: macos-15 + timeout-minutes: 90 steps: - uses: actions/checkout@v7 @@ -350,6 +352,7 @@ jobs: fail-fast: false name: ${{ matrix.os }}/${{ matrix.profile.name }} [${{ matrix.suite.name }}] runs-on: ${{ matrix.os }} + timeout-minutes: 180 steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/pr_title_check.yml b/.github/workflows/pr_title_check.yml index 65c27465234..c501b04a82d 100644 --- a/.github/workflows/pr_title_check.yml +++ b/.github/workflows/pr_title_check.yml @@ -28,6 +28,7 @@ on: jobs: check-pr-title: runs-on: ubuntu-slim + timeout-minutes: 10 steps: - uses: actions/checkout@v7 - name: Check PR title diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml index e5d8eeea1d8..c25a02fac37 100644 --- a/.github/workflows/publish_snapshot.yml +++ b/.github/workflows/publish_snapshot.yml @@ -64,6 +64,7 @@ jobs: # anywhere so the workflow can be exercised before it lands. if: github.repository == 'apache/datafusion-comet' || inputs.dry_run runs-on: ubuntu-slim + timeout-minutes: 15 outputs: publish: ${{ steps.check.outputs.publish }} steps: diff --git a/.github/workflows/pyarrow_udf_test.yml b/.github/workflows/pyarrow_udf_test.yml index f29caa052ff..2bb311f6877 100644 --- a/.github/workflows/pyarrow_udf_test.yml +++ b/.github/workflows/pyarrow_udf_test.yml @@ -35,6 +35,7 @@ jobs: pyarrow-udf: name: PyArrow UDF (${{ matrix.name }}, JDK 17, Python 3.11) runs-on: ubuntu-24.04 + timeout-minutes: 60 strategy: fail-fast: false matrix: diff --git a/.github/workflows/spark_sql_test_reusable.yml b/.github/workflows/spark_sql_test_reusable.yml index 3a775d3eedf..8274eb46560 100644 --- a/.github/workflows/spark_sql_test_reusable.yml +++ b/.github/workflows/spark_sql_test_reusable.yml @@ -80,6 +80,7 @@ jobs: build: name: Build Native + JVM Test Classes runs-on: ubuntu-24.04 + timeout-minutes: 90 container: image: amd64/rust outputs: @@ -225,6 +226,7 @@ jobs: fail-fast: false name: spark-sql-${{ matrix.module.name }}/spark-${{ inputs.spark-full }}-jdk${{ inputs.java }} runs-on: ubuntu-24.04 + timeout-minutes: 120 container: image: amd64/rust steps: @@ -349,6 +351,7 @@ jobs: name: merge-fallback-logs needs: [spark-sql-test] runs-on: ubuntu-24.04 + timeout-minutes: 15 steps: - name: Download fallback log artifacts # Not wrapped in download-artifact-retry: that local action needs a diff --git a/.github/workflows/spark_sql_writer_tests.yml b/.github/workflows/spark_sql_writer_tests.yml index 6b5fc7f6d65..a9c25aca8a6 100644 --- a/.github/workflows/spark_sql_writer_tests.yml +++ b/.github/workflows/spark_sql_writer_tests.yml @@ -57,6 +57,7 @@ jobs: writer-test: name: spark-sql-writer/spark-${{ inputs.spark-version }} runs-on: ubuntu-24.04 + timeout-minutes: 120 container: image: amd64/rust steps: diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index a22c42043c8..cc171cbcdd7 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -23,6 +23,7 @@ on: jobs: close-stale-prs: runs-on: ubuntu-slim + timeout-minutes: 15 permissions: issues: write pull-requests: write diff --git a/.github/workflows/take.yml b/.github/workflows/take.yml index 9e84da8da68..b9da6984c3d 100644 --- a/.github/workflows/take.yml +++ b/.github/workflows/take.yml @@ -26,6 +26,7 @@ permissions: jobs: issue_assign: runs-on: ubuntu-slim + timeout-minutes: 10 if: (!github.event.issue.pull_request) && (github.event.comment.body == 'take' || github.event.comment.body == 'untake') concurrency: group: ${{ github.actor }}-issue-assign