From 697cca23aab64397833d45f89d6d64e6add39619 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 22 Oct 2026 15:06:08 +0000 Subject: [PATCH 1/2] ci: the cluster-test job gets 40 minutes, not 20 The pytest cluster job runs serially and grows with every ported twin. Measured on 2026-10-22: main's job took 12 to 17 minutes (PG17 and PG18, runs on 0c5e924d, f6794929 and 9be30954), and seven of OffgridwithJD's stacked port PRs (#1681, #1683, #1684, #1685, #1687, #1688, #1689) were cancelled at 20m16s against the 20-minute limit, with no test failing. Running the files under xdist would be faster, but a port race under -n 2 is known (#1635 review), so this moves the budget first. test_harness_deps (the guards that read ci.yml): 21 passed. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71b5b7753..1169b395c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -165,7 +165,11 @@ jobs: pytest-cluster: name: pytest (cluster tests, PG ${{ matrix.pg }}) runs-on: ubuntu-latest - timeout-minutes: 20 + # 40, not 20. The run is serial and grows with every ported twin: main took 12 to 17 + # minutes on 2026-10-22, and seven stacked port PRs were cancelled at 20m16s. More + # workers (-n) would be faster, but a port race under -n 2 is known (#1635), so the + # budget moves first and the parallel run waits for that to be settled. + timeout-minutes: 40 strategy: fail-fast: false matrix: From d0e7ed26b0f1647e7587f4f8973578ad4b35cb89 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 22 Oct 2026 17:54:57 +0000 Subject: [PATCH 2/2] ci: package installs retry a stalled mirror instead of failing the job Most of the CI reds on 2026-10-22 were exit 124 from one step: `sudo timeout 600 apt-get install`, fetching LLVM and PostgreSQL packages from the runner's mirror. The nightly gate (#1729), main's run on 4adb2e7a, and the cluster jobs of #1640 and #1739 all died there, before any test ran. One stalled attempt spent the whole 600 seconds. .github/scripts/apt-retry.sh runs the same apt-get arguments up to three times, each bounded (200 s for an install, 100 s for an update), so the worst case is the same 600 and 300 seconds as before and no job's limit moves. A connection that sends nothing for 30 seconds fails at once, and partial downloads are kept for the retry. Every bounded apt-get in ci.yml (3 jobs) and nightly.yml (6 jobs) uses it, and the PG19 beta build's two unbounded calls now are bounded. Every job checks out the repository before its install step. Guards 804 passed; shellcheck -S error on the script passes. The behaviour on a stalled mirror is only provable in CI. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/apt-retry.sh | 21 +++++++++++++++++++++ .github/workflows/ci.yml | 16 ++++++++-------- .github/workflows/nightly.yml | 24 ++++++++++++------------ 3 files changed, 41 insertions(+), 20 deletions(-) create mode 100755 .github/scripts/apt-retry.sh diff --git a/.github/scripts/apt-retry.sh b/.github/scripts/apt-retry.sh new file mode 100755 index 000000000..067c55af2 --- /dev/null +++ b/.github/scripts/apt-retry.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# apt-retry.sh SECONDS ATTEMPTS APT-GET-ARGS... +# +# Run `sudo apt-get APT-GET-ARGS` up to ATTEMPTS times, each attempt bounded by +# SECONDS. A stalled package mirror used to spend a job's whole single 600-second +# attempt and fail it with exit 124 before any test ran (#1729; most of the CI reds +# on 2026-10-22). Now a stalled attempt is cut short and retried, the worst case +# stays ATTEMPTS * SECONDS, and a connection that sends nothing for 30 seconds +# fails at once instead of holding the attempt. Partial downloads are kept, so a +# retry resumes them. +set -uo pipefail +secs="$1" tries="$2" +shift 2 +for i in $(seq 1 "$tries"); do + if sudo timeout "$secs" apt-get -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 "$@"; then + exit 0 + fi + echo "apt-get $1: attempt $i of $tries failed or ran past ${secs}s" >&2 +done +exit 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1169b395c..1711fc575 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,8 +199,8 @@ jobs: echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \ https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null - sudo timeout 300 apt-get -o Acquire::Retries=5 update - sudo timeout 600 apt-get -o Acquire::Retries=5 install -y --no-install-recommends \ + .github/scripts/apt-retry.sh 100 3 update + .github/scripts/apt-retry.sh 200 3 install -y --no-install-recommends \ postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman postgresql-${{ matrix.pg }}-hll postgresql-${{ matrix.pg }}-tdigest \ liblz4-dev libzstd-dev zlib1g-dev # NOT cached, deliberately. The suites job caches because it runs a matrix for @@ -335,7 +335,7 @@ jobs: | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null # Acquire retries cover a flaky mirror; the outer timeout keeps a # wedged one from consuming the whole job budget. - sudo timeout 300 apt-get -o Acquire::Retries=5 update + .github/scripts/apt-retry.sh 100 3 update # The key has to track the package VERSIONS and not only their names. # @@ -403,7 +403,7 @@ jobs: run: | set -euo pipefail mkdir -p "$HOME/apt-archives/partial" - sudo timeout 600 apt-get -o Acquire::Retries=5 \ + .github/scripts/apt-retry.sh 200 3 \ -o Dir::Cache::archives="$HOME/apt-archives" install -y --no-install-recommends \ postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ liblz4-dev libzstd-dev zlib1g-dev 2>&1 | tee /tmp/apt-install.log @@ -484,7 +484,7 @@ jobs: - name: Install build dependencies run: | set -euo pipefail - sudo apt-get update + .github/scripts/apt-retry.sh 100 3 update # gcc-14 explicitly, so one compiler configures PostgreSQL and builds # the extension. # @@ -499,7 +499,7 @@ jobs: # project's own Makefile for PostgreSQL 19, whose headers use C23 # constructs, and the Makefile now probes for the spelling the # compiler accepts. See #294. - sudo apt-get install -y --no-install-recommends \ + .github/scripts/apt-retry.sh 200 3 install -y --no-install-recommends \ build-essential gcc-14 flex bison libreadline-dev zlib1g-dev \ liblz4-dev libzstd-dev pkg-config @@ -601,7 +601,7 @@ jobs: | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null # Acquire retries cover a flaky mirror; the outer timeout keeps a # wedged one from consuming the whole job budget. - sudo timeout 300 apt-get -o Acquire::Retries=5 update + .github/scripts/apt-retry.sh 100 3 update # See the build job for why the key hashes the resolved versions: a fixed # key freezes the archives, and the next routine PGDG minor then turns the @@ -657,7 +657,7 @@ jobs: run: | set -euo pipefail mkdir -p "$HOME/apt-archives/partial" - sudo timeout 600 apt-get -o Acquire::Retries=5 \ + .github/scripts/apt-retry.sh 200 3 \ -o Dir::Cache::archives="$HOME/apt-archives" install -y --no-install-recommends \ postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman postgresql-${{ matrix.pg }}-hll postgresql-${{ matrix.pg }}-tdigest \ liblz4-dev libzstd-dev zlib1g-dev python3-pip 2>&1 | tee /tmp/apt-install.log diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 3d48bdca2..75f95e4b8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -125,7 +125,7 @@ jobs: echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \ https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null - sudo timeout 300 apt-get -o Acquire::Retries=5 update + .github/scripts/apt-retry.sh 100 3 update # See ci.yml's build job for why the key hashes the resolved versions: a # fixed key freezes the archives, and the next routine PGDG minor then @@ -163,7 +163,7 @@ jobs: run: | set -euo pipefail mkdir -p "$HOME/apt-archives/partial" - sudo timeout 600 apt-get -o Acquire::Retries=5 \ + .github/scripts/apt-retry.sh 200 3 \ -o Dir::Cache::archives="$HOME/apt-archives" install -y --no-install-recommends \ postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman postgresql-${{ matrix.pg }}-hll postgresql-${{ matrix.pg }}-tdigest \ liblz4-dev libzstd-dev zlib1g-dev python3-pip 2>&1 | tee /tmp/apt-install.log @@ -266,8 +266,8 @@ jobs: - name: Install clang, build tools, codec headers, pyarrow; create the postgres user run: | set -euo pipefail - sudo timeout 300 apt-get -o Acquire::Retries=5 update - sudo timeout 600 apt-get -o Acquire::Retries=5 install -y --no-install-recommends \ + .github/scripts/apt-retry.sh 100 3 update + .github/scripts/apt-retry.sh 200 3 install -y --no-install-recommends \ clang llvm bison flex make perl \ liblz4-dev libzstd-dev zlib1g-dev python3-pip clang --version | head -1 @@ -418,7 +418,7 @@ jobs: echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \ https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null - sudo timeout 300 apt-get -o Acquire::Retries=5 update + .github/scripts/apt-retry.sh 100 3 update # See ci.yml's build job for why the key hashes the resolved versions: a # fixed key freezes the archives, and the next routine PGDG minor then @@ -456,7 +456,7 @@ jobs: run: | set -euo pipefail mkdir -p "$HOME/apt-archives/partial" - sudo timeout 600 apt-get -o Acquire::Retries=5 \ + .github/scripts/apt-retry.sh 200 3 \ -o Dir::Cache::archives="$HOME/apt-archives" install -y --no-install-recommends \ postgresql-18 postgresql-server-dev-18 postgresql-18-partman postgresql-18-hll postgresql-18-tdigest \ liblz4-dev libzstd-dev zlib1g-dev python3-pip lcov 2>&1 | tee /tmp/apt-install.log @@ -605,12 +605,12 @@ jobs: echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \ https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null - sudo timeout 300 apt-get -o Acquire::Retries=5 update + .github/scripts/apt-retry.sh 100 3 update - name: Install PostgreSQL 18 and the codec headers run: | set -euo pipefail - sudo timeout 600 apt-get -o Acquire::Retries=5 install -y --no-install-recommends \ + .github/scripts/apt-retry.sh 200 3 install -y --no-install-recommends \ postgresql-18 postgresql-server-dev-18 \ liblz4-dev libzstd-dev zlib1g-dev @@ -690,12 +690,12 @@ jobs: echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \ https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null - sudo timeout 300 apt-get -o Acquire::Retries=5 update + .github/scripts/apt-retry.sh 100 3 update - name: Install PostgreSQL 18, the codec headers, and pyarrow run: | set -euo pipefail - sudo timeout 600 apt-get -o Acquire::Retries=5 install -y --no-install-recommends \ + .github/scripts/apt-retry.sh 200 3 install -y --no-install-recommends \ postgresql-18 postgresql-server-dev-18 \ liblz4-dev libzstd-dev zlib1g-dev python3-pip sudo pip3 install --break-system-packages --quiet pyarrow \ @@ -759,8 +759,8 @@ jobs: echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] \ https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null - sudo timeout 300 apt-get -o Acquire::Retries=5 update - sudo timeout 600 apt-get -o Acquire::Retries=5 install -y --no-install-recommends \ + .github/scripts/apt-retry.sh 100 3 update + .github/scripts/apt-retry.sh 200 3 install -y --no-install-recommends \ postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman postgresql-${{ matrix.pg }}-hll postgresql-${{ matrix.pg }}-tdigest \ liblz4-dev libzstd-dev zlib1g-dev # NOT cached, deliberately. The suites job caches because it runs a matrix for