diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e04a09ee..04deef96 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -166,13 +166,35 @@ jobs: with: python-version: "3.14" - name: Install Microsoft ODBC Driver 18 + sqlcmd + # THIS IS THE DANGEROUS HALF OF THE apt CLASS, and it is the opposite of the one that was + # visibly hurting. The Qt sites sit in jobs carrying a job-level `timeout-minutes`, so a + # mirror hang there dies at the job cap (20-40 min, measured). THIS JOB HAS NO JOB-LEVEL + # TIMEOUT, so the same hang runs to GitHub's 360-minute default -- a whole runner-hour + # budget burned on a stalled mirror, on a leg nobody is watching because it is path-gated. + # + # The step cap bounds EVERY command here, including the two curls, which are equally + # unguarded network calls. The retry wraps only the apt pair: re-running that is idempotent, + # whereas re-fetching the signing key and repo list is not the part that hangs. + # + # THE DEEPER FIX IS A JOB-LEVEL `timeout-minutes` ON THIS JOB, which would bound every step + # rather than the one that bit. Deliberately NOT done here: choosing that number needs a + # measurement of how long this job legitimately runs, and a guessed cap on a server-DB leg + # would kill real work. Recorded as a direction, not applied as a guess. + timeout-minutes: 10 run: | - curl -fsSL https://packages.microsoft.com/keys/microsoft.asc \ + curl -fsSL --max-time 60 https://packages.microsoft.com/keys/microsoft.asc \ | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc > /dev/null - curl -fsSL "https://packages.microsoft.com/config/ubuntu/$(. /etc/os-release; echo "$VERSION_ID")/prod.list" \ + curl -fsSL --max-time 60 "https://packages.microsoft.com/config/ubuntu/$(. /etc/os-release; echo "$VERSION_ID")/prod.list" \ | sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 mssql-tools18 unixodbc-dev + for attempt in 1 2 3; do + if sudo timeout 120 apt-get update && sudo ACCEPT_EULA=Y timeout 240 apt-get install -y msodbcsql18 mssql-tools18 unixodbc-dev; then + exit 0 + fi + echo "::warning::apt attempt ${attempt}/3 failed or timed out; retrying" + sleep $((attempt * 5)) + done + echo "::error::apt-get failed 3 times. This is the UBUNTU RUNNER MIRROR, not the change under test." + exit 1 - name: Wait for SQL Server and create the database (RCSI on) run: | sqlcmd=/opt/mssql-tools18/bin/sqlcmd diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index effb795e..8737c7d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -230,9 +230,25 @@ jobs: # headless. Linux-only; Windows runners need no equivalent. - name: Install Qt offscreen system libraries if: (needs.changes.outputs.code == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch') && runner.os == 'Linux' + # BOUNDED AND RETRIED. Unguarded, this step turns an external apt-mirror hiccup into a + # blocked merge queue: measured 2026-08-18, three hangs across two attempts on three + # different jobs, one of them 27+ minutes on `test (ubuntu-latest, py3.14)` -- a REQUIRED + # context, so the queue stops. Two levers, because they answer different failures. The + # per-command `timeout` kills a HANG and lets the retry run; `timeout-minutes` is the + # backstop that stops this step ever eating a job budget again if the loop is edited wrong. + # The failure text names the cause, so the next reader is not sent hunting in their diff -- + # a check whose label points at the wrong subject is what BACKLOG #1254 is about. + timeout-minutes: 8 run: | - sudo apt-get update - sudo apt-get install -y libegl1 libgl1 libxkbcommon0 libdbus-1-3 + for attempt in 1 2 3; do + if sudo timeout 120 apt-get update && sudo timeout 180 apt-get install -y libegl1 libgl1 libxkbcommon0 libdbus-1-3; then + exit 0 + fi + echo "::warning::apt attempt ${attempt}/3 failed or timed out; retrying" + sleep $((attempt * 5)) + done + echo "::error::apt-get failed 3 times. This is the UBUNTU RUNNER MIRROR, not the change under test." + exit 1 # uv is the installer for every leg (was pip): it resolves + installs the pyproject extras far # faster than pip, into the same setup-python interpreter via `uv pip install --system`. Pinned @@ -822,9 +838,25 @@ jobs: - name: Install Qt offscreen system libraries if: (needs.changes.outputs.code == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch') && runner.os == 'Linux' + # BOUNDED AND RETRIED. Unguarded, this step turns an external apt-mirror hiccup into a + # blocked merge queue: measured 2026-08-18, three hangs across two attempts on three + # different jobs, one of them 27+ minutes on `test (ubuntu-latest, py3.14)` -- a REQUIRED + # context, so the queue stops. Two levers, because they answer different failures. The + # per-command `timeout` kills a HANG and lets the retry run; `timeout-minutes` is the + # backstop that stops this step ever eating a job budget again if the loop is edited wrong. + # The failure text names the cause, so the next reader is not sent hunting in their diff -- + # a check whose label points at the wrong subject is what BACKLOG #1254 is about. + timeout-minutes: 8 run: | - sudo apt-get update - sudo apt-get install -y libegl1 libgl1 libxkbcommon0 libdbus-1-3 + for attempt in 1 2 3; do + if sudo timeout 120 apt-get update && sudo timeout 180 apt-get install -y libegl1 libgl1 libxkbcommon0 libdbus-1-3; then + exit 0 + fi + echo "::warning::apt attempt ${attempt}/3 failed or timed out; retrying" + sleep $((attempt * 5)) + done + echo "::error::apt-get failed 3 times. This is the UBUNTU RUNNER MIRROR, not the change under test." + exit 1 - name: Set up uv if: needs.changes.outputs.code == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' @@ -967,9 +999,25 @@ jobs: # imports the whole suite regardless of what this job intends to execute. - name: Install Qt offscreen system libraries if: runner.os == 'Linux' + # BOUNDED AND RETRIED. Unguarded, this step turns an external apt-mirror hiccup into a + # blocked merge queue: measured 2026-08-18, three hangs across two attempts on three + # different jobs, one of them 27+ minutes on `test (ubuntu-latest, py3.14)` -- a REQUIRED + # context, so the queue stops. Two levers, because they answer different failures. The + # per-command `timeout` kills a HANG and lets the retry run; `timeout-minutes` is the + # backstop that stops this step ever eating a job budget again if the loop is edited wrong. + # The failure text names the cause, so the next reader is not sent hunting in their diff -- + # a check whose label points at the wrong subject is what BACKLOG #1254 is about. + timeout-minutes: 8 run: | - sudo apt-get update - sudo apt-get install -y libegl1 libgl1 libxkbcommon0 libdbus-1-3 + for attempt in 1 2 3; do + if sudo timeout 120 apt-get update && sudo timeout 180 apt-get install -y libegl1 libgl1 libxkbcommon0 libdbus-1-3; then + exit 0 + fi + echo "::warning::apt attempt ${attempt}/3 failed or timed out; retrying" + sleep $((attempt * 5)) + done + echo "::error::apt-get failed 3 times. This is the UBUNTU RUNNER MIRROR, not the change under test." + exit 1 - name: Set up uv uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 @@ -1475,13 +1523,35 @@ jobs: python-version: "3.14" - name: Install Microsoft ODBC Driver 18 + sqlcmd + # THIS IS THE DANGEROUS HALF OF THE apt CLASS, and it is the opposite of the one that was + # visibly hurting. The Qt sites sit in jobs carrying a job-level `timeout-minutes`, so a + # mirror hang there dies at the job cap (20-40 min, measured). THIS JOB HAS NO JOB-LEVEL + # TIMEOUT, so the same hang runs to GitHub's 360-minute default -- a whole runner-hour + # budget burned on a stalled mirror, on a leg nobody is watching because it is path-gated. + # + # The step cap bounds EVERY command here, including the two curls, which are equally + # unguarded network calls. The retry wraps only the apt pair: re-running that is idempotent, + # whereas re-fetching the signing key and repo list is not the part that hangs. + # + # THE DEEPER FIX IS A JOB-LEVEL `timeout-minutes` ON THIS JOB, which would bound every step + # rather than the one that bit. Deliberately NOT done here: choosing that number needs a + # measurement of how long this job legitimately runs, and a guessed cap on a server-DB leg + # would kill real work. Recorded as a direction, not applied as a guess. + timeout-minutes: 10 run: | - curl -fsSL https://packages.microsoft.com/keys/microsoft.asc \ + curl -fsSL --max-time 60 https://packages.microsoft.com/keys/microsoft.asc \ | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc > /dev/null - curl -fsSL "https://packages.microsoft.com/config/ubuntu/$(. /etc/os-release; echo "$VERSION_ID")/prod.list" \ + curl -fsSL --max-time 60 "https://packages.microsoft.com/config/ubuntu/$(. /etc/os-release; echo "$VERSION_ID")/prod.list" \ | sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 mssql-tools18 unixodbc-dev + for attempt in 1 2 3; do + if sudo timeout 120 apt-get update && sudo ACCEPT_EULA=Y timeout 240 apt-get install -y msodbcsql18 mssql-tools18 unixodbc-dev; then + exit 0 + fi + echo "::warning::apt attempt ${attempt}/3 failed or timed out; retrying" + sleep $((attempt * 5)) + done + echo "::error::apt-get failed 3 times. This is the UBUNTU RUNNER MIRROR, not the change under test." + exit 1 - name: Wait for SQL Server and create the database run: | @@ -2054,13 +2124,35 @@ jobs: python-version: "3.14" - name: Install Microsoft ODBC Driver 18 + sqlcmd + # THIS IS THE DANGEROUS HALF OF THE apt CLASS, and it is the opposite of the one that was + # visibly hurting. The Qt sites sit in jobs carrying a job-level `timeout-minutes`, so a + # mirror hang there dies at the job cap (20-40 min, measured). THIS JOB HAS NO JOB-LEVEL + # TIMEOUT, so the same hang runs to GitHub's 360-minute default -- a whole runner-hour + # budget burned on a stalled mirror, on a leg nobody is watching because it is path-gated. + # + # The step cap bounds EVERY command here, including the two curls, which are equally + # unguarded network calls. The retry wraps only the apt pair: re-running that is idempotent, + # whereas re-fetching the signing key and repo list is not the part that hangs. + # + # THE DEEPER FIX IS A JOB-LEVEL `timeout-minutes` ON THIS JOB, which would bound every step + # rather than the one that bit. Deliberately NOT done here: choosing that number needs a + # measurement of how long this job legitimately runs, and a guessed cap on a server-DB leg + # would kill real work. Recorded as a direction, not applied as a guess. + timeout-minutes: 10 run: | - curl -fsSL https://packages.microsoft.com/keys/microsoft.asc \ + curl -fsSL --max-time 60 https://packages.microsoft.com/keys/microsoft.asc \ | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc > /dev/null - curl -fsSL "https://packages.microsoft.com/config/ubuntu/$(. /etc/os-release; echo "$VERSION_ID")/prod.list" \ + curl -fsSL --max-time 60 "https://packages.microsoft.com/config/ubuntu/$(. /etc/os-release; echo "$VERSION_ID")/prod.list" \ | sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 mssql-tools18 unixodbc-dev + for attempt in 1 2 3; do + if sudo timeout 120 apt-get update && sudo ACCEPT_EULA=Y timeout 240 apt-get install -y msodbcsql18 mssql-tools18 unixodbc-dev; then + exit 0 + fi + echo "::warning::apt attempt ${attempt}/3 failed or timed out; retrying" + sleep $((attempt * 5)) + done + echo "::error::apt-get failed 3 times. This is the UBUNTU RUNNER MIRROR, not the change under test." + exit 1 - name: Wait for SQL Server and create the database (RCSI on, like the store suite) run: | diff --git a/.github/workflows/quality-advisory.yml b/.github/workflows/quality-advisory.yml index cd96c88b..acc66274 100644 --- a/.github/workflows/quality-advisory.yml +++ b/.github/workflows/quality-advisory.yml @@ -292,9 +292,25 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - name: Qt headless system deps + # BOUNDED AND RETRIED. Unguarded, this step turns an external apt-mirror hiccup into a + # blocked merge queue: measured 2026-08-18, three hangs across two attempts on three + # different jobs, one of them 27+ minutes on `test (ubuntu-latest, py3.14)` -- a REQUIRED + # context, so the queue stops. Two levers, because they answer different failures. The + # per-command `timeout` kills a HANG and lets the retry run; `timeout-minutes` is the + # backstop that stops this step ever eating a job budget again if the loop is edited wrong. + # The failure text names the cause, so the next reader is not sent hunting in their diff -- + # a check whose label points at the wrong subject is what BACKLOG #1254 is about. + timeout-minutes: 8 run: | - sudo apt-get update - sudo apt-get install -y libegl1 libgl1 libxkbcommon0 libdbus-1-3 + for attempt in 1 2 3; do + if sudo timeout 120 apt-get update && sudo timeout 180 apt-get install -y libegl1 libgl1 libxkbcommon0 libdbus-1-3; then + exit 0 + fi + echo "::warning::apt attempt ${attempt}/3 failed or timed out; retrying" + sleep $((attempt * 5)) + done + echo "::error::apt-get failed 3 times. This is the UBUNTU RUNNER MIRROR, not the change under test." + exit 1 - name: Install project + coverage tools run: | uv pip install --system --constraint constraints.lock -e ".[dev,harness,fhir,dicom,x12,xml,webauthn]" -e packaging/messagefoundry-webconsole @@ -436,9 +452,25 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - name: Qt headless system deps + # BOUNDED AND RETRIED. Unguarded, this step turns an external apt-mirror hiccup into a + # blocked merge queue: measured 2026-08-18, three hangs across two attempts on three + # different jobs, one of them 27+ minutes on `test (ubuntu-latest, py3.14)` -- a REQUIRED + # context, so the queue stops. Two levers, because they answer different failures. The + # per-command `timeout` kills a HANG and lets the retry run; `timeout-minutes` is the + # backstop that stops this step ever eating a job budget again if the loop is edited wrong. + # The failure text names the cause, so the next reader is not sent hunting in their diff -- + # a check whose label points at the wrong subject is what BACKLOG #1254 is about. + timeout-minutes: 8 run: | - sudo apt-get update - sudo apt-get install -y libegl1 libgl1 libxkbcommon0 libdbus-1-3 + for attempt in 1 2 3; do + if sudo timeout 120 apt-get update && sudo timeout 180 apt-get install -y libegl1 libgl1 libxkbcommon0 libdbus-1-3; then + exit 0 + fi + echo "::warning::apt attempt ${attempt}/3 failed or timed out; retrying" + sleep $((attempt * 5)) + done + echo "::error::apt-get failed 3 times. This is the UBUNTU RUNNER MIRROR, not the change under test." + exit 1 - name: Install project + mutmut run: | uv pip install --system --constraint constraints.lock -e ".[dev,harness,fhir,dicom,x12,xml,webauthn]" -e packaging/messagefoundry-webconsole