ci: one-shot stagger delay for CI storms

.github/actions/veristat_baseline_compare/action.yml

@@ -10,7 +10,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: actions/upload-artifact@v4
+    - uses: actions/upload-artifact@v7
       with:
         name: ${{ inputs.baseline_name }}
         if-no-files-found: error
@@ -20,7 +20,7 @@ runs:
     # - get baseline log from cache
     # - compare it to current run
     - if: ${{ github.event_name == 'pull_request' }}
-      uses: actions/cache/restore@v4
+      uses: actions/cache/restore@v5
       with:
         key: ${{ github.base_ref }}-${{ inputs.baseline_name }}-
         restore-keys: |
@@ -43,7 +43,7 @@ runs:
            "${{ github.workspace }}/${{ inputs.baseline_name }}"
 
     - if: ${{ github.event_name == 'push' }}
-      uses: actions/cache/save@v4
+      uses: actions/cache/save@v5
       with:
         key: ${{ github.ref_name }}-${{ inputs.baseline_name }}-${{ github.run_id }}
         path: '${{ github.workspace }}/${{ inputs.baseline_name }}'

.github/scripts/stagger.py

@@ -2,15 +2,16 @@
 """Stagger CI runs during KPD rebase storms.
 
 When KPD rebases all PR branches after an upstream commit, hundreds of
-workflow runs fire at once.  This script detects the storm and waits for
-it to subside before letting the expensive build jobs start.
+workflow runs fire at once.  This script detects the storm and sleeps
+for a random delay to spread the load.
 
 Storm = all of:
   1. PR synchronize event (force-push rebase, not a new PR)
   2. Base branch updated within the last 30 minutes (KPD just mirrored)
-  3. Active workflow runs (queued + in-progress) >= half of open PRs
+  3. More than 5 active workflow runs (queued + in-progress)
+  4. Active runs >= 20% of open PRs
 
-Re-checks in a loop with random 1-15 min sleeps.  Gives up after 2 hours.
+If detected, sleeps a random 1-10 minutes then proceeds.
 cancel-in-progress on the concurrency group kills sleeping runs on new pushes.
 """
 
@@ -22,10 +23,10 @@
 import requests
 
 BASE_BRANCH_RECENCY_S = 1800  # base branch "just updated" threshold
-STORM_RATIO = 0.5  # active runs / open PRs threshold
-WAIT_MIN_S = 60  # min sleep per iteration
-WAIT_MAX_S = 900  # max sleep per iteration
-MAX_TOTAL_WAIT_S = 7200  # hard cap on total wait
+STORM_RATIO = 0.2  # active runs / open PRs threshold
+STORM_MIN_ACTIVE = 5  # minimum active runs to consider a storm
+WAIT_MIN_S = 60  # min delay
+WAIT_MAX_S = 600  # max delay
 
 
 def gh_api(endpoint):
@@ -75,50 +76,44 @@ def open_pr_count(repo):
         return 0
 
 
-def is_storm(repo, base_branch):
-    age = base_branch_age_s(repo, base_branch)
+def main():
+    action = os.environ.get("GITHUB_EVENT_ACTION", "")
+    repo = os.environ.get("GITHUB_REPOSITORY", "")
+    base = os.environ.get("PR_BASE_BRANCH", "")
+
+    if action != "synchronize":
+        return
+
+    if not repo or not base:
+        return
+
+    age = base_branch_age_s(repo, base)
     if age is None or age > BASE_BRANCH_RECENCY_S:
-        print(f"Base branch {base_branch} updated {age}s ago — no storm.")
-        return False
+        print(f"Base branch {base} updated {age}s ago — no storm.")
+        return
 
     active = active_run_count(repo)
+    if active <= STORM_MIN_ACTIVE:
+        print(f"Only {active} active runs — no storm.")
+        return
+
     open_prs = open_pr_count(repo)
     if open_prs == 0:
-        return False
+        return
 
     ratio = active / open_prs
     if ratio < STORM_RATIO:
         print(f"{active} active / {open_prs} PRs ({ratio:.0%}) — no storm.")
-        return False
+        return
 
+    delay = random.randint(WAIT_MIN_S, WAIT_MAX_S)
     print(
-        f"Storm: base {base_branch} updated {age:.0f}s ago, "
-        f"{active} active / {open_prs} PRs ({ratio:.0%})."
+        f"Storm detected: base {base} updated {age:.0f}s ago, "
+        f"{active} active / {open_prs} PRs ({ratio:.0%}). "
+        f"Waiting {delay}s."
     )
-    return True
-
-
-def main():
-    action = os.environ.get("GITHUB_EVENT_ACTION", "")
-    repo = os.environ.get("GITHUB_REPOSITORY", "")
-    base = os.environ.get("PR_BASE_BRANCH", "")
-
-    if action != "synchronize":
-        return
-
-    if not repo or not base:
-        return
-
-    start = time.monotonic()
-    while is_storm(repo, base):
-        elapsed = time.monotonic() - start
-        remaining = MAX_TOTAL_WAIT_S - elapsed
-        if remaining <= 0:
-            print(f"Hit {MAX_TOTAL_WAIT_S}s cap — proceeding.")
-            break
-        delay = random.randint(WAIT_MIN_S, min(WAIT_MAX_S, int(remaining)))
-        print(f"Waiting {delay}s (elapsed: {elapsed:.0f}s)...")
-        time.sleep(delay)
+    time.sleep(delay)
+    print("Proceeding.")
 
 
 if __name__ == "__main__":

.github/workflows/ai-agent.yml

@@ -186,7 +186,7 @@ jobs:
 
       - name: Upload output artifacts
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: output
           path: output/
@@ -205,7 +205,7 @@ jobs:
     steps:
       - name: Download output artifact
         id: download
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         continue-on-error: true
         with:
           name: output

.github/workflows/gcc-bpf.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           sparse-checkout: |
             .github
@@ -57,7 +57,7 @@ jobs:
 
       - if: ${{ ! inputs.download_sources }}
         name: Checkout ${{ github.repository }} to ./src
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           path: 'src'
 
@@ -66,7 +66,7 @@ jobs:
           patches-root: '${{ github.workspace }}/ci/diffs'
           repo-root: ${{ env.REPO_ROOT }}
 
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v7
         with:
           name: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}
           path: ${{ env.REPO_ROOT }}

.github/workflows/kernel-build.yml

@@ -171,7 +171,7 @@ jobs:
           # Only on pushed changes are build artifacts actually cached, because
           # of github.com/actions/cache's cache isolation logic.
           rm -rf "${KBUILD_OUTPUT}"
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v7
         with:
           name: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}${{ inputs.release && '-release' || '' }}
           if-no-files-found: error

.github/workflows/kernel-test.yml

@@ -52,13 +52,13 @@ jobs:
       ALLOWLIST_FILE: /tmp/allowlist
       DENYLIST_FILE: /tmp/denylist
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           sparse-checkout: |
             .github
             ci
 
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v7
         with:
           name: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}
           path: .
@@ -95,7 +95,7 @@ jobs:
           kbuild-output: ${{ env.REPO_ROOT }}/kbuild-output
 
       - if: ${{ always() }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: tmon-logs-${{ inputs.arch }}-${{ inputs.toolchain_full }}-${{ inputs.test }}
           if-no-files-found: ignore

.github/workflows/lint.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Run ShellCheck
         uses: ludeeus/action-shellcheck@master
         env:
@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Run black
         uses: psf/black@stable
         with:
@@ -48,7 +48,7 @@ jobs:
         repository: ['bpf', 'vmtest', 'bar']
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: run script
         run: |
           python3 .github/scripts/matrix.py
@@ -59,7 +59,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Run unittests
         run: python3 -m unittest scripts/tests/*.py
         working-directory: .github

.github/workflows/test-progs-asan.yml

@@ -74,7 +74,7 @@ jobs:
           gcc-version: ${{ inputs.gcc_version }}
           llvm-version: ${{ inputs.llvm_version }}
 
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v7
         with:
           name: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}
           path: ${{ env.REPO_ROOT }}
@@ -107,7 +107,7 @@ jobs:
           kbuild-output: ${{ env.REPO_ROOT }}/kbuild-output
           repo-root: ${{ env.REPO_ROOT }}
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v7
         with:
           name: selftests-bpf-asan-${{ inputs.arch }}-${{ inputs.toolchain_full }}
           if-no-files-found: error
@@ -126,13 +126,13 @@ jobs:
       ALLOWLIST_FILE: /tmp/allowlist
       DENYLIST_FILE: /tmp/denylist
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           sparse-checkout: |
             .github
             ci
 
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v7
         with:
           name: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}
           path: ${{ env.REPO_ROOT }}
@@ -141,7 +141,7 @@ jobs:
         working-directory: ${{ env.REPO_ROOT }}
         run: zstd -d -T0 vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}.tar.zst --stdout | tar -xf -
 
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v7
         with:
           name: selftests-bpf-asan-${{ inputs.arch }}-${{ inputs.toolchain_full }}
           path: ${{ env.REPO_ROOT }}

.github/workflows/test.yml

@@ -20,7 +20,7 @@ jobs:
     outputs:
       build-matrix: ${{ steps.set-matrix-impl.outputs.build_matrix }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           sparse-checkout: |
             .github

.github/workflows/veristat-cilium.yml

@@ -37,14 +37,14 @@ jobs:
 
     steps:
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           sparse-checkout: |
             .github
             ci
 
       - name: Download kernel build artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           name: vmlinux-${{ env.ARCH_AND_TOOL }}
           path: .

.github/workflows/veristat-kernel.yml

@@ -34,13 +34,13 @@ jobs:
 
     steps:
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           sparse-checkout: |
             .github
             ci
 
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v7
         with:
           name: vmlinux-${{ env.ARCH_AND_TOOL }}
           path: .

.github/workflows/veristat-meta.yml

@@ -42,13 +42,13 @@ jobs:
 
     steps:
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           sparse-checkout: |
             .github
             ci
 
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v7
         with:
           name: vmlinux-${{ env.ARCH_AND_TOOL }}
           path: .

.github/workflows/veristat-scx.yml

@@ -29,7 +29,7 @@ jobs:
       SCX_BUILD_OUTPUT: ${{ github.workspace }}/scx-build-output
       SCX_REVISION: main
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           sparse-checkout: |
             .github
@@ -40,7 +40,7 @@ jobs:
           output-dir: ${{ env.SCX_BUILD_OUTPUT }}
 
       - name: Upload sched-ext build output
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: sched-ext-${{ inputs.arch }}-${{ inputs.toolchain_full }}
           if-no-files-found: error
@@ -64,14 +64,14 @@ jobs:
 
     steps:
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           sparse-checkout: |
             .github
             ci
 
       - name: Download kernel build artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           name: vmlinux-${{ env.ARCH_AND_TOOL }}
           path: .
@@ -80,7 +80,7 @@ jobs:
         run: zstd -d -T0  vmlinux-${{ env.ARCH_AND_TOOL }}.tar.zst --stdout | tar -xf -
 
       - name: Download sched-ext build output
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v7
         with:
           name: sched-ext-${{ inputs.arch }}-${{ inputs.toolchain_full }}
           path: ${{ env.SCX_BUILD_OUTPUT }}