From fc8b80c60ec597d2c2a7e4d93a5201ce5f06ea35 Mon Sep 17 00:00:00 2001
From: Thomas Carmet <8408330+tcarmet@users.noreply.github.com>
Date: Thu, 9 Jul 2026 16:01:48 -0700
Subject: [PATCH 1/3] CLDSRV-955: Parallelize docker builds into a matrix

Convert the build job into a matrix that builds the cloudserver,
cloudserver-testcoverage, pykmip and ci-mongodb images in parallel
instead of sequentially, so downstream test jobs no longer wait on a
single serial build. All image tags are preserved exactly.

The testcoverage image writes to its own cache scope to avoid racing the
production build on the shared scope while still reading it for reuse.

Federation and the dashboards push are moved to a separate
build-federation job (needs: build): they depend on the pushed
production image but are not consumed by any test job, so they now run
in parallel with the tests rather than blocking them.
---
 .github/workflows/tests.yaml | 102 ++++++++++++++++++++---------------
 1 file changed, 60 insertions(+), 42 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index d775a2b8d1..a97d6e04da 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -188,6 +188,39 @@ jobs:
     permissions:
       contents: read
       packages: write
+    strategy:
+      matrix:
+        include:
+          # Image url and tags are preserved exactly via matrix.name/matrix.tags.
+          - name: cloudserver
+            context: .
+            target: production
+            tags: ghcr.io/${{ github.repository }}:${{ github.sha }}
+            cache-from: type=gha,scope=cloudserver
+            cache-to: type=gha,mode=max,scope=cloudserver
+          # testcoverage is FROM production; it writes to its own cache scope to
+          # avoid racing the production build's cache-to on scope=cloudserver,
+          # while still reading the production scope for layer reuse.
+          - name: cloudserver-testcoverage
+            context: .
+            target: testcoverage
+            tags: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
+            cache-from: |
+              type=gha,scope=cloudserver-testcoverage
+              type=gha,scope=cloudserver
+            cache-to: type=gha,mode=max,scope=cloudserver-testcoverage
+          - name: pykmip
+            context: .github/pykmip
+            target: ''
+            tags: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
+            cache-from: type=gha,scope=pykmip
+            cache-to: type=gha,mode=max,scope=pykmip
+          - name: ci-mongodb
+            context: .github/docker/mongodb
+            target: ''
+            tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
+            cache-from: type=gha,scope=mongodb
+            cache-to: type=gha,mode=max,scope=mongodb
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -202,35 +235,42 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ github.token }}
 
-      - name: Build and push cloudserver image
+      - name: Build and push ${{ matrix.name }} image
         uses: docker/build-push-action@v5
         with:
           push: true
-          context: .
-          target: production
+          context: ${{ matrix.context }}
+          target: ${{ matrix.target }}
           provenance: false
-          tags: |
-            ghcr.io/${{ github.repository }}:${{ github.sha }}
+          tags: ${{ matrix.tags }}
           labels: |
             git.repository=${{ github.repository }}
             git.commit-sha=${{ github.sha }}
-          cache-from: type=gha,scope=cloudserver
-          cache-to: type=gha,mode=max,scope=cloudserver
+          cache-from: ${{ matrix.cache-from }}
+          cache-to: ${{ matrix.cache-to }}
 
-      - name: Build and push cloudserver image test coverage
-        uses: docker/build-push-action@v5
+  # Federation depends on the pushed cloudserver production image and is not
+  # consumed by any test job, so it (and the dashboards push) runs in parallel
+  # with the tests instead of blocking them.
+  build-federation:
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Registry
+        uses: docker/login-action@v3
         with:
-          push: true
-          context: .
-          target: testcoverage
-          provenance: false
-          tags: |
-            ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
-          labels: |
-            git.repository=${{ github.repository }}
-            git.commit-sha=${{ github.sha }}
-          cache-from: type=gha,scope=cloudserver
-          cache-to: type=gha,mode=max,scope=cloudserver
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
 
       - name: Build and push federation image
         uses: docker/build-push-action@v5
@@ -248,28 +288,6 @@ jobs:
           cache-from: type=gha,scope=federation
           cache-to: type=gha,mode=max,scope=federation
 
-      - name: Build and push pykmip image
-        uses: docker/build-push-action@v5
-        with:
-          push: true
-          context: .github/pykmip
-          tags: |
-            ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
-          labels: |
-            git.repository=${{ github.repository }}
-            git.commit-sha=${{ github.sha }}
-          cache-from: type=gha,scope=pykmip
-          cache-to: type=gha,mode=max,scope=pykmip
-
-      - name: Build and push MongoDB
-        uses: docker/build-push-action@v5
-        with:
-          push: true
-          context: .github/docker/mongodb
-          tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
-          cache-from: type=gha,scope=mongodb
-          cache-to: type=gha,mode=max,scope=mongodb
-
       - name: Install Oras
         run: |
           curl -L https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz | \

From 13359d81550c6ddb753cfea7191f6340a9a89b51 Mon Sep 17 00:00:00 2001
From: Thomas Carmet <8408330+tcarmet@users.noreply.github.com>
Date: Fri, 10 Jul 2026 07:13:45 -0700
Subject: [PATCH 2/3] CLDSRV-955: Build testcoverage in the cloudserver matrix
 entry

The testcoverage image is FROM production, so building it as a separate
parallel matrix entry forced it to rebuild the production layers (~460s).
Build it right after production in the same entry instead, sharing the
cloudserver cache scope: it reuses the freshly built production layers
(a few seconds). Sequential within one job, so there is no cache-to race.

Drops the matrix wall time from being gated by the ~460s testcoverage
rebuild to roughly the ~220s production build.
---
 .github/workflows/tests.yaml | 37 +++++++++++++++++++++++++-----------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index a97d6e04da..b5d147d9ff 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -192,35 +192,33 @@ jobs:
       matrix:
         include:
           # Image url and tags are preserved exactly via matrix.name/matrix.tags.
+          # cloudserver builds production then testcoverage (target2) in the same
+          # entry: testcoverage is FROM production, so building it right after -
+          # sharing the cloudserver cache scope - lets it reuse the production
+          # layers (a few seconds) instead of rebuilding them. Sequential within
+          # one job, so there is no cache-to race on scope=cloudserver.
           - name: cloudserver
             context: .
             target: production
             tags: ghcr.io/${{ github.repository }}:${{ github.sha }}
             cache-from: type=gha,scope=cloudserver
             cache-to: type=gha,mode=max,scope=cloudserver
-          # testcoverage is FROM production; it writes to its own cache scope to
-          # avoid racing the production build's cache-to on scope=cloudserver,
-          # while still reading the production scope for layer reuse.
-          - name: cloudserver-testcoverage
-            context: .
-            target: testcoverage
-            tags: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
-            cache-from: |
-              type=gha,scope=cloudserver-testcoverage
-              type=gha,scope=cloudserver
-            cache-to: type=gha,mode=max,scope=cloudserver-testcoverage
+            target2: testcoverage
+            tags2: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
           - name: pykmip
             context: .github/pykmip
             target: ''
             tags: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
             cache-from: type=gha,scope=pykmip
             cache-to: type=gha,mode=max,scope=pykmip
+            target2: ''
           - name: ci-mongodb
             context: .github/docker/mongodb
             target: ''
             tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
             cache-from: type=gha,scope=mongodb
             cache-to: type=gha,mode=max,scope=mongodb
+            target2: ''
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -249,6 +247,23 @@ jobs:
           cache-from: ${{ matrix.cache-from }}
           cache-to: ${{ matrix.cache-to }}
 
+      # Second build for the cloudserver entry only: testcoverage reuses the
+      # production layers just built (same cloudserver cache scope).
+      - name: Build and push cloudserver testcoverage image
+        if: matrix.target2 != ''
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: ${{ matrix.context }}
+          target: ${{ matrix.target2 }}
+          provenance: false
+          tags: ${{ matrix.tags2 }}
+          labels: |
+            git.repository=${{ github.repository }}
+            git.commit-sha=${{ github.sha }}
+          cache-from: type=gha,scope=cloudserver
+          cache-to: type=gha,mode=max,scope=cloudserver
+
   # Federation depends on the pushed cloudserver production image and is not
   # consumed by any test job, so it (and the dashboards push) runs in parallel
   # with the tests instead of blocking them.

From d496f19271902599209c039d3966b9622e239cdc Mon Sep 17 00:00:00 2001
From: Thomas Carmet <8408330+tcarmet@users.noreply.github.com>
Date: Fri, 10 Jul 2026 07:44:24 -0700
Subject: [PATCH 3/3] CLDSRV-955: Split builds into build-cloudserver job +
 build-images matrix

Replace the single build matrix (which used an awkward optional target2
field to squeeze the testcoverage build into the cloudserver entry) with
a dedicated build-cloudserver job that builds production then testcoverage
sequentially, plus a clean build-images matrix for the standalone pykmip
and ci-mongodb images.

Downstream test jobs now depend on both build jobs; build-federation
depends on build-cloudserver (the production image).
---
 .github/workflows/tests.yaml | 124 ++++++++++++++++++++---------------
 1 file changed, 70 insertions(+), 54 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index b5d147d9ff..a1b7ddbf5f 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -183,7 +183,61 @@ jobs:
           source: artifacts
         if: always()
 
-  build:
+  # cloudserver builds the production image then the testcoverage image, which
+  # is FROM production. Building them in the same job (sharing the cloudserver
+  # cache scope) lets testcoverage reuse the production layers instead of
+  # rebuilding them, with no cache-to race since they run sequentially.
+  build-cloudserver:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+
+      - name: Build and push cloudserver image
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .
+          target: production
+          provenance: false
+          tags: |
+            ghcr.io/${{ github.repository }}:${{ github.sha }}
+          labels: |
+            git.repository=${{ github.repository }}
+            git.commit-sha=${{ github.sha }}
+          cache-from: type=gha,scope=cloudserver
+          cache-to: type=gha,mode=max,scope=cloudserver
+
+      - name: Build and push cloudserver image test coverage
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .
+          target: testcoverage
+          provenance: false
+          tags: |
+            ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
+          labels: |
+            git.repository=${{ github.repository }}
+            git.commit-sha=${{ github.sha }}
+          cache-from: type=gha,scope=cloudserver
+          cache-to: type=gha,mode=max,scope=cloudserver
+
+  # pykmip and ci-mongodb are standalone images; build them in parallel.
+  build-images:
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -191,34 +245,14 @@ jobs:
     strategy:
       matrix:
         include:
-          # Image url and tags are preserved exactly via matrix.name/matrix.tags.
-          # cloudserver builds production then testcoverage (target2) in the same
-          # entry: testcoverage is FROM production, so building it right after -
-          # sharing the cloudserver cache scope - lets it reuse the production
-          # layers (a few seconds) instead of rebuilding them. Sequential within
-          # one job, so there is no cache-to race on scope=cloudserver.
-          - name: cloudserver
-            context: .
-            target: production
-            tags: ghcr.io/${{ github.repository }}:${{ github.sha }}
-            cache-from: type=gha,scope=cloudserver
-            cache-to: type=gha,mode=max,scope=cloudserver
-            target2: testcoverage
-            tags2: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
           - name: pykmip
             context: .github/pykmip
-            target: ''
             tags: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
-            cache-from: type=gha,scope=pykmip
-            cache-to: type=gha,mode=max,scope=pykmip
-            target2: ''
+            cache-scope: pykmip
           - name: ci-mongodb
             context: .github/docker/mongodb
-            target: ''
             tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
-            cache-from: type=gha,scope=mongodb
-            cache-to: type=gha,mode=max,scope=mongodb
-            target2: ''
+            cache-scope: mongodb
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -238,38 +272,20 @@ jobs:
         with:
           push: true
           context: ${{ matrix.context }}
-          target: ${{ matrix.target }}
           provenance: false
           tags: ${{ matrix.tags }}
           labels: |
             git.repository=${{ github.repository }}
             git.commit-sha=${{ github.sha }}
-          cache-from: ${{ matrix.cache-from }}
-          cache-to: ${{ matrix.cache-to }}
-
-      # Second build for the cloudserver entry only: testcoverage reuses the
-      # production layers just built (same cloudserver cache scope).
-      - name: Build and push cloudserver testcoverage image
-        if: matrix.target2 != ''
-        uses: docker/build-push-action@v5
-        with:
-          push: true
-          context: ${{ matrix.context }}
-          target: ${{ matrix.target2 }}
-          provenance: false
-          tags: ${{ matrix.tags2 }}
-          labels: |
-            git.repository=${{ github.repository }}
-            git.commit-sha=${{ github.sha }}
-          cache-from: type=gha,scope=cloudserver
-          cache-to: type=gha,mode=max,scope=cloudserver
+          cache-from: type=gha,scope=${{ matrix.cache-scope }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.cache-scope }}
 
   # Federation depends on the pushed cloudserver production image and is not
   # consumed by any test job, so it (and the dashboards push) runs in parallel
   # with the tests instead of blocking them.
   build-federation:
     runs-on: ubuntu-latest
-    needs: build
+    needs: build-cloudserver
     permissions:
       contents: read
       packages: write
@@ -319,7 +335,7 @@ jobs:
 
   multiple-backend:
     runs-on: ubuntu-24.04
-    needs: build
+    needs: [build-cloudserver, build-images]
     env:
       CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
       MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
@@ -378,7 +394,7 @@ jobs:
 
   mongo-v0-ft-tests:
     runs-on: ubuntu-24.04
-    needs: build
+    needs: [build-cloudserver, build-images]
     env:
       S3BACKEND: mem
       MPU_TESTING: 'yes'
@@ -437,7 +453,7 @@ jobs:
 
   mongo-v1-ft-tests:
     runs-on: ubuntu-24.04
-    needs: build
+    needs: [build-cloudserver, build-images]
     env:
       S3BACKEND: mem
       MPU_TESTING: 'yes'
@@ -509,7 +525,7 @@ jobs:
             job-name: file-ft-tests-null-compat
     name: ${{ matrix.job-name }}
     runs-on: ubuntu-24.04
-    needs: build
+    needs: [build-cloudserver, build-images]
     env:
       S3BACKEND: file
       S3VAULT: mem
@@ -584,7 +600,7 @@ jobs:
             job-name: s3c-ft-tests-v1
     name: ${{ matrix.job-name }}
     runs-on: ubuntu-24.04
-    needs: build
+    needs: [build-cloudserver, build-images]
     env:
       S3BACKEND: file
       S3DATA: scality
@@ -705,7 +721,7 @@ jobs:
 
   utapi-v2-tests:
     runs-on: ubuntu-24.04
-    needs: build
+    needs: [build-cloudserver, build-images]
     if: always()
     env:
       ENABLE_UTAPI_V2: t
@@ -751,7 +767,7 @@ jobs:
 
   sur-tests:
     runs-on: ubuntu-24.04
-    needs: build
+    needs: [build-cloudserver, build-images]
     strategy:
       matrix:
         inflights:
@@ -809,7 +825,7 @@ jobs:
 
   kmip-ft-tests:
     runs-on: ubuntu-24.04
-    needs: build
+    needs: [build-cloudserver, build-images]
     env:
       S3BACKEND: file
       S3VAULT: mem
@@ -869,7 +885,7 @@ jobs:
 
   kmip-cluster-ft-tests:
     runs-on: ubuntu-latest
-    needs: build
+    needs: [build-cloudserver, build-images]
     env:
       S3BACKEND: file
       S3VAULT: mem
@@ -959,7 +975,7 @@ jobs:
         matrix.opts.globalEncryptionEnabled && '-global' || ''
       }}
     runs-on: ubuntu-latest
-    needs: build
+    needs: [build-cloudserver, build-images]
     env:
       S3BACKEND: file
       S3VAULT: scality