ci(test): parallelize coverage and hosted diagnostics

.github/workflows/ci.yml

@@ -135,9 +135,6 @@ jobs:
       - name: Restore Agentstration
         if: needs.changes.outputs.dotnet == 'true'
         run: dotnet restore Agentstration.slnx -p:NuGetAudit=true -p:NuGetAuditMode=all
-      - name: Restore repository tools
-        if: needs.changes.outputs.dotnet == 'true'
-        run: dotnet tool restore
       - name: Restore complete AEP SDK
         if: needs.changes.outputs.dotnet == 'true' && needs.changes.outputs.aep == 'true'
         run: dotnet restore aep/Aep.slnx -p:NuGetAudit=true -p:NuGetAuditMode=all
@@ -148,19 +145,69 @@ jobs:
       - name: Build Agentstration
         if: needs.changes.outputs.dotnet == 'true'
         run: dotnet build Agentstration.slnx --configuration Release --no-restore
-      - name: Test functional lanes with coverage
+      - name: Test functional lanes
         if: needs.changes.outputs.dotnet == 'true'
         shell: pwsh
-        run: ./scripts/ci/run-functional-coverage.ps1 -Configuration Release -NoBuild
-      - name: Upload functional coverage report
-        if: always() && needs.changes.outputs.dotnet == 'true'
-        uses: actions/upload-artifact@v6
+        run: |
+          dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 139 --max-parallel-test-modules 4
+          if ($LASTEXITCODE -ne 0) {
+            throw 'Fast functional lane failed.'
+          }
+          dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 692 --max-parallel-test-modules 2
+          if ($LASTEXITCODE -ne 0) {
+            throw 'Integration functional lane failed.'
+          }
+      - name: Smoke-test Source Registry tool package
+        if: needs.changes.outputs.dotnet == 'true'
+        shell: pwsh
+        run: ./scripts/ci/test-source-registry-tool-package.ps1 -Configuration Release -NoBuild
+      - name: Build complete AEP SDK
+        if: needs.changes.outputs.dotnet == 'true' && needs.changes.outputs.aep == 'true'
+        run: dotnet build aep/Aep.slnx --configuration Release --no-restore
+      - name: Test complete AEP SDK
+        if: needs.changes.outputs.dotnet == 'true' && needs.changes.outputs.aep == 'true'
+        working-directory: aep
+        run: dotnet test --solution Aep.slnx --configuration Release --no-build --minimum-expected-tests 33 --max-parallel-test-modules 2
+
+  hosted-module-diagnostics:
+    name: hosted-module-diagnostics
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.dotnet == 'true'
+    timeout-minutes: 15
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v7
+      - name: Set up .NET
+        uses: actions/setup-dotnet@v6
         with:
-          name: functional-code-coverage
-          path: artifacts/coverage
-          if-no-files-found: error
+          global-json-file: global.json
+      - name: Cache NuGet packages
+        uses: actions/cache@v5
+        with:
+          path: ~/.nuget/packages
+          key: nuget-${{ runner.os }}-${{ hashFiles('global.json', 'Directory.Build.props', 'Directory.Packages.props', 'aep/Directory.Build.props', 'aep/Directory.Packages.props', '**/*.csproj') }}
+          restore-keys: |
+            nuget-${{ runner.os }}-
+      - name: Restore Agentstration
+        run: dotnet restore Agentstration.slnx -p:NuGetAudit=true -p:NuGetAuditMode=all
+      - name: Build hosted test modules
+        shell: pwsh
+        run: |
+          $modules = @(
+            'Agentstration.Management.Api.Tests',
+            'Agentstration.Management.Bootstrap.Tests',
+            'Agentstration.Management.Security.Tests',
+            'Agentstration.Management.Aep.Tests',
+            'Agentstration.Management.Sources.Tests'
+          )
+          foreach ($module in $modules) {
+            dotnet build "tests/$module/$module.csproj" --configuration Release --no-restore
+            if ($LASTEXITCODE -ne 0) {
+              throw "Could not build hosted test module '$module'."
+            }
+          }
       - name: Capture hosted module diagnostics
-        if: needs.changes.outputs.dotnet == 'true'
         shell: pwsh
         run: |
           @(
@@ -185,23 +232,93 @@ jobs:
               -FailureWorkingSetMiB $module.Failure
           }
       - name: Upload hosted module diagnostics
-        if: always() && needs.changes.outputs.dotnet == 'true'
+        if: always()
         uses: actions/upload-artifact@v6
         with:
           name: hosted-test-module-diagnostics
           path: artifacts/test-diagnostics/*.json
           if-no-files-found: warn
-      - name: Smoke-test Source Registry tool package
-        if: needs.changes.outputs.dotnet == 'true'
+
+  functional-coverage:
+    name: functional-coverage-${{ matrix.shard }}
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.dotnet == 'true'
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [a, b]
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v7
+      - name: Set up .NET
+        uses: actions/setup-dotnet@v6
+        with:
+          global-json-file: global.json
+      - name: Cache NuGet packages
+        uses: actions/cache@v5
+        with:
+          path: ~/.nuget/packages
+          key: nuget-${{ runner.os }}-${{ hashFiles('global.json', 'Directory.Build.props', 'Directory.Packages.props', '.config/dotnet-tools.json', 'aep/Directory.Build.props', 'aep/Directory.Packages.props', '**/*.csproj') }}
+          restore-keys: |
+            nuget-${{ runner.os }}-
+      - name: Restore Agentstration
+        run: dotnet restore Agentstration.slnx -p:NuGetAudit=true -p:NuGetAuditMode=all
+      - name: Build Agentstration
+        run: dotnet build Agentstration.slnx --configuration Release --no-restore
+      - name: Collect functional coverage shard ${{ matrix.shard }}
         shell: pwsh
-        run: ./scripts/ci/test-source-registry-tool-package.ps1 -Configuration Release -NoBuild
-      - name: Build complete AEP SDK
-        if: needs.changes.outputs.dotnet == 'true' && needs.changes.outputs.aep == 'true'
-        run: dotnet build aep/Aep.slnx --configuration Release --no-restore
-      - name: Test complete AEP SDK
-        if: needs.changes.outputs.dotnet == 'true' && needs.changes.outputs.aep == 'true'
-        working-directory: aep
-        run: dotnet test --solution Aep.slnx --configuration Release --no-build --minimum-expected-tests 33 --max-parallel-test-modules 2
+        run: ./scripts/ci/run-functional-coverage.ps1 -Configuration Release -NoBuild -Shard ${{ matrix.shard }} -SkipPublish
+      - name: Upload raw functional coverage shard ${{ matrix.shard }}
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: functional-code-coverage-raw-${{ matrix.shard }}
+          path: artifacts/coverage/raw/${{ matrix.shard }}
+          if-no-files-found: error
+
+  publish-functional-coverage:
+    name: publish-functional-coverage
+    runs-on: ubuntu-latest
+    needs: [changes, functional-coverage]
+    if: needs.changes.outputs.dotnet == 'true' && needs.functional-coverage.result == 'success'
+    timeout-minutes: 10
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v7
+      - name: Set up .NET
+        uses: actions/setup-dotnet@v6
+        with:
+          global-json-file: global.json
+      - name: Cache NuGet packages
+        uses: actions/cache@v5
+        with:
+          path: ~/.nuget/packages
+          key: nuget-${{ runner.os }}-${{ hashFiles('global.json', 'Directory.Build.props', 'Directory.Packages.props', '.config/dotnet-tools.json', 'aep/Directory.Build.props', 'aep/Directory.Packages.props', '**/*.csproj') }}
+          restore-keys: |
+            nuget-${{ runner.os }}-
+      - name: Restore repository tools
+        run: dotnet tool restore
+      - name: Download raw functional coverage shard a
+        uses: actions/download-artifact@v8
+        with:
+          name: functional-code-coverage-raw-a
+          path: artifacts/coverage/raw/a
+      - name: Download raw functional coverage shard b
+        uses: actions/download-artifact@v8
+        with:
+          name: functional-code-coverage-raw-b
+          path: artifacts/coverage/raw/b
+      - name: Publish consolidated functional coverage
+        shell: pwsh
+        run: ./scripts/ci/publish-functional-coverage.ps1
+      - name: Upload functional coverage report
+        uses: actions/upload-artifact@v6
+        with:
+          name: functional-code-coverage
+          path: artifacts/coverage
+          if-no-files-found: error
 
   postgresql-storage:
     name: postgresql-storage

Agentstration.Tests.CoverageA.slnx

@@ -0,0 +1,10 @@
+<Solution>
+  <Folder Name="/tests/coverage-a/">
+    <Project Path="tests/Agentstration.Application.Tests/Agentstration.Application.Tests.csproj" />
+    <Project Path="tests/Agentstration.Management.Api.Tests/Agentstration.Management.Api.Tests.csproj" />
+    <Project Path="tests/Agentstration.Management.Bootstrap.Tests/Agentstration.Management.Bootstrap.Tests.csproj" />
+    <Project Path="tests/Agentstration.Management.Aep.Tests/Agentstration.Management.Aep.Tests.csproj" />
+    <Project Path="tests/Agentstration.ModelProviders.Tests/Agentstration.ModelProviders.Tests.csproj" />
+    <Project Path="tests/Agentstration.Workplace.Web.Tests/Agentstration.Workplace.Web.Tests.csproj" />
+  </Folder>
+</Solution>

Agentstration.Tests.CoverageB.slnx

@@ -0,0 +1,11 @@
+<Solution>
+  <Folder Name="/tests/coverage-b/">
+    <Project Path="tests/Agentstration.Management.Storage.Tests/Agentstration.Management.Storage.Tests.csproj" />
+    <Project Path="tests/Agentstration.Management.Sources.Tests/Agentstration.Management.Sources.Tests.csproj" />
+    <Project Path="tests/Agentstration.Management.Security.Tests/Agentstration.Management.Security.Tests.csproj" />
+    <Project Path="tests/Agentstration.Runtime.Tests/Agentstration.Runtime.Tests.csproj" />
+    <Project Path="tests/Agentstration.SourceProviders.Git.Tests/Agentstration.SourceProviders.Git.Tests.csproj" />
+    <Project Path="tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj" />
+    <Project Path="tests/Agentstration.Work.Api.Tests/Agentstration.Work.Api.Tests.csproj" />
+  </Folder>
+</Solution>

docs/contributing/testing.md

@@ -22,7 +22,7 @@ Run both fast and integration solutions for complete required functional validat
 
 ## Functional coverage
 
-CI collects managed-code coverage while running the required Fast and Integration lanes. Performance workloads, live-provider scenarios, test assemblies, generated sources, and files outside product `src` directories are excluded. Coverage is initially report-only: collection or report-generation failures fail CI, but the measured percentage does not.
+CI collects managed-code coverage from the required Fast and Integration inventory on two balanced runners while the uninstrumented functional lanes remain in `build-and-test`. The raw shard reports are merged into one consolidated result after both runners complete. Performance workloads, live-provider scenarios, test assemblies, generated sources, and files outside product `src` directories are excluded. Coverage is initially report-only: collection or report-generation failures fail CI, but the measured percentage does not.
 
 After restoring dependencies and building the root solution, reproduce the CI report locally with:
 
@@ -47,7 +47,7 @@ The report records workload parameters, provider, elapsed time, runtime and OS m
 
 ## CI concurrency and memory diagnostics
 
-The fast, integration, and performance lanes cap concurrent test modules at 4, 2, and 1 respectively. Host-heavy Management modules also use one class worker per assembly. CI reruns the designated hosted modules sequentially through `scripts/ci/run-test-module-with-diagnostics.ps1`; each JSON artifact contains the discovered count, duration, process peak working set, process peak private memory, aggregate peak working set for active `dotnet` processes, runtime, and OS. The diagnostic artifact deliberately excludes test output and payloads.
+The fast, integration, and performance lanes cap concurrent test modules at 4, 2, and 1 respectively. Host-heavy Management modules also use one class worker per assembly. CI reruns the designated hosted modules sequentially through `scripts/ci/run-test-module-with-diagnostics.ps1` in the dedicated `hosted-module-diagnostics` job. That job runs in parallel with functional coverage and compiles only the designated test projects after restoring the repository graph, so memory safeguards do not extend the `build-and-test` critical path. Each JSON artifact contains the discovered count, duration, process peak working set, process peak private memory, aggregate peak working set for active `dotnet` processes, runtime, and OS. The diagnostic artifact deliberately excludes test output and payloads.
 
 The initial budgets below use Release runs on Windows 11 10.0.26200 with .NET 10.0.10/10.0.11, collected during #298. A warning is evidence to review the Linux and Windows trend; a failure protects constrained runners from returning to the original greater-than-1-GiB process. Adjust these values only after retaining representative artifacts from both runner families.
 

scripts/ci/publish-functional-coverage.ps1

@@ -19,7 +19,13 @@ if (-not $resolvedCoverageRoot.StartsWith($artifactsPrefix, [StringComparison]::
 
 $rawRoot = Join-Path $resolvedCoverageRoot 'raw'
 $reportRoot = Join-Path $resolvedCoverageRoot 'report'
-$coverageFiles = @(Get-ChildItem -LiteralPath $rawRoot -Filter '*.cobertura.xml' -File -Recurse -ErrorAction SilentlyContinue)
+$shardRoots = @('a', 'b') |
+    ForEach-Object { Join-Path $rawRoot $_ } |
+    Where-Object { Test-Path -LiteralPath $_ -PathType Container }
+$inputRoots = if ($shardRoots.Count -gt 0) { $shardRoots } else { @($rawRoot) }
+$coverageFiles = @($inputRoots | ForEach-Object {
+    Get-ChildItem -LiteralPath $_ -Filter '*.cobertura.xml' -File -Recurse -ErrorAction SilentlyContinue
+})
 if ($coverageFiles.Count -eq 0) {
     throw "No Cobertura inputs were found below '$rawRoot'."
 }

scripts/ci/run-functional-coverage.ps1

@@ -3,7 +3,12 @@ param(
     [ValidateSet('Debug', 'Release')]
     [string] $Configuration = 'Release',
 
-    [switch] $NoBuild
+    [switch] $NoBuild,
+
+    [ValidateSet('All', 'a', 'b')]
+    [string] $Shard = 'All',
+
+    [switch] $SkipPublish
 )
 
 $ErrorActionPreference = 'Stop'
@@ -15,27 +20,66 @@ $resolvedCoverageRoot = [System.IO.Path]::GetFullPath($coverageRoot)
 if (-not $resolvedCoverageRoot.StartsWith($coveragePrefix, [StringComparison]::OrdinalIgnoreCase)) {
     throw "Coverage output '$resolvedCoverageRoot' must be inside '$artifactsRoot'."
 }
-if (Test-Path -LiteralPath $resolvedCoverageRoot) {
-    Remove-Item -LiteralPath $resolvedCoverageRoot -Recurse -Force
+$cleanupRoot = if ($Shard -eq 'All') {
+    $resolvedCoverageRoot
+}
+else {
+    Join-Path $resolvedCoverageRoot "raw/$($Shard.ToLowerInvariant())"
+}
+if (Test-Path -LiteralPath $cleanupRoot) {
+    Remove-Item -LiteralPath $cleanupRoot -Recurse -Force
 }
 
 $settingsPath = Join-Path $PSScriptRoot 'coverage.settings.xml'
-$lanes = @(
-    @{
-        Name = 'Fast'
-        Solution = 'Agentstration.Tests.Fast.slnx'
-        MinimumTests = 139
-        ParallelModules = 4
-        ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/fast'
-    },
-    @{
-        Name = 'Integration'
-        Solution = 'Agentstration.Tests.Integration.slnx'
-        MinimumTests = 692
-        ParallelModules = 2
-        ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/integration'
+$lanes = switch ($Shard.ToLowerInvariant()) {
+    'a' {
+        @(
+            @{
+                Name = 'Fast'
+                Solution = 'Agentstration.Tests.Fast.slnx'
+                MinimumTests = 139
+                ParallelModules = 4
+                ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/a/fast'
+            },
+            @{
+                Name = 'Integration coverage shard A'
+                Solution = 'Agentstration.Tests.CoverageA.slnx'
+                MinimumTests = 221
+                ParallelModules = 2
+                ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/a/integration'
+            }
+        )
     }
-)
+    'b' {
+        @(
+            @{
+                Name = 'Integration coverage shard B'
+                Solution = 'Agentstration.Tests.CoverageB.slnx'
+                MinimumTests = 471
+                ParallelModules = 2
+                ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/b/integration'
+            }
+        )
+    }
+    default {
+        @(
+            @{
+                Name = 'Fast'
+                Solution = 'Agentstration.Tests.Fast.slnx'
+                MinimumTests = 139
+                ParallelModules = 4
+                ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/fast'
+            },
+            @{
+                Name = 'Integration'
+                Solution = 'Agentstration.Tests.Integration.slnx'
+                MinimumTests = 692
+                ParallelModules = 2
+                ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/integration'
+            }
+        )
+    }
+}
 
 Push-Location $repositoryRoot
 try {
@@ -63,7 +107,9 @@ try {
         }
     }
 
-    & (Join-Path $PSScriptRoot 'publish-functional-coverage.ps1') -CoverageRoot $resolvedCoverageRoot
+    if (-not $SkipPublish) {
+        & (Join-Path $PSScriptRoot 'publish-functional-coverage.ps1') -CoverageRoot $resolvedCoverageRoot
+    }
 }
 finally {
     Pop-Location