diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aee30719..35a43a07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,9 +135,6 @@ jobs: - name: Restore Agentstration if: needs.changes.outputs.dotnet == 'true' run: dotnet restore Agentstration.slnx -p:NuGetAudit=true -p:NuGetAuditMode=all - - name: Restore repository tools - if: needs.changes.outputs.dotnet == 'true' - run: dotnet tool restore - name: Restore complete AEP SDK if: needs.changes.outputs.dotnet == 'true' && needs.changes.outputs.aep == 'true' run: dotnet restore aep/Aep.slnx -p:NuGetAudit=true -p:NuGetAuditMode=all @@ -148,19 +145,69 @@ jobs: - name: Build Agentstration if: needs.changes.outputs.dotnet == 'true' run: dotnet build Agentstration.slnx --configuration Release --no-restore - - name: Test functional lanes with coverage + - name: Test functional lanes if: needs.changes.outputs.dotnet == 'true' shell: pwsh - run: ./scripts/ci/run-functional-coverage.ps1 -Configuration Release -NoBuild - - name: Upload functional coverage report - if: always() && needs.changes.outputs.dotnet == 'true' - uses: actions/upload-artifact@v6 + run: | + dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 139 --max-parallel-test-modules 4 + if ($LASTEXITCODE -ne 0) { + throw 'Fast functional lane failed.' + } + dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 692 --max-parallel-test-modules 2 + if ($LASTEXITCODE -ne 0) { + throw 'Integration functional lane failed.' + } + - name: Smoke-test Source Registry tool package + if: needs.changes.outputs.dotnet == 'true' + shell: pwsh + run: ./scripts/ci/test-source-registry-tool-package.ps1 -Configuration Release -NoBuild + - name: Build complete AEP SDK + if: needs.changes.outputs.dotnet == 'true' && needs.changes.outputs.aep == 'true' + run: dotnet build aep/Aep.slnx --configuration Release --no-restore + - name: Test complete AEP SDK + if: needs.changes.outputs.dotnet == 'true' && needs.changes.outputs.aep == 'true' + working-directory: aep + run: dotnet test --solution Aep.slnx --configuration Release --no-build --minimum-expected-tests 33 --max-parallel-test-modules 2 + + hosted-module-diagnostics: + name: hosted-module-diagnostics + runs-on: ubuntu-latest + needs: changes + if: needs.changes.outputs.dotnet == 'true' + timeout-minutes: 15 + steps: + - name: Check out repository + uses: actions/checkout@v7 + - name: Set up .NET + uses: actions/setup-dotnet@v6 with: - name: functional-code-coverage - path: artifacts/coverage - if-no-files-found: error + global-json-file: global.json + - name: Cache NuGet packages + uses: actions/cache@v5 + with: + path: ~/.nuget/packages + key: nuget-${{ runner.os }}-${{ hashFiles('global.json', 'Directory.Build.props', 'Directory.Packages.props', 'aep/Directory.Build.props', 'aep/Directory.Packages.props', '**/*.csproj') }} + restore-keys: | + nuget-${{ runner.os }}- + - name: Restore Agentstration + run: dotnet restore Agentstration.slnx -p:NuGetAudit=true -p:NuGetAuditMode=all + - name: Build hosted test modules + shell: pwsh + run: | + $modules = @( + 'Agentstration.Management.Api.Tests', + 'Agentstration.Management.Bootstrap.Tests', + 'Agentstration.Management.Security.Tests', + 'Agentstration.Management.Aep.Tests', + 'Agentstration.Management.Sources.Tests' + ) + foreach ($module in $modules) { + dotnet build "tests/$module/$module.csproj" --configuration Release --no-restore + if ($LASTEXITCODE -ne 0) { + throw "Could not build hosted test module '$module'." + } + } - name: Capture hosted module diagnostics - if: needs.changes.outputs.dotnet == 'true' shell: pwsh run: | @( @@ -185,23 +232,93 @@ jobs: -FailureWorkingSetMiB $module.Failure } - name: Upload hosted module diagnostics - if: always() && needs.changes.outputs.dotnet == 'true' + if: always() uses: actions/upload-artifact@v6 with: name: hosted-test-module-diagnostics path: artifacts/test-diagnostics/*.json if-no-files-found: warn - - name: Smoke-test Source Registry tool package - if: needs.changes.outputs.dotnet == 'true' + + functional-coverage: + name: functional-coverage-${{ matrix.shard }} + runs-on: ubuntu-latest + needs: changes + if: needs.changes.outputs.dotnet == 'true' + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + shard: [a, b] + steps: + - name: Check out repository + uses: actions/checkout@v7 + - name: Set up .NET + uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + - name: Cache NuGet packages + uses: actions/cache@v5 + with: + path: ~/.nuget/packages + key: nuget-${{ runner.os }}-${{ hashFiles('global.json', 'Directory.Build.props', 'Directory.Packages.props', '.config/dotnet-tools.json', 'aep/Directory.Build.props', 'aep/Directory.Packages.props', '**/*.csproj') }} + restore-keys: | + nuget-${{ runner.os }}- + - name: Restore Agentstration + run: dotnet restore Agentstration.slnx -p:NuGetAudit=true -p:NuGetAuditMode=all + - name: Build Agentstration + run: dotnet build Agentstration.slnx --configuration Release --no-restore + - name: Collect functional coverage shard ${{ matrix.shard }} shell: pwsh - run: ./scripts/ci/test-source-registry-tool-package.ps1 -Configuration Release -NoBuild - - name: Build complete AEP SDK - if: needs.changes.outputs.dotnet == 'true' && needs.changes.outputs.aep == 'true' - run: dotnet build aep/Aep.slnx --configuration Release --no-restore - - name: Test complete AEP SDK - if: needs.changes.outputs.dotnet == 'true' && needs.changes.outputs.aep == 'true' - working-directory: aep - run: dotnet test --solution Aep.slnx --configuration Release --no-build --minimum-expected-tests 33 --max-parallel-test-modules 2 + run: ./scripts/ci/run-functional-coverage.ps1 -Configuration Release -NoBuild -Shard ${{ matrix.shard }} -SkipPublish + - name: Upload raw functional coverage shard ${{ matrix.shard }} + if: always() + uses: actions/upload-artifact@v6 + with: + name: functional-code-coverage-raw-${{ matrix.shard }} + path: artifacts/coverage/raw/${{ matrix.shard }} + if-no-files-found: error + + publish-functional-coverage: + name: publish-functional-coverage + runs-on: ubuntu-latest + needs: [changes, functional-coverage] + if: needs.changes.outputs.dotnet == 'true' && needs.functional-coverage.result == 'success' + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@v7 + - name: Set up .NET + uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + - name: Cache NuGet packages + uses: actions/cache@v5 + with: + path: ~/.nuget/packages + key: nuget-${{ runner.os }}-${{ hashFiles('global.json', 'Directory.Build.props', 'Directory.Packages.props', '.config/dotnet-tools.json', 'aep/Directory.Build.props', 'aep/Directory.Packages.props', '**/*.csproj') }} + restore-keys: | + nuget-${{ runner.os }}- + - name: Restore repository tools + run: dotnet tool restore + - name: Download raw functional coverage shard a + uses: actions/download-artifact@v8 + with: + name: functional-code-coverage-raw-a + path: artifacts/coverage/raw/a + - name: Download raw functional coverage shard b + uses: actions/download-artifact@v8 + with: + name: functional-code-coverage-raw-b + path: artifacts/coverage/raw/b + - name: Publish consolidated functional coverage + shell: pwsh + run: ./scripts/ci/publish-functional-coverage.ps1 + - name: Upload functional coverage report + uses: actions/upload-artifact@v6 + with: + name: functional-code-coverage + path: artifacts/coverage + if-no-files-found: error postgresql-storage: name: postgresql-storage diff --git a/Agentstration.Tests.CoverageA.slnx b/Agentstration.Tests.CoverageA.slnx new file mode 100644 index 00000000..b155ef14 --- /dev/null +++ b/Agentstration.Tests.CoverageA.slnx @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/Agentstration.Tests.CoverageB.slnx b/Agentstration.Tests.CoverageB.slnx new file mode 100644 index 00000000..a17ee294 --- /dev/null +++ b/Agentstration.Tests.CoverageB.slnx @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index afe2b174..a41d6765 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -22,7 +22,7 @@ Run both fast and integration solutions for complete required functional validat ## Functional coverage -CI collects managed-code coverage while running the required Fast and Integration lanes. Performance workloads, live-provider scenarios, test assemblies, generated sources, and files outside product `src` directories are excluded. Coverage is initially report-only: collection or report-generation failures fail CI, but the measured percentage does not. +CI collects managed-code coverage from the required Fast and Integration inventory on two balanced runners while the uninstrumented functional lanes remain in `build-and-test`. The raw shard reports are merged into one consolidated result after both runners complete. Performance workloads, live-provider scenarios, test assemblies, generated sources, and files outside product `src` directories are excluded. Coverage is initially report-only: collection or report-generation failures fail CI, but the measured percentage does not. After restoring dependencies and building the root solution, reproduce the CI report locally with: @@ -47,7 +47,7 @@ The report records workload parameters, provider, elapsed time, runtime and OS m ## CI concurrency and memory diagnostics -The fast, integration, and performance lanes cap concurrent test modules at 4, 2, and 1 respectively. Host-heavy Management modules also use one class worker per assembly. CI reruns the designated hosted modules sequentially through `scripts/ci/run-test-module-with-diagnostics.ps1`; each JSON artifact contains the discovered count, duration, process peak working set, process peak private memory, aggregate peak working set for active `dotnet` processes, runtime, and OS. The diagnostic artifact deliberately excludes test output and payloads. +The fast, integration, and performance lanes cap concurrent test modules at 4, 2, and 1 respectively. Host-heavy Management modules also use one class worker per assembly. CI reruns the designated hosted modules sequentially through `scripts/ci/run-test-module-with-diagnostics.ps1` in the dedicated `hosted-module-diagnostics` job. That job runs in parallel with functional coverage and compiles only the designated test projects after restoring the repository graph, so memory safeguards do not extend the `build-and-test` critical path. Each JSON artifact contains the discovered count, duration, process peak working set, process peak private memory, aggregate peak working set for active `dotnet` processes, runtime, and OS. The diagnostic artifact deliberately excludes test output and payloads. The initial budgets below use Release runs on Windows 11 10.0.26200 with .NET 10.0.10/10.0.11, collected during #298. A warning is evidence to review the Linux and Windows trend; a failure protects constrained runners from returning to the original greater-than-1-GiB process. Adjust these values only after retaining representative artifacts from both runner families. diff --git a/scripts/ci/publish-functional-coverage.ps1 b/scripts/ci/publish-functional-coverage.ps1 index ae0deb65..6d1d17df 100644 --- a/scripts/ci/publish-functional-coverage.ps1 +++ b/scripts/ci/publish-functional-coverage.ps1 @@ -19,7 +19,13 @@ if (-not $resolvedCoverageRoot.StartsWith($artifactsPrefix, [StringComparison]:: $rawRoot = Join-Path $resolvedCoverageRoot 'raw' $reportRoot = Join-Path $resolvedCoverageRoot 'report' -$coverageFiles = @(Get-ChildItem -LiteralPath $rawRoot -Filter '*.cobertura.xml' -File -Recurse -ErrorAction SilentlyContinue) +$shardRoots = @('a', 'b') | + ForEach-Object { Join-Path $rawRoot $_ } | + Where-Object { Test-Path -LiteralPath $_ -PathType Container } +$inputRoots = if ($shardRoots.Count -gt 0) { $shardRoots } else { @($rawRoot) } +$coverageFiles = @($inputRoots | ForEach-Object { + Get-ChildItem -LiteralPath $_ -Filter '*.cobertura.xml' -File -Recurse -ErrorAction SilentlyContinue +}) if ($coverageFiles.Count -eq 0) { throw "No Cobertura inputs were found below '$rawRoot'." } diff --git a/scripts/ci/run-functional-coverage.ps1 b/scripts/ci/run-functional-coverage.ps1 index 2a95037a..d5e9479e 100644 --- a/scripts/ci/run-functional-coverage.ps1 +++ b/scripts/ci/run-functional-coverage.ps1 @@ -3,7 +3,12 @@ param( [ValidateSet('Debug', 'Release')] [string] $Configuration = 'Release', - [switch] $NoBuild + [switch] $NoBuild, + + [ValidateSet('All', 'a', 'b')] + [string] $Shard = 'All', + + [switch] $SkipPublish ) $ErrorActionPreference = 'Stop' @@ -15,27 +20,66 @@ $resolvedCoverageRoot = [System.IO.Path]::GetFullPath($coverageRoot) if (-not $resolvedCoverageRoot.StartsWith($coveragePrefix, [StringComparison]::OrdinalIgnoreCase)) { throw "Coverage output '$resolvedCoverageRoot' must be inside '$artifactsRoot'." } -if (Test-Path -LiteralPath $resolvedCoverageRoot) { - Remove-Item -LiteralPath $resolvedCoverageRoot -Recurse -Force +$cleanupRoot = if ($Shard -eq 'All') { + $resolvedCoverageRoot +} +else { + Join-Path $resolvedCoverageRoot "raw/$($Shard.ToLowerInvariant())" +} +if (Test-Path -LiteralPath $cleanupRoot) { + Remove-Item -LiteralPath $cleanupRoot -Recurse -Force } $settingsPath = Join-Path $PSScriptRoot 'coverage.settings.xml' -$lanes = @( - @{ - Name = 'Fast' - Solution = 'Agentstration.Tests.Fast.slnx' - MinimumTests = 139 - ParallelModules = 4 - ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/fast' - }, - @{ - Name = 'Integration' - Solution = 'Agentstration.Tests.Integration.slnx' - MinimumTests = 692 - ParallelModules = 2 - ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/integration' +$lanes = switch ($Shard.ToLowerInvariant()) { + 'a' { + @( + @{ + Name = 'Fast' + Solution = 'Agentstration.Tests.Fast.slnx' + MinimumTests = 139 + ParallelModules = 4 + ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/a/fast' + }, + @{ + Name = 'Integration coverage shard A' + Solution = 'Agentstration.Tests.CoverageA.slnx' + MinimumTests = 221 + ParallelModules = 2 + ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/a/integration' + } + ) } -) + 'b' { + @( + @{ + Name = 'Integration coverage shard B' + Solution = 'Agentstration.Tests.CoverageB.slnx' + MinimumTests = 471 + ParallelModules = 2 + ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/b/integration' + } + ) + } + default { + @( + @{ + Name = 'Fast' + Solution = 'Agentstration.Tests.Fast.slnx' + MinimumTests = 139 + ParallelModules = 4 + ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/fast' + }, + @{ + Name = 'Integration' + Solution = 'Agentstration.Tests.Integration.slnx' + MinimumTests = 692 + ParallelModules = 2 + ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/integration' + } + ) + } +} Push-Location $repositoryRoot try { @@ -63,7 +107,9 @@ try { } } - & (Join-Path $PSScriptRoot 'publish-functional-coverage.ps1') -CoverageRoot $resolvedCoverageRoot + if (-not $SkipPublish) { + & (Join-Path $PSScriptRoot 'publish-functional-coverage.ps1') -CoverageRoot $resolvedCoverageRoot + } } finally { Pop-Location