diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json new file mode 100644 index 00000000..59ea6b8a --- /dev/null +++ b/.config/dotnet-tools.json @@ -0,0 +1,13 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "dotnet-reportgenerator-globaltool": { + "version": "5.5.11", + "commands": [ + "reportgenerator" + ], + "rollForward": false + } + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b3f8a71..aee30719 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,12 +129,15 @@ jobs: uses: actions/cache@v5 with: path: ~/.nuget/packages - key: nuget-${{ runner.os }}-${{ hashFiles('global.json', 'Directory.Build.props', 'Directory.Packages.props', 'aep/Directory.Build.props', 'aep/Directory.Packages.props', '**/*.csproj') }} + key: nuget-${{ runner.os }}-${{ hashFiles('global.json', 'Directory.Build.props', 'Directory.Packages.props', '.config/dotnet-tools.json', 'aep/Directory.Build.props', 'aep/Directory.Packages.props', '**/*.csproj') }} restore-keys: | nuget-${{ runner.os }}- - name: Restore Agentstration if: needs.changes.outputs.dotnet == 'true' run: dotnet restore Agentstration.slnx -p:NuGetAudit=true -p:NuGetAuditMode=all + - name: Restore repository tools + if: needs.changes.outputs.dotnet == 'true' + run: dotnet tool restore - name: Restore complete AEP SDK if: needs.changes.outputs.dotnet == 'true' && needs.changes.outputs.aep == 'true' run: dotnet restore aep/Aep.slnx -p:NuGetAudit=true -p:NuGetAuditMode=all @@ -145,12 +148,17 @@ jobs: - name: Build Agentstration if: needs.changes.outputs.dotnet == 'true' run: dotnet build Agentstration.slnx --configuration Release --no-restore - - name: Test fast lane + - name: Test functional lanes with coverage if: needs.changes.outputs.dotnet == 'true' - run: dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 139 --max-parallel-test-modules 4 - - name: Test integration lane - if: needs.changes.outputs.dotnet == 'true' - run: dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 692 --max-parallel-test-modules 2 + shell: pwsh + run: ./scripts/ci/run-functional-coverage.ps1 -Configuration Release -NoBuild + - name: Upload functional coverage report + if: always() && needs.changes.outputs.dotnet == 'true' + uses: actions/upload-artifact@v6 + with: + name: functional-code-coverage + path: artifacts/coverage + if-no-files-found: error - name: Capture hosted module diagnostics if: needs.changes.outputs.dotnet == 'true' shell: pwsh diff --git a/AGENTS.md b/AGENTS.md index 2243ad2b..116ce84d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -193,6 +193,13 @@ dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Rele The two test solutions together form the required deterministic, offline functional suite. Performance and live-provider workloads are explicit opt-ins documented in `docs/contributing/testing.md`. For a focused iteration, run the affected test project first, then run both functional lanes before handoff. Do not suppress warnings or disable analyzers to make a change pass. +When changing coverage collection or its CI workflow, restore the pinned repository tool and reproduce the consolidated report after the Release build: + +```powershell +dotnet tool restore +./scripts/ci/run-functional-coverage.ps1 -Configuration Release -NoBuild +``` + To smoke-test the executable default with the Development bootstrap when startup behavior changes: ```powershell diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index e167cb38..afe2b174 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -20,6 +20,19 @@ dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Rele Run both fast and integration solutions for complete required functional validation. Their union is the functional test inventory represented by the root solution. +## Functional coverage + +CI collects managed-code coverage while running the required Fast and Integration lanes. Performance workloads, live-provider scenarios, test assemblies, generated sources, and files outside product `src` directories are excluded. Coverage is initially report-only: collection or report-generation failures fail CI, but the measured percentage does not. + +After restoring dependencies and building the root solution, reproduce the CI report locally with: + +```powershell +dotnet tool restore +./scripts/ci/run-functional-coverage.ps1 -Configuration Release -NoBuild +``` + +The script merges every module result into `artifacts/coverage/report/Cobertura.xml`, writes the line and branch totals to `artifacts/coverage/report/summary.md`, and generates the browsable report at `artifacts/coverage/report/index.html`. The same summary appears in the GitHub Actions run, and the complete raw and consolidated output is retained as the `functional-code-coverage` artifact. + ## Performance lane Performance workloads live in a dedicated project and are never discovered by the fast or integration lanes: diff --git a/scripts/ci/coverage.settings.xml b/scripts/ci/coverage.settings.xml new file mode 100644 index 00000000..ac05d575 --- /dev/null +++ b/scripts/ci/coverage.settings.xml @@ -0,0 +1,31 @@ + + + False + + + + .*[\\/][Aa]gentstration[^\\/]*\.(dll|exe)$ + + + .*\.Tests\.(dll|exe)$ + + + + + ^System\.Diagnostics\.CodeAnalysis\.ExcludeFromCodeCoverageAttribute$ + ^System\.CodeDom\.Compiler\.GeneratedCodeAttribute$ + ^System\.Runtime\.CompilerServices\.CompilerGeneratedAttribute$ + + + + + .*[\\/]src[\\/].*\.cs$ + + + .*[\\/]obj[\\/].* + .*\.g\.cs$ + .*\.generated\.cs$ + + + + diff --git a/scripts/ci/publish-functional-coverage.ps1 b/scripts/ci/publish-functional-coverage.ps1 new file mode 100644 index 00000000..ae0deb65 --- /dev/null +++ b/scripts/ci/publish-functional-coverage.ps1 @@ -0,0 +1,88 @@ +[CmdletBinding()] +param( + [string] $CoverageRoot = "artifacts/coverage" +) + +$ErrorActionPreference = 'Stop' +$repositoryRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '../..')).Path +$artifactsRoot = [System.IO.Path]::GetFullPath((Join-Path $repositoryRoot 'artifacts')) +$resolvedCoverageRoot = if ([System.IO.Path]::IsPathRooted($CoverageRoot)) { + [System.IO.Path]::GetFullPath($CoverageRoot) +} +else { + [System.IO.Path]::GetFullPath((Join-Path $repositoryRoot $CoverageRoot)) +} +$artifactsPrefix = $artifactsRoot.TrimEnd([System.IO.Path]::DirectorySeparatorChar) + [System.IO.Path]::DirectorySeparatorChar +if (-not $resolvedCoverageRoot.StartsWith($artifactsPrefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "Coverage output '$resolvedCoverageRoot' must be inside '$artifactsRoot'." +} + +$rawRoot = Join-Path $resolvedCoverageRoot 'raw' +$reportRoot = Join-Path $resolvedCoverageRoot 'report' +$coverageFiles = @(Get-ChildItem -LiteralPath $rawRoot -Filter '*.cobertura.xml' -File -Recurse -ErrorAction SilentlyContinue) +if ($coverageFiles.Count -eq 0) { + throw "No Cobertura inputs were found below '$rawRoot'." +} + +New-Item -ItemType Directory -Force -Path $reportRoot | Out-Null +$reports = ($coverageFiles.FullName -join ';') +& dotnet reportgenerator "-reports:$reports" "-targetdir:$reportRoot" '-reporttypes:Html;Cobertura' +if ($LASTEXITCODE -ne 0) { + throw "ReportGenerator exited with code $LASTEXITCODE." +} + +$coberturaPath = Join-Path $reportRoot 'Cobertura.xml' +$htmlPath = Join-Path $reportRoot 'index.html' +if (-not (Test-Path -LiteralPath $coberturaPath) -or -not (Test-Path -LiteralPath $htmlPath)) { + throw "The consolidated Cobertura and HTML reports were not both produced." +} + +[xml] $coverage = Get-Content -LiteralPath $coberturaPath -Raw +$coverageNode = $coverage.SelectSingleNode("/*[local-name()='coverage']") +if ($null -eq $coverageNode) { + throw "The consolidated report does not contain a Cobertura coverage root." +} + +$invariant = [System.Globalization.CultureInfo]::InvariantCulture +$linesCovered = [int]::Parse($coverageNode.GetAttribute('lines-covered'), $invariant) +$linesValid = [int]::Parse($coverageNode.GetAttribute('lines-valid'), $invariant) +$branchesCovered = [int]::Parse($coverageNode.GetAttribute('branches-covered'), $invariant) +$branchesValid = [int]::Parse($coverageNode.GetAttribute('branches-valid'), $invariant) +if ($linesValid -eq 0 -or $branchesValid -eq 0) { + throw "The consolidated report must contain line and branch coverage." +} + +$packages = @($coverage.SelectNodes("//*[local-name()='package']")) +$invalidPackages = @($packages | Where-Object { + $_.GetAttribute('name') -match '(?i)(^|\.)Tests($|\.)' -or + $_.GetAttribute('name') -match '(?i)Performance\.Tests' +}) +$invalidSources = @($coverage.SelectNodes("//*[local-name()='class']") | Where-Object { + $_.GetAttribute('filename') -match '(?i)[\\/]tests[\\/]' +}) +if ($invalidPackages.Count -gt 0 -or $invalidSources.Count -gt 0) { + throw "The consolidated report unexpectedly contains test or performance sources." +} + +function Format-Rate([int] $covered, [int] $valid) { + return (($covered / $valid).ToString('P2', $invariant)) +} + +$summary = @( + '## Functional code coverage', + '', + '| Metric | Covered | Total | Rate |', + '| --- | ---: | ---: | ---: |', + "| Lines | $linesCovered | $linesValid | $(Format-Rate $linesCovered $linesValid) |", + "| Branches | $branchesCovered | $branchesValid | $(Format-Rate $branchesCovered $branchesValid) |", + '', + "Merged from $($coverageFiles.Count) Fast and Integration module reports. Performance and test assemblies are excluded.", + '', + 'Coverage is report-only; no percentage threshold is enforced.' +) +$summaryPath = Join-Path $reportRoot 'summary.md' +$summary | Set-Content -LiteralPath $summaryPath -Encoding utf8 +$summary | ForEach-Object { Write-Host $_ } +if (-not [string]::IsNullOrWhiteSpace($env:GITHUB_STEP_SUMMARY)) { + $summary | Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Encoding utf8 +} diff --git a/scripts/ci/run-functional-coverage.ps1 b/scripts/ci/run-functional-coverage.ps1 new file mode 100644 index 00000000..2a95037a --- /dev/null +++ b/scripts/ci/run-functional-coverage.ps1 @@ -0,0 +1,70 @@ +[CmdletBinding()] +param( + [ValidateSet('Debug', 'Release')] + [string] $Configuration = 'Release', + + [switch] $NoBuild +) + +$ErrorActionPreference = 'Stop' +$repositoryRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '../..')).Path +$coverageRoot = Join-Path $repositoryRoot 'artifacts/coverage' +$artifactsRoot = [System.IO.Path]::GetFullPath((Join-Path $repositoryRoot 'artifacts')) +$coveragePrefix = $artifactsRoot.TrimEnd([System.IO.Path]::DirectorySeparatorChar) + [System.IO.Path]::DirectorySeparatorChar +$resolvedCoverageRoot = [System.IO.Path]::GetFullPath($coverageRoot) +if (-not $resolvedCoverageRoot.StartsWith($coveragePrefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "Coverage output '$resolvedCoverageRoot' must be inside '$artifactsRoot'." +} +if (Test-Path -LiteralPath $resolvedCoverageRoot) { + Remove-Item -LiteralPath $resolvedCoverageRoot -Recurse -Force +} + +$settingsPath = Join-Path $PSScriptRoot 'coverage.settings.xml' +$lanes = @( + @{ + Name = 'Fast' + Solution = 'Agentstration.Tests.Fast.slnx' + MinimumTests = 139 + ParallelModules = 4 + ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/fast' + }, + @{ + Name = 'Integration' + Solution = 'Agentstration.Tests.Integration.slnx' + MinimumTests = 692 + ParallelModules = 2 + ResultsDirectory = Join-Path $resolvedCoverageRoot 'raw/integration' + } +) + +Push-Location $repositoryRoot +try { + foreach ($lane in $lanes) { + Write-Host "Collecting $($lane.Name) lane coverage..." + $arguments = @( + 'test', + '--solution', $lane.Solution, + '--configuration', $Configuration + ) + if ($NoBuild) { + $arguments += '--no-build' + } + $arguments += @( + '--minimum-expected-tests', $lane.MinimumTests, + '--max-parallel-test-modules', $lane.ParallelModules, + '--coverage', + '--coverage-output-format', 'cobertura', + '--coverage-settings', $settingsPath, + '--results-directory', $lane.ResultsDirectory + ) + & dotnet @arguments + if ($LASTEXITCODE -ne 0) { + throw "$($lane.Name) coverage execution exited with code $LASTEXITCODE." + } + } + + & (Join-Path $PSScriptRoot 'publish-functional-coverage.ps1') -CoverageRoot $resolvedCoverageRoot +} +finally { + Pop-Location +}