diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e759d615..2b3f8a71 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,7 @@ jobs: dotnet: ${{ steps.classify.outputs.dotnet }} aep: ${{ steps.classify.outputs.aep }} container: ${{ steps.classify.outputs.container }} + storage: ${{ steps.classify.outputs.storage }} steps: - name: Check out repository uses: actions/checkout@v7 @@ -62,15 +63,20 @@ jobs: $container = $runEverything -or @($changedFiles | Where-Object { $_ -match '^(src/|aep/src/|deploy/|Dockerfile$|\.dockerignore$|global\.json$|Agentstration\.slnx$|Directory\.(Build|Packages)\.(props|targets)$|\.github/workflows/ci\.yml$)' }).Count -gt 0 + $storage = $runEverything -or @($changedFiles | Where-Object { + $_ -match '^(src/Agentstration\.(Application|Flow|Infrastructure|Management|Runtime|Web|Work)|tests/Agentstration\.(Application|Performance|Web)\.Tests/|Agentstration\.Tests\.Performance\.slnx$|Directory\.(Build|Packages)\.(props|targets)$|global\.json$|\.github/workflows/(ci|storage-performance)\.yml$)' + }).Count -gt 0 "dotnet=$($dotnet.ToString().ToLowerInvariant())" >> $env:GITHUB_OUTPUT "aep=$($aep.ToString().ToLowerInvariant())" >> $env:GITHUB_OUTPUT "container=$($container.ToString().ToLowerInvariant())" >> $env:GITHUB_OUTPUT + "storage=$($storage.ToString().ToLowerInvariant())" >> $env:GITHUB_OUTPUT Write-Host "Changed files: $($changedFiles.Count)" Write-Host "Run .NET validation: $dotnet" Write-Host "Run complete AEP validation: $aep" Write-Host "Build container: $container" + Write-Host "Run storage smoke: $storage" windows-host-lifecycle: name: windows-host-lifecycle @@ -97,7 +103,7 @@ jobs: - name: Build web tests run: dotnet build tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj --configuration Release --no-restore - name: Verify Windows host lifecycle - run: dotnet tests/Agentstration.Web.Tests/bin/Release/net10.0/Agentstration.Web.Tests.dll --filter "FullyQualifiedName~QuartzHostLifecycleTests|FullyQualifiedName~StartupDoesNotCreateLegacyDataJson" --progress off + run: dotnet tests/Agentstration.Web.Tests/bin/Release/net10.0/Agentstration.Web.Tests.dll --filter "FullyQualifiedName~QuartzHostLifecycleTests|FullyQualifiedName~StartupDoesNotCreateLegacyDataJson" --minimum-expected-tests 2 --progress off build-and-test: name: build-and-test @@ -139,9 +145,44 @@ jobs: - name: Build Agentstration if: needs.changes.outputs.dotnet == 'true' run: dotnet build Agentstration.slnx --configuration Release --no-restore - - name: Test Agentstration + - name: Test fast lane + if: needs.changes.outputs.dotnet == 'true' + run: dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 139 --max-parallel-test-modules 4 + - name: Test integration lane if: needs.changes.outputs.dotnet == 'true' - run: dotnet test Agentstration.slnx --configuration Release --no-build + run: dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 692 --max-parallel-test-modules 2 + - name: Capture hosted module diagnostics + if: needs.changes.outputs.dotnet == 'true' + shell: pwsh + run: | + @( + '## Hosted test module diagnostics', + '', + '| Module | Tests | Duration (s) | Peak working set (MiB) | Peak private memory (MiB) | Aggregate dotnet working set (MiB) | Status |', + '| --- | ---: | ---: | ---: | ---: | ---: | --- |' + ) | Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Encoding utf8 + $modules = @( + @{ Name = 'Agentstration.Management.Api.Tests'; Tests = 43; Warning = 900; Failure = 1024 }, + @{ Name = 'Agentstration.Management.Bootstrap.Tests'; Tests = 23; Warning = 500; Failure = 700 }, + @{ Name = 'Agentstration.Management.Security.Tests'; Tests = 38; Warning = 800; Failure = 1024 }, + @{ Name = 'Agentstration.Management.Aep.Tests'; Tests = 11; Warning = 450; Failure = 700 }, + @{ Name = 'Agentstration.Management.Sources.Tests'; Tests = 93; Warning = 550; Failure = 900 } + ) + foreach ($module in $modules) { + ./scripts/ci/run-test-module-with-diagnostics.ps1 ` + -AssemblyPath "tests/$($module.Name)/bin/Release/net10.0/$($module.Name).dll" ` + -ReportPath "artifacts/test-diagnostics/$($module.Name).json" ` + -MinimumExpectedTests $module.Tests ` + -WarningWorkingSetMiB $module.Warning ` + -FailureWorkingSetMiB $module.Failure + } + - name: Upload hosted module diagnostics + if: always() && needs.changes.outputs.dotnet == 'true' + uses: actions/upload-artifact@v6 + with: + name: hosted-test-module-diagnostics + path: artifacts/test-diagnostics/*.json + if-no-files-found: warn - name: Smoke-test Source Registry tool package if: needs.changes.outputs.dotnet == 'true' shell: pwsh @@ -151,11 +192,14 @@ jobs: run: dotnet build aep/Aep.slnx --configuration Release --no-restore - name: Test complete AEP SDK if: needs.changes.outputs.dotnet == 'true' && needs.changes.outputs.aep == 'true' - run: dotnet test aep/Aep.slnx --configuration Release --no-build + working-directory: aep + run: dotnet test --solution Aep.slnx --configuration Release --no-build --minimum-expected-tests 33 --max-parallel-test-modules 2 postgresql-storage: name: postgresql-storage runs-on: ubuntu-latest + needs: changes + if: needs.changes.outputs.storage == 'true' timeout-minutes: 20 services: postgres: @@ -181,11 +225,15 @@ jobs: with: global-json-file: global.json - name: Restore PostgreSQL integration tests - run: dotnet restore tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj -p:NuGetAudit=true -p:NuGetAuditMode=all + run: | + dotnet restore tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj -p:NuGetAudit=true -p:NuGetAuditMode=all + dotnet restore tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj -p:NuGetAudit=true -p:NuGetAuditMode=all - name: Build PostgreSQL integration tests - run: dotnet build tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj --configuration Release --no-restore + run: | + dotnet build tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj --configuration Release --no-restore + dotnet build tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj --configuration Release --no-restore - name: Run PostgreSQL migration and restart test - run: dotnet tests/Agentstration.Web.Tests/bin/Release/net10.0/Agentstration.Web.Tests.dll --filter "Name=EmptyDatabaseMigratesAndRemainsReadyAfterRestart" --progress off + run: dotnet tests/Agentstration.Web.Tests/bin/Release/net10.0/Agentstration.Web.Tests.dll --filter "Name=EmptyDatabaseMigratesAndRemainsReadyAfterRestart" --minimum-expected-tests 1 --progress off - name: Run PostgreSQL storage concurrency benchmark env: AGENTSTRATION_STORAGE_BENCHMARK_PROVIDER: PostgreSql @@ -193,8 +241,15 @@ jobs: AGENTSTRATION_STORAGE_BENCHMARK_CONCURRENCY: 8 AGENTSTRATION_STORAGE_BENCHMARK_REPORT: ${{ runner.temp }}/agentstration-storage-benchmark-postgresql.json run: | - dotnet tests/Agentstration.Web.Tests/bin/Release/net10.0/Agentstration.Web.Tests.dll --filter "Name=ReportsConcurrentRelationalWriteMetrics" --progress off + dotnet tests/Agentstration.Performance.Tests/bin/Release/net10.0/Agentstration.Performance.Tests.dll --filter "Name=ReportsConcurrentRelationalWriteMetrics" --minimum-expected-tests 1 --progress off cat "$AGENTSTRATION_STORAGE_BENCHMARK_REPORT" + - name: Upload PostgreSQL storage smoke report + if: always() + uses: actions/upload-artifact@v6 + with: + name: storage-benchmark-postgresql-smoke + path: ${{ runner.temp }}/agentstration-storage-benchmark-postgresql.json + if-no-files-found: error container: name: container diff --git a/.github/workflows/release-source-registry-tool.yml b/.github/workflows/release-source-registry-tool.yml index 89682066..daa7a053 100644 --- a/.github/workflows/release-source-registry-tool.yml +++ b/.github/workflows/release-source-registry-tool.yml @@ -87,9 +87,10 @@ jobs: -p:Version=${{ steps.version.outputs.package-version }} - name: Test tool run: >- - dotnet test tests/Agentstration.Tools.SourceRegistry.Tests/Agentstration.Tools.SourceRegistry.Tests.csproj + dotnet test --project tests/Agentstration.Tools.SourceRegistry.Tests/Agentstration.Tools.SourceRegistry.Tests.csproj --configuration Release --no-build + --minimum-expected-tests 1 -p:Version=${{ steps.version.outputs.package-version }} - name: Smoke-test package shell: pwsh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index acc72688..7c683dec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,14 +54,17 @@ jobs: run: dotnet restore Agentstration.slnx -p:NuGetAudit=true -p:NuGetAuditMode=all - name: Build run: dotnet build Agentstration.slnx --configuration Release --no-restore - - name: Test - run: dotnet test Agentstration.slnx --configuration Release --no-build + - name: Test fast lane + run: dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 139 --max-parallel-test-modules 4 + - name: Test integration lane + run: dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 692 --max-parallel-test-modules 2 - name: Restore autonomous AEP SDK run: dotnet restore aep/Aep.slnx -p:NuGetAudit=true -p:NuGetAuditMode=all - name: Build autonomous AEP SDK run: dotnet build aep/Aep.slnx --configuration Release --no-restore - name: Test autonomous AEP SDK - run: dotnet test aep/Aep.slnx --configuration Release --no-build + working-directory: aep + run: dotnet test --solution Aep.slnx --configuration Release --no-build --minimum-expected-tests 33 --max-parallel-test-modules 2 - name: Publish framework-dependent applications shell: pwsh run: | diff --git a/.github/workflows/storage-performance.yml b/.github/workflows/storage-performance.yml new file mode 100644 index 00000000..c509453c --- /dev/null +++ b/.github/workflows/storage-performance.yml @@ -0,0 +1,68 @@ +name: Storage performance + +on: + workflow_dispatch: + schedule: + - cron: "23 3 * * 1" + +permissions: + contents: read + +concurrency: + group: storage-performance-${{ github.ref }} + cancel-in-progress: false + +jobs: + benchmark: + name: ${{ matrix.provider }} full workload + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + provider: [Sqlite, PostgreSql] + services: + postgres: + image: postgres:17 + env: + POSTGRES_DB: agentstration + POSTGRES_USER: agentstration + POSTGRES_PASSWORD: agentstration-ci-only + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U agentstration -d agentstration" + --health-interval 5s + --health-timeout 5s + --health-retries 12 + env: + AGENTSTRATION_TEST_POSTGRES: Host=127.0.0.1;Port=5432;Database=agentstration;Username=agentstration;Password=agentstration-ci-only + AGENTSTRATION_STORAGE_BENCHMARK_PROVIDER: ${{ matrix.provider }} + AGENTSTRATION_STORAGE_BENCHMARK_OPERATIONS: 500 + AGENTSTRATION_STORAGE_BENCHMARK_CONCURRENCY: 16 + AGENTSTRATION_STORAGE_BENCHMARK_REPORT: ${{ runner.temp }}/storage-benchmark-${{ matrix.provider }}.json + steps: + - name: Check out repository + uses: actions/checkout@v7 + - name: Set up .NET + uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + - name: Restore performance tests + run: dotnet restore tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj -p:NuGetAudit=true -p:NuGetAuditMode=all + - name: Build performance tests + run: dotnet build tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj --configuration Release --no-restore + - name: Run full storage workload + run: dotnet tests/Agentstration.Performance.Tests/bin/Release/net10.0/Agentstration.Performance.Tests.dll --filter "Name=ReportsConcurrentRelationalWriteMetrics" --minimum-expected-tests 1 --progress off + - name: Display report + if: always() + shell: pwsh + run: if (Test-Path -LiteralPath $env:AGENTSTRATION_STORAGE_BENCHMARK_REPORT) { Get-Content -LiteralPath $env:AGENTSTRATION_STORAGE_BENCHMARK_REPORT } + - name: Upload storage report + if: always() + uses: actions/upload-artifact@v6 + with: + name: storage-benchmark-${{ matrix.provider }}-full + path: ${{ runner.temp }}/storage-benchmark-${{ matrix.provider }}.json + if-no-files-found: error diff --git a/AGENTS.md b/AGENTS.md index 3a124359..2243ad2b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,7 +48,14 @@ src/ tests/ Agentstration.Application.Tests/ Agentstration.ArchitectureTests/ - Agentstration.Management.Tests/ + Agentstration.Management.Core.Tests/ + Agentstration.Management.Storage.Tests/ + Agentstration.Management.Sources.Tests/ + Agentstration.Management.Api.Tests/ + Agentstration.Management.Bootstrap.Tests/ + Agentstration.Management.Security.Tests/ + Agentstration.Management.Aep.Tests/ + Agentstration.Performance.Tests/ docs/ architecture.md decisions/ @@ -180,10 +187,11 @@ Run from the repository root: ```powershell dotnet restore Agentstration.slnx dotnet build Agentstration.slnx --configuration Release --no-restore -dotnet test Agentstration.slnx --configuration Release --no-build +dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 139 --max-parallel-test-modules 4 +dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 692 --max-parallel-test-modules 2 ``` -For a focused iteration, run the affected test project first, then run the full build and test suite before handoff. Do not suppress warnings or disable analyzers to make a change pass. +The two test solutions together form the required deterministic, offline functional suite. Performance and live-provider workloads are explicit opt-ins documented in `docs/contributing/testing.md`. For a focused iteration, run the affected test project first, then run both functional lanes before handoff. Do not suppress warnings or disable analyzers to make a change pass. To smoke-test the executable default with the Development bootstrap when startup behavior changes: diff --git a/Agentstration.Tests.Fast.slnx b/Agentstration.Tests.Fast.slnx new file mode 100644 index 00000000..cab470cd --- /dev/null +++ b/Agentstration.Tests.Fast.slnx @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/Agentstration.Tests.Integration.slnx b/Agentstration.Tests.Integration.slnx new file mode 100644 index 00000000..c502a14a --- /dev/null +++ b/Agentstration.Tests.Integration.slnx @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/Agentstration.Tests.Performance.slnx b/Agentstration.Tests.Performance.slnx new file mode 100644 index 00000000..e9b6a9af --- /dev/null +++ b/Agentstration.Tests.Performance.slnx @@ -0,0 +1,5 @@ + + + + + diff --git a/Agentstration.slnx b/Agentstration.slnx index 7eea6f3d..b4d70dd5 100644 --- a/Agentstration.slnx +++ b/Agentstration.slnx @@ -55,8 +55,15 @@ - + + + + + + + + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d149d195..32ffdd2a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,11 +22,12 @@ Use MSTest for behavior changes and keep the default test suite deterministic an ```powershell dotnet restore Agentstration.slnx dotnet build Agentstration.slnx --configuration Release --no-restore -dotnet test Agentstration.slnx --configuration Release --no-build +dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 1 +dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 1 ./scripts/ci/verify-dotnet-format.ps1 -BaseRevision "origin/main" ``` -The format script checks changed C# and Razor files in both solutions. The autonomous AEP subtree has a larger standalone solution; when it changes, also restore, build, and test `aep/Aep.slnx`. +The fast and integration lanes together cover the complete required functional suite. Performance and live-provider workloads are explicit opt-ins described in the [test lane guide](docs/contributing/testing.md). The format script checks changed C# and Razor files in both product solutions. The autonomous AEP subtree has a larger standalone solution; when it changes, also restore, build, and test `aep/Aep.slnx`. If documentation changed, run: diff --git a/README.md b/README.md index 72265af3..fc700ee7 100644 --- a/README.md +++ b/README.md @@ -201,10 +201,11 @@ Aspire starts Agentstration's AEP extensions against existing inference servers. ```powershell dotnet build Agentstration.slnx --configuration Release -dotnet test Agentstration.slnx --configuration Release +dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 1 +dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 1 ``` -Warnings are treated as errors, .NET analyzers are enabled and NuGet audit findings fail restore. The default tests are designed to remain offline and cost-free; real-provider smoke tests are opt-in. +Warnings are treated as errors, .NET analyzers are enabled and NuGet audit findings fail restore. The fast and integration lanes together provide complete required functional validation while remaining offline and cost-free; real-provider and performance workloads are opt-in. See the [test lane guide](docs/contributing/testing.md) for project classification and focused commands. ## Documentation diff --git a/aep/README.md b/aep/README.md index ef563b3a..32937974 100644 --- a/aep/README.md +++ b/aep/README.md @@ -16,7 +16,7 @@ AEP is an autonomous, versioned protocol and .NET SDK for discovering, validatin ```powershell dotnet restore Aep.slnx dotnet build Aep.slnx --configuration Release --no-restore -dotnet test Aep.slnx --configuration Release --no-build +dotnet test --solution Aep.slnx --configuration Release --no-build --minimum-expected-tests 1 ``` Run a generic sample and Inspector independently: diff --git a/docs/architecture.md b/docs/architecture.md index 040e62ee..89c109e4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -46,7 +46,14 @@ src/ tests/ Agentstration.Application.Tests/ Agentstration.ArchitectureTests/ - Agentstration.Management.Tests/ + Agentstration.Management.Core.Tests/ + Agentstration.Management.Storage.Tests/ + Agentstration.Management.Sources.Tests/ + Agentstration.Management.Api.Tests/ + Agentstration.Management.Bootstrap.Tests/ + Agentstration.Management.Security.Tests/ + Agentstration.Management.Aep.Tests/ + Agentstration.Performance.Tests/ Agentstration.Web.Tests/ Agentstration.Web.Components.Tests/ Agentstration.Web.FlowDesigner.Tests/ diff --git a/docs/concepts/model-providers.md b/docs/concepts/model-providers.md index 79ef701a..5ce10e70 100644 --- a/docs/concepts/model-providers.md +++ b/docs/concepts/model-providers.md @@ -49,7 +49,7 @@ The default tests use fake HTTP. To run the optional real-server smoke test: $env:AGENTSTRATION_LOCALAI_ENDPOINT = "http://localhost:8081" $env:AGENTSTRATION_LOCALAI_MODEL = "your-chat-model" # Optional: $env:AGENTSTRATION_LOCALAI_API_KEY = "..." -dotnet test tests/Agentstration.ModelProviders.Tests --filter TestCategory=Integration +dotnet test --project tests/Agentstration.ModelProviders.Tests/Agentstration.ModelProviders.Tests.csproj --filter TestCategory=Integration --minimum-expected-tests 1 ``` ## llama.cpp @@ -120,5 +120,5 @@ The default tests use fake HTTP and require no model. To run the optional real-s ```powershell $env:AGENTSTRATION_LLAMA_CPP_ENDPOINT = "http://localhost:8080" $env:AGENTSTRATION_LLAMA_CPP_MODEL = "local-gguf" -dotnet test tests/Agentstration.ModelProviders.Tests --filter TestCategory=Integration +dotnet test --project tests/Agentstration.ModelProviders.Tests/Agentstration.ModelProviders.Tests.csproj --filter TestCategory=Integration --minimum-expected-tests 1 ``` diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md new file mode 100644 index 00000000..e167cb38 --- /dev/null +++ b/docs/contributing/testing.md @@ -0,0 +1,74 @@ +# Test lanes + +Agentstration separates test execution by runtime cost and dependency type. The root `Agentstration.slnx` remains the complete build graph, while the checked-in test solutions provide explicit execution boundaries. + +## Fast lane + +Fast tests do not start an ASP.NET Core host, open a relational database, spawn a child process, or contact a remote provider. They cover architecture rules, component behavior, mapping, validation, and in-process command behavior. + +```powershell +dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 139 --max-parallel-test-modules 4 +``` + +## Integration lane + +Integration tests exercise a real boundary such as `WebApplicationFactory`, SQLite, Git, persistent Identity, or runtime reconstruction. This lane remains deterministic and offline by default. Tests marked `Integration` for a live model provider are opt-in and report inconclusive unless their documented environment variables are supplied. + +```powershell +dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 692 --max-parallel-test-modules 2 +``` + +Run both fast and integration solutions for complete required functional validation. Their union is the functional test inventory represented by the root solution. + +## Performance lane + +Performance workloads live in a dedicated project and are never discovered by the fast or integration lanes: + +```powershell +$env:AGENTSTRATION_STORAGE_BENCHMARK_PROVIDER = "Sqlite" +$env:AGENTSTRATION_STORAGE_BENCHMARK_REPORT = "artifacts/storage-benchmark-sqlite.json" +dotnet test --solution Agentstration.Tests.Performance.slnx --configuration Release --no-build --filter TestCategory=Benchmark --minimum-expected-tests 1 --max-parallel-test-modules 1 +``` + +The report records workload parameters, provider, elapsed time, runtime and OS metadata, throughput, median, p95, errors, conflicts, and retries. SQLite remains the local default; set `AGENTSTRATION_TEST_POSTGRES` for an opt-in PostgreSQL run. Pull requests execute only a bounded contention smoke for relevant storage paths. Full workloads run from the scheduled or manual `Storage performance` workflow and publish their JSON reports. Latency and throughput gates remain disabled until representative baselines have been collected; zero storage errors is always required. + +## CI concurrency and memory diagnostics + +The fast, integration, and performance lanes cap concurrent test modules at 4, 2, and 1 respectively. Host-heavy Management modules also use one class worker per assembly. CI reruns the designated hosted modules sequentially through `scripts/ci/run-test-module-with-diagnostics.ps1`; each JSON artifact contains the discovered count, duration, process peak working set, process peak private memory, aggregate peak working set for active `dotnet` processes, runtime, and OS. The diagnostic artifact deliberately excludes test output and payloads. + +The initial budgets below use Release runs on Windows 11 10.0.26200 with .NET 10.0.10/10.0.11, collected during #298. A warning is evidence to review the Linux and Windows trend; a failure protects constrained runners from returning to the original greater-than-1-GiB process. Adjust these values only after retaining representative artifacts from both runner families. + +| Module | Baseline peak (MiB) | Warning (MiB) | Failure (MiB) | Minimum tests | +| --- | ---: | ---: | ---: | ---: | +| `Agentstration.Management.Api.Tests` | 749.3 | 900 | 1024 | 43 | +| `Agentstration.Management.Bootstrap.Tests` | 355.3 | 500 | 700 | 23 | +| `Agentstration.Management.Security.Tests` | 730.1 | 800 | 1024 | 38 | +| `Agentstration.Management.Aep.Tests` | 322.1 | 450 | 700 | 11 | +| `Agentstration.Management.Sources.Tests` | 431.5 | 550 | 900 | 93 | + +## Project classification + +| Test project | Lane | Boundary or rationale | +| --- | --- | --- | +| `Agentstration.ArchitectureTests` | Fast | Assembly dependency rules | +| `Agentstration.Management.Core.Tests` | Fast | Pure Management validation and in-memory use cases | +| `Agentstration.Tools.SourceRegistry.Tests` | Fast | In-process CLI and manifest validation | +| `Agentstration.Web.Components.Tests` | Fast | bUnit component behavior | +| `Agentstration.Web.FlowDesigner.Tests` | Fast | bUnit and graph projection behavior | +| `Agentstration.Workplace.Components.Tests` | Fast | bUnit component behavior | +| `Agentstration.Application.Tests` | Integration | Includes SQLite Flow and Work storage contracts | +| `Agentstration.Management.Storage.Tests` | Integration | SQLite control-plane, Identity persistence, secrets, audit, and trigger storage | +| `Agentstration.Management.Sources.Tests` | Integration | Source, registry, provider, and Pack distribution boundaries | +| `Agentstration.Management.Api.Tests` | Integration | Hosted Management API tests using the API-only test profile | +| `Agentstration.Management.Bootstrap.Tests` | Integration | Declarative bootstrap catalog, application, and hosted startup scenarios using the API-only test profile | +| `Agentstration.Management.Security.Tests` | Integration | Identity, authorization, local-account, and interactive Security boundaries | +| `Agentstration.Management.Aep.Tests` | Integration | AEP enrollment lifecycle and extension inventory boundaries using the API-only test profile | +| `Agentstration.ModelProviders.Tests` | Integration, provider-optional | AEP test hosts plus opt-in live-provider checks | +| `Agentstration.Runtime.Tests` | Integration | SQLite reconstruction and hosted runtime endpoints | +| `Agentstration.SourceProviders.Git.Tests` | Integration | Real Git processes and file-system repositories | +| `Agentstration.Web.Tests` | Integration | Full Web host plus a bounded deterministic SQLite contention correctness smoke | +| `Agentstration.Performance.Tests` | Performance, opt-in | SQLite/PostgreSQL relational storage concurrency workloads and machine-readable reports | +| `Agentstration.Work.Api.Tests` | Integration | Full Work API host | +| `Agentstration.Workplace.Web.Tests` | Integration | Workplace HTTP host | + +The autonomous AEP SDK keeps its own `aep/Aep.slnx` validation because it can be built and released independently from the product solution. diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index e3e412e7..c203cb67 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -149,18 +149,18 @@ The Workplace depends on the Console API. If the Console is running but the Work ## Storage concurrency benchmark -The opt-in benchmark uses the same workload for both providers: each operation creates and updates a Work Item, appends one Flow event, appends one Runtime event, and stores a Runtime checkpoint. It reports throughput, median and p95 latency, errors, concurrency conflicts, and retries. It has no pass/fail timing threshold and is skipped by the standard test suite. +The opt-in benchmark uses the same workload for both providers: each operation creates and updates a Work Item, appends one Flow event, appends one Runtime event, and stores a Runtime checkpoint. It reports workload and environment metadata, throughput, median and p95 latency, errors, concurrency conflicts, and retries. It has no pass/fail timing threshold and is absent from the functional test lanes. ```powershell $env:AGENTSTRATION_STORAGE_BENCHMARK_PROVIDER = "Sqlite" $env:AGENTSTRATION_STORAGE_BENCHMARK_OPERATIONS = "100" $env:AGENTSTRATION_STORAGE_BENCHMARK_CONCURRENCY = "8" $env:AGENTSTRATION_STORAGE_BENCHMARK_REPORT = "$env:TEMP\agentstration-storage-benchmark.json" -dotnet test tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj --configuration Release --filter "Name=ReportsConcurrentRelationalWriteMetrics" --logger "console;verbosity=detailed" +dotnet test --project tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj --configuration Release --filter "Name=ReportsConcurrentRelationalWriteMetrics" --minimum-expected-tests 1 --max-parallel-test-modules 1 --logger "console;verbosity=detailed" $env:AGENTSTRATION_STORAGE_BENCHMARK_PROVIDER = "PostgreSql" $env:AGENTSTRATION_TEST_POSTGRES = "Host=localhost;Database=agentstration;Username=agentstration;Password=" -dotnet test tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj --configuration Release --filter "Name=ReportsConcurrentRelationalWriteMetrics" --logger "console;verbosity=detailed" +dotnet test --project tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj --configuration Release --filter "Name=ReportsConcurrentRelationalWriteMetrics" --minimum-expected-tests 1 --max-parallel-test-modules 1 --logger "console;verbosity=detailed" ``` ## Backup and restore diff --git a/docs/reference/current-capabilities.md b/docs/reference/current-capabilities.md index efec5eb5..94acc4e1 100644 --- a/docs/reference/current-capabilities.md +++ b/docs/reference/current-capabilities.md @@ -430,7 +430,7 @@ The capture creates a correlated `gen_ai.http.payload_capture` span between the ```powershell dotnet build Agentstration.slnx --configuration Release -dotnet test Agentstration.slnx --configuration Release +dotnet test --solution Agentstration.slnx --configuration Release --minimum-expected-tests 1 ``` Warnings are errors, .NET analyzers are enabled, and NuGet audit findings fail restore. The test suite covers Management, Work, Workplace, Flow, Runtime, Triggers, Packs, Agents, workspace isolation, MCP infrastructure, REST startup, and dependency rules. diff --git a/docs/reference/identity-and-authorization.md b/docs/reference/identity-and-authorization.md index 67d0d820..9d93af24 100644 --- a/docs/reference/identity-and-authorization.md +++ b/docs/reference/identity-and-authorization.md @@ -426,7 +426,7 @@ The validated baseline for this implementation is: ```text dotnet build Agentstration.slnx --configuration Release --no-restore -dotnet test Agentstration.slnx --configuration Release --no-build +dotnet test --solution Agentstration.slnx --configuration Release --no-build --minimum-expected-tests 1 Build: 0 warnings, 0 errors Tests: 461 passed, 0 failed, 2 optional provider integration tests skipped diff --git a/docs/site/sidebars.js b/docs/site/sidebars.js index 30e0f093..26f5f98e 100644 --- a/docs/site/sidebars.js +++ b/docs/site/sidebars.js @@ -100,6 +100,7 @@ const sidebars = { items: [ 'contributing/overview', 'contributing/development-slots', + 'contributing/testing', 'contributing/documentation', 'contributing/github-governance', ], diff --git a/scripts/ci/run-test-module-with-diagnostics.ps1 b/scripts/ci/run-test-module-with-diagnostics.ps1 new file mode 100644 index 00000000..d1dbccae --- /dev/null +++ b/scripts/ci/run-test-module-with-diagnostics.ps1 @@ -0,0 +1,146 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [string] $AssemblyPath, + + [Parameter(Mandatory)] + [string] $ReportPath, + + [Parameter(Mandatory)] + [ValidateRange(1, [int]::MaxValue)] + [int] $MinimumExpectedTests, + + [ValidateRange(0, [int]::MaxValue)] + [int] $WarningWorkingSetMiB = 0, + + [ValidateRange(0, [int]::MaxValue)] + [int] $FailureWorkingSetMiB = 0 +) + +$ErrorActionPreference = 'Stop' +$resolvedAssembly = (Resolve-Path -LiteralPath $AssemblyPath).Path +$resolvedReport = [System.IO.Path]::GetFullPath($ReportPath) +$reportDirectory = Split-Path -Parent $resolvedReport +if (-not [string]::IsNullOrWhiteSpace($reportDirectory)) { + New-Item -ItemType Directory -Force -Path $reportDirectory | Out-Null +} +$trxFileName = ".test-results-$([Guid]::NewGuid().ToString('N')).trx" +$trxPath = Join-Path $reportDirectory $trxFileName + +$startInfo = [System.Diagnostics.ProcessStartInfo]::new('dotnet') +$startInfo.ArgumentList.Add($resolvedAssembly) +$startInfo.ArgumentList.Add('--minimum-expected-tests') +$startInfo.ArgumentList.Add($MinimumExpectedTests.ToString([System.Globalization.CultureInfo]::InvariantCulture)) +$startInfo.ArgumentList.Add('--progress') +$startInfo.ArgumentList.Add('off') +$startInfo.ArgumentList.Add('--report-trx') +$startInfo.ArgumentList.Add('--report-trx-filename') +$startInfo.ArgumentList.Add($trxFileName) +$startInfo.ArgumentList.Add('--results-directory') +$startInfo.ArgumentList.Add($reportDirectory) +$startInfo.UseShellExecute = $false +$startInfo.CreateNoWindow = $true +$startInfo.RedirectStandardOutput = $true +$startInfo.RedirectStandardError = $true + +$process = [System.Diagnostics.Process]::new() +$process.StartInfo = $startInfo +$stopwatch = [System.Diagnostics.Stopwatch]::StartNew() +if (-not $process.Start()) { + throw "Could not start test module '$resolvedAssembly'." +} + +$standardOutput = $process.StandardOutput.ReadToEndAsync() +$standardError = $process.StandardError.ReadToEndAsync() +$peakWorkingSetBytes = 0L +$peakPrivateBytes = 0L +$peakAggregateDotnetWorkingSetBytes = 0L +while (-not $process.HasExited) { + $process.Refresh() + $peakWorkingSetBytes = [Math]::Max($peakWorkingSetBytes, $process.WorkingSet64) + $peakPrivateBytes = [Math]::Max($peakPrivateBytes, $process.PrivateMemorySize64) + $aggregateDotnetWorkingSetBytes = 0L + foreach ($dotnetProcess in [System.Diagnostics.Process]::GetProcessesByName('dotnet')) { + try { + $dotnetProcess.Refresh() + $aggregateDotnetWorkingSetBytes += $dotnetProcess.WorkingSet64 + } + catch [System.InvalidOperationException] { + # A process can exit between enumeration and sampling. + } + finally { + $dotnetProcess.Dispose() + } + } + $peakAggregateDotnetWorkingSetBytes = [Math]::Max($peakAggregateDotnetWorkingSetBytes, $aggregateDotnetWorkingSetBytes) + Start-Sleep -Milliseconds 100 +} +$process.WaitForExit() +$stopwatch.Stop() +$null = $standardOutput.GetAwaiter().GetResult() +$null = $standardError.GetAwaiter().GetResult() + +$testCount = 0 +if (Test-Path -LiteralPath $trxPath) { + try { + [xml] $trx = Get-Content -LiteralPath $trxPath -Raw + $counters = $trx.SelectSingleNode("//*[local-name()='Counters']") + if ($null -ne $counters) { + $testCount = [int]$counters.GetAttribute('total') + } + } + finally { + [System.IO.File]::Delete($trxPath) + } +} +$workingSetMiB = [Math]::Round($peakWorkingSetBytes / 1MB, 1) +$privateMiB = [Math]::Round($peakPrivateBytes / 1MB, 1) +$aggregateDotnetWorkingSetMiB = [Math]::Round($peakAggregateDotnetWorkingSetBytes / 1MB, 1) +$moduleName = [System.IO.Path]::GetFileNameWithoutExtension($resolvedAssembly) +$status = if ($process.ExitCode -ne 0 -or + $testCount -lt $MinimumExpectedTests -or + ($FailureWorkingSetMiB -gt 0 -and $workingSetMiB -ge $FailureWorkingSetMiB)) { + 'failed' +} +elseif ($WarningWorkingSetMiB -gt 0 -and $workingSetMiB -ge $WarningWorkingSetMiB) { + 'warning' +} +else { + 'passed' +} + +[ordered]@{ + module = $moduleName + status = $status + testCount = $testCount + minimumExpectedTests = $MinimumExpectedTests + durationSeconds = [Math]::Round($stopwatch.Elapsed.TotalSeconds, 3) + peakWorkingSetMiB = $workingSetMiB + peakPrivateMemoryMiB = $privateMiB + peakAggregateDotnetWorkingSetMiB = $aggregateDotnetWorkingSetMiB + warningWorkingSetMiB = $WarningWorkingSetMiB + failureWorkingSetMiB = $FailureWorkingSetMiB + exitCode = $process.ExitCode + runtime = [System.Runtime.InteropServices.RuntimeInformation]::FrameworkDescription + os = [System.Runtime.InteropServices.RuntimeInformation]::OSDescription +} | ConvertTo-Json | Set-Content -LiteralPath $resolvedReport -Encoding utf8 + +$summary = "${moduleName}: $testCount tests, $([Math]::Round($stopwatch.Elapsed.TotalSeconds, 1)) s, $workingSetMiB MiB working set, $privateMiB MiB private, $aggregateDotnetWorkingSetMiB MiB aggregate dotnet, status $status" +Write-Host $summary +if (-not [string]::IsNullOrWhiteSpace($env:GITHUB_STEP_SUMMARY)) { + "| $moduleName | $testCount | $([Math]::Round($stopwatch.Elapsed.TotalSeconds, 1)) | $workingSetMiB | $privateMiB | $aggregateDotnetWorkingSetMiB | $status |" | + Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Encoding utf8 +} + +if ($process.ExitCode -ne 0) { + throw "Test module '$moduleName' exited with code $($process.ExitCode). Rerun it directly for failure details." +} +if ($testCount -lt $MinimumExpectedTests) { + throw "Test module '$moduleName' discovered $testCount tests; expected at least $MinimumExpectedTests." +} +if ($FailureWorkingSetMiB -gt 0 -and $workingSetMiB -ge $FailureWorkingSetMiB) { + throw "Test module '$moduleName' peaked at $workingSetMiB MiB working set; failure budget is $FailureWorkingSetMiB MiB." +} +if ($status -eq 'warning') { + Write-Warning "Test module '$moduleName' peaked at $workingSetMiB MiB working set; warning budget is $WarningWorkingSetMiB MiB." +} diff --git a/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj b/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj index 14c23eeb..51ee7809 100644 --- a/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj +++ b/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj @@ -20,7 +20,11 @@ - + + + + + diff --git a/src/Agentstration.Web/Program.cs b/src/Agentstration.Web/Program.cs index 06819ce6..7ea5af8b 100644 --- a/src/Agentstration.Web/Program.cs +++ b/src/Agentstration.Web/Program.cs @@ -45,6 +45,8 @@ toolExecutionCapture.Validate(); builder.Services.AddSingleton(toolExecutionCapture); var isTesting = builder.Environment.IsEnvironment("Testing"); +var apiOnlyTesting = isTesting + && builder.Configuration.GetValue("Agentstration:Testing:ApiOnly", false); var hostedServicesEnabled = !isTesting || builder.Configuration.GetValue("Agentstration:Testing:HostedServicesEnabled", false); var openTelemetryEnabled = !isTesting @@ -140,8 +142,11 @@ await context.HttpContext.Response.WriteAsJsonAsync( })); }); builder.Services.AddAgentstrationOpenApi(); -builder.Services.AddRazorPages(); -builder.Services.AddRazorComponents().AddInteractiveServerComponents(); +if (!apiOnlyTesting) +{ + builder.Services.AddRazorPages(); + builder.Services.AddRazorComponents().AddInteractiveServerComponents(); +} builder.Services.AddAgentstrationLocalization(builder.Configuration); builder.Services.AddSignalR(); if (storageProvider == AgentstrationStorageProvider.PostgreSql) @@ -261,7 +266,7 @@ await context.HttpContext.Response.WriteAsJsonAsync( app.UseMiddleware(); app.UseAuthorization(); if (app.Environment.IsDevelopment() || app.Environment.IsEnvironment("Testing")) app.MapAgentstrationOpenApi(); -app.UseAntiforgery(); +if (!apiOnlyTesting) app.UseAntiforgery(); app.MapGet("/health", () => Results.Ok(new { status = "healthy" })).AllowAnonymous(); app.MapGet("/health/ready", (IAgentstrationStorageInitializer storage) => storage.IsReady ? Results.Ok(new { status = "ready" }) @@ -284,10 +289,13 @@ await context.HttpContext.Response.WriteAsJsonAsync( app.MapHub("/hubs/workplace").RequireAuthorization(Agentstration.Web.Security.AgentstrationPolicies.CanReadRuns); if (app.Environment.IsDevelopment()) app.MapOllamaDiagnostics(); app.MapMcp("/mcp").RequireAuthorization(Agentstration.Web.Security.AgentstrationPolicies.CanExecuteRuns); -app.MapStaticAssets().AllowAnonymous(); -app.MapRazorPages(); -app.MapRazorComponents().AddAdditionalAssemblies(typeof(MainLayout).Assembly).AddInteractiveServerRenderMode() - .RequireAuthorization(Agentstration.Web.Security.AgentstrationPolicies.Authenticated); +if (!apiOnlyTesting) +{ + app.MapStaticAssets().AllowAnonymous(); + app.MapRazorPages(); + app.MapRazorComponents().AddAdditionalAssemblies(typeof(MainLayout).Assembly).AddInteractiveServerRenderMode() + .RequireAuthorization(Agentstration.Web.Security.AgentstrationPolicies.Authenticated); +} try { RequestContext? bootstrapContext = null; diff --git a/tests/Agentstration.Application.Tests/FlowStoragePerformanceTests.cs b/tests/Agentstration.Application.Tests/FlowStorageQueryContractTests.cs similarity index 100% rename from tests/Agentstration.Application.Tests/FlowStoragePerformanceTests.cs rename to tests/Agentstration.Application.Tests/FlowStorageQueryContractTests.cs diff --git a/tests/Agentstration.Management.Tests/AepEnrollmentTests.cs b/tests/Agentstration.Management.Aep.Tests/AepEnrollmentTests.cs similarity index 92% rename from tests/Agentstration.Management.Tests/AepEnrollmentTests.cs rename to tests/Agentstration.Management.Aep.Tests/AepEnrollmentTests.cs index f2c32bac..1a53dde6 100644 --- a/tests/Agentstration.Management.Tests/AepEnrollmentTests.cs +++ b/tests/Agentstration.Management.Aep.Tests/AepEnrollmentTests.cs @@ -4,6 +4,7 @@ using Agentstration.Aep.Abstractions; using Agentstration.Management.Abstractions; using Agentstration.Management.Core; +using Agentstration.ModelProviders; using Agentstration.Resources; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; @@ -12,14 +13,56 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class AepEnrollmentTests { - private static WebApplicationFactory EnrollmentFactory() => Factory().WithWebHostBuilder(builder => - builder.ConfigureServices(services => + private static WebApplicationFactory EnrollmentFactory() => new AepEnrollmentTestFactory(); + + private static Task GetBootstrapContextAsync(WebApplicationFactory factory) => + factory.Services + .GetRequiredService() + .EnsureInitializedAsync(default); + + private sealed class AepEnrollmentTestFactory : WebApplicationFactory + { + protected override void ConfigureWebHost(IWebHostBuilder builder) { - services.RemoveAll(); - services.AddSingleton(); - })); + builder.UseEnvironment("Testing"); + builder.UseSetting("Agentstration:Testing:ApiOnly", "true"); + builder.UseSetting("Logging:LogLevel:Default", "Warning"); + builder.ConfigureServices(services => + { + services.RemoveAll(); + services.RemoveAll(); + services.AddSingleton(); + services.AddSingleton(); + }); + } + } + + private sealed class UnavailableExtensionInspector : IExtensionInspector + { + public bool CanHandle(string providerType) => true; + public bool CanInspectEndpoint(Uri endpoint) => true; + + public ValueTask InspectAsync( + ModelProviderConfiguration provider, + CancellationToken cancellationToken = default) => + InspectAsync(provider.Name, provider.Endpoint, cancellationToken); + + public ValueTask InspectAsync( + string registrationName, + Uri endpoint, + CancellationToken cancellationToken = default) => + ValueTask.FromResult(new ExtensionInspection( + registrationName, + endpoint, + "unavailable", + null, + [], + [], + "The test extension is intentionally unavailable.")); + } [TestMethod] public async Task PairingCodeCanBeDisabledAndConfigurationCanLockTheMode() diff --git a/tests/Agentstration.Management.Tests/Agentstration.Management.Tests.csproj b/tests/Agentstration.Management.Aep.Tests/Agentstration.Management.Aep.Tests.csproj similarity index 100% rename from tests/Agentstration.Management.Tests/Agentstration.Management.Tests.csproj rename to tests/Agentstration.Management.Aep.Tests/Agentstration.Management.Aep.Tests.csproj diff --git a/tests/Agentstration.Management.Aep.Tests/AssemblyInfo.cs b/tests/Agentstration.Management.Aep.Tests/AssemblyInfo.cs new file mode 100644 index 00000000..c53e6b4a --- /dev/null +++ b/tests/Agentstration.Management.Aep.Tests/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: Parallelize(Workers = 1, Scope = ExecutionScope.ClassLevel)] diff --git a/tests/Agentstration.Management.Api.Tests/Agentstration.Management.Api.Tests.csproj b/tests/Agentstration.Management.Api.Tests/Agentstration.Management.Api.Tests.csproj new file mode 100644 index 00000000..29044df4 --- /dev/null +++ b/tests/Agentstration.Management.Api.Tests/Agentstration.Management.Api.Tests.csproj @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/tests/Agentstration.Management.Api.Tests/AssemblyInfo.cs b/tests/Agentstration.Management.Api.Tests/AssemblyInfo.cs new file mode 100644 index 00000000..c53e6b4a --- /dev/null +++ b/tests/Agentstration.Management.Api.Tests/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: Parallelize(Workers = 1, Scope = ExecutionScope.ClassLevel)] diff --git a/tests/Agentstration.Management.Tests/ModelManagementApiTests.Extensions.cs b/tests/Agentstration.Management.Api.Tests/ExtensionRegistrationApiTests.cs similarity index 96% rename from tests/Agentstration.Management.Tests/ModelManagementApiTests.Extensions.cs rename to tests/Agentstration.Management.Api.Tests/ExtensionRegistrationApiTests.cs index 9aee9b52..ddbbfc59 100644 --- a/tests/Agentstration.Management.Tests/ModelManagementApiTests.Extensions.cs +++ b/tests/Agentstration.Management.Api.Tests/ExtensionRegistrationApiTests.cs @@ -15,12 +15,13 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class ExtensionRegistrationApiTests : ModelManagementApiTestBase { [TestMethod] public async Task SeededOllamaProviderUsesAepExtensionEndpointInsteadOfNativeOllamaEndpoint() { - await using var factory = new WebApplicationFactory().WithWebHostBuilder(builder => + await using var factory = new ApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("AI:Provider", "Managed"); @@ -42,7 +43,7 @@ public async Task SeededOllamaProviderUsesAepExtensionEndpointInsteadOfNativeOll [TestMethod] public async Task SeededLlamaCppProviderUsesItsAepExtensionEndpoint() { - await using var factory = new WebApplicationFactory().WithWebHostBuilder(builder => + await using var factory = new ApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("AI:Provider", "Managed"); @@ -64,7 +65,7 @@ public async Task SeededLlamaCppProviderUsesItsAepExtensionEndpoint() [TestMethod] public async Task SeededLocalAiProviderUsesItsAepExtensionEndpoint() { - await using var factory = new WebApplicationFactory().WithWebHostBuilder(builder => + await using var factory = new ApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("AI:Provider", "Managed"); @@ -85,7 +86,7 @@ public async Task SeededLocalAiProviderUsesItsAepExtensionEndpoint() [TestMethod] public async Task GitSourceExtensionConfigurationUsesTheAspireRegistrationIdentity() { - await using var factory = new WebApplicationFactory().WithWebHostBuilder(builder => + await using var factory = new ApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("Agentstration:Extensions:Agentstration.Extensions.Git:Endpoint", "http://localhost:5295"); @@ -143,7 +144,7 @@ public async Task ExtensionsApiDoesNotAcceptManualDiscoveryCommand() using var client = factory.CreateClient(); using var discoveryResponse = await client.PostAsync("/api/extensions/discover", null); - Assert.AreEqual(HttpStatusCode.MethodNotAllowed, discoveryResponse.StatusCode); + Assert.AreEqual(HttpStatusCode.NotFound, discoveryResponse.StatusCode); } [TestMethod] diff --git a/tests/Agentstration.Management.Tests/ModelManagementApiTests.cs b/tests/Agentstration.Management.Api.Tests/ModelManagementApiTestBase.cs similarity index 89% rename from tests/Agentstration.Management.Tests/ModelManagementApiTests.cs rename to tests/Agentstration.Management.Api.Tests/ModelManagementApiTestBase.cs index 437ecf99..e352d782 100644 --- a/tests/Agentstration.Management.Tests/ModelManagementApiTests.cs +++ b/tests/Agentstration.Management.Api.Tests/ModelManagementApiTestBase.cs @@ -15,28 +15,11 @@ namespace Agentstration.Management.Tests; -[TestClass] -public sealed partial class ModelManagementApiTests +public abstract class ModelManagementApiTestBase { - private static WebApplicationFactory Factory() => - new WebApplicationFactory().WithWebHostBuilder(builder => - { - builder.UseEnvironment("Testing"); - builder.UseSetting("ConnectionStrings:ollama-extension", "Endpoint=http://127.0.0.1:1"); - builder.UseSetting("Logging:LogLevel:Default", "Warning"); - builder.ConfigureServices(services => - { - services.RemoveAll(); - services.RemoveAll(); - services.RemoveAll(); - services.AddSingleton(); - services.AddSingleton(provider => provider.GetRequiredService()); - services.AddSingleton(provider => provider.GetRequiredService()); - services.AddSingleton(provider => provider.GetRequiredService()); - }); - }); + protected static WebApplicationFactory Factory() => new ManagementApiTestFactory(); - private static WebApplicationFactory DiagnosticFactory() => Factory().WithWebHostBuilder(builder => + protected static WebApplicationFactory DiagnosticFactory() => Factory().WithWebHostBuilder(builder => builder.ConfigureServices(services => { services.RemoveAll(); @@ -45,7 +28,7 @@ private static WebApplicationFactory DiagnosticFactory() => Factory().W services.AddSingleton(); })); - private static Task GetBootstrapContextAsync(WebApplicationFactory factory) => + protected static Task GetBootstrapContextAsync(WebApplicationFactory factory) => factory.Services .GetRequiredService() .EnsureInitializedAsync(default); @@ -123,7 +106,7 @@ public ValueTask InspectAsync( UnavailableDetails)); } - private sealed class ConfiguredEndpointInspector : IExtensionInspector + protected sealed class ConfiguredEndpointInspector : IExtensionInspector { public bool CanHandle(string providerType) => true; public bool CanInspectEndpoint(Uri endpoint) => true; @@ -144,7 +127,7 @@ [new ExtensionContribution("model-provider", "discovered")], [])); } - private sealed class MigrationExtensionAdapter : IExtensionInspector, IExtensionOptionsMigrator + protected sealed class MigrationExtensionAdapter : IExtensionInspector, IExtensionOptionsMigrator { private static readonly JsonElement SourceSchema = JsonSerializer.SerializeToElement(new { @@ -202,7 +185,7 @@ public ValueTask MigrateAsync( } } - private static CreateModelProfileRequest Request(string name, string model) => new( + protected static CreateModelProfileRequest Request(string name, string model) => new( name, new ModelProfileProperties { @@ -212,4 +195,33 @@ public ValueTask MigrateAsync( Model = new ModelSelection { Name = model }, Generation = new ModelGenerationOptions { Temperature = 0.3, MaxOutputTokens = 512 } }); + + private sealed class ManagementApiTestFactory : ApiOnlyWebApplicationFactory + { + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + base.ConfigureWebHost(builder); + builder.UseSetting("ConnectionStrings:ollama-extension", "Endpoint=http://127.0.0.1:1"); + builder.ConfigureServices(services => + { + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.AddSingleton(); + services.AddSingleton(provider => provider.GetRequiredService()); + services.AddSingleton(provider => provider.GetRequiredService()); + services.AddSingleton(provider => provider.GetRequiredService()); + }); + } + } +} + +internal class ApiOnlyWebApplicationFactory : WebApplicationFactory +{ + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment("Testing"); + builder.UseSetting("Agentstration:Testing:ApiOnly", "true"); + builder.UseSetting("Logging:LogLevel:Default", "Warning"); + } } diff --git a/tests/Agentstration.Management.Tests/ModelManagementApiTests.ProfilesAndResolution.cs b/tests/Agentstration.Management.Api.Tests/ModelProfileApiTests.cs similarity index 99% rename from tests/Agentstration.Management.Tests/ModelManagementApiTests.ProfilesAndResolution.cs rename to tests/Agentstration.Management.Api.Tests/ModelProfileApiTests.cs index 3a19b3b8..b4fc16a6 100644 --- a/tests/Agentstration.Management.Tests/ModelManagementApiTests.ProfilesAndResolution.cs +++ b/tests/Agentstration.Management.Api.Tests/ModelProfileApiTests.cs @@ -15,12 +15,13 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class ModelProfileApiTests : ModelManagementApiTestBase { [TestMethod] public async Task ManagedHostCompositionUsesPersistedModelProfileResolver() { - await using var factory = new WebApplicationFactory().WithWebHostBuilder(builder => + await using var factory = new ApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("AI:Provider", "Managed"); diff --git a/tests/Agentstration.Management.Tests/ModelManagementApiTests.Providers.cs b/tests/Agentstration.Management.Api.Tests/ModelProviderApiTests.cs similarity index 97% rename from tests/Agentstration.Management.Tests/ModelManagementApiTests.Providers.cs rename to tests/Agentstration.Management.Api.Tests/ModelProviderApiTests.cs index 35fa7617..412b1a02 100644 --- a/tests/Agentstration.Management.Tests/ModelManagementApiTests.Providers.cs +++ b/tests/Agentstration.Management.Api.Tests/ModelProviderApiTests.cs @@ -15,7 +15,8 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class ModelProviderApiTests : ModelManagementApiTestBase { [TestMethod] public async Task ReadOnlyProviderApisExposeConfiguredProviderAndUnavailableDiscovery() diff --git a/tests/Agentstration.Management.Tests/NotificationDeliveryTests.cs b/tests/Agentstration.Management.Api.Tests/NotificationDeliveryApiTests.cs similarity index 99% rename from tests/Agentstration.Management.Tests/NotificationDeliveryTests.cs rename to tests/Agentstration.Management.Api.Tests/NotificationDeliveryApiTests.cs index 7d632fad..fd4e5585 100644 --- a/tests/Agentstration.Management.Tests/NotificationDeliveryTests.cs +++ b/tests/Agentstration.Management.Api.Tests/NotificationDeliveryApiTests.cs @@ -16,7 +16,8 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class NotificationDeliveryApiTests : ModelManagementApiTestBase { [TestMethod] public void NotificationDeliverySamplesUseOnlyGenericFlowAndToolSteps() diff --git a/tests/Agentstration.Management.Tests/ModelManagementApiTests.ResourceScopes.cs b/tests/Agentstration.Management.Api.Tests/ResourceScopeApiTests.cs similarity index 98% rename from tests/Agentstration.Management.Tests/ModelManagementApiTests.ResourceScopes.cs rename to tests/Agentstration.Management.Api.Tests/ResourceScopeApiTests.cs index 301bbbe2..58b260cf 100644 --- a/tests/Agentstration.Management.Tests/ModelManagementApiTests.ResourceScopes.cs +++ b/tests/Agentstration.Management.Api.Tests/ResourceScopeApiTests.cs @@ -7,7 +7,8 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class ResourceScopeApiTests : ModelManagementApiTestBase { [TestMethod] public void ResourceScopePolicyMatchesTheInitialOwnershipModel() diff --git a/tests/Agentstration.Management.Tests/ModelManagementApiTests.RuntimeAndTools.cs b/tests/Agentstration.Management.Api.Tests/RuntimeProfileAndToolApiTests.cs similarity index 99% rename from tests/Agentstration.Management.Tests/ModelManagementApiTests.RuntimeAndTools.cs rename to tests/Agentstration.Management.Api.Tests/RuntimeProfileAndToolApiTests.cs index 41ce8524..daf26186 100644 --- a/tests/Agentstration.Management.Tests/ModelManagementApiTests.RuntimeAndTools.cs +++ b/tests/Agentstration.Management.Api.Tests/RuntimeProfileAndToolApiTests.cs @@ -15,7 +15,8 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class RuntimeProfileAndToolApiTests : ModelManagementApiTestBase { [TestMethod] public async Task RuntimeProfileIsPersistedAsAnIndependentManagementResource() diff --git a/tests/Agentstration.Management.Tests/ToolDefinitionTests.cs b/tests/Agentstration.Management.Api.Tests/ToolDefinitionApiTests.cs similarity index 99% rename from tests/Agentstration.Management.Tests/ToolDefinitionTests.cs rename to tests/Agentstration.Management.Api.Tests/ToolDefinitionApiTests.cs index 5b5846b6..a458f72f 100644 --- a/tests/Agentstration.Management.Tests/ToolDefinitionTests.cs +++ b/tests/Agentstration.Management.Api.Tests/ToolDefinitionApiTests.cs @@ -15,7 +15,8 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class ToolDefinitionApiTests : ModelManagementApiTestBase { [TestMethod] public async Task ToolDefinitionCrudMaterializesInternalProviderAndGovernedTool() diff --git a/tests/Agentstration.Management.Bootstrap.Tests/Agentstration.Management.Bootstrap.Tests.csproj b/tests/Agentstration.Management.Bootstrap.Tests/Agentstration.Management.Bootstrap.Tests.csproj new file mode 100644 index 00000000..29044df4 --- /dev/null +++ b/tests/Agentstration.Management.Bootstrap.Tests/Agentstration.Management.Bootstrap.Tests.csproj @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/tests/Agentstration.Management.Bootstrap.Tests/AssemblyInfo.cs b/tests/Agentstration.Management.Bootstrap.Tests/AssemblyInfo.cs new file mode 100644 index 00000000..c53e6b4a --- /dev/null +++ b/tests/Agentstration.Management.Bootstrap.Tests/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: Parallelize(Workers = 1, Scope = ExecutionScope.ClassLevel)] diff --git a/tests/Agentstration.Management.Tests/DeclarativeBootstrapTests.cs b/tests/Agentstration.Management.Bootstrap.Tests/DeclarativeBootstrapTests.cs similarity index 99% rename from tests/Agentstration.Management.Tests/DeclarativeBootstrapTests.cs rename to tests/Agentstration.Management.Bootstrap.Tests/DeclarativeBootstrapTests.cs index d81321bd..4c20b000 100644 --- a/tests/Agentstration.Management.Tests/DeclarativeBootstrapTests.cs +++ b/tests/Agentstration.Management.Bootstrap.Tests/DeclarativeBootstrapTests.cs @@ -777,7 +777,7 @@ private static WebApplicationFactory Factory( bool configureOllamaExtension = false) { EnsureInstanceProfileDescriptor(path); - return new WebApplicationFactory().WithWebHostBuilder(builder => + return new BootstrapApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("Agentstration:Authentication:Mode", "Local"); @@ -794,7 +794,7 @@ private static WebApplicationFactory Factory( private static WebApplicationFactory FactoryWithExtensionPresenceHandler(string path, bool discoverOnStartup) { EnsureInstanceProfileDescriptor(path); - return new WebApplicationFactory().WithWebHostBuilder(builder => + return new BootstrapApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("Agentstration:Authentication:Mode", "Development"); @@ -1163,3 +1163,13 @@ public TemporaryDirectory() public void Dispose() => Directory.Delete(Path, recursive: true); } } + +internal sealed class BootstrapApiOnlyWebApplicationFactory : WebApplicationFactory +{ + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment("Testing"); + builder.UseSetting("Agentstration:Testing:ApiOnly", "true"); + builder.UseSetting("Logging:LogLevel:Default", "Warning"); + } +} diff --git a/tests/Agentstration.Management.Core.Tests/Agentstration.Management.Core.Tests.csproj b/tests/Agentstration.Management.Core.Tests/Agentstration.Management.Core.Tests.csproj new file mode 100644 index 00000000..3d428411 --- /dev/null +++ b/tests/Agentstration.Management.Core.Tests/Agentstration.Management.Core.Tests.csproj @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/tests/Agentstration.Management.Tests/ToolDiscoveryTests.cs b/tests/Agentstration.Management.Core.Tests/ToolDiscoveryTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/ToolDiscoveryTests.cs rename to tests/Agentstration.Management.Core.Tests/ToolDiscoveryTests.cs diff --git a/tests/Agentstration.Management.Tests/ToolExecutionHookManagementTests.cs b/tests/Agentstration.Management.Core.Tests/ToolExecutionHookManagementTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/ToolExecutionHookManagementTests.cs rename to tests/Agentstration.Management.Core.Tests/ToolExecutionHookManagementTests.cs diff --git a/tests/Agentstration.Management.Tests/ToolManagementTests.cs b/tests/Agentstration.Management.Core.Tests/ToolManagementTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/ToolManagementTests.cs rename to tests/Agentstration.Management.Core.Tests/ToolManagementTests.cs diff --git a/tests/Agentstration.Management.Security.Tests/Agentstration.Management.Security.Tests.csproj b/tests/Agentstration.Management.Security.Tests/Agentstration.Management.Security.Tests.csproj new file mode 100644 index 00000000..29044df4 --- /dev/null +++ b/tests/Agentstration.Management.Security.Tests/Agentstration.Management.Security.Tests.csproj @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/tests/Agentstration.Management.Security.Tests/AssemblyInfo.cs b/tests/Agentstration.Management.Security.Tests/AssemblyInfo.cs new file mode 100644 index 00000000..c53e6b4a --- /dev/null +++ b/tests/Agentstration.Management.Security.Tests/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: Parallelize(Workers = 1, Scope = ExecutionScope.ClassLevel)] diff --git a/tests/Agentstration.Management.Tests/IdentityFoundationTests.cs b/tests/Agentstration.Management.Security.Tests/IdentityFoundationTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/IdentityFoundationTests.cs rename to tests/Agentstration.Management.Security.Tests/IdentityFoundationTests.cs diff --git a/tests/Agentstration.Management.Tests/SecurityApiTests.Authorization.cs b/tests/Agentstration.Management.Security.Tests/SecurityApiTests.Authorization.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SecurityApiTests.Authorization.cs rename to tests/Agentstration.Management.Security.Tests/SecurityApiTests.Authorization.cs diff --git a/tests/Agentstration.Management.Tests/SecurityApiTests.BoundariesAndTokens.cs b/tests/Agentstration.Management.Security.Tests/SecurityApiTests.BoundariesAndTokens.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SecurityApiTests.BoundariesAndTokens.cs rename to tests/Agentstration.Management.Security.Tests/SecurityApiTests.BoundariesAndTokens.cs diff --git a/tests/Agentstration.Management.Tests/SecurityApiTests.LocalAccounts.cs b/tests/Agentstration.Management.Security.Tests/SecurityApiTests.LocalAccounts.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SecurityApiTests.LocalAccounts.cs rename to tests/Agentstration.Management.Security.Tests/SecurityApiTests.LocalAccounts.cs diff --git a/tests/Agentstration.Management.Tests/SecurityApiTests.PlatformAndMembership.cs b/tests/Agentstration.Management.Security.Tests/SecurityApiTests.PlatformAndMembership.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SecurityApiTests.PlatformAndMembership.cs rename to tests/Agentstration.Management.Security.Tests/SecurityApiTests.PlatformAndMembership.cs diff --git a/tests/Agentstration.Management.Tests/SecurityApiTests.cs b/tests/Agentstration.Management.Security.Tests/SecurityApiTests.cs similarity index 94% rename from tests/Agentstration.Management.Tests/SecurityApiTests.cs rename to tests/Agentstration.Management.Security.Tests/SecurityApiTests.cs index 9cd97025..4320ddfc 100644 --- a/tests/Agentstration.Management.Tests/SecurityApiTests.cs +++ b/tests/Agentstration.Management.Security.Tests/SecurityApiTests.cs @@ -23,16 +23,18 @@ public sealed partial class SecurityApiTests private const string LocalPassword = "A-strong-local-password-42!"; private const string ChangedLocalPassword = "A-changed-local-password-84!"; - private static WebApplicationFactory Factory(string mode) + private static WebApplicationFactory Factory(string mode) => new SecurityWebApplicationFactory(mode); + + private sealed class SecurityWebApplicationFactory(string mode) : WebApplicationFactory { - WebApplicationFactory? factory = null; - factory = new WebApplicationFactory().WithWebHostBuilder(builder => + protected override void ConfigureWebHost(IWebHostBuilder builder) { builder.UseEnvironment("Testing"); builder.UseSetting("Agentstration:Authentication:Mode", mode); + builder.UseSetting("Logging:LogLevel:Default", "Warning"); builder.ConfigureTestServices(services => { - Func handlerFactory = () => factory!.Server.CreateHandler(); + Func handlerFactory = () => Server.CreateHandler(); RouteThroughTestServer(services, handlerFactory); RouteThroughTestServer(services, handlerFactory); RouteThroughTestServer(services, handlerFactory); @@ -40,8 +42,7 @@ private static WebApplicationFactory Factory(string mode) RouteThroughTestServer(services, handlerFactory); RouteThroughTestServer(services, handlerFactory); }); - }); - return factory; + } } private static void RouteThroughTestServer( diff --git a/tests/Agentstration.Management.Sources.Tests/Agentstration.Management.Sources.Tests.csproj b/tests/Agentstration.Management.Sources.Tests/Agentstration.Management.Sources.Tests.csproj new file mode 100644 index 00000000..29044df4 --- /dev/null +++ b/tests/Agentstration.Management.Sources.Tests/Agentstration.Management.Sources.Tests.csproj @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/tests/Agentstration.Management.Tests/PackCompositionTests.cs b/tests/Agentstration.Management.Sources.Tests/PackCompositionTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/PackCompositionTests.cs rename to tests/Agentstration.Management.Sources.Tests/PackCompositionTests.cs diff --git a/tests/Agentstration.Management.Tests/PackTests.cs b/tests/Agentstration.Management.Sources.Tests/PackTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/PackTests.cs rename to tests/Agentstration.Management.Sources.Tests/PackTests.cs diff --git a/tests/Agentstration.Management.Tests/SourceProviderManagementTests.cs b/tests/Agentstration.Management.Sources.Tests/SourceProviderManagementTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SourceProviderManagementTests.cs rename to tests/Agentstration.Management.Sources.Tests/SourceProviderManagementTests.cs diff --git a/tests/Agentstration.Management.Tests/SourceRegistryInfrastructureTests.cs b/tests/Agentstration.Management.Sources.Tests/SourceRegistryInfrastructureTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SourceRegistryInfrastructureTests.cs rename to tests/Agentstration.Management.Sources.Tests/SourceRegistryInfrastructureTests.cs diff --git a/tests/Agentstration.Management.Tests/SourceRegistryManagementTests.cs b/tests/Agentstration.Management.Sources.Tests/SourceRegistryManagementTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SourceRegistryManagementTests.cs rename to tests/Agentstration.Management.Sources.Tests/SourceRegistryManagementTests.cs diff --git a/tests/Agentstration.Management.Tests/SourceTests.cs b/tests/Agentstration.Management.Sources.Tests/SourceTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SourceTests.cs rename to tests/Agentstration.Management.Sources.Tests/SourceTests.cs diff --git a/tests/Agentstration.Management.Storage.Tests/Agentstration.Management.Storage.Tests.csproj b/tests/Agentstration.Management.Storage.Tests/Agentstration.Management.Storage.Tests.csproj new file mode 100644 index 00000000..4e2edfbb --- /dev/null +++ b/tests/Agentstration.Management.Storage.Tests/Agentstration.Management.Storage.Tests.csproj @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/tests/Agentstration.Management.Tests/ControlPlaneStoreHardeningTests.cs b/tests/Agentstration.Management.Storage.Tests/ControlPlaneStoreHardeningTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/ControlPlaneStoreHardeningTests.cs rename to tests/Agentstration.Management.Storage.Tests/ControlPlaneStoreHardeningTests.cs diff --git a/tests/Agentstration.Management.Tests/LocalIdentityPersistenceTests.cs b/tests/Agentstration.Management.Storage.Tests/LocalIdentityPersistenceTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/LocalIdentityPersistenceTests.cs rename to tests/Agentstration.Management.Storage.Tests/LocalIdentityPersistenceTests.cs diff --git a/tests/Agentstration.Management.Tests/LocalSecretVaultTests.cs b/tests/Agentstration.Management.Storage.Tests/LocalSecretVaultTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/LocalSecretVaultTests.cs rename to tests/Agentstration.Management.Storage.Tests/LocalSecretVaultTests.cs diff --git a/tests/Agentstration.Management.Tests/ManagementPlaneTests.cs b/tests/Agentstration.Management.Storage.Tests/ManagementPlaneTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/ManagementPlaneTests.cs rename to tests/Agentstration.Management.Storage.Tests/ManagementPlaneTests.cs diff --git a/tests/Agentstration.Management.Tests/SecurityAuditPersistenceTests.cs b/tests/Agentstration.Management.Storage.Tests/SecurityAuditPersistenceTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SecurityAuditPersistenceTests.cs rename to tests/Agentstration.Management.Storage.Tests/SecurityAuditPersistenceTests.cs diff --git a/tests/Agentstration.Management.Tests/TriggerTests.cs b/tests/Agentstration.Management.Storage.Tests/TriggerTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/TriggerTests.cs rename to tests/Agentstration.Management.Storage.Tests/TriggerTests.cs diff --git a/tests/Agentstration.Management.Tests/AssemblyInfo.cs b/tests/Agentstration.Management.Tests/AssemblyInfo.cs deleted file mode 100644 index 9dc8a785..00000000 --- a/tests/Agentstration.Management.Tests/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; - -[assembly: Parallelize(Workers = 4, Scope = ExecutionScope.ClassLevel)] diff --git a/tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj b/tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj new file mode 100644 index 00000000..d51864ee --- /dev/null +++ b/tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/Agentstration.Performance.Tests/AssemblyInfo.cs b/tests/Agentstration.Performance.Tests/AssemblyInfo.cs new file mode 100644 index 00000000..c53e6b4a --- /dev/null +++ b/tests/Agentstration.Performance.Tests/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: Parallelize(Workers = 1, Scope = ExecutionScope.ClassLevel)] diff --git a/tests/Agentstration.Web.Tests/StorageConcurrencyBenchmarkTests.cs b/tests/Agentstration.Performance.Tests/StorageConcurrencyBenchmarkTests.cs similarity index 93% rename from tests/Agentstration.Web.Tests/StorageConcurrencyBenchmarkTests.cs rename to tests/Agentstration.Performance.Tests/StorageConcurrencyBenchmarkTests.cs index 8775c816..94cd2883 100644 --- a/tests/Agentstration.Web.Tests/StorageConcurrencyBenchmarkTests.cs +++ b/tests/Agentstration.Performance.Tests/StorageConcurrencyBenchmarkTests.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using System.Diagnostics; using System.Net; +using System.Runtime.InteropServices; using System.Text.Json; using Agentstration.Flow; using Agentstration.Flow.Storage.Abstractions; @@ -13,7 +14,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -namespace Agentstration.Web.Tests; +namespace Agentstration.Performance.Tests; [TestClass] public sealed class StorageConcurrencyBenchmarkTests @@ -73,8 +74,8 @@ await Task.WhenAll(Enumerable.Range(0, operations).Select(async index => var workItem = WorkItem.Create(WorkItemId.New(), workspaceId, "benchmark", $"Operation {index}", now); var storedWorkItem = await workItems.CreateAsync(workItem, default); var expectedVersion = storedWorkItem.Value.Version; - workItem.AddMessage("update", "benchmark", Guid.NewGuid(), now.AddMilliseconds(1)); - await workItems.SaveAsync(workItem, expectedVersion, default); + storedWorkItem.Value.AddMessage("update", "benchmark", Guid.NewGuid(), now.AddMilliseconds(1)); + await workItems.SaveAsync(storedWorkItem.Value, expectedVersion, default); var flowId = new FlowId("benchmark-flow"); var flowRunId = $"flow-{Guid.NewGuid():N}"; @@ -148,6 +149,12 @@ await checkpoints.StoreAsync(new RuntimeExecutionState( provider, operations, concurrency, + writesPerOperation = 7, + elapsedMilliseconds = elapsed.TotalMilliseconds, + runtime = RuntimeInformation.FrameworkDescription, + runtimeVersion = Environment.Version.ToString(), + os = RuntimeInformation.OSDescription, + processArchitecture = RuntimeInformation.ProcessArchitecture.ToString(), throughputPerSecond = operations / elapsed.TotalSeconds, medianMilliseconds = Percentile(ordered, 0.50), p95Milliseconds = Percentile(ordered, 0.95), diff --git a/tests/Agentstration.Web.Tests/StorageConcurrencySmokeTests.cs b/tests/Agentstration.Web.Tests/StorageConcurrencySmokeTests.cs new file mode 100644 index 00000000..7f68a735 --- /dev/null +++ b/tests/Agentstration.Web.Tests/StorageConcurrencySmokeTests.cs @@ -0,0 +1,53 @@ +using Agentstration.Resources; +using Agentstration.Work; +using Agentstration.Work.Storage.Abstractions; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Agentstration.Web.Tests; + +[TestClass] +public sealed class StorageConcurrencySmokeTests +{ + [TestMethod] + public async Task SqliteAcceptsBoundedConcurrentWorkItemWritesWithoutErrors() + { + var dataDirectory = Path.Combine(Path.GetTempPath(), $"agentstration-storage-smoke-{Guid.NewGuid():N}"); + try + { + await using var host = new WebApplicationFactory().WithWebHostBuilder(builder => + { + builder.UseEnvironment("Testing"); + builder.UseSetting("Agentstration:Testing:ApiOnly", "true"); + builder.UseSetting("Agentstration:Storage:Provider", "Sqlite"); + builder.UseSetting("Data:Directory", dataDirectory); + builder.UseSetting("Agentstration:Bootstrap:InitialBootstrapEnabled", "false"); + builder.ConfigureLogging(logging => logging.ClearProviders()); + }); + _ = host.CreateClient(); + var repository = host.Services.GetRequiredService(); + var workspaceId = new WorkspaceId(Guid.NewGuid()); + + var stored = await Task.WhenAll(Enumerable.Range(0, 8).Select(async index => + { + var id = WorkItemId.New(); + var now = DateTimeOffset.UtcNow; + var item = WorkItem.Create(id, workspaceId, "content", $"Smoke {index}", now); + var created = await repository.CreateAsync(item, default); + var expectedVersion = created.Value.Version; + created.Value.AddMessage("update", "smoke", Guid.NewGuid(), now.AddMilliseconds(1)); + await repository.SaveAsync(created.Value, expectedVersion, default); + return await repository.GetAsync(workspaceId, id, default); + })); + + Assert.HasCount(8, stored); + Assert.IsTrue(stored.All(item => item is not null && item.Value.Messages.Count == 1)); + } + finally + { + if (Directory.Exists(dataDirectory)) Directory.Delete(dataDirectory, true); + } + } +}