From 1f390c36a0029c22eba08fd42d96f1cc66bf918f Mon Sep 17 00:00:00 2001 From: gbaudrit Date: Fri, 11 Sep 2026 11:40:27 +0200 Subject: [PATCH 1/9] fix(test): enforce Microsoft Testing Platform inputs (#299) (#306) --- .github/workflows/ci.yml | 11 ++++++----- .github/workflows/release-source-registry-tool.yml | 3 ++- .github/workflows/release.yml | 5 +++-- AGENTS.md | 2 +- CONTRIBUTING.md | 2 +- README.md | 2 +- aep/README.md | 2 +- docs/concepts/model-providers.md | 4 ++-- docs/getting-started/configuration.md | 4 ++-- docs/reference/current-capabilities.md | 2 +- docs/reference/identity-and-authorization.md | 2 +- 11 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e759d615..6fae1e48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,7 +97,7 @@ jobs: - name: Build web tests run: dotnet build tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj --configuration Release --no-restore - name: Verify Windows host lifecycle - run: dotnet tests/Agentstration.Web.Tests/bin/Release/net10.0/Agentstration.Web.Tests.dll --filter "FullyQualifiedName~QuartzHostLifecycleTests|FullyQualifiedName~StartupDoesNotCreateLegacyDataJson" --progress off + run: dotnet tests/Agentstration.Web.Tests/bin/Release/net10.0/Agentstration.Web.Tests.dll --filter "FullyQualifiedName~QuartzHostLifecycleTests|FullyQualifiedName~StartupDoesNotCreateLegacyDataJson" --minimum-expected-tests 2 --progress off build-and-test: name: build-and-test @@ -141,7 +141,7 @@ jobs: run: dotnet build Agentstration.slnx --configuration Release --no-restore - name: Test Agentstration if: needs.changes.outputs.dotnet == 'true' - run: dotnet test Agentstration.slnx --configuration Release --no-build + run: dotnet test --solution Agentstration.slnx --configuration Release --no-build --minimum-expected-tests 1 - name: Smoke-test Source Registry tool package if: needs.changes.outputs.dotnet == 'true' shell: pwsh @@ -151,7 +151,8 @@ jobs: run: dotnet build aep/Aep.slnx --configuration Release --no-restore - name: Test complete AEP SDK if: needs.changes.outputs.dotnet == 'true' && needs.changes.outputs.aep == 'true' - run: dotnet test aep/Aep.slnx --configuration Release --no-build + working-directory: aep + run: dotnet test --solution Aep.slnx --configuration Release --no-build --minimum-expected-tests 1 postgresql-storage: name: postgresql-storage @@ -185,7 +186,7 @@ jobs: - name: Build PostgreSQL integration tests run: dotnet build tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj --configuration Release --no-restore - name: Run PostgreSQL migration and restart test - run: dotnet tests/Agentstration.Web.Tests/bin/Release/net10.0/Agentstration.Web.Tests.dll --filter "Name=EmptyDatabaseMigratesAndRemainsReadyAfterRestart" --progress off + run: dotnet tests/Agentstration.Web.Tests/bin/Release/net10.0/Agentstration.Web.Tests.dll --filter "Name=EmptyDatabaseMigratesAndRemainsReadyAfterRestart" --minimum-expected-tests 1 --progress off - name: Run PostgreSQL storage concurrency benchmark env: AGENTSTRATION_STORAGE_BENCHMARK_PROVIDER: PostgreSql @@ -193,7 +194,7 @@ jobs: AGENTSTRATION_STORAGE_BENCHMARK_CONCURRENCY: 8 AGENTSTRATION_STORAGE_BENCHMARK_REPORT: ${{ runner.temp }}/agentstration-storage-benchmark-postgresql.json run: | - dotnet tests/Agentstration.Web.Tests/bin/Release/net10.0/Agentstration.Web.Tests.dll --filter "Name=ReportsConcurrentRelationalWriteMetrics" --progress off + dotnet tests/Agentstration.Web.Tests/bin/Release/net10.0/Agentstration.Web.Tests.dll --filter "Name=ReportsConcurrentRelationalWriteMetrics" --minimum-expected-tests 1 --progress off cat "$AGENTSTRATION_STORAGE_BENCHMARK_REPORT" container: diff --git a/.github/workflows/release-source-registry-tool.yml b/.github/workflows/release-source-registry-tool.yml index 89682066..daa7a053 100644 --- a/.github/workflows/release-source-registry-tool.yml +++ b/.github/workflows/release-source-registry-tool.yml @@ -87,9 +87,10 @@ jobs: -p:Version=${{ steps.version.outputs.package-version }} - name: Test tool run: >- - dotnet test tests/Agentstration.Tools.SourceRegistry.Tests/Agentstration.Tools.SourceRegistry.Tests.csproj + dotnet test --project tests/Agentstration.Tools.SourceRegistry.Tests/Agentstration.Tools.SourceRegistry.Tests.csproj --configuration Release --no-build + --minimum-expected-tests 1 -p:Version=${{ steps.version.outputs.package-version }} - name: Smoke-test package shell: pwsh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index acc72688..15c9a02c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -55,13 +55,14 @@ jobs: - name: Build run: dotnet build Agentstration.slnx --configuration Release --no-restore - name: Test - run: dotnet test Agentstration.slnx --configuration Release --no-build + run: dotnet test --solution Agentstration.slnx --configuration Release --no-build --minimum-expected-tests 1 - name: Restore autonomous AEP SDK run: dotnet restore aep/Aep.slnx -p:NuGetAudit=true -p:NuGetAuditMode=all - name: Build autonomous AEP SDK run: dotnet build aep/Aep.slnx --configuration Release --no-restore - name: Test autonomous AEP SDK - run: dotnet test aep/Aep.slnx --configuration Release --no-build + working-directory: aep + run: dotnet test --solution Aep.slnx --configuration Release --no-build --minimum-expected-tests 1 - name: Publish framework-dependent applications shell: pwsh run: | diff --git a/AGENTS.md b/AGENTS.md index 3a124359..12b0ffee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -180,7 +180,7 @@ Run from the repository root: ```powershell dotnet restore Agentstration.slnx dotnet build Agentstration.slnx --configuration Release --no-restore -dotnet test Agentstration.slnx --configuration Release --no-build +dotnet test --solution Agentstration.slnx --configuration Release --no-build --minimum-expected-tests 1 ``` For a focused iteration, run the affected test project first, then run the full build and test suite before handoff. Do not suppress warnings or disable analyzers to make a change pass. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d149d195..b703406c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,7 +22,7 @@ Use MSTest for behavior changes and keep the default test suite deterministic an ```powershell dotnet restore Agentstration.slnx dotnet build Agentstration.slnx --configuration Release --no-restore -dotnet test Agentstration.slnx --configuration Release --no-build +dotnet test --solution Agentstration.slnx --configuration Release --no-build --minimum-expected-tests 1 ./scripts/ci/verify-dotnet-format.ps1 -BaseRevision "origin/main" ``` diff --git a/README.md b/README.md index 72265af3..32008233 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,7 @@ Aspire starts Agentstration's AEP extensions against existing inference servers. ```powershell dotnet build Agentstration.slnx --configuration Release -dotnet test Agentstration.slnx --configuration Release +dotnet test --solution Agentstration.slnx --configuration Release --minimum-expected-tests 1 ``` Warnings are treated as errors, .NET analyzers are enabled and NuGet audit findings fail restore. The default tests are designed to remain offline and cost-free; real-provider smoke tests are opt-in. diff --git a/aep/README.md b/aep/README.md index ef563b3a..32937974 100644 --- a/aep/README.md +++ b/aep/README.md @@ -16,7 +16,7 @@ AEP is an autonomous, versioned protocol and .NET SDK for discovering, validatin ```powershell dotnet restore Aep.slnx dotnet build Aep.slnx --configuration Release --no-restore -dotnet test Aep.slnx --configuration Release --no-build +dotnet test --solution Aep.slnx --configuration Release --no-build --minimum-expected-tests 1 ``` Run a generic sample and Inspector independently: diff --git a/docs/concepts/model-providers.md b/docs/concepts/model-providers.md index 79ef701a..5ce10e70 100644 --- a/docs/concepts/model-providers.md +++ b/docs/concepts/model-providers.md @@ -49,7 +49,7 @@ The default tests use fake HTTP. To run the optional real-server smoke test: $env:AGENTSTRATION_LOCALAI_ENDPOINT = "http://localhost:8081" $env:AGENTSTRATION_LOCALAI_MODEL = "your-chat-model" # Optional: $env:AGENTSTRATION_LOCALAI_API_KEY = "..." -dotnet test tests/Agentstration.ModelProviders.Tests --filter TestCategory=Integration +dotnet test --project tests/Agentstration.ModelProviders.Tests/Agentstration.ModelProviders.Tests.csproj --filter TestCategory=Integration --minimum-expected-tests 1 ``` ## llama.cpp @@ -120,5 +120,5 @@ The default tests use fake HTTP and require no model. To run the optional real-s ```powershell $env:AGENTSTRATION_LLAMA_CPP_ENDPOINT = "http://localhost:8080" $env:AGENTSTRATION_LLAMA_CPP_MODEL = "local-gguf" -dotnet test tests/Agentstration.ModelProviders.Tests --filter TestCategory=Integration +dotnet test --project tests/Agentstration.ModelProviders.Tests/Agentstration.ModelProviders.Tests.csproj --filter TestCategory=Integration --minimum-expected-tests 1 ``` diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index e3e412e7..4006ab2f 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -156,11 +156,11 @@ $env:AGENTSTRATION_STORAGE_BENCHMARK_PROVIDER = "Sqlite" $env:AGENTSTRATION_STORAGE_BENCHMARK_OPERATIONS = "100" $env:AGENTSTRATION_STORAGE_BENCHMARK_CONCURRENCY = "8" $env:AGENTSTRATION_STORAGE_BENCHMARK_REPORT = "$env:TEMP\agentstration-storage-benchmark.json" -dotnet test tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj --configuration Release --filter "Name=ReportsConcurrentRelationalWriteMetrics" --logger "console;verbosity=detailed" +dotnet test --project tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj --configuration Release --filter "Name=ReportsConcurrentRelationalWriteMetrics" --minimum-expected-tests 1 --logger "console;verbosity=detailed" $env:AGENTSTRATION_STORAGE_BENCHMARK_PROVIDER = "PostgreSql" $env:AGENTSTRATION_TEST_POSTGRES = "Host=localhost;Database=agentstration;Username=agentstration;Password=" -dotnet test tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj --configuration Release --filter "Name=ReportsConcurrentRelationalWriteMetrics" --logger "console;verbosity=detailed" +dotnet test --project tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj --configuration Release --filter "Name=ReportsConcurrentRelationalWriteMetrics" --minimum-expected-tests 1 --logger "console;verbosity=detailed" ``` ## Backup and restore diff --git a/docs/reference/current-capabilities.md b/docs/reference/current-capabilities.md index efec5eb5..94acc4e1 100644 --- a/docs/reference/current-capabilities.md +++ b/docs/reference/current-capabilities.md @@ -430,7 +430,7 @@ The capture creates a correlated `gen_ai.http.payload_capture` span between the ```powershell dotnet build Agentstration.slnx --configuration Release -dotnet test Agentstration.slnx --configuration Release +dotnet test --solution Agentstration.slnx --configuration Release --minimum-expected-tests 1 ``` Warnings are errors, .NET analyzers are enabled, and NuGet audit findings fail restore. The test suite covers Management, Work, Workplace, Flow, Runtime, Triggers, Packs, Agents, workspace isolation, MCP infrastructure, REST startup, and dependency rules. diff --git a/docs/reference/identity-and-authorization.md b/docs/reference/identity-and-authorization.md index 67d0d820..9d93af24 100644 --- a/docs/reference/identity-and-authorization.md +++ b/docs/reference/identity-and-authorization.md @@ -426,7 +426,7 @@ The validated baseline for this implementation is: ```text dotnet build Agentstration.slnx --configuration Release --no-restore -dotnet test Agentstration.slnx --configuration Release --no-build +dotnet test --solution Agentstration.slnx --configuration Release --no-build --minimum-expected-tests 1 Build: 0 warnings, 0 errors Tests: 461 passed, 0 failed, 2 optional provider integration tests skipped From d19c053ae34a00652a7bb2eb8f9327a5ee8d3468 Mon Sep 17 00:00:00 2001 From: gbaudrit Date: Fri, 11 Sep 2026 11:48:09 +0200 Subject: [PATCH 2/9] test: define independent validation lanes (#300) (#307) --- .github/workflows/ci.yml | 7 ++-- AGENTS.md | 5 +-- Agentstration.Tests.Fast.slnx | 9 +++++ Agentstration.Tests.Integration.slnx | 12 +++++++ Agentstration.Tests.Performance.slnx | 5 +++ CONTRIBUTING.md | 5 +-- README.md | 5 +-- docs/contributing/testing.md | 53 ++++++++++++++++++++++++++++ docs/site/sidebars.js | 1 + 9 files changed, 94 insertions(+), 8 deletions(-) create mode 100644 Agentstration.Tests.Fast.slnx create mode 100644 Agentstration.Tests.Integration.slnx create mode 100644 Agentstration.Tests.Performance.slnx create mode 100644 docs/contributing/testing.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fae1e48..8fe722f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,9 +139,12 @@ jobs: - name: Build Agentstration if: needs.changes.outputs.dotnet == 'true' run: dotnet build Agentstration.slnx --configuration Release --no-restore - - name: Test Agentstration + - name: Test fast lane if: needs.changes.outputs.dotnet == 'true' - run: dotnet test --solution Agentstration.slnx --configuration Release --no-build --minimum-expected-tests 1 + run: dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 1 + - name: Test integration lane + if: needs.changes.outputs.dotnet == 'true' + run: dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 1 - name: Smoke-test Source Registry tool package if: needs.changes.outputs.dotnet == 'true' shell: pwsh diff --git a/AGENTS.md b/AGENTS.md index 12b0ffee..c0faf5ab 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -180,10 +180,11 @@ Run from the repository root: ```powershell dotnet restore Agentstration.slnx dotnet build Agentstration.slnx --configuration Release --no-restore -dotnet test --solution Agentstration.slnx --configuration Release --no-build --minimum-expected-tests 1 +dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 1 +dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 1 ``` -For a focused iteration, run the affected test project first, then run the full build and test suite before handoff. Do not suppress warnings or disable analyzers to make a change pass. +The two test solutions together form the required deterministic, offline functional suite. Performance and live-provider workloads are explicit opt-ins documented in `docs/contributing/testing.md`. For a focused iteration, run the affected test project first, then run both functional lanes before handoff. Do not suppress warnings or disable analyzers to make a change pass. To smoke-test the executable default with the Development bootstrap when startup behavior changes: diff --git a/Agentstration.Tests.Fast.slnx b/Agentstration.Tests.Fast.slnx new file mode 100644 index 00000000..e94d2744 --- /dev/null +++ b/Agentstration.Tests.Fast.slnx @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/Agentstration.Tests.Integration.slnx b/Agentstration.Tests.Integration.slnx new file mode 100644 index 00000000..bc4af4cd --- /dev/null +++ b/Agentstration.Tests.Integration.slnx @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/Agentstration.Tests.Performance.slnx b/Agentstration.Tests.Performance.slnx new file mode 100644 index 00000000..82b5e516 --- /dev/null +++ b/Agentstration.Tests.Performance.slnx @@ -0,0 +1,5 @@ + + + + + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b703406c..32ffdd2a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,11 +22,12 @@ Use MSTest for behavior changes and keep the default test suite deterministic an ```powershell dotnet restore Agentstration.slnx dotnet build Agentstration.slnx --configuration Release --no-restore -dotnet test --solution Agentstration.slnx --configuration Release --no-build --minimum-expected-tests 1 +dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 1 +dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 1 ./scripts/ci/verify-dotnet-format.ps1 -BaseRevision "origin/main" ``` -The format script checks changed C# and Razor files in both solutions. The autonomous AEP subtree has a larger standalone solution; when it changes, also restore, build, and test `aep/Aep.slnx`. +The fast and integration lanes together cover the complete required functional suite. Performance and live-provider workloads are explicit opt-ins described in the [test lane guide](docs/contributing/testing.md). The format script checks changed C# and Razor files in both product solutions. The autonomous AEP subtree has a larger standalone solution; when it changes, also restore, build, and test `aep/Aep.slnx`. If documentation changed, run: diff --git a/README.md b/README.md index 32008233..fc700ee7 100644 --- a/README.md +++ b/README.md @@ -201,10 +201,11 @@ Aspire starts Agentstration's AEP extensions against existing inference servers. ```powershell dotnet build Agentstration.slnx --configuration Release -dotnet test --solution Agentstration.slnx --configuration Release --minimum-expected-tests 1 +dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 1 +dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 1 ``` -Warnings are treated as errors, .NET analyzers are enabled and NuGet audit findings fail restore. The default tests are designed to remain offline and cost-free; real-provider smoke tests are opt-in. +Warnings are treated as errors, .NET analyzers are enabled and NuGet audit findings fail restore. The fast and integration lanes together provide complete required functional validation while remaining offline and cost-free; real-provider and performance workloads are opt-in. See the [test lane guide](docs/contributing/testing.md) for project classification and focused commands. ## Documentation diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md new file mode 100644 index 00000000..e1d45fd8 --- /dev/null +++ b/docs/contributing/testing.md @@ -0,0 +1,53 @@ +# Test lanes + +Agentstration separates test execution by runtime cost and dependency type. The root `Agentstration.slnx` remains the complete build graph, while the checked-in test solutions provide explicit execution boundaries. + +## Fast lane + +Fast tests do not start an ASP.NET Core host, open a relational database, spawn a child process, or contact a remote provider. They cover architecture rules, component behavior, mapping, validation, and in-process command behavior. + +```powershell +dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 1 +``` + +## Integration lane + +Integration tests exercise a real boundary such as `WebApplicationFactory`, SQLite, Git, persistent Identity, or runtime reconstruction. This lane remains deterministic and offline by default. Tests marked `Integration` for a live model provider are opt-in and report inconclusive unless their documented environment variables are supplied. + +```powershell +dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 1 +``` + +Run both fast and integration solutions for complete required functional validation. Their union is the functional test inventory represented by the root solution. + +## Performance lane + +Performance workloads are selected explicitly and are never part of the fast lane: + +```powershell +$env:AGENTSTRATION_STORAGE_BENCHMARK_PROVIDER = "Sqlite" +$env:AGENTSTRATION_STORAGE_BENCHMARK_REPORT = "artifacts/storage-benchmark-sqlite.json" +dotnet test --solution Agentstration.Tests.Performance.slnx --configuration Release --no-build --filter TestCategory=Benchmark --minimum-expected-tests 1 +``` + +The current Web storage workload remains a mixed-project boundary until #303 moves it to a dedicated performance project. PostgreSQL is optional and runs only in its service-backed CI job or an explicitly configured local environment. + +## Project classification + +| Test project | Lane | Boundary or rationale | +| --- | --- | --- | +| `Agentstration.ArchitectureTests` | Fast | Assembly dependency rules | +| `Agentstration.Tools.SourceRegistry.Tests` | Fast | In-process CLI and manifest validation | +| `Agentstration.Web.Components.Tests` | Fast | bUnit component behavior | +| `Agentstration.Web.FlowDesigner.Tests` | Fast | bUnit and graph projection behavior | +| `Agentstration.Workplace.Components.Tests` | Fast | bUnit component behavior | +| `Agentstration.Application.Tests` | Integration | Includes SQLite Flow and Work storage contracts | +| `Agentstration.Management.Tests` | Integration, mixed | Hosted, SQLite, Identity, source, and pure tests; split by #301 and #302 | +| `Agentstration.ModelProviders.Tests` | Integration, provider-optional | AEP test hosts plus opt-in live-provider checks | +| `Agentstration.Runtime.Tests` | Integration | SQLite reconstruction and hosted runtime endpoints | +| `Agentstration.SourceProviders.Git.Tests` | Integration | Real Git processes and file-system repositories | +| `Agentstration.Web.Tests` | Integration, mixed | Full Web host plus the temporary benchmark boundary tracked by #303 | +| `Agentstration.Work.Api.Tests` | Integration | Full Work API host | +| `Agentstration.Workplace.Web.Tests` | Integration | Workplace HTTP host | + +The autonomous AEP SDK keeps its own `aep/Aep.slnx` validation because it can be built and released independently from the product solution. diff --git a/docs/site/sidebars.js b/docs/site/sidebars.js index 30e0f093..26f5f98e 100644 --- a/docs/site/sidebars.js +++ b/docs/site/sidebars.js @@ -100,6 +100,7 @@ const sidebars = { items: [ 'contributing/overview', 'contributing/development-slots', + 'contributing/testing', 'contributing/documentation', 'contributing/github-governance', ], From e2d0382fdfe277a0d8e0b71fee4bb690d5a156c1 Mon Sep 17 00:00:00 2001 From: gbaudrit Date: Fri, 11 Sep 2026 12:02:47 +0200 Subject: [PATCH 3/9] test(management): split core storage and source suites (#301) (#309) --- AGENTS.md | 3 +++ Agentstration.Tests.Fast.slnx | 1 + Agentstration.Tests.Integration.slnx | 2 ++ Agentstration.slnx | 3 +++ docs/architecture.md | 3 +++ docs/contributing/testing.md | 5 ++++- .../Agentstration.Infrastructure.csproj | 1 + .../Agentstration.Management.Core.Tests.csproj | 8 ++++++++ .../ToolDiscoveryTests.cs | 0 .../ToolExecutionHookManagementTests.cs | 0 .../ToolManagementTests.cs | 0 ...entstration.Management.Sources.Tests.csproj | 16 ++++++++++++++++ .../PackCompositionTests.cs | 0 .../PackTests.cs | 0 .../SourceProviderManagementTests.cs | 0 .../SourceRegistryInfrastructureTests.cs | 0 .../SourceRegistryManagementTests.cs | 0 .../SourceTests.cs | 0 ...entstration.Management.Storage.Tests.csproj | 18 ++++++++++++++++++ .../ControlPlaneStoreHardeningTests.cs | 0 .../LocalIdentityPersistenceTests.cs | 0 .../LocalSecretVaultTests.cs | 0 .../ManagementPlaneTests.cs | 0 .../SecurityAuditPersistenceTests.cs | 0 .../TriggerTests.cs | 0 25 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 tests/Agentstration.Management.Core.Tests/Agentstration.Management.Core.Tests.csproj rename tests/{Agentstration.Management.Tests => Agentstration.Management.Core.Tests}/ToolDiscoveryTests.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Core.Tests}/ToolExecutionHookManagementTests.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Core.Tests}/ToolManagementTests.cs (100%) create mode 100644 tests/Agentstration.Management.Sources.Tests/Agentstration.Management.Sources.Tests.csproj rename tests/{Agentstration.Management.Tests => Agentstration.Management.Sources.Tests}/PackCompositionTests.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Sources.Tests}/PackTests.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Sources.Tests}/SourceProviderManagementTests.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Sources.Tests}/SourceRegistryInfrastructureTests.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Sources.Tests}/SourceRegistryManagementTests.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Sources.Tests}/SourceTests.cs (100%) create mode 100644 tests/Agentstration.Management.Storage.Tests/Agentstration.Management.Storage.Tests.csproj rename tests/{Agentstration.Management.Tests => Agentstration.Management.Storage.Tests}/ControlPlaneStoreHardeningTests.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Storage.Tests}/LocalIdentityPersistenceTests.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Storage.Tests}/LocalSecretVaultTests.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Storage.Tests}/ManagementPlaneTests.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Storage.Tests}/SecurityAuditPersistenceTests.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Storage.Tests}/TriggerTests.cs (100%) diff --git a/AGENTS.md b/AGENTS.md index c0faf5ab..bffb3268 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,6 +48,9 @@ src/ tests/ Agentstration.Application.Tests/ Agentstration.ArchitectureTests/ + Agentstration.Management.Core.Tests/ + Agentstration.Management.Storage.Tests/ + Agentstration.Management.Sources.Tests/ Agentstration.Management.Tests/ docs/ architecture.md diff --git a/Agentstration.Tests.Fast.slnx b/Agentstration.Tests.Fast.slnx index e94d2744..cab470cd 100644 --- a/Agentstration.Tests.Fast.slnx +++ b/Agentstration.Tests.Fast.slnx @@ -1,6 +1,7 @@ + diff --git a/Agentstration.Tests.Integration.slnx b/Agentstration.Tests.Integration.slnx index bc4af4cd..36808a46 100644 --- a/Agentstration.Tests.Integration.slnx +++ b/Agentstration.Tests.Integration.slnx @@ -1,6 +1,8 @@ + + diff --git a/Agentstration.slnx b/Agentstration.slnx index 7eea6f3d..c3b02545 100644 --- a/Agentstration.slnx +++ b/Agentstration.slnx @@ -55,6 +55,9 @@ + + + diff --git a/docs/architecture.md b/docs/architecture.md index 040e62ee..e101b68f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -46,6 +46,9 @@ src/ tests/ Agentstration.Application.Tests/ Agentstration.ArchitectureTests/ + Agentstration.Management.Core.Tests/ + Agentstration.Management.Storage.Tests/ + Agentstration.Management.Sources.Tests/ Agentstration.Management.Tests/ Agentstration.Web.Tests/ Agentstration.Web.Components.Tests/ diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index e1d45fd8..ae48645a 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -37,12 +37,15 @@ The current Web storage workload remains a mixed-project boundary until #303 mov | Test project | Lane | Boundary or rationale | | --- | --- | --- | | `Agentstration.ArchitectureTests` | Fast | Assembly dependency rules | +| `Agentstration.Management.Core.Tests` | Fast | Pure Management validation and in-memory use cases | | `Agentstration.Tools.SourceRegistry.Tests` | Fast | In-process CLI and manifest validation | | `Agentstration.Web.Components.Tests` | Fast | bUnit component behavior | | `Agentstration.Web.FlowDesigner.Tests` | Fast | bUnit and graph projection behavior | | `Agentstration.Workplace.Components.Tests` | Fast | bUnit component behavior | | `Agentstration.Application.Tests` | Integration | Includes SQLite Flow and Work storage contracts | -| `Agentstration.Management.Tests` | Integration, mixed | Hosted, SQLite, Identity, source, and pure tests; split by #301 and #302 | +| `Agentstration.Management.Storage.Tests` | Integration | SQLite control-plane, Identity persistence, secrets, audit, and trigger storage | +| `Agentstration.Management.Sources.Tests` | Integration | Source, registry, provider, and Pack distribution boundaries | +| `Agentstration.Management.Tests` | Integration, mixed | Remaining hosted API, Security, bootstrap, and AEP tests; split by #302 | | `Agentstration.ModelProviders.Tests` | Integration, provider-optional | AEP test hosts plus opt-in live-provider checks | | `Agentstration.Runtime.Tests` | Integration | SQLite reconstruction and hosted runtime endpoints | | `Agentstration.SourceProviders.Git.Tests` | Integration | Real Git processes and file-system repositories | diff --git a/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj b/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj index 14c23eeb..9d5d4bd2 100644 --- a/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj +++ b/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj @@ -21,6 +21,7 @@ + diff --git a/tests/Agentstration.Management.Core.Tests/Agentstration.Management.Core.Tests.csproj b/tests/Agentstration.Management.Core.Tests/Agentstration.Management.Core.Tests.csproj new file mode 100644 index 00000000..3d428411 --- /dev/null +++ b/tests/Agentstration.Management.Core.Tests/Agentstration.Management.Core.Tests.csproj @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/tests/Agentstration.Management.Tests/ToolDiscoveryTests.cs b/tests/Agentstration.Management.Core.Tests/ToolDiscoveryTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/ToolDiscoveryTests.cs rename to tests/Agentstration.Management.Core.Tests/ToolDiscoveryTests.cs diff --git a/tests/Agentstration.Management.Tests/ToolExecutionHookManagementTests.cs b/tests/Agentstration.Management.Core.Tests/ToolExecutionHookManagementTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/ToolExecutionHookManagementTests.cs rename to tests/Agentstration.Management.Core.Tests/ToolExecutionHookManagementTests.cs diff --git a/tests/Agentstration.Management.Tests/ToolManagementTests.cs b/tests/Agentstration.Management.Core.Tests/ToolManagementTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/ToolManagementTests.cs rename to tests/Agentstration.Management.Core.Tests/ToolManagementTests.cs diff --git a/tests/Agentstration.Management.Sources.Tests/Agentstration.Management.Sources.Tests.csproj b/tests/Agentstration.Management.Sources.Tests/Agentstration.Management.Sources.Tests.csproj new file mode 100644 index 00000000..29044df4 --- /dev/null +++ b/tests/Agentstration.Management.Sources.Tests/Agentstration.Management.Sources.Tests.csproj @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/tests/Agentstration.Management.Tests/PackCompositionTests.cs b/tests/Agentstration.Management.Sources.Tests/PackCompositionTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/PackCompositionTests.cs rename to tests/Agentstration.Management.Sources.Tests/PackCompositionTests.cs diff --git a/tests/Agentstration.Management.Tests/PackTests.cs b/tests/Agentstration.Management.Sources.Tests/PackTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/PackTests.cs rename to tests/Agentstration.Management.Sources.Tests/PackTests.cs diff --git a/tests/Agentstration.Management.Tests/SourceProviderManagementTests.cs b/tests/Agentstration.Management.Sources.Tests/SourceProviderManagementTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SourceProviderManagementTests.cs rename to tests/Agentstration.Management.Sources.Tests/SourceProviderManagementTests.cs diff --git a/tests/Agentstration.Management.Tests/SourceRegistryInfrastructureTests.cs b/tests/Agentstration.Management.Sources.Tests/SourceRegistryInfrastructureTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SourceRegistryInfrastructureTests.cs rename to tests/Agentstration.Management.Sources.Tests/SourceRegistryInfrastructureTests.cs diff --git a/tests/Agentstration.Management.Tests/SourceRegistryManagementTests.cs b/tests/Agentstration.Management.Sources.Tests/SourceRegistryManagementTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SourceRegistryManagementTests.cs rename to tests/Agentstration.Management.Sources.Tests/SourceRegistryManagementTests.cs diff --git a/tests/Agentstration.Management.Tests/SourceTests.cs b/tests/Agentstration.Management.Sources.Tests/SourceTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SourceTests.cs rename to tests/Agentstration.Management.Sources.Tests/SourceTests.cs diff --git a/tests/Agentstration.Management.Storage.Tests/Agentstration.Management.Storage.Tests.csproj b/tests/Agentstration.Management.Storage.Tests/Agentstration.Management.Storage.Tests.csproj new file mode 100644 index 00000000..4e2edfbb --- /dev/null +++ b/tests/Agentstration.Management.Storage.Tests/Agentstration.Management.Storage.Tests.csproj @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/tests/Agentstration.Management.Tests/ControlPlaneStoreHardeningTests.cs b/tests/Agentstration.Management.Storage.Tests/ControlPlaneStoreHardeningTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/ControlPlaneStoreHardeningTests.cs rename to tests/Agentstration.Management.Storage.Tests/ControlPlaneStoreHardeningTests.cs diff --git a/tests/Agentstration.Management.Tests/LocalIdentityPersistenceTests.cs b/tests/Agentstration.Management.Storage.Tests/LocalIdentityPersistenceTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/LocalIdentityPersistenceTests.cs rename to tests/Agentstration.Management.Storage.Tests/LocalIdentityPersistenceTests.cs diff --git a/tests/Agentstration.Management.Tests/LocalSecretVaultTests.cs b/tests/Agentstration.Management.Storage.Tests/LocalSecretVaultTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/LocalSecretVaultTests.cs rename to tests/Agentstration.Management.Storage.Tests/LocalSecretVaultTests.cs diff --git a/tests/Agentstration.Management.Tests/ManagementPlaneTests.cs b/tests/Agentstration.Management.Storage.Tests/ManagementPlaneTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/ManagementPlaneTests.cs rename to tests/Agentstration.Management.Storage.Tests/ManagementPlaneTests.cs diff --git a/tests/Agentstration.Management.Tests/SecurityAuditPersistenceTests.cs b/tests/Agentstration.Management.Storage.Tests/SecurityAuditPersistenceTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SecurityAuditPersistenceTests.cs rename to tests/Agentstration.Management.Storage.Tests/SecurityAuditPersistenceTests.cs diff --git a/tests/Agentstration.Management.Tests/TriggerTests.cs b/tests/Agentstration.Management.Storage.Tests/TriggerTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/TriggerTests.cs rename to tests/Agentstration.Management.Storage.Tests/TriggerTests.cs From 85c09b8d3519e9410885a504a42f42a076bb7533 Mon Sep 17 00:00:00 2001 From: gbaudrit Date: Fri, 11 Sep 2026 12:30:36 +0200 Subject: [PATCH 4/9] test: split hosted management suites (#311) --- AGENTS.md | 4 +- Agentstration.Tests.Integration.slnx | 4 +- Agentstration.slnx | 4 +- docs/architecture.md | 4 +- docs/contributing/testing.md | 4 +- .../Agentstration.Infrastructure.csproj | 4 +- src/Agentstration.Web/Program.cs | 22 ++++--- .../AepEnrollmentTests.cs | 55 +++++++++++++++-- ...Agentstration.Management.Aep.Tests.csproj} | 0 .../AssemblyInfo.cs | 3 + .../Agentstration.Management.Api.Tests.csproj | 16 +++++ .../AssemblyInfo.cs | 3 + .../DeclarativeBootstrapTests.cs | 4 +- .../ExtensionRegistrationApiTests.cs} | 13 ++-- .../ModelManagementApiTestBase.cs} | 60 +++++++++++-------- .../ModelProfileApiTests.cs} | 5 +- .../ModelProviderApiTests.cs} | 3 +- .../NotificationDeliveryApiTests.cs} | 3 +- .../ResourceScopeApiTests.cs} | 3 +- .../RuntimeProfileAndToolApiTests.cs} | 3 +- .../ToolDefinitionApiTests.cs} | 3 +- ...tstration.Management.Security.Tests.csproj | 16 +++++ .../AssemblyInfo.cs | 3 + .../IdentityFoundationTests.cs | 0 .../SecurityApiTests.Authorization.cs | 0 .../SecurityApiTests.BoundariesAndTokens.cs | 0 .../SecurityApiTests.LocalAccounts.cs | 0 .../SecurityApiTests.PlatformAndMembership.cs | 0 .../SecurityApiTests.cs | 13 ++-- .../AssemblyInfo.cs | 3 - 30 files changed, 188 insertions(+), 67 deletions(-) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Aep.Tests}/AepEnrollmentTests.cs (92%) rename tests/{Agentstration.Management.Tests/Agentstration.Management.Tests.csproj => Agentstration.Management.Aep.Tests/Agentstration.Management.Aep.Tests.csproj} (100%) create mode 100644 tests/Agentstration.Management.Aep.Tests/AssemblyInfo.cs create mode 100644 tests/Agentstration.Management.Api.Tests/Agentstration.Management.Api.Tests.csproj create mode 100644 tests/Agentstration.Management.Api.Tests/AssemblyInfo.cs rename tests/{Agentstration.Management.Tests => Agentstration.Management.Api.Tests}/DeclarativeBootstrapTests.cs (99%) rename tests/{Agentstration.Management.Tests/ModelManagementApiTests.Extensions.cs => Agentstration.Management.Api.Tests/ExtensionRegistrationApiTests.cs} (96%) rename tests/{Agentstration.Management.Tests/ModelManagementApiTests.cs => Agentstration.Management.Api.Tests/ModelManagementApiTestBase.cs} (89%) rename tests/{Agentstration.Management.Tests/ModelManagementApiTests.ProfilesAndResolution.cs => Agentstration.Management.Api.Tests/ModelProfileApiTests.cs} (99%) rename tests/{Agentstration.Management.Tests/ModelManagementApiTests.Providers.cs => Agentstration.Management.Api.Tests/ModelProviderApiTests.cs} (97%) rename tests/{Agentstration.Management.Tests/NotificationDeliveryTests.cs => Agentstration.Management.Api.Tests/NotificationDeliveryApiTests.cs} (99%) rename tests/{Agentstration.Management.Tests/ModelManagementApiTests.ResourceScopes.cs => Agentstration.Management.Api.Tests/ResourceScopeApiTests.cs} (98%) rename tests/{Agentstration.Management.Tests/ModelManagementApiTests.RuntimeAndTools.cs => Agentstration.Management.Api.Tests/RuntimeProfileAndToolApiTests.cs} (99%) rename tests/{Agentstration.Management.Tests/ToolDefinitionTests.cs => Agentstration.Management.Api.Tests/ToolDefinitionApiTests.cs} (99%) create mode 100644 tests/Agentstration.Management.Security.Tests/Agentstration.Management.Security.Tests.csproj create mode 100644 tests/Agentstration.Management.Security.Tests/AssemblyInfo.cs rename tests/{Agentstration.Management.Tests => Agentstration.Management.Security.Tests}/IdentityFoundationTests.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Security.Tests}/SecurityApiTests.Authorization.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Security.Tests}/SecurityApiTests.BoundariesAndTokens.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Security.Tests}/SecurityApiTests.LocalAccounts.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Security.Tests}/SecurityApiTests.PlatformAndMembership.cs (100%) rename tests/{Agentstration.Management.Tests => Agentstration.Management.Security.Tests}/SecurityApiTests.cs (94%) delete mode 100644 tests/Agentstration.Management.Tests/AssemblyInfo.cs diff --git a/AGENTS.md b/AGENTS.md index bffb3268..5c2cd86a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,7 +51,9 @@ tests/ Agentstration.Management.Core.Tests/ Agentstration.Management.Storage.Tests/ Agentstration.Management.Sources.Tests/ - Agentstration.Management.Tests/ + Agentstration.Management.Api.Tests/ + Agentstration.Management.Security.Tests/ + Agentstration.Management.Aep.Tests/ docs/ architecture.md decisions/ diff --git a/Agentstration.Tests.Integration.slnx b/Agentstration.Tests.Integration.slnx index 36808a46..de77682c 100644 --- a/Agentstration.Tests.Integration.slnx +++ b/Agentstration.Tests.Integration.slnx @@ -3,7 +3,9 @@ - + + + diff --git a/Agentstration.slnx b/Agentstration.slnx index c3b02545..7d7cb154 100644 --- a/Agentstration.slnx +++ b/Agentstration.slnx @@ -58,7 +58,9 @@ - + + + diff --git a/docs/architecture.md b/docs/architecture.md index e101b68f..55199bfc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -49,7 +49,9 @@ tests/ Agentstration.Management.Core.Tests/ Agentstration.Management.Storage.Tests/ Agentstration.Management.Sources.Tests/ - Agentstration.Management.Tests/ + Agentstration.Management.Api.Tests/ + Agentstration.Management.Security.Tests/ + Agentstration.Management.Aep.Tests/ Agentstration.Web.Tests/ Agentstration.Web.Components.Tests/ Agentstration.Web.FlowDesigner.Tests/ diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index ae48645a..15228e86 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -45,7 +45,9 @@ The current Web storage workload remains a mixed-project boundary until #303 mov | `Agentstration.Application.Tests` | Integration | Includes SQLite Flow and Work storage contracts | | `Agentstration.Management.Storage.Tests` | Integration | SQLite control-plane, Identity persistence, secrets, audit, and trigger storage | | `Agentstration.Management.Sources.Tests` | Integration | Source, registry, provider, and Pack distribution boundaries | -| `Agentstration.Management.Tests` | Integration, mixed | Remaining hosted API, Security, bootstrap, and AEP tests; split by #302 | +| `Agentstration.Management.Api.Tests` | Integration | Hosted Management API and declarative bootstrap tests using the API-only test profile | +| `Agentstration.Management.Security.Tests` | Integration | Identity, authorization, local-account, and interactive Security boundaries | +| `Agentstration.Management.Aep.Tests` | Integration | AEP enrollment lifecycle and extension inventory boundaries using the API-only test profile | | `Agentstration.ModelProviders.Tests` | Integration, provider-optional | AEP test hosts plus opt-in live-provider checks | | `Agentstration.Runtime.Tests` | Integration | SQLite reconstruction and hosted runtime endpoints | | `Agentstration.SourceProviders.Git.Tests` | Integration | Real Git processes and file-system repositories | diff --git a/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj b/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj index 9d5d4bd2..eecdc5a2 100644 --- a/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj +++ b/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj @@ -20,7 +20,9 @@ - + + + diff --git a/src/Agentstration.Web/Program.cs b/src/Agentstration.Web/Program.cs index 06819ce6..7ea5af8b 100644 --- a/src/Agentstration.Web/Program.cs +++ b/src/Agentstration.Web/Program.cs @@ -45,6 +45,8 @@ toolExecutionCapture.Validate(); builder.Services.AddSingleton(toolExecutionCapture); var isTesting = builder.Environment.IsEnvironment("Testing"); +var apiOnlyTesting = isTesting + && builder.Configuration.GetValue("Agentstration:Testing:ApiOnly", false); var hostedServicesEnabled = !isTesting || builder.Configuration.GetValue("Agentstration:Testing:HostedServicesEnabled", false); var openTelemetryEnabled = !isTesting @@ -140,8 +142,11 @@ await context.HttpContext.Response.WriteAsJsonAsync( })); }); builder.Services.AddAgentstrationOpenApi(); -builder.Services.AddRazorPages(); -builder.Services.AddRazorComponents().AddInteractiveServerComponents(); +if (!apiOnlyTesting) +{ + builder.Services.AddRazorPages(); + builder.Services.AddRazorComponents().AddInteractiveServerComponents(); +} builder.Services.AddAgentstrationLocalization(builder.Configuration); builder.Services.AddSignalR(); if (storageProvider == AgentstrationStorageProvider.PostgreSql) @@ -261,7 +266,7 @@ await context.HttpContext.Response.WriteAsJsonAsync( app.UseMiddleware(); app.UseAuthorization(); if (app.Environment.IsDevelopment() || app.Environment.IsEnvironment("Testing")) app.MapAgentstrationOpenApi(); -app.UseAntiforgery(); +if (!apiOnlyTesting) app.UseAntiforgery(); app.MapGet("/health", () => Results.Ok(new { status = "healthy" })).AllowAnonymous(); app.MapGet("/health/ready", (IAgentstrationStorageInitializer storage) => storage.IsReady ? Results.Ok(new { status = "ready" }) @@ -284,10 +289,13 @@ await context.HttpContext.Response.WriteAsJsonAsync( app.MapHub("/hubs/workplace").RequireAuthorization(Agentstration.Web.Security.AgentstrationPolicies.CanReadRuns); if (app.Environment.IsDevelopment()) app.MapOllamaDiagnostics(); app.MapMcp("/mcp").RequireAuthorization(Agentstration.Web.Security.AgentstrationPolicies.CanExecuteRuns); -app.MapStaticAssets().AllowAnonymous(); -app.MapRazorPages(); -app.MapRazorComponents().AddAdditionalAssemblies(typeof(MainLayout).Assembly).AddInteractiveServerRenderMode() - .RequireAuthorization(Agentstration.Web.Security.AgentstrationPolicies.Authenticated); +if (!apiOnlyTesting) +{ + app.MapStaticAssets().AllowAnonymous(); + app.MapRazorPages(); + app.MapRazorComponents().AddAdditionalAssemblies(typeof(MainLayout).Assembly).AddInteractiveServerRenderMode() + .RequireAuthorization(Agentstration.Web.Security.AgentstrationPolicies.Authenticated); +} try { RequestContext? bootstrapContext = null; diff --git a/tests/Agentstration.Management.Tests/AepEnrollmentTests.cs b/tests/Agentstration.Management.Aep.Tests/AepEnrollmentTests.cs similarity index 92% rename from tests/Agentstration.Management.Tests/AepEnrollmentTests.cs rename to tests/Agentstration.Management.Aep.Tests/AepEnrollmentTests.cs index f2c32bac..1a53dde6 100644 --- a/tests/Agentstration.Management.Tests/AepEnrollmentTests.cs +++ b/tests/Agentstration.Management.Aep.Tests/AepEnrollmentTests.cs @@ -4,6 +4,7 @@ using Agentstration.Aep.Abstractions; using Agentstration.Management.Abstractions; using Agentstration.Management.Core; +using Agentstration.ModelProviders; using Agentstration.Resources; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; @@ -12,14 +13,56 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class AepEnrollmentTests { - private static WebApplicationFactory EnrollmentFactory() => Factory().WithWebHostBuilder(builder => - builder.ConfigureServices(services => + private static WebApplicationFactory EnrollmentFactory() => new AepEnrollmentTestFactory(); + + private static Task GetBootstrapContextAsync(WebApplicationFactory factory) => + factory.Services + .GetRequiredService() + .EnsureInitializedAsync(default); + + private sealed class AepEnrollmentTestFactory : WebApplicationFactory + { + protected override void ConfigureWebHost(IWebHostBuilder builder) { - services.RemoveAll(); - services.AddSingleton(); - })); + builder.UseEnvironment("Testing"); + builder.UseSetting("Agentstration:Testing:ApiOnly", "true"); + builder.UseSetting("Logging:LogLevel:Default", "Warning"); + builder.ConfigureServices(services => + { + services.RemoveAll(); + services.RemoveAll(); + services.AddSingleton(); + services.AddSingleton(); + }); + } + } + + private sealed class UnavailableExtensionInspector : IExtensionInspector + { + public bool CanHandle(string providerType) => true; + public bool CanInspectEndpoint(Uri endpoint) => true; + + public ValueTask InspectAsync( + ModelProviderConfiguration provider, + CancellationToken cancellationToken = default) => + InspectAsync(provider.Name, provider.Endpoint, cancellationToken); + + public ValueTask InspectAsync( + string registrationName, + Uri endpoint, + CancellationToken cancellationToken = default) => + ValueTask.FromResult(new ExtensionInspection( + registrationName, + endpoint, + "unavailable", + null, + [], + [], + "The test extension is intentionally unavailable.")); + } [TestMethod] public async Task PairingCodeCanBeDisabledAndConfigurationCanLockTheMode() diff --git a/tests/Agentstration.Management.Tests/Agentstration.Management.Tests.csproj b/tests/Agentstration.Management.Aep.Tests/Agentstration.Management.Aep.Tests.csproj similarity index 100% rename from tests/Agentstration.Management.Tests/Agentstration.Management.Tests.csproj rename to tests/Agentstration.Management.Aep.Tests/Agentstration.Management.Aep.Tests.csproj diff --git a/tests/Agentstration.Management.Aep.Tests/AssemblyInfo.cs b/tests/Agentstration.Management.Aep.Tests/AssemblyInfo.cs new file mode 100644 index 00000000..c53e6b4a --- /dev/null +++ b/tests/Agentstration.Management.Aep.Tests/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: Parallelize(Workers = 1, Scope = ExecutionScope.ClassLevel)] diff --git a/tests/Agentstration.Management.Api.Tests/Agentstration.Management.Api.Tests.csproj b/tests/Agentstration.Management.Api.Tests/Agentstration.Management.Api.Tests.csproj new file mode 100644 index 00000000..29044df4 --- /dev/null +++ b/tests/Agentstration.Management.Api.Tests/Agentstration.Management.Api.Tests.csproj @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/tests/Agentstration.Management.Api.Tests/AssemblyInfo.cs b/tests/Agentstration.Management.Api.Tests/AssemblyInfo.cs new file mode 100644 index 00000000..c53e6b4a --- /dev/null +++ b/tests/Agentstration.Management.Api.Tests/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: Parallelize(Workers = 1, Scope = ExecutionScope.ClassLevel)] diff --git a/tests/Agentstration.Management.Tests/DeclarativeBootstrapTests.cs b/tests/Agentstration.Management.Api.Tests/DeclarativeBootstrapTests.cs similarity index 99% rename from tests/Agentstration.Management.Tests/DeclarativeBootstrapTests.cs rename to tests/Agentstration.Management.Api.Tests/DeclarativeBootstrapTests.cs index d81321bd..e90d55e7 100644 --- a/tests/Agentstration.Management.Tests/DeclarativeBootstrapTests.cs +++ b/tests/Agentstration.Management.Api.Tests/DeclarativeBootstrapTests.cs @@ -777,7 +777,7 @@ private static WebApplicationFactory Factory( bool configureOllamaExtension = false) { EnsureInstanceProfileDescriptor(path); - return new WebApplicationFactory().WithWebHostBuilder(builder => + return new ApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("Agentstration:Authentication:Mode", "Local"); @@ -794,7 +794,7 @@ private static WebApplicationFactory Factory( private static WebApplicationFactory FactoryWithExtensionPresenceHandler(string path, bool discoverOnStartup) { EnsureInstanceProfileDescriptor(path); - return new WebApplicationFactory().WithWebHostBuilder(builder => + return new ApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("Agentstration:Authentication:Mode", "Development"); diff --git a/tests/Agentstration.Management.Tests/ModelManagementApiTests.Extensions.cs b/tests/Agentstration.Management.Api.Tests/ExtensionRegistrationApiTests.cs similarity index 96% rename from tests/Agentstration.Management.Tests/ModelManagementApiTests.Extensions.cs rename to tests/Agentstration.Management.Api.Tests/ExtensionRegistrationApiTests.cs index 9aee9b52..ddbbfc59 100644 --- a/tests/Agentstration.Management.Tests/ModelManagementApiTests.Extensions.cs +++ b/tests/Agentstration.Management.Api.Tests/ExtensionRegistrationApiTests.cs @@ -15,12 +15,13 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class ExtensionRegistrationApiTests : ModelManagementApiTestBase { [TestMethod] public async Task SeededOllamaProviderUsesAepExtensionEndpointInsteadOfNativeOllamaEndpoint() { - await using var factory = new WebApplicationFactory().WithWebHostBuilder(builder => + await using var factory = new ApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("AI:Provider", "Managed"); @@ -42,7 +43,7 @@ public async Task SeededOllamaProviderUsesAepExtensionEndpointInsteadOfNativeOll [TestMethod] public async Task SeededLlamaCppProviderUsesItsAepExtensionEndpoint() { - await using var factory = new WebApplicationFactory().WithWebHostBuilder(builder => + await using var factory = new ApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("AI:Provider", "Managed"); @@ -64,7 +65,7 @@ public async Task SeededLlamaCppProviderUsesItsAepExtensionEndpoint() [TestMethod] public async Task SeededLocalAiProviderUsesItsAepExtensionEndpoint() { - await using var factory = new WebApplicationFactory().WithWebHostBuilder(builder => + await using var factory = new ApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("AI:Provider", "Managed"); @@ -85,7 +86,7 @@ public async Task SeededLocalAiProviderUsesItsAepExtensionEndpoint() [TestMethod] public async Task GitSourceExtensionConfigurationUsesTheAspireRegistrationIdentity() { - await using var factory = new WebApplicationFactory().WithWebHostBuilder(builder => + await using var factory = new ApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("Agentstration:Extensions:Agentstration.Extensions.Git:Endpoint", "http://localhost:5295"); @@ -143,7 +144,7 @@ public async Task ExtensionsApiDoesNotAcceptManualDiscoveryCommand() using var client = factory.CreateClient(); using var discoveryResponse = await client.PostAsync("/api/extensions/discover", null); - Assert.AreEqual(HttpStatusCode.MethodNotAllowed, discoveryResponse.StatusCode); + Assert.AreEqual(HttpStatusCode.NotFound, discoveryResponse.StatusCode); } [TestMethod] diff --git a/tests/Agentstration.Management.Tests/ModelManagementApiTests.cs b/tests/Agentstration.Management.Api.Tests/ModelManagementApiTestBase.cs similarity index 89% rename from tests/Agentstration.Management.Tests/ModelManagementApiTests.cs rename to tests/Agentstration.Management.Api.Tests/ModelManagementApiTestBase.cs index 437ecf99..e352d782 100644 --- a/tests/Agentstration.Management.Tests/ModelManagementApiTests.cs +++ b/tests/Agentstration.Management.Api.Tests/ModelManagementApiTestBase.cs @@ -15,28 +15,11 @@ namespace Agentstration.Management.Tests; -[TestClass] -public sealed partial class ModelManagementApiTests +public abstract class ModelManagementApiTestBase { - private static WebApplicationFactory Factory() => - new WebApplicationFactory().WithWebHostBuilder(builder => - { - builder.UseEnvironment("Testing"); - builder.UseSetting("ConnectionStrings:ollama-extension", "Endpoint=http://127.0.0.1:1"); - builder.UseSetting("Logging:LogLevel:Default", "Warning"); - builder.ConfigureServices(services => - { - services.RemoveAll(); - services.RemoveAll(); - services.RemoveAll(); - services.AddSingleton(); - services.AddSingleton(provider => provider.GetRequiredService()); - services.AddSingleton(provider => provider.GetRequiredService()); - services.AddSingleton(provider => provider.GetRequiredService()); - }); - }); + protected static WebApplicationFactory Factory() => new ManagementApiTestFactory(); - private static WebApplicationFactory DiagnosticFactory() => Factory().WithWebHostBuilder(builder => + protected static WebApplicationFactory DiagnosticFactory() => Factory().WithWebHostBuilder(builder => builder.ConfigureServices(services => { services.RemoveAll(); @@ -45,7 +28,7 @@ private static WebApplicationFactory DiagnosticFactory() => Factory().W services.AddSingleton(); })); - private static Task GetBootstrapContextAsync(WebApplicationFactory factory) => + protected static Task GetBootstrapContextAsync(WebApplicationFactory factory) => factory.Services .GetRequiredService() .EnsureInitializedAsync(default); @@ -123,7 +106,7 @@ public ValueTask InspectAsync( UnavailableDetails)); } - private sealed class ConfiguredEndpointInspector : IExtensionInspector + protected sealed class ConfiguredEndpointInspector : IExtensionInspector { public bool CanHandle(string providerType) => true; public bool CanInspectEndpoint(Uri endpoint) => true; @@ -144,7 +127,7 @@ [new ExtensionContribution("model-provider", "discovered")], [])); } - private sealed class MigrationExtensionAdapter : IExtensionInspector, IExtensionOptionsMigrator + protected sealed class MigrationExtensionAdapter : IExtensionInspector, IExtensionOptionsMigrator { private static readonly JsonElement SourceSchema = JsonSerializer.SerializeToElement(new { @@ -202,7 +185,7 @@ public ValueTask MigrateAsync( } } - private static CreateModelProfileRequest Request(string name, string model) => new( + protected static CreateModelProfileRequest Request(string name, string model) => new( name, new ModelProfileProperties { @@ -212,4 +195,33 @@ public ValueTask MigrateAsync( Model = new ModelSelection { Name = model }, Generation = new ModelGenerationOptions { Temperature = 0.3, MaxOutputTokens = 512 } }); + + private sealed class ManagementApiTestFactory : ApiOnlyWebApplicationFactory + { + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + base.ConfigureWebHost(builder); + builder.UseSetting("ConnectionStrings:ollama-extension", "Endpoint=http://127.0.0.1:1"); + builder.ConfigureServices(services => + { + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.AddSingleton(); + services.AddSingleton(provider => provider.GetRequiredService()); + services.AddSingleton(provider => provider.GetRequiredService()); + services.AddSingleton(provider => provider.GetRequiredService()); + }); + } + } +} + +internal class ApiOnlyWebApplicationFactory : WebApplicationFactory +{ + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment("Testing"); + builder.UseSetting("Agentstration:Testing:ApiOnly", "true"); + builder.UseSetting("Logging:LogLevel:Default", "Warning"); + } } diff --git a/tests/Agentstration.Management.Tests/ModelManagementApiTests.ProfilesAndResolution.cs b/tests/Agentstration.Management.Api.Tests/ModelProfileApiTests.cs similarity index 99% rename from tests/Agentstration.Management.Tests/ModelManagementApiTests.ProfilesAndResolution.cs rename to tests/Agentstration.Management.Api.Tests/ModelProfileApiTests.cs index 3a19b3b8..b4fc16a6 100644 --- a/tests/Agentstration.Management.Tests/ModelManagementApiTests.ProfilesAndResolution.cs +++ b/tests/Agentstration.Management.Api.Tests/ModelProfileApiTests.cs @@ -15,12 +15,13 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class ModelProfileApiTests : ModelManagementApiTestBase { [TestMethod] public async Task ManagedHostCompositionUsesPersistedModelProfileResolver() { - await using var factory = new WebApplicationFactory().WithWebHostBuilder(builder => + await using var factory = new ApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("AI:Provider", "Managed"); diff --git a/tests/Agentstration.Management.Tests/ModelManagementApiTests.Providers.cs b/tests/Agentstration.Management.Api.Tests/ModelProviderApiTests.cs similarity index 97% rename from tests/Agentstration.Management.Tests/ModelManagementApiTests.Providers.cs rename to tests/Agentstration.Management.Api.Tests/ModelProviderApiTests.cs index 35fa7617..412b1a02 100644 --- a/tests/Agentstration.Management.Tests/ModelManagementApiTests.Providers.cs +++ b/tests/Agentstration.Management.Api.Tests/ModelProviderApiTests.cs @@ -15,7 +15,8 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class ModelProviderApiTests : ModelManagementApiTestBase { [TestMethod] public async Task ReadOnlyProviderApisExposeConfiguredProviderAndUnavailableDiscovery() diff --git a/tests/Agentstration.Management.Tests/NotificationDeliveryTests.cs b/tests/Agentstration.Management.Api.Tests/NotificationDeliveryApiTests.cs similarity index 99% rename from tests/Agentstration.Management.Tests/NotificationDeliveryTests.cs rename to tests/Agentstration.Management.Api.Tests/NotificationDeliveryApiTests.cs index 7d632fad..fd4e5585 100644 --- a/tests/Agentstration.Management.Tests/NotificationDeliveryTests.cs +++ b/tests/Agentstration.Management.Api.Tests/NotificationDeliveryApiTests.cs @@ -16,7 +16,8 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class NotificationDeliveryApiTests : ModelManagementApiTestBase { [TestMethod] public void NotificationDeliverySamplesUseOnlyGenericFlowAndToolSteps() diff --git a/tests/Agentstration.Management.Tests/ModelManagementApiTests.ResourceScopes.cs b/tests/Agentstration.Management.Api.Tests/ResourceScopeApiTests.cs similarity index 98% rename from tests/Agentstration.Management.Tests/ModelManagementApiTests.ResourceScopes.cs rename to tests/Agentstration.Management.Api.Tests/ResourceScopeApiTests.cs index 301bbbe2..58b260cf 100644 --- a/tests/Agentstration.Management.Tests/ModelManagementApiTests.ResourceScopes.cs +++ b/tests/Agentstration.Management.Api.Tests/ResourceScopeApiTests.cs @@ -7,7 +7,8 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class ResourceScopeApiTests : ModelManagementApiTestBase { [TestMethod] public void ResourceScopePolicyMatchesTheInitialOwnershipModel() diff --git a/tests/Agentstration.Management.Tests/ModelManagementApiTests.RuntimeAndTools.cs b/tests/Agentstration.Management.Api.Tests/RuntimeProfileAndToolApiTests.cs similarity index 99% rename from tests/Agentstration.Management.Tests/ModelManagementApiTests.RuntimeAndTools.cs rename to tests/Agentstration.Management.Api.Tests/RuntimeProfileAndToolApiTests.cs index 41ce8524..daf26186 100644 --- a/tests/Agentstration.Management.Tests/ModelManagementApiTests.RuntimeAndTools.cs +++ b/tests/Agentstration.Management.Api.Tests/RuntimeProfileAndToolApiTests.cs @@ -15,7 +15,8 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class RuntimeProfileAndToolApiTests : ModelManagementApiTestBase { [TestMethod] public async Task RuntimeProfileIsPersistedAsAnIndependentManagementResource() diff --git a/tests/Agentstration.Management.Tests/ToolDefinitionTests.cs b/tests/Agentstration.Management.Api.Tests/ToolDefinitionApiTests.cs similarity index 99% rename from tests/Agentstration.Management.Tests/ToolDefinitionTests.cs rename to tests/Agentstration.Management.Api.Tests/ToolDefinitionApiTests.cs index 5b5846b6..a458f72f 100644 --- a/tests/Agentstration.Management.Tests/ToolDefinitionTests.cs +++ b/tests/Agentstration.Management.Api.Tests/ToolDefinitionApiTests.cs @@ -15,7 +15,8 @@ namespace Agentstration.Management.Tests; -public sealed partial class ModelManagementApiTests +[TestClass] +public sealed class ToolDefinitionApiTests : ModelManagementApiTestBase { [TestMethod] public async Task ToolDefinitionCrudMaterializesInternalProviderAndGovernedTool() diff --git a/tests/Agentstration.Management.Security.Tests/Agentstration.Management.Security.Tests.csproj b/tests/Agentstration.Management.Security.Tests/Agentstration.Management.Security.Tests.csproj new file mode 100644 index 00000000..29044df4 --- /dev/null +++ b/tests/Agentstration.Management.Security.Tests/Agentstration.Management.Security.Tests.csproj @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/tests/Agentstration.Management.Security.Tests/AssemblyInfo.cs b/tests/Agentstration.Management.Security.Tests/AssemblyInfo.cs new file mode 100644 index 00000000..c53e6b4a --- /dev/null +++ b/tests/Agentstration.Management.Security.Tests/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: Parallelize(Workers = 1, Scope = ExecutionScope.ClassLevel)] diff --git a/tests/Agentstration.Management.Tests/IdentityFoundationTests.cs b/tests/Agentstration.Management.Security.Tests/IdentityFoundationTests.cs similarity index 100% rename from tests/Agentstration.Management.Tests/IdentityFoundationTests.cs rename to tests/Agentstration.Management.Security.Tests/IdentityFoundationTests.cs diff --git a/tests/Agentstration.Management.Tests/SecurityApiTests.Authorization.cs b/tests/Agentstration.Management.Security.Tests/SecurityApiTests.Authorization.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SecurityApiTests.Authorization.cs rename to tests/Agentstration.Management.Security.Tests/SecurityApiTests.Authorization.cs diff --git a/tests/Agentstration.Management.Tests/SecurityApiTests.BoundariesAndTokens.cs b/tests/Agentstration.Management.Security.Tests/SecurityApiTests.BoundariesAndTokens.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SecurityApiTests.BoundariesAndTokens.cs rename to tests/Agentstration.Management.Security.Tests/SecurityApiTests.BoundariesAndTokens.cs diff --git a/tests/Agentstration.Management.Tests/SecurityApiTests.LocalAccounts.cs b/tests/Agentstration.Management.Security.Tests/SecurityApiTests.LocalAccounts.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SecurityApiTests.LocalAccounts.cs rename to tests/Agentstration.Management.Security.Tests/SecurityApiTests.LocalAccounts.cs diff --git a/tests/Agentstration.Management.Tests/SecurityApiTests.PlatformAndMembership.cs b/tests/Agentstration.Management.Security.Tests/SecurityApiTests.PlatformAndMembership.cs similarity index 100% rename from tests/Agentstration.Management.Tests/SecurityApiTests.PlatformAndMembership.cs rename to tests/Agentstration.Management.Security.Tests/SecurityApiTests.PlatformAndMembership.cs diff --git a/tests/Agentstration.Management.Tests/SecurityApiTests.cs b/tests/Agentstration.Management.Security.Tests/SecurityApiTests.cs similarity index 94% rename from tests/Agentstration.Management.Tests/SecurityApiTests.cs rename to tests/Agentstration.Management.Security.Tests/SecurityApiTests.cs index 9cd97025..4320ddfc 100644 --- a/tests/Agentstration.Management.Tests/SecurityApiTests.cs +++ b/tests/Agentstration.Management.Security.Tests/SecurityApiTests.cs @@ -23,16 +23,18 @@ public sealed partial class SecurityApiTests private const string LocalPassword = "A-strong-local-password-42!"; private const string ChangedLocalPassword = "A-changed-local-password-84!"; - private static WebApplicationFactory Factory(string mode) + private static WebApplicationFactory Factory(string mode) => new SecurityWebApplicationFactory(mode); + + private sealed class SecurityWebApplicationFactory(string mode) : WebApplicationFactory { - WebApplicationFactory? factory = null; - factory = new WebApplicationFactory().WithWebHostBuilder(builder => + protected override void ConfigureWebHost(IWebHostBuilder builder) { builder.UseEnvironment("Testing"); builder.UseSetting("Agentstration:Authentication:Mode", mode); + builder.UseSetting("Logging:LogLevel:Default", "Warning"); builder.ConfigureTestServices(services => { - Func handlerFactory = () => factory!.Server.CreateHandler(); + Func handlerFactory = () => Server.CreateHandler(); RouteThroughTestServer(services, handlerFactory); RouteThroughTestServer(services, handlerFactory); RouteThroughTestServer(services, handlerFactory); @@ -40,8 +42,7 @@ private static WebApplicationFactory Factory(string mode) RouteThroughTestServer(services, handlerFactory); RouteThroughTestServer(services, handlerFactory); }); - }); - return factory; + } } private static void RouteThroughTestServer( diff --git a/tests/Agentstration.Management.Tests/AssemblyInfo.cs b/tests/Agentstration.Management.Tests/AssemblyInfo.cs deleted file mode 100644 index 9dc8a785..00000000 --- a/tests/Agentstration.Management.Tests/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; - -[assembly: Parallelize(Workers = 4, Scope = ExecutionScope.ClassLevel)] From 3897eaa024ba4879a0509ab824784400648f577c Mon Sep 17 00:00:00 2001 From: gbaudrit Date: Fri, 11 Sep 2026 12:36:45 +0200 Subject: [PATCH 5/9] test: isolate storage performance workloads (#312) --- .github/workflows/ci.yml | 25 ++++++- .github/workflows/storage-performance.yml | 68 +++++++++++++++++++ AGENTS.md | 1 + Agentstration.Tests.Performance.slnx | 2 +- Agentstration.slnx | 1 + docs/architecture.md | 1 + docs/contributing/testing.md | 7 +- ...ts.cs => FlowStorageQueryContractTests.cs} | 0 .../Agentstration.Performance.Tests.csproj | 6 ++ .../AssemblyInfo.cs | 3 + .../StorageConcurrencyBenchmarkTests.cs | 13 +++- .../StorageConcurrencySmokeTests.cs | 53 +++++++++++++++ 12 files changed, 170 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/storage-performance.yml rename tests/Agentstration.Application.Tests/{FlowStoragePerformanceTests.cs => FlowStorageQueryContractTests.cs} (100%) create mode 100644 tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj create mode 100644 tests/Agentstration.Performance.Tests/AssemblyInfo.cs rename tests/{Agentstration.Web.Tests => Agentstration.Performance.Tests}/StorageConcurrencyBenchmarkTests.cs (93%) create mode 100644 tests/Agentstration.Web.Tests/StorageConcurrencySmokeTests.cs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8fe722f5..afbcfcee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,7 @@ jobs: dotnet: ${{ steps.classify.outputs.dotnet }} aep: ${{ steps.classify.outputs.aep }} container: ${{ steps.classify.outputs.container }} + storage: ${{ steps.classify.outputs.storage }} steps: - name: Check out repository uses: actions/checkout@v7 @@ -62,15 +63,20 @@ jobs: $container = $runEverything -or @($changedFiles | Where-Object { $_ -match '^(src/|aep/src/|deploy/|Dockerfile$|\.dockerignore$|global\.json$|Agentstration\.slnx$|Directory\.(Build|Packages)\.(props|targets)$|\.github/workflows/ci\.yml$)' }).Count -gt 0 + $storage = $runEverything -or @($changedFiles | Where-Object { + $_ -match '^(src/Agentstration\.(Application|Flow|Infrastructure|Management|Runtime|Web|Work)|tests/Agentstration\.(Application|Performance|Web)\.Tests/|Agentstration\.Tests\.Performance\.slnx$|Directory\.(Build|Packages)\.(props|targets)$|global\.json$|\.github/workflows/(ci|storage-performance)\.yml$)' + }).Count -gt 0 "dotnet=$($dotnet.ToString().ToLowerInvariant())" >> $env:GITHUB_OUTPUT "aep=$($aep.ToString().ToLowerInvariant())" >> $env:GITHUB_OUTPUT "container=$($container.ToString().ToLowerInvariant())" >> $env:GITHUB_OUTPUT + "storage=$($storage.ToString().ToLowerInvariant())" >> $env:GITHUB_OUTPUT Write-Host "Changed files: $($changedFiles.Count)" Write-Host "Run .NET validation: $dotnet" Write-Host "Run complete AEP validation: $aep" Write-Host "Build container: $container" + Write-Host "Run storage smoke: $storage" windows-host-lifecycle: name: windows-host-lifecycle @@ -160,6 +166,8 @@ jobs: postgresql-storage: name: postgresql-storage runs-on: ubuntu-latest + needs: changes + if: needs.changes.outputs.storage == 'true' timeout-minutes: 20 services: postgres: @@ -185,9 +193,13 @@ jobs: with: global-json-file: global.json - name: Restore PostgreSQL integration tests - run: dotnet restore tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj -p:NuGetAudit=true -p:NuGetAuditMode=all + run: | + dotnet restore tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj -p:NuGetAudit=true -p:NuGetAuditMode=all + dotnet restore tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj -p:NuGetAudit=true -p:NuGetAuditMode=all - name: Build PostgreSQL integration tests - run: dotnet build tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj --configuration Release --no-restore + run: | + dotnet build tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj --configuration Release --no-restore + dotnet build tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj --configuration Release --no-restore - name: Run PostgreSQL migration and restart test run: dotnet tests/Agentstration.Web.Tests/bin/Release/net10.0/Agentstration.Web.Tests.dll --filter "Name=EmptyDatabaseMigratesAndRemainsReadyAfterRestart" --minimum-expected-tests 1 --progress off - name: Run PostgreSQL storage concurrency benchmark @@ -197,8 +209,15 @@ jobs: AGENTSTRATION_STORAGE_BENCHMARK_CONCURRENCY: 8 AGENTSTRATION_STORAGE_BENCHMARK_REPORT: ${{ runner.temp }}/agentstration-storage-benchmark-postgresql.json run: | - dotnet tests/Agentstration.Web.Tests/bin/Release/net10.0/Agentstration.Web.Tests.dll --filter "Name=ReportsConcurrentRelationalWriteMetrics" --minimum-expected-tests 1 --progress off + dotnet tests/Agentstration.Performance.Tests/bin/Release/net10.0/Agentstration.Performance.Tests.dll --filter "Name=ReportsConcurrentRelationalWriteMetrics" --minimum-expected-tests 1 --progress off cat "$AGENTSTRATION_STORAGE_BENCHMARK_REPORT" + - name: Upload PostgreSQL storage smoke report + if: always() + uses: actions/upload-artifact@v6 + with: + name: storage-benchmark-postgresql-smoke + path: ${{ runner.temp }}/agentstration-storage-benchmark-postgresql.json + if-no-files-found: error container: name: container diff --git a/.github/workflows/storage-performance.yml b/.github/workflows/storage-performance.yml new file mode 100644 index 00000000..c509453c --- /dev/null +++ b/.github/workflows/storage-performance.yml @@ -0,0 +1,68 @@ +name: Storage performance + +on: + workflow_dispatch: + schedule: + - cron: "23 3 * * 1" + +permissions: + contents: read + +concurrency: + group: storage-performance-${{ github.ref }} + cancel-in-progress: false + +jobs: + benchmark: + name: ${{ matrix.provider }} full workload + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + provider: [Sqlite, PostgreSql] + services: + postgres: + image: postgres:17 + env: + POSTGRES_DB: agentstration + POSTGRES_USER: agentstration + POSTGRES_PASSWORD: agentstration-ci-only + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U agentstration -d agentstration" + --health-interval 5s + --health-timeout 5s + --health-retries 12 + env: + AGENTSTRATION_TEST_POSTGRES: Host=127.0.0.1;Port=5432;Database=agentstration;Username=agentstration;Password=agentstration-ci-only + AGENTSTRATION_STORAGE_BENCHMARK_PROVIDER: ${{ matrix.provider }} + AGENTSTRATION_STORAGE_BENCHMARK_OPERATIONS: 500 + AGENTSTRATION_STORAGE_BENCHMARK_CONCURRENCY: 16 + AGENTSTRATION_STORAGE_BENCHMARK_REPORT: ${{ runner.temp }}/storage-benchmark-${{ matrix.provider }}.json + steps: + - name: Check out repository + uses: actions/checkout@v7 + - name: Set up .NET + uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + - name: Restore performance tests + run: dotnet restore tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj -p:NuGetAudit=true -p:NuGetAuditMode=all + - name: Build performance tests + run: dotnet build tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj --configuration Release --no-restore + - name: Run full storage workload + run: dotnet tests/Agentstration.Performance.Tests/bin/Release/net10.0/Agentstration.Performance.Tests.dll --filter "Name=ReportsConcurrentRelationalWriteMetrics" --minimum-expected-tests 1 --progress off + - name: Display report + if: always() + shell: pwsh + run: if (Test-Path -LiteralPath $env:AGENTSTRATION_STORAGE_BENCHMARK_REPORT) { Get-Content -LiteralPath $env:AGENTSTRATION_STORAGE_BENCHMARK_REPORT } + - name: Upload storage report + if: always() + uses: actions/upload-artifact@v6 + with: + name: storage-benchmark-${{ matrix.provider }}-full + path: ${{ runner.temp }}/storage-benchmark-${{ matrix.provider }}.json + if-no-files-found: error diff --git a/AGENTS.md b/AGENTS.md index 5c2cd86a..831007d9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,6 +54,7 @@ tests/ Agentstration.Management.Api.Tests/ Agentstration.Management.Security.Tests/ Agentstration.Management.Aep.Tests/ + Agentstration.Performance.Tests/ docs/ architecture.md decisions/ diff --git a/Agentstration.Tests.Performance.slnx b/Agentstration.Tests.Performance.slnx index 82b5e516..e9b6a9af 100644 --- a/Agentstration.Tests.Performance.slnx +++ b/Agentstration.Tests.Performance.slnx @@ -1,5 +1,5 @@ - + diff --git a/Agentstration.slnx b/Agentstration.slnx index 7d7cb154..f2672806 100644 --- a/Agentstration.slnx +++ b/Agentstration.slnx @@ -62,6 +62,7 @@ + diff --git a/docs/architecture.md b/docs/architecture.md index 55199bfc..75c191f6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -52,6 +52,7 @@ tests/ Agentstration.Management.Api.Tests/ Agentstration.Management.Security.Tests/ Agentstration.Management.Aep.Tests/ + Agentstration.Performance.Tests/ Agentstration.Web.Tests/ Agentstration.Web.Components.Tests/ Agentstration.Web.FlowDesigner.Tests/ diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index 15228e86..0660633d 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -22,7 +22,7 @@ Run both fast and integration solutions for complete required functional validat ## Performance lane -Performance workloads are selected explicitly and are never part of the fast lane: +Performance workloads live in a dedicated project and are never discovered by the fast or integration lanes: ```powershell $env:AGENTSTRATION_STORAGE_BENCHMARK_PROVIDER = "Sqlite" @@ -30,7 +30,7 @@ $env:AGENTSTRATION_STORAGE_BENCHMARK_REPORT = "artifacts/storage-benchmark-sqlit dotnet test --solution Agentstration.Tests.Performance.slnx --configuration Release --no-build --filter TestCategory=Benchmark --minimum-expected-tests 1 ``` -The current Web storage workload remains a mixed-project boundary until #303 moves it to a dedicated performance project. PostgreSQL is optional and runs only in its service-backed CI job or an explicitly configured local environment. +The report records workload parameters, provider, elapsed time, runtime and OS metadata, throughput, median, p95, errors, conflicts, and retries. SQLite remains the local default; set `AGENTSTRATION_TEST_POSTGRES` for an opt-in PostgreSQL run. Pull requests execute only a bounded contention smoke for relevant storage paths. Full workloads run from the scheduled or manual `Storage performance` workflow and publish their JSON reports. Latency and throughput gates remain disabled until representative baselines have been collected; zero storage errors is always required. ## Project classification @@ -51,7 +51,8 @@ The current Web storage workload remains a mixed-project boundary until #303 mov | `Agentstration.ModelProviders.Tests` | Integration, provider-optional | AEP test hosts plus opt-in live-provider checks | | `Agentstration.Runtime.Tests` | Integration | SQLite reconstruction and hosted runtime endpoints | | `Agentstration.SourceProviders.Git.Tests` | Integration | Real Git processes and file-system repositories | -| `Agentstration.Web.Tests` | Integration, mixed | Full Web host plus the temporary benchmark boundary tracked by #303 | +| `Agentstration.Web.Tests` | Integration | Full Web host plus a bounded deterministic SQLite contention correctness smoke | +| `Agentstration.Performance.Tests` | Performance, opt-in | SQLite/PostgreSQL relational storage concurrency workloads and machine-readable reports | | `Agentstration.Work.Api.Tests` | Integration | Full Work API host | | `Agentstration.Workplace.Web.Tests` | Integration | Workplace HTTP host | diff --git a/tests/Agentstration.Application.Tests/FlowStoragePerformanceTests.cs b/tests/Agentstration.Application.Tests/FlowStorageQueryContractTests.cs similarity index 100% rename from tests/Agentstration.Application.Tests/FlowStoragePerformanceTests.cs rename to tests/Agentstration.Application.Tests/FlowStorageQueryContractTests.cs diff --git a/tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj b/tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj new file mode 100644 index 00000000..d51864ee --- /dev/null +++ b/tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/Agentstration.Performance.Tests/AssemblyInfo.cs b/tests/Agentstration.Performance.Tests/AssemblyInfo.cs new file mode 100644 index 00000000..c53e6b4a --- /dev/null +++ b/tests/Agentstration.Performance.Tests/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: Parallelize(Workers = 1, Scope = ExecutionScope.ClassLevel)] diff --git a/tests/Agentstration.Web.Tests/StorageConcurrencyBenchmarkTests.cs b/tests/Agentstration.Performance.Tests/StorageConcurrencyBenchmarkTests.cs similarity index 93% rename from tests/Agentstration.Web.Tests/StorageConcurrencyBenchmarkTests.cs rename to tests/Agentstration.Performance.Tests/StorageConcurrencyBenchmarkTests.cs index 8775c816..94cd2883 100644 --- a/tests/Agentstration.Web.Tests/StorageConcurrencyBenchmarkTests.cs +++ b/tests/Agentstration.Performance.Tests/StorageConcurrencyBenchmarkTests.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using System.Diagnostics; using System.Net; +using System.Runtime.InteropServices; using System.Text.Json; using Agentstration.Flow; using Agentstration.Flow.Storage.Abstractions; @@ -13,7 +14,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -namespace Agentstration.Web.Tests; +namespace Agentstration.Performance.Tests; [TestClass] public sealed class StorageConcurrencyBenchmarkTests @@ -73,8 +74,8 @@ await Task.WhenAll(Enumerable.Range(0, operations).Select(async index => var workItem = WorkItem.Create(WorkItemId.New(), workspaceId, "benchmark", $"Operation {index}", now); var storedWorkItem = await workItems.CreateAsync(workItem, default); var expectedVersion = storedWorkItem.Value.Version; - workItem.AddMessage("update", "benchmark", Guid.NewGuid(), now.AddMilliseconds(1)); - await workItems.SaveAsync(workItem, expectedVersion, default); + storedWorkItem.Value.AddMessage("update", "benchmark", Guid.NewGuid(), now.AddMilliseconds(1)); + await workItems.SaveAsync(storedWorkItem.Value, expectedVersion, default); var flowId = new FlowId("benchmark-flow"); var flowRunId = $"flow-{Guid.NewGuid():N}"; @@ -148,6 +149,12 @@ await checkpoints.StoreAsync(new RuntimeExecutionState( provider, operations, concurrency, + writesPerOperation = 7, + elapsedMilliseconds = elapsed.TotalMilliseconds, + runtime = RuntimeInformation.FrameworkDescription, + runtimeVersion = Environment.Version.ToString(), + os = RuntimeInformation.OSDescription, + processArchitecture = RuntimeInformation.ProcessArchitecture.ToString(), throughputPerSecond = operations / elapsed.TotalSeconds, medianMilliseconds = Percentile(ordered, 0.50), p95Milliseconds = Percentile(ordered, 0.95), diff --git a/tests/Agentstration.Web.Tests/StorageConcurrencySmokeTests.cs b/tests/Agentstration.Web.Tests/StorageConcurrencySmokeTests.cs new file mode 100644 index 00000000..7f68a735 --- /dev/null +++ b/tests/Agentstration.Web.Tests/StorageConcurrencySmokeTests.cs @@ -0,0 +1,53 @@ +using Agentstration.Resources; +using Agentstration.Work; +using Agentstration.Work.Storage.Abstractions; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Agentstration.Web.Tests; + +[TestClass] +public sealed class StorageConcurrencySmokeTests +{ + [TestMethod] + public async Task SqliteAcceptsBoundedConcurrentWorkItemWritesWithoutErrors() + { + var dataDirectory = Path.Combine(Path.GetTempPath(), $"agentstration-storage-smoke-{Guid.NewGuid():N}"); + try + { + await using var host = new WebApplicationFactory().WithWebHostBuilder(builder => + { + builder.UseEnvironment("Testing"); + builder.UseSetting("Agentstration:Testing:ApiOnly", "true"); + builder.UseSetting("Agentstration:Storage:Provider", "Sqlite"); + builder.UseSetting("Data:Directory", dataDirectory); + builder.UseSetting("Agentstration:Bootstrap:InitialBootstrapEnabled", "false"); + builder.ConfigureLogging(logging => logging.ClearProviders()); + }); + _ = host.CreateClient(); + var repository = host.Services.GetRequiredService(); + var workspaceId = new WorkspaceId(Guid.NewGuid()); + + var stored = await Task.WhenAll(Enumerable.Range(0, 8).Select(async index => + { + var id = WorkItemId.New(); + var now = DateTimeOffset.UtcNow; + var item = WorkItem.Create(id, workspaceId, "content", $"Smoke {index}", now); + var created = await repository.CreateAsync(item, default); + var expectedVersion = created.Value.Version; + created.Value.AddMessage("update", "smoke", Guid.NewGuid(), now.AddMilliseconds(1)); + await repository.SaveAsync(created.Value, expectedVersion, default); + return await repository.GetAsync(workspaceId, id, default); + })); + + Assert.HasCount(8, stored); + Assert.IsTrue(stored.All(item => item is not null && item.Value.Messages.Count == 1)); + } + finally + { + if (Directory.Exists(dataDirectory)) Directory.Delete(dataDirectory, true); + } + } +} From 0d1c7c03de149fa381d51869b81f8287212e548d Mon Sep 17 00:00:00 2001 From: gbaudrit Date: Fri, 11 Sep 2026 12:47:41 +0200 Subject: [PATCH 6/9] ci: bound test parallelism and report memory (#334) --- .github/workflows/ci.yml | 37 ++++- .github/workflows/release.yml | 8 +- docs/contributing/testing.md | 19 ++- docs/getting-started/configuration.md | 6 +- .../ci/run-test-module-with-diagnostics.ps1 | 128 ++++++++++++++++++ 5 files changed, 186 insertions(+), 12 deletions(-) create mode 100644 scripts/ci/run-test-module-with-diagnostics.ps1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afbcfcee..59e60d48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,10 +147,41 @@ jobs: run: dotnet build Agentstration.slnx --configuration Release --no-restore - name: Test fast lane if: needs.changes.outputs.dotnet == 'true' - run: dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 1 + run: dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 139 --max-parallel-test-modules 4 - name: Test integration lane if: needs.changes.outputs.dotnet == 'true' - run: dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 1 + run: dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 692 --max-parallel-test-modules 2 + - name: Capture hosted module diagnostics + if: needs.changes.outputs.dotnet == 'true' + shell: pwsh + run: | + @( + '## Hosted test module diagnostics', + '', + '| Module | Tests | Duration (s) | Peak working set (MiB) | Peak private memory (MiB) | Aggregate dotnet working set (MiB) | Status |', + '| --- | ---: | ---: | ---: | ---: | ---: | --- |' + ) | Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Encoding utf8 + $modules = @( + @{ Name = 'Agentstration.Management.Api.Tests'; Tests = 66; Warning = 900; Failure = 1024 }, + @{ Name = 'Agentstration.Management.Security.Tests'; Tests = 38; Warning = 800; Failure = 1024 }, + @{ Name = 'Agentstration.Management.Aep.Tests'; Tests = 11; Warning = 450; Failure = 700 }, + @{ Name = 'Agentstration.Management.Sources.Tests'; Tests = 93; Warning = 550; Failure = 900 } + ) + foreach ($module in $modules) { + ./scripts/ci/run-test-module-with-diagnostics.ps1 ` + -AssemblyPath "tests/$($module.Name)/bin/Release/net10.0/$($module.Name).dll" ` + -ReportPath "artifacts/test-diagnostics/$($module.Name).json" ` + -MinimumExpectedTests $module.Tests ` + -WarningWorkingSetMiB $module.Warning ` + -FailureWorkingSetMiB $module.Failure + } + - name: Upload hosted module diagnostics + if: always() && needs.changes.outputs.dotnet == 'true' + uses: actions/upload-artifact@v6 + with: + name: hosted-test-module-diagnostics + path: artifacts/test-diagnostics/*.json + if-no-files-found: warn - name: Smoke-test Source Registry tool package if: needs.changes.outputs.dotnet == 'true' shell: pwsh @@ -161,7 +192,7 @@ jobs: - name: Test complete AEP SDK if: needs.changes.outputs.dotnet == 'true' && needs.changes.outputs.aep == 'true' working-directory: aep - run: dotnet test --solution Aep.slnx --configuration Release --no-build --minimum-expected-tests 1 + run: dotnet test --solution Aep.slnx --configuration Release --no-build --minimum-expected-tests 33 --max-parallel-test-modules 2 postgresql-storage: name: postgresql-storage diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 15c9a02c..7c683dec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,15 +54,17 @@ jobs: run: dotnet restore Agentstration.slnx -p:NuGetAudit=true -p:NuGetAuditMode=all - name: Build run: dotnet build Agentstration.slnx --configuration Release --no-restore - - name: Test - run: dotnet test --solution Agentstration.slnx --configuration Release --no-build --minimum-expected-tests 1 + - name: Test fast lane + run: dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 139 --max-parallel-test-modules 4 + - name: Test integration lane + run: dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 692 --max-parallel-test-modules 2 - name: Restore autonomous AEP SDK run: dotnet restore aep/Aep.slnx -p:NuGetAudit=true -p:NuGetAuditMode=all - name: Build autonomous AEP SDK run: dotnet build aep/Aep.slnx --configuration Release --no-restore - name: Test autonomous AEP SDK working-directory: aep - run: dotnet test --solution Aep.slnx --configuration Release --no-build --minimum-expected-tests 1 + run: dotnet test --solution Aep.slnx --configuration Release --no-build --minimum-expected-tests 33 --max-parallel-test-modules 2 - name: Publish framework-dependent applications shell: pwsh run: | diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index 0660633d..51890a5b 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -7,7 +7,7 @@ Agentstration separates test execution by runtime cost and dependency type. The Fast tests do not start an ASP.NET Core host, open a relational database, spawn a child process, or contact a remote provider. They cover architecture rules, component behavior, mapping, validation, and in-process command behavior. ```powershell -dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 1 +dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 139 --max-parallel-test-modules 4 ``` ## Integration lane @@ -15,7 +15,7 @@ dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --n Integration tests exercise a real boundary such as `WebApplicationFactory`, SQLite, Git, persistent Identity, or runtime reconstruction. This lane remains deterministic and offline by default. Tests marked `Integration` for a live model provider are opt-in and report inconclusive unless their documented environment variables are supplied. ```powershell -dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 1 +dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 692 --max-parallel-test-modules 2 ``` Run both fast and integration solutions for complete required functional validation. Their union is the functional test inventory represented by the root solution. @@ -27,11 +27,24 @@ Performance workloads live in a dedicated project and are never discovered by th ```powershell $env:AGENTSTRATION_STORAGE_BENCHMARK_PROVIDER = "Sqlite" $env:AGENTSTRATION_STORAGE_BENCHMARK_REPORT = "artifacts/storage-benchmark-sqlite.json" -dotnet test --solution Agentstration.Tests.Performance.slnx --configuration Release --no-build --filter TestCategory=Benchmark --minimum-expected-tests 1 +dotnet test --solution Agentstration.Tests.Performance.slnx --configuration Release --no-build --filter TestCategory=Benchmark --minimum-expected-tests 1 --max-parallel-test-modules 1 ``` The report records workload parameters, provider, elapsed time, runtime and OS metadata, throughput, median, p95, errors, conflicts, and retries. SQLite remains the local default; set `AGENTSTRATION_TEST_POSTGRES` for an opt-in PostgreSQL run. Pull requests execute only a bounded contention smoke for relevant storage paths. Full workloads run from the scheduled or manual `Storage performance` workflow and publish their JSON reports. Latency and throughput gates remain disabled until representative baselines have been collected; zero storage errors is always required. +## CI concurrency and memory diagnostics + +The fast, integration, and performance lanes cap concurrent test modules at 4, 2, and 1 respectively. Host-heavy Management modules also use one class worker per assembly. CI reruns the designated hosted modules sequentially through `scripts/ci/run-test-module-with-diagnostics.ps1`; each JSON artifact contains the discovered count, duration, process peak working set, process peak private memory, aggregate peak working set for active `dotnet` processes, runtime, and OS. The diagnostic artifact deliberately excludes test output and payloads. + +The initial budgets below use Release runs on Windows 11 10.0.26200 with .NET 10.0.10/10.0.11, collected during #298. A warning is evidence to review the Linux and Windows trend; a failure protects constrained runners from returning to the original greater-than-1-GiB process. Adjust these values only after retaining representative artifacts from both runner families. + +| Module | Baseline peak (MiB) | Warning (MiB) | Failure (MiB) | Minimum tests | +| --- | ---: | ---: | ---: | ---: | +| `Agentstration.Management.Api.Tests` | 887.0 | 900 | 1024 | 66 | +| `Agentstration.Management.Security.Tests` | 730.1 | 800 | 1024 | 38 | +| `Agentstration.Management.Aep.Tests` | 322.1 | 450 | 700 | 11 | +| `Agentstration.Management.Sources.Tests` | 431.5 | 550 | 900 | 93 | + ## Project classification | Test project | Lane | Boundary or rationale | diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index 4006ab2f..c203cb67 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -149,18 +149,18 @@ The Workplace depends on the Console API. If the Console is running but the Work ## Storage concurrency benchmark -The opt-in benchmark uses the same workload for both providers: each operation creates and updates a Work Item, appends one Flow event, appends one Runtime event, and stores a Runtime checkpoint. It reports throughput, median and p95 latency, errors, concurrency conflicts, and retries. It has no pass/fail timing threshold and is skipped by the standard test suite. +The opt-in benchmark uses the same workload for both providers: each operation creates and updates a Work Item, appends one Flow event, appends one Runtime event, and stores a Runtime checkpoint. It reports workload and environment metadata, throughput, median and p95 latency, errors, concurrency conflicts, and retries. It has no pass/fail timing threshold and is absent from the functional test lanes. ```powershell $env:AGENTSTRATION_STORAGE_BENCHMARK_PROVIDER = "Sqlite" $env:AGENTSTRATION_STORAGE_BENCHMARK_OPERATIONS = "100" $env:AGENTSTRATION_STORAGE_BENCHMARK_CONCURRENCY = "8" $env:AGENTSTRATION_STORAGE_BENCHMARK_REPORT = "$env:TEMP\agentstration-storage-benchmark.json" -dotnet test --project tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj --configuration Release --filter "Name=ReportsConcurrentRelationalWriteMetrics" --minimum-expected-tests 1 --logger "console;verbosity=detailed" +dotnet test --project tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj --configuration Release --filter "Name=ReportsConcurrentRelationalWriteMetrics" --minimum-expected-tests 1 --max-parallel-test-modules 1 --logger "console;verbosity=detailed" $env:AGENTSTRATION_STORAGE_BENCHMARK_PROVIDER = "PostgreSql" $env:AGENTSTRATION_TEST_POSTGRES = "Host=localhost;Database=agentstration;Username=agentstration;Password=" -dotnet test --project tests/Agentstration.Web.Tests/Agentstration.Web.Tests.csproj --configuration Release --filter "Name=ReportsConcurrentRelationalWriteMetrics" --minimum-expected-tests 1 --logger "console;verbosity=detailed" +dotnet test --project tests/Agentstration.Performance.Tests/Agentstration.Performance.Tests.csproj --configuration Release --filter "Name=ReportsConcurrentRelationalWriteMetrics" --minimum-expected-tests 1 --max-parallel-test-modules 1 --logger "console;verbosity=detailed" ``` ## Backup and restore diff --git a/scripts/ci/run-test-module-with-diagnostics.ps1 b/scripts/ci/run-test-module-with-diagnostics.ps1 new file mode 100644 index 00000000..688960ea --- /dev/null +++ b/scripts/ci/run-test-module-with-diagnostics.ps1 @@ -0,0 +1,128 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [string] $AssemblyPath, + + [Parameter(Mandatory)] + [string] $ReportPath, + + [Parameter(Mandatory)] + [ValidateRange(1, [int]::MaxValue)] + [int] $MinimumExpectedTests, + + [ValidateRange(0, [int]::MaxValue)] + [int] $WarningWorkingSetMiB = 0, + + [ValidateRange(0, [int]::MaxValue)] + [int] $FailureWorkingSetMiB = 0 +) + +$ErrorActionPreference = 'Stop' +$resolvedAssembly = (Resolve-Path -LiteralPath $AssemblyPath).Path +$resolvedReport = [System.IO.Path]::GetFullPath($ReportPath) +$reportDirectory = Split-Path -Parent $resolvedReport +if (-not [string]::IsNullOrWhiteSpace($reportDirectory)) { + New-Item -ItemType Directory -Force -Path $reportDirectory | Out-Null +} + +$startInfo = [System.Diagnostics.ProcessStartInfo]::new('dotnet') +$startInfo.ArgumentList.Add($resolvedAssembly) +$startInfo.ArgumentList.Add('--minimum-expected-tests') +$startInfo.ArgumentList.Add($MinimumExpectedTests.ToString([System.Globalization.CultureInfo]::InvariantCulture)) +$startInfo.ArgumentList.Add('--progress') +$startInfo.ArgumentList.Add('off') +$startInfo.UseShellExecute = $false +$startInfo.CreateNoWindow = $true +$startInfo.RedirectStandardOutput = $true +$startInfo.RedirectStandardError = $true + +$process = [System.Diagnostics.Process]::new() +$process.StartInfo = $startInfo +$stopwatch = [System.Diagnostics.Stopwatch]::StartNew() +if (-not $process.Start()) { + throw "Could not start test module '$resolvedAssembly'." +} + +$standardOutput = $process.StandardOutput.ReadToEndAsync() +$standardError = $process.StandardError.ReadToEndAsync() +$peakWorkingSetBytes = 0L +$peakPrivateBytes = 0L +$peakAggregateDotnetWorkingSetBytes = 0L +while (-not $process.HasExited) { + $process.Refresh() + $peakWorkingSetBytes = [Math]::Max($peakWorkingSetBytes, $process.WorkingSet64) + $peakPrivateBytes = [Math]::Max($peakPrivateBytes, $process.PrivateMemorySize64) + $aggregateDotnetWorkingSetBytes = 0L + foreach ($dotnetProcess in [System.Diagnostics.Process]::GetProcessesByName('dotnet')) { + try { + $dotnetProcess.Refresh() + $aggregateDotnetWorkingSetBytes += $dotnetProcess.WorkingSet64 + } + catch [System.InvalidOperationException] { + # A process can exit between enumeration and sampling. + } + finally { + $dotnetProcess.Dispose() + } + } + $peakAggregateDotnetWorkingSetBytes = [Math]::Max($peakAggregateDotnetWorkingSetBytes, $aggregateDotnetWorkingSetBytes) + Start-Sleep -Milliseconds 100 +} +$process.WaitForExit() +$stopwatch.Stop() +$output = $standardOutput.GetAwaiter().GetResult() +$null = $standardError.GetAwaiter().GetResult() + +$countMatches = [regex]::Matches($output, '(?im)^\s*total\s*:\s*(\d+)\s*$') +$testCount = if ($countMatches.Count -gt 0) { [int]$countMatches[$countMatches.Count - 1].Groups[1].Value } else { 0 } +$workingSetMiB = [Math]::Round($peakWorkingSetBytes / 1MB, 1) +$privateMiB = [Math]::Round($peakPrivateBytes / 1MB, 1) +$aggregateDotnetWorkingSetMiB = [Math]::Round($peakAggregateDotnetWorkingSetBytes / 1MB, 1) +$moduleName = [System.IO.Path]::GetFileNameWithoutExtension($resolvedAssembly) +$status = if ($process.ExitCode -ne 0 -or + $testCount -lt $MinimumExpectedTests -or + ($FailureWorkingSetMiB -gt 0 -and $workingSetMiB -ge $FailureWorkingSetMiB)) { + 'failed' +} +elseif ($WarningWorkingSetMiB -gt 0 -and $workingSetMiB -ge $WarningWorkingSetMiB) { + 'warning' +} +else { + 'passed' +} + +[ordered]@{ + module = $moduleName + status = $status + testCount = $testCount + minimumExpectedTests = $MinimumExpectedTests + durationSeconds = [Math]::Round($stopwatch.Elapsed.TotalSeconds, 3) + peakWorkingSetMiB = $workingSetMiB + peakPrivateMemoryMiB = $privateMiB + peakAggregateDotnetWorkingSetMiB = $aggregateDotnetWorkingSetMiB + warningWorkingSetMiB = $WarningWorkingSetMiB + failureWorkingSetMiB = $FailureWorkingSetMiB + exitCode = $process.ExitCode + runtime = [System.Runtime.InteropServices.RuntimeInformation]::FrameworkDescription + os = [System.Runtime.InteropServices.RuntimeInformation]::OSDescription +} | ConvertTo-Json | Set-Content -LiteralPath $resolvedReport -Encoding utf8 + +$summary = "${moduleName}: $testCount tests, $([Math]::Round($stopwatch.Elapsed.TotalSeconds, 1)) s, $workingSetMiB MiB working set, $privateMiB MiB private, $aggregateDotnetWorkingSetMiB MiB aggregate dotnet, status $status" +Write-Host $summary +if (-not [string]::IsNullOrWhiteSpace($env:GITHUB_STEP_SUMMARY)) { + "| $moduleName | $testCount | $([Math]::Round($stopwatch.Elapsed.TotalSeconds, 1)) | $workingSetMiB | $privateMiB | $aggregateDotnetWorkingSetMiB | $status |" | + Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Encoding utf8 +} + +if ($process.ExitCode -ne 0) { + throw "Test module '$moduleName' exited with code $($process.ExitCode). Rerun it directly for failure details." +} +if ($testCount -lt $MinimumExpectedTests) { + throw "Test module '$moduleName' discovered $testCount tests; expected at least $MinimumExpectedTests." +} +if ($FailureWorkingSetMiB -gt 0 -and $workingSetMiB -ge $FailureWorkingSetMiB) { + throw "Test module '$moduleName' peaked at $workingSetMiB MiB working set; failure budget is $FailureWorkingSetMiB MiB." +} +if ($status -eq 'warning') { + Write-Warning "Test module '$moduleName' peaked at $workingSetMiB MiB working set; warning budget is $WarningWorkingSetMiB MiB." +} From f37e5f62ee56a3522473b504747cb7ad5270ec9d Mon Sep 17 00:00:00 2001 From: gbaudrit Date: Fri, 11 Sep 2026 13:01:58 +0200 Subject: [PATCH 7/9] fix(ci): read diagnostic test counts from TRX --- .../ci/run-test-module-with-diagnostics.ps1 | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/scripts/ci/run-test-module-with-diagnostics.ps1 b/scripts/ci/run-test-module-with-diagnostics.ps1 index 688960ea..d1dbccae 100644 --- a/scripts/ci/run-test-module-with-diagnostics.ps1 +++ b/scripts/ci/run-test-module-with-diagnostics.ps1 @@ -24,6 +24,8 @@ $reportDirectory = Split-Path -Parent $resolvedReport if (-not [string]::IsNullOrWhiteSpace($reportDirectory)) { New-Item -ItemType Directory -Force -Path $reportDirectory | Out-Null } +$trxFileName = ".test-results-$([Guid]::NewGuid().ToString('N')).trx" +$trxPath = Join-Path $reportDirectory $trxFileName $startInfo = [System.Diagnostics.ProcessStartInfo]::new('dotnet') $startInfo.ArgumentList.Add($resolvedAssembly) @@ -31,6 +33,11 @@ $startInfo.ArgumentList.Add('--minimum-expected-tests') $startInfo.ArgumentList.Add($MinimumExpectedTests.ToString([System.Globalization.CultureInfo]::InvariantCulture)) $startInfo.ArgumentList.Add('--progress') $startInfo.ArgumentList.Add('off') +$startInfo.ArgumentList.Add('--report-trx') +$startInfo.ArgumentList.Add('--report-trx-filename') +$startInfo.ArgumentList.Add($trxFileName) +$startInfo.ArgumentList.Add('--results-directory') +$startInfo.ArgumentList.Add($reportDirectory) $startInfo.UseShellExecute = $false $startInfo.CreateNoWindow = $true $startInfo.RedirectStandardOutput = $true @@ -70,11 +77,22 @@ while (-not $process.HasExited) { } $process.WaitForExit() $stopwatch.Stop() -$output = $standardOutput.GetAwaiter().GetResult() +$null = $standardOutput.GetAwaiter().GetResult() $null = $standardError.GetAwaiter().GetResult() -$countMatches = [regex]::Matches($output, '(?im)^\s*total\s*:\s*(\d+)\s*$') -$testCount = if ($countMatches.Count -gt 0) { [int]$countMatches[$countMatches.Count - 1].Groups[1].Value } else { 0 } +$testCount = 0 +if (Test-Path -LiteralPath $trxPath) { + try { + [xml] $trx = Get-Content -LiteralPath $trxPath -Raw + $counters = $trx.SelectSingleNode("//*[local-name()='Counters']") + if ($null -ne $counters) { + $testCount = [int]$counters.GetAttribute('total') + } + } + finally { + [System.IO.File]::Delete($trxPath) + } +} $workingSetMiB = [Math]::Round($peakWorkingSetBytes / 1MB, 1) $privateMiB = [Math]::Round($peakPrivateBytes / 1MB, 1) $aggregateDotnetWorkingSetMiB = [Math]::Round($peakAggregateDotnetWorkingSetBytes / 1MB, 1) From 13b7e71f24c1b8b54a7b639776bedf1d63249547 Mon Sep 17 00:00:00 2001 From: gbaudrit Date: Fri, 11 Sep 2026 13:22:30 +0200 Subject: [PATCH 8/9] docs(test): align agent validation commands --- AGENTS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 831007d9..edaab139 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -186,8 +186,8 @@ Run from the repository root: ```powershell dotnet restore Agentstration.slnx dotnet build Agentstration.slnx --configuration Release --no-restore -dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 1 -dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 1 +dotnet test --solution Agentstration.Tests.Fast.slnx --configuration Release --no-build --minimum-expected-tests 139 --max-parallel-test-modules 4 +dotnet test --solution Agentstration.Tests.Integration.slnx --configuration Release --no-build --minimum-expected-tests 692 --max-parallel-test-modules 2 ``` The two test solutions together form the required deterministic, offline functional suite. Performance and live-provider workloads are explicit opt-ins documented in `docs/contributing/testing.md`. For a focused iteration, run the affected test project first, then run both functional lanes before handoff. Do not suppress warnings or disable analyzers to make a change pass. From 771224739b22449417dfc8c5a2627dfa58f2ae39 Mon Sep 17 00:00:00 2001 From: gbaudrit Date: Fri, 11 Sep 2026 13:59:51 +0200 Subject: [PATCH 9/9] test(management): isolate bootstrap host tests (#353) --- .github/workflows/ci.yml | 3 ++- AGENTS.md | 1 + Agentstration.Tests.Integration.slnx | 1 + Agentstration.slnx | 1 + docs/architecture.md | 1 + docs/contributing/testing.md | 6 ++++-- .../Agentstration.Infrastructure.csproj | 1 + ...entstration.Management.Bootstrap.Tests.csproj | 16 ++++++++++++++++ .../AssemblyInfo.cs | 3 +++ .../DeclarativeBootstrapTests.cs | 14 ++++++++++++-- 10 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 tests/Agentstration.Management.Bootstrap.Tests/Agentstration.Management.Bootstrap.Tests.csproj create mode 100644 tests/Agentstration.Management.Bootstrap.Tests/AssemblyInfo.cs rename tests/{Agentstration.Management.Api.Tests => Agentstration.Management.Bootstrap.Tests}/DeclarativeBootstrapTests.cs (99%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59e60d48..2b3f8a71 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -162,7 +162,8 @@ jobs: '| --- | ---: | ---: | ---: | ---: | ---: | --- |' ) | Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Encoding utf8 $modules = @( - @{ Name = 'Agentstration.Management.Api.Tests'; Tests = 66; Warning = 900; Failure = 1024 }, + @{ Name = 'Agentstration.Management.Api.Tests'; Tests = 43; Warning = 900; Failure = 1024 }, + @{ Name = 'Agentstration.Management.Bootstrap.Tests'; Tests = 23; Warning = 500; Failure = 700 }, @{ Name = 'Agentstration.Management.Security.Tests'; Tests = 38; Warning = 800; Failure = 1024 }, @{ Name = 'Agentstration.Management.Aep.Tests'; Tests = 11; Warning = 450; Failure = 700 }, @{ Name = 'Agentstration.Management.Sources.Tests'; Tests = 93; Warning = 550; Failure = 900 } diff --git a/AGENTS.md b/AGENTS.md index edaab139..2243ad2b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,6 +52,7 @@ tests/ Agentstration.Management.Storage.Tests/ Agentstration.Management.Sources.Tests/ Agentstration.Management.Api.Tests/ + Agentstration.Management.Bootstrap.Tests/ Agentstration.Management.Security.Tests/ Agentstration.Management.Aep.Tests/ Agentstration.Performance.Tests/ diff --git a/Agentstration.Tests.Integration.slnx b/Agentstration.Tests.Integration.slnx index de77682c..c502a14a 100644 --- a/Agentstration.Tests.Integration.slnx +++ b/Agentstration.Tests.Integration.slnx @@ -4,6 +4,7 @@ + diff --git a/Agentstration.slnx b/Agentstration.slnx index f2672806..b4d70dd5 100644 --- a/Agentstration.slnx +++ b/Agentstration.slnx @@ -59,6 +59,7 @@ + diff --git a/docs/architecture.md b/docs/architecture.md index 75c191f6..89c109e4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -50,6 +50,7 @@ tests/ Agentstration.Management.Storage.Tests/ Agentstration.Management.Sources.Tests/ Agentstration.Management.Api.Tests/ + Agentstration.Management.Bootstrap.Tests/ Agentstration.Management.Security.Tests/ Agentstration.Management.Aep.Tests/ Agentstration.Performance.Tests/ diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index 51890a5b..e167cb38 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -40,7 +40,8 @@ The initial budgets below use Release runs on Windows 11 10.0.26200 with .NET 10 | Module | Baseline peak (MiB) | Warning (MiB) | Failure (MiB) | Minimum tests | | --- | ---: | ---: | ---: | ---: | -| `Agentstration.Management.Api.Tests` | 887.0 | 900 | 1024 | 66 | +| `Agentstration.Management.Api.Tests` | 749.3 | 900 | 1024 | 43 | +| `Agentstration.Management.Bootstrap.Tests` | 355.3 | 500 | 700 | 23 | | `Agentstration.Management.Security.Tests` | 730.1 | 800 | 1024 | 38 | | `Agentstration.Management.Aep.Tests` | 322.1 | 450 | 700 | 11 | | `Agentstration.Management.Sources.Tests` | 431.5 | 550 | 900 | 93 | @@ -58,7 +59,8 @@ The initial budgets below use Release runs on Windows 11 10.0.26200 with .NET 10 | `Agentstration.Application.Tests` | Integration | Includes SQLite Flow and Work storage contracts | | `Agentstration.Management.Storage.Tests` | Integration | SQLite control-plane, Identity persistence, secrets, audit, and trigger storage | | `Agentstration.Management.Sources.Tests` | Integration | Source, registry, provider, and Pack distribution boundaries | -| `Agentstration.Management.Api.Tests` | Integration | Hosted Management API and declarative bootstrap tests using the API-only test profile | +| `Agentstration.Management.Api.Tests` | Integration | Hosted Management API tests using the API-only test profile | +| `Agentstration.Management.Bootstrap.Tests` | Integration | Declarative bootstrap catalog, application, and hosted startup scenarios using the API-only test profile | | `Agentstration.Management.Security.Tests` | Integration | Identity, authorization, local-account, and interactive Security boundaries | | `Agentstration.Management.Aep.Tests` | Integration | AEP enrollment lifecycle and extension inventory boundaries using the API-only test profile | | `Agentstration.ModelProviders.Tests` | Integration, provider-optional | AEP test hosts plus opt-in live-provider checks | diff --git a/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj b/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj index eecdc5a2..51ee7809 100644 --- a/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj +++ b/src/Agentstration.Infrastructure/Agentstration.Infrastructure.csproj @@ -21,6 +21,7 @@ + diff --git a/tests/Agentstration.Management.Bootstrap.Tests/Agentstration.Management.Bootstrap.Tests.csproj b/tests/Agentstration.Management.Bootstrap.Tests/Agentstration.Management.Bootstrap.Tests.csproj new file mode 100644 index 00000000..29044df4 --- /dev/null +++ b/tests/Agentstration.Management.Bootstrap.Tests/Agentstration.Management.Bootstrap.Tests.csproj @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/tests/Agentstration.Management.Bootstrap.Tests/AssemblyInfo.cs b/tests/Agentstration.Management.Bootstrap.Tests/AssemblyInfo.cs new file mode 100644 index 00000000..c53e6b4a --- /dev/null +++ b/tests/Agentstration.Management.Bootstrap.Tests/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: Parallelize(Workers = 1, Scope = ExecutionScope.ClassLevel)] diff --git a/tests/Agentstration.Management.Api.Tests/DeclarativeBootstrapTests.cs b/tests/Agentstration.Management.Bootstrap.Tests/DeclarativeBootstrapTests.cs similarity index 99% rename from tests/Agentstration.Management.Api.Tests/DeclarativeBootstrapTests.cs rename to tests/Agentstration.Management.Bootstrap.Tests/DeclarativeBootstrapTests.cs index e90d55e7..4c20b000 100644 --- a/tests/Agentstration.Management.Api.Tests/DeclarativeBootstrapTests.cs +++ b/tests/Agentstration.Management.Bootstrap.Tests/DeclarativeBootstrapTests.cs @@ -777,7 +777,7 @@ private static WebApplicationFactory Factory( bool configureOllamaExtension = false) { EnsureInstanceProfileDescriptor(path); - return new ApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => + return new BootstrapApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("Agentstration:Authentication:Mode", "Local"); @@ -794,7 +794,7 @@ private static WebApplicationFactory Factory( private static WebApplicationFactory FactoryWithExtensionPresenceHandler(string path, bool discoverOnStartup) { EnsureInstanceProfileDescriptor(path); - return new ApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => + return new BootstrapApiOnlyWebApplicationFactory().WithWebHostBuilder(builder => { builder.UseEnvironment("Testing"); builder.UseSetting("Agentstration:Authentication:Mode", "Development"); @@ -1163,3 +1163,13 @@ public TemporaryDirectory() public void Dispose() => Directory.Delete(Path, recursive: true); } } + +internal sealed class BootstrapApiOnlyWebApplicationFactory : WebApplicationFactory +{ + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment("Testing"); + builder.UseSetting("Agentstration:Testing:ApiOnly", "true"); + builder.UseSetting("Logging:LogLevel:Default", "Warning"); + } +}