diff --git a/.github/workflows/idempotency-release.yml b/.github/workflows/idempotency-release.yml new file mode 100644 index 0000000..6f9669c --- /dev/null +++ b/.github/workflows/idempotency-release.yml @@ -0,0 +1,130 @@ +name: Idempotency release + +# Publishes BuildingBlocks.Idempotency when an idempotency-v* tag is pushed. +# Tag format: idempotency-vX.Y.Z must match in BuildingBlocks.Idempotency.csproj. +# Example: git tag idempotency-v1.0.0 && git push origin idempotency-v1.0.0 + +on: + release: + types: [published] + push: + tags: + - "idempotency-v*" + workflow_dispatch: + inputs: + publish_nuget: + description: "Push to nuget.org (requires Trusted Publishing + NUGET_USER)" + type: boolean + default: false + +jobs: + release: + if: ${{ github.event_name == 'workflow_dispatch' || startsWith(github.ref_name, 'idempotency-v') }} + runs-on: ubuntu-latest + environment: nuget + permissions: + id-token: write + contents: write + env: + NUGET_USER: ${{ secrets.NUGET_USER }} + + steps: + - uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + 8.0.x + 9.0.x + 10.0.x + + - name: Resolve version + id: ver + shell: bash + run: | + set -euo pipefail + PROJECT_VERSION=$(sed -n 's:.*\(.*\).*:\1:p' src/BuildingBlocks/Idempotency/BuildingBlocks.Idempotency.csproj | head -1) + echo "project=$PROJECT_VERSION" >> "$GITHUB_OUTPUT" + REF="${GITHUB_REF_NAME}" + if [[ "$GITHUB_REF" == refs/tags/* ]]; then + TAG="$REF" + STRIPPED="${TAG#idempotency-v}" + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "from_tag=$STRIPPED" >> "$GITHUB_OUTPUT" + if [[ "$STRIPPED" != "$PROJECT_VERSION" ]]; then + echo "::error::Tag version ($STRIPPED) does not match project ($PROJECT_VERSION)" + exit 1 + fi + else + echo "tag=" >> "$GITHUB_OUTPUT" + echo "from_tag=" >> "$GITHUB_OUTPUT" + fi + + - name: Test library (net8 / net9 / net10) + run: dotnet test tests/BuildingBlocks/Idempotency.Tests/BuildingBlocks.Idempotency.Tests.csproj -c Release --verbosity minimal + + - name: Pack + run: dotnet pack src/BuildingBlocks/Idempotency/BuildingBlocks.Idempotency.csproj -c Release -o artifacts/nuget -p:ContinuousIntegrationBuild=true + + - name: Upload workflow artifacts + uses: actions/upload-artifact@v4 + with: + name: nuget-packages-idempotency-release + path: | + artifacts/nuget/*.nupkg + artifacts/nuget/*.snupkg + if-no-files-found: error + + - name: Attach packages to GitHub Release + if: github.event_name == 'release' + continue-on-error: true + uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2 + with: + tag_name: ${{ github.ref_name }} + files: | + artifacts/nuget/*.nupkg + artifacts/nuget/*.snupkg + + - name: Create GitHub Release for tag + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2 + with: + tag_name: ${{ github.ref_name }} + name: BuildingBlocks.Idempotency ${{ steps.ver.outputs.project }} + generate_release_notes: true + files: | + artifacts/nuget/*.nupkg + artifacts/nuget/*.snupkg + + - name: NuGet login (Trusted Publishing OIDC) + if: ${{ env.NUGET_USER != '' && (github.event_name != 'workflow_dispatch' || inputs.publish_nuget) }} + id: login + uses: NuGet/login@v1 + with: + user: ${{ env.NUGET_USER }} + + - name: Push to nuget.org + if: ${{ env.NUGET_USER != '' && (github.event_name != 'workflow_dispatch' || inputs.publish_nuget) && steps.login.outcome == 'success' }} + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + for pkg in artifacts/nuget/*.nupkg; do + dotnet nuget push "$pkg" \ + --api-key "${{ steps.login.outputs.NUGET_API_KEY }}" \ + --source https://api.nuget.org/v3/index.json \ + --skip-duplicate + done + for spkg in artifacts/nuget/*.snupkg; do + dotnet nuget push "$spkg" \ + --api-key "${{ steps.login.outputs.NUGET_API_KEY }}" \ + --source https://api.nuget.org/v3/index.json \ + --skip-duplicate || true + done + + - name: Dry-run notice + if: ${{ env.NUGET_USER == '' || (github.event_name == 'workflow_dispatch' && !inputs.publish_nuget) }} + run: | + echo "Skipped nuget.org push (NUGET_USER unset or publish_nuget=false)." + echo "GitHub Release and workflow artifacts are still produced for tag/release triggers." diff --git a/.github/workflows/idempotency.yml b/.github/workflows/idempotency.yml new file mode 100644 index 0000000..aab609d --- /dev/null +++ b/.github/workflows/idempotency.yml @@ -0,0 +1,49 @@ +name: Idempotency CI + +on: + push: + paths: + - "src/BuildingBlocks/Idempotency/**" + - "tests/BuildingBlocks/Idempotency.Tests/**" + - ".github/workflows/idempotency.yml" + pull_request: + paths: + - "src/BuildingBlocks/Idempotency/**" + - "tests/BuildingBlocks/Idempotency.Tests/**" + - ".github/workflows/idempotency.yml" + workflow_dispatch: + +concurrency: + group: idempotency-ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + test-and-pack: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + 8.0.x + 9.0.x + 10.0.x + + - name: Test library (net8 / net9 / net10) + run: dotnet test tests/BuildingBlocks/Idempotency.Tests/BuildingBlocks.Idempotency.Tests.csproj -c Release --verbosity minimal + + - name: Pack + run: dotnet pack src/BuildingBlocks/Idempotency/BuildingBlocks.Idempotency.csproj -c Release -o artifacts/nuget -p:ContinuousIntegrationBuild=true + + - name: Upload nupkg + uses: actions/upload-artifact@v4 + with: + name: nuget-packages-idempotency + path: | + artifacts/nuget/*.nupkg + artifacts/nuget/*.snupkg + if-no-files-found: error diff --git a/.gitignore b/.gitignore index a03d1e0..416b49b 100644 --- a/.gitignore +++ b/.gitignore @@ -190,6 +190,8 @@ docs/** !docs/building-blocks/** !docs/adr/ !docs/adr/** +!docs/lab/ +!docs/lab/** # Local-only polluted/autogen solutions src/src.sln diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..f9ab01c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,49 @@ +# FeatureFusion — agent notes + +Public **.NET lab** plus extracted MIT **BuildingBlocks**. The application is the laboratory. A package is a reusable result that earned a boundary. + +Do not treat FeatureFusion as a framework. Install a BuildingBlock only when it solves a problem the host actually has. + +## Lab vs package + +| Kind | What it is | Agent docs | +|------|------------|------------| +| **Lab** | Runnable API + Aspire AppHost (`src/Lab/`) | This file + [docs/lab](docs/lab/README.md) + root README | +| **Packable BuildingBlock** | NuGet on nuget.org | `src/BuildingBlocks//AGENTS.md` + `PACKAGE_README.md` | +| **In-repo sibling** | Pagination IR / Dapper — not packed | `Pagination/AGENTS.md`, `Pagination.Dapper/AGENTS.md` | +| **Lab-only** | EventBus, YARP rate limit, feature flags | Code + [linkedin catalog](docs/linkedin-posts.md) — no package `AGENTS.md` | + +**Lab charter (evidence families, extraction rules, roadmap):** [docs/lab/README.md](docs/lab/README.md). Behavioral experiment catalog: [Experiments README](tests/Lab/IntegrationTests/Experiments/README.md) (Exp 1–18 + MAF prototype). + +## Packages (read these first) + +| Package | When to choose | AGENTS.md | +|---------|----------------|-----------| +| `BuildingBlocks.Mediator` | CQRS Send + ordered pipeline | [Mediator](src/BuildingBlocks/Mediator/AGENTS.md) | +| `BuildingBlocks.Pagination.EntityFrameworkCore` | Keyset (cursor) paging for EF Core | [Pagination.EF](src/BuildingBlocks/Pagination.EntityFrameworkCore/AGENTS.md) | +| `BuildingBlocks.Mcp` | Same logic as HTTP as deny-by-default MCP tools | [Mcp](src/BuildingBlocks/Mcp/AGENTS.md) | +| `BuildingBlocks.Idempotency` | HTTP Idempotency-Key + optional Redis lock | [Idempotency](src/BuildingBlocks/Idempotency/AGENTS.md) | +| `BuildingBlocks.Telemetry` | Config-driven OpenTelemetry (not a SigNoz SDK) | [Telemetry](src/BuildingBlocks/Telemetry/AGENTS.md) | +| `BuildingBlocks.Aspire.Hosting.SigNoz` | Local AppHost SigNoz only (net10) | [SigNoz](src/BuildingBlocks/Aspire.Hosting.SigNoz/AGENTS.md) | + +Pagination IR (`BuildingBlocks.Pagination`) is bundled into the EF Core nupkg. Do not pack it. Dapper pagination is a lab project. + +## Human docs + +- [README](README.md) — lab + package overview +- [docs/lab](docs/lab/README.md) — lab charter, evidence families, extraction rules +- [docs/building-blocks](docs/building-blocks/) — getting started, mediator, pagination, MCP, telemetry +- [docs/linkedin-posts.md](docs/linkedin-posts.md) — post ↔ code map +- [tests/Lab/IntegrationTests/Experiments/README.md](tests/Lab/IntegrationTests/Experiments/README.md) — behavioral integration experiments (Lab convention + catalog Exp 1–18) +- [CONTRIBUTING.md](CONTRIBUTING.md) +- [llms.txt](llms.txt) — URL index for crawlers + +## Lab-only (not NuGet) + +- RabbitMQ EventBus + outbox/inbox — `src/Lab/EventBus` +- YARP + Memcached rate limiting — `src/Lab/FeatureFusion.ApiGateway` +- Feature-management filters — lab FeatureFusion + +## Related + +Writing map: [docs/linkedin-posts.md](docs/linkedin-posts.md). Do not put LinkedIn URLs in NuGet XML docs or `PACKAGE_README.md`. diff --git a/CHANGELOG.md b/CHANGELOG.md index e311ffd..a482f20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,28 @@ All notable changes to **BuildingBlocks** packages in this repository are docume The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## BuildingBlocks.Idempotency [1.0.0] - 2026-09-04 + +### Added + +- ASP.NET Core HTTP **Idempotency-Key** filter via shared `IdempotencyGate`: MVC `[Idempotent]` / `IdempotentAttributeFilter` and Minimal API `WithIdempotency` / `IdempotentEndpointFilter` +- Response envelope replay on Completed keys: status code, content-type, body, plus configurable replay header (default `X-Idempotent-Response`) +- Cache **all HTTP 2xx** successes (not only 200) +- Processing state returns ProblemDetails (default 409) while the lease is active; expired Processing is treated as a miss +- Separate `ProcessingTtl` / `EntryTtl` with optional per-endpoint seconds on attribute / `WithIdempotency` +- Optional Redis SET NX lock via fluent `.UseRedisLock()` / `AddRedisIdempotencyLock` when `UseLock` is true (lock covers GetOrCreate) +- Opt-in request fingerprinting (`EnableRequestFingerprint`): `SHA-256(method + "\n" + path + "\n" + body)`; mismatch → ProblemDetails (default 422); default off (Exp 3 compatible) +- `DuplicateCompletedBehavior` (Replay default / Conflict), configurable processing/duplicate/fingerprint status codes, `MaxKeyLength` + control-character validation +- RFC 9457 `ProblemDetails` errors with stable `type` URIs under `https://buildingblocks.dev/errors/idempotency/` +- Optional ActivitySource via `.UseTelemetry()` / `AddIdempotencyTelemetry` (no cache key tag by default; no BuildingBlocks.Telemetry package dependency) +- Fluent DI: `AddBuildingBlocksIdempotency(...).UseRedisLock().UseTelemetry()` +- Multi-target `net8.0` / `net9.0` / `net10.0`; Lab FeatureFusion hosts MVC on `POST /api/v2/Order/order` and Minimal API smoke on `POST /api/v2/idempotency-smoke` (provenance Exp 3 / 4 / 12) + +### Notes + +- Distinct from MCP write idempotency (`UseMemoryIdempotency` / `IMcpIdempotencyStore` in BuildingBlocks.Mcp) +- Package depends on `Newtonsoft.Json` (MVC ObjectResult capture), `Ulid`, and `StackExchange.Redis` (Redis lock); hosts that use the lock must register `IConnectionMultiplexer` + ## BuildingBlocks.Pagination.EntityFrameworkCore [1.0.0] - 2026-08-29 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2d95c6d..48c7866 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,12 +12,15 @@ Formerly published as `Maxofpower/FeatureManagement`; GitHub redirects the old U - `dotnet test tests/BuildingBlocks/Mediator.Tests` - `dotnet test tests/BuildingBlocks/Mediator.Analyzers.Tests` - `dotnet test tests/BuildingBlocks/Mcp.Tests` + - `dotnet test tests/BuildingBlocks/Idempotency.Tests` - `dotnet test tests/BuildingBlocks/Pagination.Tests` - `dotnet test tests/BuildingBlocks/Pagination.EntityFrameworkCore.Tests` - `dotnet test tests/BuildingBlocks/Pagination.Dapper.Tests` - `dotnet test tests/BuildingBlocks/Telemetry.Tests` - `dotnet test tests/BuildingBlocks/Aspire.Hosting.SigNoz.Tests` -4. Pack: `dotnet pack src/BuildingBlocks/Mediator -c Release -o artifacts/nuget` (same for Telemetry, Mcp, `Pagination.EntityFrameworkCore`) +4. Pack: `dotnet pack src/BuildingBlocks/Mediator -c Release -o artifacts/nuget` (same for Telemetry, Mcp, Idempotency, `Pagination.EntityFrameworkCore`) + +Release tags (must match the project ``): `mediator-v*`, `mcp-v*`, `idempotency-v*`, `telemetry-v*`, `pagination-v*`, `telemetry-signoz-v*` (see `.github/workflows/*-release.yml`). ## Guidelines @@ -38,6 +41,12 @@ Formerly published as `Maxofpower/FeatureManagement`; GitHub redirects the old U - Public API changes: update `PublicAPI.Shipped.txt` / XML docs. - Benchmarks: `dotnet run -c Release --project benchmarks/BuildingBlocks/Pagination.EntityFrameworkCore.Benchmarks -- --filter "*" --job Dry` to smoke; drop `--job Dry` for measurement. +### BuildingBlocks.Idempotency + +- HTTP `Idempotency-Key` only — do not fold MCP `IMcpIdempotencyStore` into this package. +- Host owns `IDistributedCache` (and `IConnectionMultiplexer` when using the Redis lock). +- Lab provenance: Experiments 3 / 4 / 12 under `tests/Lab/IntegrationTests/Experiments/`. + ### BuildingBlocks.Telemetry / SigNoz hosting - Keep OpenTelemetry packages on stable versions (no prerelease deps). @@ -47,6 +56,8 @@ Formerly published as `Maxofpower/FeatureManagement`; GitHub redirects the old U - Prefer vertical-slice feature folders under `Features/{Name}/`. - Leave `Microsoft.FeatureManagement` and `FeatureManagement` JSON as the original feature-flag demo — not the product identity. +- Lab charter and evidence families: [docs/lab/README.md](docs/lab/README.md). Do not invent a Scenario/ExperimentEngine NuGet without crossing the extraction rule there. +- Integration experiments: [Experiments README](tests/Lab/IntegrationTests/Experiments/README.md). Prefer observation helpers over production changes. Contributions are accepted under the project **MIT** license (inbound = outbound). @@ -54,4 +65,5 @@ Contributions are accepted under the project **MIT** license (inbound = outbound - Prefer focused PRs with tests. - Run the relevant test project(s) green before requesting review. -- When you publish a LinkedIn post tied to this repo, add a row in [docs/linkedin-posts.md](docs/linkedin-posts.md). +- New packable BuildingBlock: add `AGENTS.md` + `PACKAGE_README.md`, a row in [docs/linkedin-posts.md](docs/linkedin-posts.md), and an entry in the root [AGENTS.md](AGENTS.md) / [llms.txt](llms.txt). Do not put LinkedIn URLs in `PACKAGE_README.md` or packed XML docs. +- When I publish a LinkedIn or Medium post tied to this repo, I'll add a row in [docs/linkedin-posts.md](docs/linkedin-posts.md) and a line in the root README LinkedIn catalog. diff --git a/FeatureFusion.sln b/FeatureFusion.sln index f9329dd..2058870 100644 --- a/FeatureFusion.sln +++ b/FeatureFusion.sln @@ -97,6 +97,14 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Pagination.EntityFrameworkC EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BuildingBlocks.Pagination.EntityFrameworkCore.SqlServer.Tests", "tests\BuildingBlocks\Pagination.EntityFrameworkCore.SqlServer.Tests\BuildingBlocks.Pagination.EntityFrameworkCore.SqlServer.Tests.csproj", "{426AF793-4437-4F21-9333-9CD6656CF70B}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Idempotency", "Idempotency", "{3B6DFA28-E423-47BD-1DC7-709D0430AE1C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BuildingBlocks.Idempotency", "src\BuildingBlocks\Idempotency\BuildingBlocks.Idempotency.csproj", "{C0B899A7-8C72-4D54-8A9A-E5C65A43F108}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Idempotency.Tests", "Idempotency.Tests", "{E409DBCD-B9E3-14BD-B95F-04CE755E3F82}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BuildingBlocks.Idempotency.Tests", "tests\BuildingBlocks\Idempotency.Tests\BuildingBlocks.Idempotency.Tests.csproj", "{2DBA7F51-AC3B-4464-8348-0374FD62FDA0}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -479,6 +487,30 @@ Global {426AF793-4437-4F21-9333-9CD6656CF70B}.Release|x64.Build.0 = Release|Any CPU {426AF793-4437-4F21-9333-9CD6656CF70B}.Release|x86.ActiveCfg = Release|Any CPU {426AF793-4437-4F21-9333-9CD6656CF70B}.Release|x86.Build.0 = Release|Any CPU + {C0B899A7-8C72-4D54-8A9A-E5C65A43F108}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C0B899A7-8C72-4D54-8A9A-E5C65A43F108}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C0B899A7-8C72-4D54-8A9A-E5C65A43F108}.Debug|x64.ActiveCfg = Debug|Any CPU + {C0B899A7-8C72-4D54-8A9A-E5C65A43F108}.Debug|x64.Build.0 = Debug|Any CPU + {C0B899A7-8C72-4D54-8A9A-E5C65A43F108}.Debug|x86.ActiveCfg = Debug|Any CPU + {C0B899A7-8C72-4D54-8A9A-E5C65A43F108}.Debug|x86.Build.0 = Debug|Any CPU + {C0B899A7-8C72-4D54-8A9A-E5C65A43F108}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C0B899A7-8C72-4D54-8A9A-E5C65A43F108}.Release|Any CPU.Build.0 = Release|Any CPU + {C0B899A7-8C72-4D54-8A9A-E5C65A43F108}.Release|x64.ActiveCfg = Release|Any CPU + {C0B899A7-8C72-4D54-8A9A-E5C65A43F108}.Release|x64.Build.0 = Release|Any CPU + {C0B899A7-8C72-4D54-8A9A-E5C65A43F108}.Release|x86.ActiveCfg = Release|Any CPU + {C0B899A7-8C72-4D54-8A9A-E5C65A43F108}.Release|x86.Build.0 = Release|Any CPU + {2DBA7F51-AC3B-4464-8348-0374FD62FDA0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2DBA7F51-AC3B-4464-8348-0374FD62FDA0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2DBA7F51-AC3B-4464-8348-0374FD62FDA0}.Debug|x64.ActiveCfg = Debug|Any CPU + {2DBA7F51-AC3B-4464-8348-0374FD62FDA0}.Debug|x64.Build.0 = Debug|Any CPU + {2DBA7F51-AC3B-4464-8348-0374FD62FDA0}.Debug|x86.ActiveCfg = Debug|Any CPU + {2DBA7F51-AC3B-4464-8348-0374FD62FDA0}.Debug|x86.Build.0 = Debug|Any CPU + {2DBA7F51-AC3B-4464-8348-0374FD62FDA0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2DBA7F51-AC3B-4464-8348-0374FD62FDA0}.Release|Any CPU.Build.0 = Release|Any CPU + {2DBA7F51-AC3B-4464-8348-0374FD62FDA0}.Release|x64.ActiveCfg = Release|Any CPU + {2DBA7F51-AC3B-4464-8348-0374FD62FDA0}.Release|x64.Build.0 = Release|Any CPU + {2DBA7F51-AC3B-4464-8348-0374FD62FDA0}.Release|x86.ActiveCfg = Release|Any CPU + {2DBA7F51-AC3B-4464-8348-0374FD62FDA0}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -528,5 +560,9 @@ Global {76EF4366-66C3-473E-A70A-8F21C3A9BBFD} = {65FCD0DC-6806-16AE-E6D5-351A72620A08} {8B381FE9-7898-C0DF-E266-E94C56A9360F} = {8A1B2C3D-4E5F-6789-ABCD-333333333333} {426AF793-4437-4F21-9333-9CD6656CF70B} = {8B381FE9-7898-C0DF-E266-E94C56A9360F} + {3B6DFA28-E423-47BD-1DC7-709D0430AE1C} = {8A1B2C3D-4E5F-6789-ABCD-111111111111} + {C0B899A7-8C72-4D54-8A9A-E5C65A43F108} = {3B6DFA28-E423-47BD-1DC7-709D0430AE1C} + {E409DBCD-B9E3-14BD-B95F-04CE755E3F82} = {8A1B2C3D-4E5F-6789-ABCD-333333333333} + {2DBA7F51-AC3B-4464-8348-0374FD62FDA0} = {E409DBCD-B9E3-14BD-B95F-04CE755E3F82} EndGlobalSection EndGlobal diff --git a/README.md b/README.md index 36baa96..a592f10 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # FeatureFusion -**BuildingBlocks for .NET** — CQRS Send + pipeline, config-driven OpenTelemetry, MCP tools, keyset pagination, and a local Aspire SigNoz stack — plus a runnable lab that uses them. +**BuildingBlocks for .NET** — CQRS Send + pipeline, HTTP Idempotency-Key, config-driven OpenTelemetry, MCP tools, keyset pagination, and a local Aspire SigNoz stack — plus a runnable lab that uses them. Formerly [FeatureManagement](https://github.com/Maxofpower/FeatureManagement) (GitHub redirects). @@ -10,6 +10,7 @@ Formerly [FeatureManagement](https://github.com/Maxofpower/FeatureManagement) (G [![Aspire](https://img.shields.io/badge/Aspire-13.4-C3002F?logo=dotnet&logoColor=white)](https://learn.microsoft.com/dotnet/aspire/) [![NuGet · BuildingBlocks.Mediator](https://img.shields.io/nuget/v/BuildingBlocks.Mediator.svg?label=NuGet%20·%20Mediator&logo=nuget)](https://www.nuget.org/packages/BuildingBlocks.Mediator) [![NuGet · BuildingBlocks.Mcp](https://img.shields.io/nuget/v/BuildingBlocks.Mcp.svg?label=NuGet%20·%20MCP&logo=nuget)](https://www.nuget.org/packages/BuildingBlocks.Mcp) +[![NuGet · BuildingBlocks.Idempotency](https://img.shields.io/nuget/v/BuildingBlocks.Idempotency.svg?label=NuGet%20·%20Idempotency&logo=nuget)](https://www.nuget.org/packages/BuildingBlocks.Idempotency) [![NuGet · BuildingBlocks.Pagination.EntityFrameworkCore](https://img.shields.io/nuget/v/BuildingBlocks.Pagination.EntityFrameworkCore.svg?label=NuGet%20·%20Pagination&logo=nuget)](https://www.nuget.org/packages/BuildingBlocks.Pagination.EntityFrameworkCore) [![NuGet · BuildingBlocks.Telemetry](https://img.shields.io/nuget/v/BuildingBlocks.Telemetry.svg?label=NuGet%20·%20Telemetry&logo=nuget)](https://www.nuget.org/packages/BuildingBlocks.Telemetry) [![NuGet · BuildingBlocks.Aspire.Hosting.SigNoz](https://img.shields.io/nuget/v/BuildingBlocks.Aspire.Hosting.SigNoz.svg?label=NuGet%20·%20SigNoz%20hosting&logo=nuget)](https://www.nuget.org/packages/BuildingBlocks.Aspire.Hosting.SigNoz) @@ -31,6 +32,7 @@ Formerly [FeatureManagement](https://github.com/Maxofpower/FeatureManagement) (G - [How they work together](#how-they-work-together) - [BuildingBlocks.Mediator](#buildingblocksmediator) - [BuildingBlocks.Mcp](#buildingblocksmcp) + - [BuildingBlocks.Idempotency](#buildingblocksidempotency) - [BuildingBlocks.Pagination.EntityFrameworkCore](#buildingblockspaginationentityframeworkcore) - [BuildingBlocks.Telemetry](#buildingblockstelemetry) - [BuildingBlocks.Aspire.Hosting.SigNoz](#buildingblocksaspirehostingsignoz) @@ -58,6 +60,7 @@ NuGet packages you can install in **your** hosts. The FeatureFusion API is a sho |---------|------|------| | **[BuildingBlocks.Mediator](https://www.nuget.org/packages/BuildingBlocks.Mediator)** | CQRS **Send** + ordered pipeline (`ICommand` / `IQuery`, typed behaviors, opt-in traces + metrics) | net8 / net9 / net10 | | **[BuildingBlocks.Mcp](https://www.nuget.org/packages/BuildingBlocks.Mcp)** | Message types → MCP tools on the official SDK (deny-by-default, `McpResult`, HTTP + opt-in stdio) | net8 / net9 / net10 | +| **[BuildingBlocks.Idempotency](https://www.nuget.org/packages/BuildingBlocks.Idempotency)** | HTTP **Idempotency-Key** — MVC + Minimal API, 2xx envelope replay, ProblemDetails, optional Redis lock, fingerprint, ActivitySource | net8 / net9 / net10 | | **[BuildingBlocks.Pagination.EntityFrameworkCore](https://www.nuget.org/packages/BuildingBlocks.Pagination.EntityFrameworkCore)** | Typed keyset (cursor) pagination for EF Core (IR bundled) | net8 / net9 / net10 | | **[BuildingBlocks.Telemetry](https://www.nuget.org/packages/BuildingBlocks.Telemetry)** | Config-driven OpenTelemetry (traces, metrics, logs) + `IntegrateMediator` / opt-in `IntegrateMcp` | net8 / net9 / net10 | | **[BuildingBlocks.Aspire.Hosting.SigNoz](https://www.nuget.org/packages/BuildingBlocks.Aspire.Hosting.SigNoz)** | Local-dev Aspire `AddSigNoz()` + `WithSigNozOtlpExporter` | net10 (AppHost) | @@ -350,6 +353,40 @@ Cursor HTTP: --- +### BuildingBlocks.Idempotency + +[![NuGet](https://img.shields.io/nuget/v/BuildingBlocks.Idempotency.svg?logo=nuget)](https://www.nuget.org/packages/BuildingBlocks.Idempotency) + +ASP.NET Core HTTP **Idempotency-Key** for MVC and Minimal API. Host-owned `IDistributedCache`, **2xx** envelope replay, ProblemDetails on conflicts, optional Redis SET NX lock, opt-in method/path/body fingerprint, per-endpoint TTL, optional ActivitySource. Distinct from MCP write idempotency (`UseMemoryIdempotency` / `IMcpIdempotencyStore` above). + +**Version 1.0.0** is in this repo; the NuGet badge resolves after `idempotency-v1.0.0` is tagged and published. + +```bash +dotnet add package BuildingBlocks.Idempotency +``` + +```csharp +builder.Services.AddBuildingBlocksIdempotency(o => +{ + o.ProcessingTtl = TimeSpan.FromMinutes(2); // longer than worst-case handler + // o.EnableRequestFingerprint = true; // method+path+body; mismatch → 422 +}) +.UseRedisLock() +.UseTelemetry(); // optional — AddSource("BuildingBlocks.Idempotency") + +[HttpPost] +[Idempotent(useLock: true)] +public async Task> Create([FromBody] CreateOrder request) { ... } + +app.MapPost("/orders", CreateAsync).WithIdempotency(useLock: true); +``` + +- Package README: [`src/BuildingBlocks/Idempotency/PACKAGE_README.md`](src/BuildingBlocks/Idempotency/PACKAGE_README.md) · agent notes: [`AGENTS.md`](src/BuildingBlocks/Idempotency/AGENTS.md) +- Docs: [`docs/building-blocks/idempotency.md`](docs/building-blocks/idempotency.md) +- Lab: MVC `POST /api/v2/Order/order`, Minimal API smoke `POST /api/v2/idempotency-smoke`. Provenance: Experiments **3**, **4**, **12** ([catalog](tests/Lab/IntegrationTests/Experiments/README.md)). + +--- + ### BuildingBlocks.Pagination.EntityFrameworkCore [![NuGet](https://img.shields.io/nuget/v/BuildingBlocks.Pagination.EntityFrameworkCore.svg?logo=nuget)](https://www.nuget.org/packages/BuildingBlocks.Pagination.EntityFrameworkCore) @@ -670,7 +707,7 @@ Install the packages above in your own hosts, **or** clone this repo and run **F | Telemetry | **`BuildingBlocks.Telemetry`** in ServiceDefaults; **`BuildingBlocks.Aspire.Hosting.SigNoz`** on AppHost | | Event bus | RabbitMQ + transactional outbox/inbox, DLQ, dedup hooks | | Aspire lab | AppHost orchestration for Postgres, Redis, RabbitMQ, Memcached, SigNoz | -| IdempotentFusion | ULID `Idempotency-Key` + Redis status tracking + optional lock | +| HTTP idempotency | **`BuildingBlocks.Idempotency`** — MVC + Minimal API, 2xx envelope replay, optional Redis lock (`POST /api/v2/Order/order`) | | Feature flags (demo) | ASP.NET Core Feature Management + custom filters (claims / VIP) | | API surface | Versioned controllers + Minimal APIs, FluentValidation patterns | | Gateway | YARP reverse proxy + Memcached distributed rate limiting | @@ -893,12 +930,12 @@ docker compose up -d Conditional features via Microsoft.FeatureManagement and custom filters (e.g. VIP claims). Versioned controllers and Minimal APIs under `/api/v1|v2/...`. -### IdempotentFusion +### HTTP idempotency (BuildingBlocks.Idempotency) -REST idempotency with ULID keys and Redis status tracking (`POST /api/v2/Order/order`). +REST idempotency with `IDistributedCache` status tracking, MVC `[Idempotent]` / Minimal API `WithIdempotency`, and optional Redis lock (`POST /api/v2/Order/order`, smoke `POST /api/v2/idempotency-smoke`). See [BuildingBlocks.Idempotency](#buildingblocksidempotency). - [Idempotency with CQRS](https://www.linkedin.com/feed/update/urn:li:activity:7303686809891356676/) -- [IdempotentFusion project](https://www.linkedin.com/feed/update/urn:li:activity:7309149985307029504/) +- [IdempotentFusion project](https://www.linkedin.com/feed/update/urn:li:activity:7309149985307029504/) (historical Lab name) ### API versioning & validation @@ -938,7 +975,7 @@ See [Pagination showcase](#pagination-showcase) for the FeatureFusion catalog (` | **Polling publisher** | `OutBoxWorker` background poll → publish | | **Dead letter queue** | EventBus DLX / DLQ topology | | **Message deduplication** | Inbox + `MessageDeduplicationService` | -| **Idempotency** | `IdempotentAttribute` + Redis status tracking | +| **Idempotency** | `BuildingBlocks.Idempotency` — `[Idempotent]` / `WithIdempotency`, cache envelope, optional Redis lock | | **Feature toggle** | ASP.NET Core Feature Management + custom filters | | **Rate limiting** | ApiGateway Memcached fixed-window limiter | | **Circuit breaker / resilience** | Polly `ResiliencePipelineFactory` | diff --git a/docs/building-blocks/idempotency.md b/docs/building-blocks/idempotency.md new file mode 100644 index 0000000..8f4dfe1 --- /dev/null +++ b/docs/building-blocks/idempotency.md @@ -0,0 +1,65 @@ +# BuildingBlocks.Idempotency + +ASP.NET Core HTTP **Idempotency-Key** for MVC and Minimal API: host-owned `IDistributedCache`, optional Redis lock, opt-in fingerprint, ProblemDetails on conflicts, optional ActivitySource. + +Canonical consumer docs: [package README](../../src/BuildingBlocks/Idempotency/PACKAGE_README.md). Agent notes: [AGENTS.md](../../src/BuildingBlocks/Idempotency/AGENTS.md). + +## Install + +```bash +dotnet add package BuildingBlocks.Idempotency +``` + +Targets **.NET 8 / 9 / 10**. Register `IDistributedCache` on the host. For locked endpoints, register `IConnectionMultiplexer` and call `.UseRedisLock()`. + +## Register + +```csharp +builder.Services.AddBuildingBlocksIdempotency(o => +{ + o.ProcessingTtl = TimeSpan.FromMinutes(2); // longer than worst-case handler + // o.EnableRequestFingerprint = true; + // o.UserIdFallback = "anonymous"; +}) +.UseRedisLock() +.UseTelemetry(); // optional — also AddSource("BuildingBlocks.Idempotency") +``` + +```csharp +// MVC +[HttpPost] +[Idempotent(useLock: true)] +public async Task> Create([FromBody] CreateOrder request) { ... } + +// Minimal API +app.MapPost("/orders", CreateAsync).WithIdempotency(useLock: true); +``` + +## Behavior (summary) + +| Situation | Result | +|-----------|--------| +| New key | Execute; cache **2xx** envelope | +| Completed (default) | Replay + `X-Idempotent-Response` | +| Processing (active lease) | ProblemDetails 409 | +| Fingerprint mismatch | ProblemDetails 422 | +| Bad key / no identity / lock fail | ProblemDetails 400 / 401 / 500 | +| Non-2xx or exception | Drop cache entry | + +Full tables (TTL, options, ProblemDetails `type` URIs): [PACKAGE_README](../../src/BuildingBlocks/Idempotency/PACKAGE_README.md). + +## FeatureFusion lab + +| Surface | Path | +|---------|------| +| MVC (locked) | `POST /api/v2/Order/order` — `[Idempotent(useLock: true)]` | +| Minimal API smoke | `POST /api/v2/idempotency-smoke` — `.WithIdempotency(useLock: true)` | +| DI | `AddBuildingBlocksIdempotency` → `.UseRedisLock().UseTelemetry()`; OTel `AddSource("BuildingBlocks.Idempotency")` | + +Behavioral provenance (regression gates): Experiments **3** (cache vs production), **4** (concurrency / lock), **12** (fingerprint) — [Experiments README](../../tests/Lab/IntegrationTests/Experiments/README.md). Exp **15** documents ProcessingTtl lease overlap. + +Do not reintroduce Lab-local idempotency filter copies; use the package. + +## Not this package + +MCP write idempotency (`UseMemoryIdempotency`) lives in **BuildingBlocks.Mcp**. diff --git a/docs/building-blocks/mcp.md b/docs/building-blocks/mcp.md index 228e2ed..f5025f8 100644 --- a/docs/building-blocks/mcp.md +++ b/docs/building-blocks/mcp.md @@ -14,7 +14,7 @@ Requires .NET 8 / 9 / 10. HTTP is the default: `MapBuildingBlocksMcp()` → `/mc Cursor talks to a **running** API. After you add or rename tools, restart the host **and** reload the MCP server in Cursor. Aspire restart does not refresh Cursor’s cached `tools/list`. -Host OpenTelemetry: `IntegrateMcp = true` on `AddServiceDefaults` / `AddTelemetry`, plus `o.UseTelemetry()` on the MCP builder. +Host OpenTelemetry: `IntegrateMcp = true` on `AddServiceDefaults` / `AddTelemetry`, plus `o.UseTelemetry()` on the MCP builder. `UseTelemetry(t => t.IncludeExceptionDetails = true)` is an explicit Development-only opt-in for exception text on `McpErrorCode.Internal`; it is not enabled automatically from the environment. Without `IntegrateMcp`, `mcp.tool` spans stay in-process. ## What this package is diff --git a/docs/building-blocks/mediator.md b/docs/building-blocks/mediator.md index 0d7d9b6..3dc4fbe 100644 --- a/docs/building-blocks/mediator.md +++ b/docs/building-blocks/mediator.md @@ -2,9 +2,7 @@ **Status:** NuGet 1.1.0 (CQRS-first Send + pipeline; typed command/query behaviors; UseTelemetry traces + metrics) **Date:** 2026-08-27 -**Author:** Mohammad Hasan Hosseini -**LinkedIn:** [BuildingBlocks.Mediator v1.0.1](https://lnkd.in/p/eU5TsuR4) · [Mediator Pattern + Pipeline Behavior](https://www.linkedin.com/feed/update/urn:li:activity:7311311587372367873/) (prior) -**Catalog:** `docs/linkedin-posts.md` → `mediator` / `mediator-building-blocks` +**Author:** Mohammad Hasan Hosseini Docs: [getting-started](getting-started.md) · [design](mediator-design.md) · [TEST_MATRIX](TEST_MATRIX.md) diff --git a/docs/building-blocks/pagination.md b/docs/building-blocks/pagination.md index 8742f91..253cf35 100644 --- a/docs/building-blocks/pagination.md +++ b/docs/building-blocks/pagination.md @@ -12,6 +12,8 @@ dotnet add package BuildingBlocks.Pagination.EntityFrameworkCore Requires .NET 8 / 9 / 10. The nupkg includes `BuildingBlocks.Pagination.dll` (`SortKey`, cursors — the shared **intermediate representation** that adapters execute). Core and Dapper projects in this repo are **not** packed. PackageRef `Microsoft.EntityFrameworkCore` matches the TFM (8 / 9 / 10). +![OFFSET skip growing with page number versus keyset seeking from the last Price and Id.](../medium/images/04-offset-vs-keyset.png) + ## SortKey ```csharp @@ -70,6 +72,8 @@ GET /api/v2/products-page?limit=20&sortBy=Price&sortDirection=Ascending&cursor=< Empty cursor + `pageDirection=Backward` starts at the last page (`GET /api/v2/products-page?limit=20&pageDirection=Backward`). `sortBy`: `Id` · `Name` · `Price` · `CreatedAt` (each composite key ends with unique `Id`). `sortDirection`: `Ascending` · `Descending`. +![First page, next cursor, previous cursor, last page via pageDirection=Backward.](../medium/images/04b-cursor-flow.png) + Same query on: - `POST /api/v2/products-page` — Minimal API (compatibility) diff --git a/docs/building-blocks/telemetry.md b/docs/building-blocks/telemetry.md index fb4605f..9c236ba 100644 --- a/docs/building-blocks/telemetry.md +++ b/docs/building-blocks/telemetry.md @@ -212,6 +212,12 @@ Vertical slices under `Internal/`: `Pipeline/`, `Exporters/`, `Instrumentations/ ## SigNoz click-through (local Aspire) +![SigNoz ASP .NET Core Metrics for FeatureFusion in Development after catalog load.](../medium/images/07-signoz-dashboard.png) + +![SigNoz metrics explorer: HTTP server duration, mediator.send.duration, Npgsql connection create time.](../medium/images/07b-signoz-metrics-explorer.png) + +Crops from a Development session for service `FeatureFusion` after catalog traffic — not an SLA. + AppHost: `AddSigNoz().WithUi().WithDashboards()` + `WithSigNozOtlpExporter`. FeatureFusion AppHost uses `WithUiFromConfiguration` (`src/Lab/FeatureFusion.AppHost/appsettings.Development.json`): **`dev@local.test` / `DevPassword123!`**. Override with `SigNoz__AdminEmail`, `SigNoz__AdminPassword`, `SigNoz__UiPort`, …. The UI still requires login; root-user env only skips signup. Custom lab credentials are not shown on the Aspire connection panel. Omits `WithDataVolume()` (writable layer still persists when `Lifetime = Persistent`; sqlite volume always persists). After traffic: **Services** → **Traces** (waterfall) → **Metrics** / **Logs**. Filter spans with `telemetry.component`. Dashboard tiles use SigNoz **View in Traces**. The seeded **BuildingBlocks Telemetry** dashboard has four sections — Service RED, Components (grouped by diff --git a/docs/lab/MCP-EVENTBUS-RESEARCH-REPORT.md b/docs/lab/MCP-EVENTBUS-RESEARCH-REPORT.md new file mode 100644 index 0000000..b92517f --- /dev/null +++ b/docs/lab/MCP-EVENTBUS-RESEARCH-REPORT.md @@ -0,0 +1,146 @@ +# MCP / EventBus research branch — final report + +**Status:** Closed after Exp 20. Do not add Exp 21 in this line. +**Scope:** FeatureFusion Lab experiments 1–20 + MAF prototype (non-numbered). +**Date:** 2026-09-04 + +This document ends the MCP/EventBus research branch and marks the transition point to FeatureFusion **capability development** (capability choice is a separate next step). + +--- + +## 1. What Exp 1–20 established + +| # | Finding (short) | +|---|-----------------| +| 1–2 | Careless keyset pagination clients (HTTP/MCP) are observable and abusable | +| 3–4, 12, 15 | HTTP Redis idempotency: miss/hit, concurrency, fingerprint, ProcessingTtl lease overlap | +| 5, 8, 10 | Outbox → worker → RabbitMQ → inbox → handler works for HTTP and MCP | +| 6, 13, 14, 16 | MCP write semantics: confirm+replay, key regeneration amplifies, same-key concurrency safe, rate limit bounds storms | +| 7, 17 | Duplicate delivery: inbox dedup (flag off) vs `processed_messages` (flag on) | +| 9, 11 | Consumer failure → retry / DLQ / handler-throw outcomes characterized | +| 18 | **W3C Trace Context does not cross RabbitMQ**; correlate by `IntegrationEvent.Id` / `OrderId` | +| 19 | Lab observation seam journals happy-path stages (Observed vs Inferred) | +| 20 | Publish-then-crash (after broker publish / before outbox MarkProcessed) leaves outbox pending, may republish; inbox can still keep handler once | + +--- + +## 2. Hypotheses confirmed + +- HTTP Idempotency-Key + Redis is a reusable, shippable contract → **BuildingBlocks.Idempotency**. +- MCP tools can share Mediator handlers with HTTP when confirmation/idempotency are explicit. +- `IMcpRateLimiter` bounds distinct-key tool storms before invoke. +- Outbox/inbox pipeline delivers `OrderCreated` end-to-end under Aspire. +- Inbox (and optional `processed_messages`) suppress duplicate **handler** work for the same message identity. +- Agent regenerated MCP keys amplify business side effects (not an idempotency bug). +- Lab seam can observe publish-before-mark and processor enter/exit without changing default semantics when faults are disarmed. +- Crash-after-publish leaves outbox pending and risks a second `PublishDirect` (characterized, not fixed). + +--- + +## 3. Hypotheses rejected / falsified + +- “W3C automatically correlates async consumer work” — **rejected** (Exp 18). +- “Global `ProcessedEvents.Count` is experiment-owned” — **rejected** (Exp 13 flake; scoped by OrderId). +- “Same logical agent intent is safe under new MCP keys” — **rejected as safety claim**; amplification is expected under the key contract (Exp 13). +- “Processing lease covers full handler duration by default” — **rejected** (Exp 15 lease overlap). + +--- + +## 4. Real production concerns + +1. **Publish-then-crash / MarkProcessed gap** (Exp 8 named; Exp 20 characterized): broker may receive a message while outbox stays pending → republish risk. +2. **No W3C across RabbitMQ** (Exp 18): ops must use business ids for async correlation. +3. **MCP idempotency ≠ HTTP Redis idempotency**: agents that regenerate keys create real duplicate orders. +4. **ProcessingTtl** without renewal allows overlapping production under the same Idempotency-Key. +5. Consumer retry/DLQ semantics must be measured, not assumed from `RetryCount` alone (Exp 9/11). + +--- + +## 5. Reusable BuildingBlock potential + +| Already shipped / justified | Not justified as NuGet | +|-----------------------------|-------------------------| +| Mediator, Mcp, Idempotency, Pagination.EF, Telemetry, Aspire.Hosting.SigNoz | EventBus (lab-only) | +| | EventBus Lab seam / journal / fault controller | +| | Scenario / ExperimentEngine | +| | Fixed-permit MCP rate limiter test double | +| | Lease-hold handler gates | + +--- + +## 6. Interesting but NOT worth implementing (now) + +- Full W3C RabbitMQ propagation as a “quick fix” without product requirement. +- Generic EventBus framework or Scenario DSL. +- Packaging the Lab observation seam. +- Exhaustive fault matrix (delay, drop-before-ack, etc.) — Exp 20 is the deliberate end of this branch. +- Fixing publish-then-crash inside this research branch (product decision). + +--- + +## 7. What MCP contributed + +- Second surface over the same handlers (`orders.create`, `products.list`, …). +- Agent-relevant contracts: confirmation, memory idempotency, rate limiting. +- Evidence that agent clients need different failure modes than HTTP Redis keys (Exp 13/16). + +--- + +## 8. What Aspire contributed + +- Realistic local topology: Postgres, Redis, RabbitMQ, Memcached (+ SigNoz OTLP in AppHost). +- Shared `AspireFixture` for IntegrationTests without a parallel Testcontainers stack. +- Made outbox/inbox/DLQ experiments honest (real broker, real worker poll). + +--- + +## 9. What telemetry contributed + +- In-process `Activity` capture for Mediator / MCP / Npgsql / EventBus `ProcessMessage`. +- Proof that sync paths are correlatable; async path needs business ids (Exp 18). +- Lab journal complements (does not replace) OpenTelemetry for experiment evidence. + +--- + +## 10. Lab infrastructure to retain + +- `AspireFixture` + experiment convention (hypothesis → labeled calls → artifact → asserts). +- Phase 2 helpers: `HttpOrderCreate`, `OrderOutboxObserver`, `McpToolResults`, `LabTrace`, `LabRunInfo`. +- Lab EventBus seam: `IEventBusLabHook` (null no-op), journal, processor decorator, fault controller (Exp 19/20). +- Numbered Exp 1–20 catalog + `docs/lab/README.md` charter. +- Packable BuildingBlocks with proven extraction provenance. + +--- + +## 11. Lab infrastructure that should remain disposable + +- Per-experiment gates (lease hold, fixed-permit limiter). +- MAF/Ollama prototype wiring. +- Exp-specific JSON artifact shapes and call records. +- One-off fault arms beyond the closed branch’s Exp 20. + +--- + +## 12. What should explicitly NOT be built (from this branch) + +- Exp 21+ in the MCP/EventBus research line. +- Scenario/Actor/ExperimentEngine product. +- EventBus NuGet / generic messaging framework. +- Silent W3C propagation “to make traces pretty.” +- Production behavior changes solely to green tests. + +--- + +## Product transition + +**Research (this branch) ends at Exp 20.** + +**Next step (separate session):** choose the next FeatureFusion **product capability** using this report — not by inventing another messaging experiment. + +Transition criteria met when: + +1. Evidence families 1–20 are documented and Exp 20 closed the EventBus seam question for this branch. +2. BuildingBlocks extraction rules remain intact. +3. Remaining EventBus gaps (crash recovery product fix, W3C) are logged as **product decisions**, not open-ended lab expansion. + +Capability selection is intentionally deferred. diff --git a/docs/lab/README.md b/docs/lab/README.md new file mode 100644 index 0000000..609f985 --- /dev/null +++ b/docs/lab/README.md @@ -0,0 +1,104 @@ +# FeatureFusion Lab + +The **application is the laboratory**. A NuGet BuildingBlock is a reusable result that earned a boundary through real Aspire-backed experiments — not a framework dumped into the monorepo. + +This document is the lab charter. Experiment procedures live in [`tests/Lab/IntegrationTests/Experiments/README.md`](../../tests/Lab/IntegrationTests/Experiments/README.md). Package install docs live under [`docs/building-blocks/`](../building-blocks/). + +## What the Lab is + +| Layer | Role | +|-------|------| +| **FeatureFusion API** (`src/Lab/FeatureFusion`) | Vertical-slice host: HTTP + MCP tools share Mediator handlers | +| **Aspire AppHost** (`src/Lab/FeatureFusion.AppHost`) | Local orchestration: Postgres, Redis, RabbitMQ, Memcached, SigNoz OTLP | +| **Integration experiments** | Falsifiable hypotheses against real infrastructure (no mocks for the path under test) | +| **BuildingBlocks** (`src/BuildingBlocks/*`) | MIT NuGet packages extracted only when a contract is proven and reusable | + +## What the Lab is not + +- Not a Scenario/Actor/ExperimentEngine product (yet — see roadmap) +- Not a kitchen-sink framework: do not install BuildingBlocks “just because they exist” +- Not where LinkedIn URLs belong in NuGet XML / `PACKAGE_README.md` (keep those in [`docs/linkedin-posts.md`](../linkedin-posts.md)) + +## Architecture (current) + +``` +Cursor / Claude / MAF prototype (optional) + │ MCP HTTP /mcp + ▼ +FeatureFusion (WAF in tests / AppHost in local) + ├── BuildingBlocks.Mcp → tools catalog + invoker + ├── BuildingBlocks.Mediator → Send + pipeline + ├── BuildingBlocks.Idempotency → HTTP Idempotency-Key (Redis) + ├── BuildingBlocks.Pagination.* → products.list / products-page + ├── BuildingBlocks.Telemetry → OTel (IntegrateMediator / IntegrateMcp / EventBus source) + └── Lab EventBus (not packed) → outbox → RabbitMQ → inbox → handlers + ▲ +Aspire: Postgres │ Redis │ RabbitMQ │ Memcached │ SigNoz +``` + +### Production / reusable vs experiment-only + +| Kind | Location | Packed? | +|------|----------|---------| +| BuildingBlocks (Mediator, Mcp, Idempotency, Pagination.EF, Telemetry, Aspire.Hosting.SigNoz) | `src/BuildingBlocks/` | Yes (per-package tags / workflows) | +| Pagination IR / Dapper | `src/BuildingBlocks/Pagination*` | IR bundled; Dapper lab-only | +| EventBus outbox/inbox/DLQ | `src/Lab/EventBus` | **No** | +| YARP + Memcached rate limit | `src/Lab/FeatureFusion.ApiGateway` | **No** | +| Feature-flag demos | FeatureFusion | **No** | +| Experiment helpers | `tests/Lab/IntegrationTests/Infrastructure/` | **No** (test observation) | +| Numbered experiments Exp 1–18 + MAF prototype | `tests/.../Experiments/` | **No** | + +## Evidence families (Exp 1–18) + +| Family | Experiments | Outcome | +|--------|-------------|---------| +| Keyset pagination abuse | 1–2 | Careless HTTP/MCP cursor clients characterized | +| HTTP Redis idempotency | 3, 4, 12, 15 | Miss/hit, concurrency, fingerprint, ProcessingTtl lease overlap → **BuildingBlocks.Idempotency 1.0.0** | +| MCP write / agent-client semantics | 6, 13, 14, 16 | Confirm+replay, regenerated keys amplify, concurrent same-key safe, **IMcpRateLimiter** bounds distinct-key storms | +| Outbox → bus → inbox happy path | 5, 8, 10 | HTTP + MCP parity; outbox row lifecycle | +| Consumer dedup / failure | 7, 9, 11, 17 | Inbox dedup; retry/DLQ; `EnableDeduplication` + `processed_messages` | +| Sync + async telemetry | TraceEvidence, 18 | In-process capture works; **W3C does not cross RabbitMQ** — correlate by `IntegrationEvent.Id` / `OrderId` | +| EventBus Lab seam | 19–20 | Observation baseline + publish-then-crash characterization (**branch end**) | +| MAF + Ollama + MCP | Prototype only | Research spike; not a numbered experiment | + +## Extraction rule + +Extract to BuildingBlocks only when **all** hold: + +1. A falsifiable experiment (or package test matrix) proved the contract on real infra. +2. At least one host outside FeatureFusion would install it without the lab. +3. The public API can stay stable without pulling EventBus / YARP / FeatureManagement. +4. CI pack + release workflow exists (or is added in the same change). + +Do **not** extract: Scenario DSL, one-off test gates, fixed-permit rate limiters used only in Exp 16, lease-hold handler wrappers, or EventBus until crash/propagation seams are deliberate product work. + +## Package / versioning (current) + +- Each packable project owns `` in its `.csproj` (e.g. `BuildingBlocks.Mcp` `1.0.0`). +- Release tags are package-prefixed (`mcp-v*`, `mediator-v*`, `telemetry-v*`, …) and must match ``. +- Per-package GitHub Actions: `*-yml` CI + `*-release.yml` pack / Trusted Publishing to nuget.org (`idempotency.yml` / `idempotency-release.yml` for this package). +- IntegrationTests / Exp 1–18 are **not** in CI (local Aspire/Docker). + +## Roadmap (lab architecture — not experiment numbers) + +| Phase | Intent | Status | +|-------|--------|--------| +| **1** | Lab charter + evidence-family map | **Done** | +| **2** | Collapse stable IntegrationTests observation helpers (≥3 identical) | **Done** | +| **3** | Lab EventBus observation seam + Exp 19 baseline + Exp 20 publish-then-crash | **Done** — MCP/EventBus research branch **closed** (see `docs/lab/MCP-EVENTBUS-RESEARCH-REPORT.md`) | + +## MCP surface (lab today) + +| Tool | Kind | Notes | +|------|------|-------| +| `orders.create` | Command | Confirmation + MCP memory idempotency | +| `products.list` | Query | Keyset pagination | +| `demo.echo` | Command | `Idempotent = false` smoke | +| `lab.ping` | Query | Minimal API → MCP | + +## Related + +- [Experiments catalog](../../tests/Lab/IntegrationTests/Experiments/README.md) +- [AGENTS.md](../../AGENTS.md) +- [BuildingBlocks getting started](../building-blocks/getting-started.md) +- [LinkedIn / Medium catalog](../linkedin-posts.md) diff --git a/docs/linkedin-posts.md b/docs/linkedin-posts.md index dcc0106..c3525f2 100644 --- a/docs/linkedin-posts.md +++ b/docs/linkedin-posts.md @@ -1,21 +1,29 @@ # LinkedIn posts catalog -Canonical map from [FeatureFusion](https://github.com/Maxofpower/FeatureFusion) (formerly FeatureManagement) features to LinkedIn deep-dives by [Mohammad Hasan Hosseini](https://www.linkedin.com/in/mhhoseini/). +Canonical map from [FeatureFusion](https://github.com/Maxofpower/FeatureFusion) (formerly FeatureManagement) features to LinkedIn / Medium writing by [Mohammad Hasan Hosseini](https://www.linkedin.com/in/mhhoseini/). -Primary links are also mirrored on public types via XML `` (e.g. `IMediator`, `IdempotentAttribute`) so IDEs show them without opening this file. +Keep LinkedIn and Medium URLs here and on the [root README](../README.md) catalog. Do **not** put them in NuGet `PACKAGE_README.md` files or packed XML remarks. -| Id | Title | Status | URL | Code map | Summary | -|----|-------|--------|-----|----------|---------| -| `mediator` | Manual Mediator + pipeline behaviors | published | https://www.linkedin.com/feed/update/urn:li:activity:7311311587372367873/ | was `Infrastructure/CQRS`; now `src/BuildingBlocks/Mediator` | Custom CQRS Mediator with cached wrappers, void via `ICommand : ICommand`, OTel-friendly behaviors. Follow-up NuGet post → `mediator-building-blocks`. | -| `mediator-building-blocks` | BuildingBlocks.Mediator NuGet (v1.0.1) | published | https://lnkd.in/p/eU5TsuR4 | `src/BuildingBlocks/Mediator` + `docs/building-blocks/mediator.md` | Packaged CQRS Send + ordered pipeline (void via `ICommand : ICommand`). Publish/notifications out of v1. Prior deep-dive → `mediator`. | -| `cursor-pagination` | Reusable generic bidirectional keyset (cursor) pagination | published | https://www.linkedin.com/feed/update/urn:li:activity:7325068550614708225/ | `src/BuildingBlocks/Pagination.EntityFrameworkCore` + lab `ProductService` | Typed `SortKey`, opaque cursors, one EF Core nupkg. Optional `QueryHint` (default none). Lab: MVC `POST /api/v2/Product/products`, Minimal API `GET /api/v2/products-page` (POST kept), Dapper `products-dapper`, MCP `products.list` — one `GetProductsQuery`. | -| `idempotency-cqrs` | Idempotency with CQRS commands | published | https://www.linkedin.com/feed/update/urn:li:activity:7303686809891356676/ | (pattern lab) | Command-level idempotency via reusable `IdentifiedCommand` + ULID `Idempotency-Key`, intercepting before handlers. Complementary to Mediator building blocks — not core v1. | -| `idempotentfusion` | IdempotentFusion (REST API) | published | https://www.linkedin.com/feed/update/urn:li:activity:7309149985307029504/ | `src/Lab/FeatureFusion/Infrastructure/Filters/Idempotent*` | Redis-backed request status tracking (Processing/Completed/Failed), optional distributed lock, ULID keys. | -| `mcp-message-tools` | Message types as MCP tools | planned | _add permalink when published_ | `src/BuildingBlocks/Mcp` + `docs/building-blocks/mcp.md` | Map `[McpTool]` commands/queries **or** public static endpoint methods **or** `MapTool` handlers to the official MCP SDK. Same logic as HTTP. Deny-by-default catalog. Not OpenAPI, not a SOLID linter. | -| `pubsub` | In-process EventPublisher | planned | _add permalink_ | sibling lab `pub-sub pattern` | In-process publish to consumers — **not** part of Mediator Send/pipeline v1. | +Umbrella narrative (lab → packages): [Medium](https://medium.com/@m2hweb86/featurefusion-an-engineering-lab-that-extracts-reusable-net-building-blocks-28152cafafb9) · [LinkedIn](https://lnkd.in/p/e4-FrM44). + +| Id | Title | Status | URL | Code map | Related | Summary | +|----|-------|--------|-----|----------|---------|---------| +| `medium-engineering-lab` | FeatureFusion: an engineering lab that extracts reusable .NET building blocks | published | https://medium.com/@m2hweb86/featurefusion-an-engineering-lab-that-extracts-reusable-net-building-blocks-28152cafafb9 | repo + `src/BuildingBlocks/*` + lab `GetProductsQuery` | `building-blocks-overview`, `mediator-building-blocks`, `cursor-pagination`, `mcp-message-tools` | Long-form lab-vs-package story. Same `GetProductsQuery` on GET / POST / MVC / Dapper / MCP. | +| `building-blocks-overview` | BuildingBlocks + NuGet packages (Mediator 1.1.0, keyset, MCP, Telemetry, SigNoz) | published | https://lnkd.in/p/e4-FrM44 | `src/BuildingBlocks/*` | `medium-engineering-lab`, `mediator-building-blocks`, `cursor-pagination`, `mcp-message-tools` | Umbrella post: extract only when the contract earned a boundary. Not a SigNoz SDK. Dapper pagination stays in-repo. | +| `mediator` | Manual Mediator + pipeline behaviors | published | https://www.linkedin.com/feed/update/urn:li:activity:7311311587372367873/ | was `Infrastructure/CQRS`; now `src/BuildingBlocks/Mediator` | `mediator-building-blocks` | Custom CQRS Mediator with cached wrappers, void via `ICommand : ICommand`, OTel-friendly behaviors. Follow-up NuGet post → `mediator-building-blocks`. | +| `mediator-building-blocks` | BuildingBlocks.Mediator NuGet (v1.0.1 launch; current 1.1.0) | published | https://lnkd.in/p/eU5TsuR4 | `src/BuildingBlocks/Mediator` + `docs/building-blocks/mediator.md` | `mediator`, `building-blocks-overview` | Packaged CQRS Send + ordered pipeline (void via `ICommand : ICommand`). Publish/notifications out of v1. 1.1.0 adds typed command/query behaviors + Send metrics. Prior deep-dive → `mediator`. | +| `cursor-pagination` | Reusable generic bidirectional keyset (cursor) pagination | published | https://www.linkedin.com/feed/update/urn:li:activity:7325068550614708225/ | `src/BuildingBlocks/Pagination.EntityFrameworkCore` + lab `ProductService` | `building-blocks-overview` | Typed `SortKey`, opaque cursors, one EF Core nupkg. Optional `QueryHint` (default none). Lab: MVC `POST /api/v2/Product/products`, Minimal API `GET /api/v2/products-page` (POST kept), Dapper `products-dapper`, MCP `products.list` — one `GetProductsQuery`. | +| `mcp-message-tools` | Message types as MCP tools | published | https://lnkd.in/p/e4-FrM44 | `src/BuildingBlocks/Mcp` + `docs/building-blocks/mcp.md` | `building-blocks-overview` | Deny-by-default `[McpTool]` / `WithMcp` / `MapTool` on the official SDK. Same logic as HTTP. Interim permalink is the umbrella post until a dedicated MCP post ships. Not OpenAPI, not a SOLID linter. | +| `idempotency-cqrs` | Idempotency with CQRS commands | published | https://www.linkedin.com/feed/update/urn:li:activity:7303686809891356676/ | (pattern lab) | `idempotentfusion` | **Lab only, not NuGet.** Command-level idempotency via reusable `IdentifiedCommand` + ULID `Idempotency-Key`, intercepting before handlers. Complementary to Mediator — not core v1. | +| `idempotentfusion` | IdempotentFusion (REST API) | published | https://www.linkedin.com/feed/update/urn:li:activity:7309149985307029504/ | `src/BuildingBlocks/Idempotency` + Lab `OrderController` | `idempotency-cqrs` | HTTP Idempotency-Key via `BuildingBlocks.Idempotency` (Redis cache host-owned, optional SET NX lock). Lab sets `UserIdFallback = "123"`. | +| `eventbus-outbox-inbox` | Lightweight modular EventBus (outbox, inbox, DLQ) | published | https://www.linkedin.com/posts/mhhoseini_a-lightweight-modular-eventbus-implementation-activity-7316381126535663631-6FiI | `src/Lab/EventBus` | `aspire-integration-testing` | **Lab only, not NuGet.** Sibling of Mediator Send — not `INotification`. | +| `aspire-integration-testing` | Integration testing microservices with Aspire | published | https://www.linkedin.com/posts/mhhoseini_integration-testing-microservices-with-aspire-activity-7321088554565029888-Z0a8 | `tests/Lab/IntegrationTests` + `src/Lab/FeatureFusion.AppHost` | `building-blocks-overview` | **Lab only, not NuGet.** Aspire-hosted fixture against real Postgres / Redis / RabbitMQ / Memcached. | +| `ddd-architecture` | Diving into DDD | published | https://www.linkedin.com/posts/mhhoseini_diving-into-ddd-activity-7427983383126896640-Eaaw | lab vertical slices under `src/Lab/FeatureFusion/Features` | `medium-engineering-lab` | **Lab only, not NuGet.** Architecture notes; FeatureFusion is the showcase host, not a DDD framework. | +| `pubsub` | In-process EventPublisher | planned | _add permalink_ | sibling lab `pub-sub pattern` | | In-process publish to consumers — **not** part of Mediator Send/pipeline v1. | ## How to maintain -1. When you publish a LinkedIn post tied to this repo, add a row here **in the same PR** that ships or documents the feature. -2. Link the same URL from the matching README section under **Related LinkedIn**. +1. When you publish a LinkedIn or Medium post tied to this repo, add a row here **in the same PR** that ships or documents the feature. +2. Link the same URL from the root [README](../README.md) **LinkedIn catalog** — not from NuGet `PACKAGE_README.md` or packed XML docs. 3. Prefer activity permalinks (`urn:li:activity:…`) or `/posts/…` URLs — do not invent links. +4. New packable BuildingBlock: add `AGENTS.md` + `PACKAGE_README.md` (no LinkedIn URLs in those files). diff --git a/llms.txt b/llms.txt new file mode 100644 index 0000000..5f0a9ba --- /dev/null +++ b/llms.txt @@ -0,0 +1,45 @@ +# FeatureFusion + +> .NET engineering lab that extracts focused MIT BuildingBlocks (Mediator, Idempotency, keyset pagination, MCP, OpenTelemetry, Aspire SigNoz). The application is the laboratory; NuGet is the reusable result. + +## Canonical + +- [README](https://github.com/Maxofpower/FeatureFusion/blob/main/README.md): lab + package overview +- [AGENTS.md](https://github.com/Maxofpower/FeatureFusion/blob/main/AGENTS.md): repo map for coding agents +- [Medium](https://medium.com/@m2hweb86/featurefusion-an-engineering-lab-that-extracts-reusable-net-building-blocks-28152cafafb9): lab-to-package narrative +- [Writing catalog](https://github.com/Maxofpower/FeatureFusion/blob/main/docs/linkedin-posts.md): LinkedIn / Medium post ↔ code map +- [CONTRIBUTING](https://github.com/Maxofpower/FeatureFusion/blob/main/CONTRIBUTING.md) + +## BuildingBlocks — NuGet READMEs + +- [Mediator PACKAGE_README](https://github.com/Maxofpower/FeatureFusion/blob/main/src/BuildingBlocks/Mediator/PACKAGE_README.md) +- [Idempotency PACKAGE_README](https://github.com/Maxofpower/FeatureFusion/blob/main/src/BuildingBlocks/Idempotency/PACKAGE_README.md) +- [Pagination.EntityFrameworkCore PACKAGE_README](https://github.com/Maxofpower/FeatureFusion/blob/main/src/BuildingBlocks/Pagination.EntityFrameworkCore/PACKAGE_README.md) +- [Mcp PACKAGE_README](https://github.com/Maxofpower/FeatureFusion/blob/main/src/BuildingBlocks/Mcp/PACKAGE_README.md) +- [Telemetry PACKAGE_README](https://github.com/Maxofpower/FeatureFusion/blob/main/src/BuildingBlocks/Telemetry/PACKAGE_README.md) +- [Aspire.Hosting.SigNoz PACKAGE_README](https://github.com/Maxofpower/FeatureFusion/blob/main/src/BuildingBlocks/Aspire.Hosting.SigNoz/PACKAGE_README.md) + +## BuildingBlocks — agent notes + +- [Mediator AGENTS](https://github.com/Maxofpower/FeatureFusion/blob/main/src/BuildingBlocks/Mediator/AGENTS.md) +- [Idempotency AGENTS](https://github.com/Maxofpower/FeatureFusion/blob/main/src/BuildingBlocks/Idempotency/AGENTS.md) +- [Pagination.EntityFrameworkCore AGENTS](https://github.com/Maxofpower/FeatureFusion/blob/main/src/BuildingBlocks/Pagination.EntityFrameworkCore/AGENTS.md) +- [Pagination IR AGENTS](https://github.com/Maxofpower/FeatureFusion/blob/main/src/BuildingBlocks/Pagination/AGENTS.md) +- [Pagination.Dapper AGENTS](https://github.com/Maxofpower/FeatureFusion/blob/main/src/BuildingBlocks/Pagination.Dapper/AGENTS.md) +- [Mcp AGENTS](https://github.com/Maxofpower/FeatureFusion/blob/main/src/BuildingBlocks/Mcp/AGENTS.md) +- [Telemetry AGENTS](https://github.com/Maxofpower/FeatureFusion/blob/main/src/BuildingBlocks/Telemetry/AGENTS.md) +- [Aspire.Hosting.SigNoz AGENTS](https://github.com/Maxofpower/FeatureFusion/blob/main/src/BuildingBlocks/Aspire.Hosting.SigNoz/AGENTS.md) + +## Guides + +- [Getting started](https://github.com/Maxofpower/FeatureFusion/blob/main/docs/building-blocks/getting-started.md) +- [Mediator](https://github.com/Maxofpower/FeatureFusion/blob/main/docs/building-blocks/mediator.md) +- [Idempotency](https://github.com/Maxofpower/FeatureFusion/blob/main/docs/building-blocks/idempotency.md) +- [Pagination](https://github.com/Maxofpower/FeatureFusion/blob/main/docs/building-blocks/pagination.md) +- [MCP](https://github.com/Maxofpower/FeatureFusion/blob/main/docs/building-blocks/mcp.md) +- [Telemetry](https://github.com/Maxofpower/FeatureFusion/blob/main/docs/building-blocks/telemetry.md) + +## Lab integration experiments + +- [Lab charter](https://github.com/Maxofpower/FeatureFusion/blob/main/docs/lab/README.md): evidence families, extraction rules, roadmap +- [Experiments README](https://github.com/Maxofpower/FeatureFusion/blob/main/tests/Lab/IntegrationTests/Experiments/README.md): behavioral experiment catalog Exp 1–18 + MAF prototype, evidence model, conventions diff --git a/src/BuildingBlocks/Aspire.Hosting.SigNoz/PACKAGE_README.md b/src/BuildingBlocks/Aspire.Hosting.SigNoz/PACKAGE_README.md index acae2e2..47c3128 100644 --- a/src/BuildingBlocks/Aspire.Hosting.SigNoz/PACKAGE_README.md +++ b/src/BuildingBlocks/Aspire.Hosting.SigNoz/PACKAGE_README.md @@ -34,6 +34,12 @@ builder.AddProject("api") Run the AppHost **https** profile. Add `.WithDataVolume()` if you need ClickHouse/ZooKeeper history across restarts. Do not use `WithReference` for OTLP. +![SigNoz ASP .NET Core Metrics for FeatureFusion in Development after catalog load.](https://raw.githubusercontent.com/Maxofpower/FeatureFusion/main/docs/medium/images/07-signoz-dashboard.png) + +![SigNoz metrics explorer: HTTP server duration, mediator.send.duration, Npgsql connection create time.](https://raw.githubusercontent.com/Maxofpower/FeatureFusion/main/docs/medium/images/07b-signoz-metrics-explorer.png) + +These screenshots are a Development lab session after catalog traffic — not an SLA. + ## Quick start — all options Local-dev AppHost only. Production still uses `BuildingBlocks.Telemetry` + a real OTLP endpoint. Run the AppHost **https** profile. Do not use `WithReference` for OTLP. diff --git a/src/BuildingBlocks/Idempotency/AGENTS.md b/src/BuildingBlocks/Idempotency/AGENTS.md new file mode 100644 index 0000000..f6050bb --- /dev/null +++ b/src/BuildingBlocks/Idempotency/AGENTS.md @@ -0,0 +1,50 @@ +# BuildingBlocks.Idempotency — agent notes + +**Status: COMPLETE — 1.0.0** + +HTTP **Idempotency-Key** for ASP.NET Core. Consumer docs: [PACKAGE_README.md](PACKAGE_README.md). Lab guide: [docs/building-blocks/idempotency.md](../../../docs/building-blocks/idempotency.md). + +Install: `dotnet add package BuildingBlocks.Idempotency`. + +Lab FeatureFusion hosts this package — **do not** reintroduce Lab-local filter copies. Unit tests: `tests/BuildingBlocks/Idempotency.Tests`. Provenance Exp **3 / 4 / 12** (and lease Exp **15**). + +## Register (Lab shape) + +```csharp +builder.Services.AddBuildingBlocksIdempotency(o => +{ + o.UserIdFallback = "123"; // Lab-only; not a package default + o.ProcessingTtl = TimeSpan.FromMinutes(2); +}) +.UseRedisLock() +.UseTelemetry(); + +// host OTel: +telemetry.AddSource("BuildingBlocks.Idempotency"); + +[Idempotent(useLock: true)] +public async Task Create(...) { } + +app.MapPost("/path", handler).WithIdempotency(useLock: true); +``` + +## Semantics (do not regress) + +- Shared `IdempotencyGate` for MVC + Minimal API +- Cache all **2xx**; replay envelope + configurable replay header +- Errors: ProblemDetails (`https://buildingblocks.dev/errors/idempotency/...`) +- Fingerprint default **off** (Exp 3); on → method+path+body SHA-256 (Exp 12) +- Lock only around GetOrCreate when `UseLock` (Exp 4) +- MVC ObjectResult body: Newtonsoft.Json; Minimal API `IResult`: System.Text.Json +- Telemetry optional; no cache-key tag by default; no BuildingBlocks.Telemetry package ref + +## Lab evidence + +| Exp | Role | +|-----|------| +| **3** CacheVsProduction | Miss → production, hit → replay; fingerprint-off body tolerance | +| **4** ConcurrencyRace | Redis SET NX + Processing/Completed under concurrent same-key clients | +| **12** Fingerprint | Opt-in mismatch → 422 vs same-body replay | +| **15** ProcessingLease | Expired ProcessingTtl can admit overlapping production | + +Catalog: `tests/Lab/IntegrationTests/Experiments/README.md`. diff --git a/src/BuildingBlocks/Idempotency/Abstractions/DuplicateCompletedBehavior.cs b/src/BuildingBlocks/Idempotency/Abstractions/DuplicateCompletedBehavior.cs new file mode 100644 index 0000000..ef54f8a --- /dev/null +++ b/src/BuildingBlocks/Idempotency/Abstractions/DuplicateCompletedBehavior.cs @@ -0,0 +1,17 @@ +namespace BuildingBlocks.Idempotency; + +/// +/// Strategy when the same Idempotency-Key is reused after a Completed (or Failed) cache entry. +/// +public enum DuplicateCompletedBehavior +{ + /// + /// Replay the stored status, content-type, and body (default; Exp 3 compatible). + /// + Replay = 0, + + /// + /// Reject the reuse with (default 409). + /// + Conflict = 1 +} diff --git a/src/BuildingBlocks/Idempotency/Abstractions/IIdempotencyLock.cs b/src/BuildingBlocks/Idempotency/Abstractions/IIdempotencyLock.cs new file mode 100644 index 0000000..d30ceac --- /dev/null +++ b/src/BuildingBlocks/Idempotency/Abstractions/IIdempotencyLock.cs @@ -0,0 +1,14 @@ +namespace BuildingBlocks.Idempotency; + +/// +/// Optional distributed lock used only around GetOrCreate of the cache entry when +/// or Minimal API WithIdempotency(useLock: true) is set. +/// +public interface IIdempotencyLock +{ + /// Attempts to acquire a lock. Returns false if another owner holds it. + Task AcquireAsync(string key, string value, TimeSpan expiry, CancellationToken cancellationToken = default); + + /// Releases the lock only when matches the owner. + Task ReleaseAsync(string key, string value, CancellationToken cancellationToken = default); +} diff --git a/src/BuildingBlocks/Idempotency/Abstractions/IdempotencyCacheEntry.cs b/src/BuildingBlocks/Idempotency/Abstractions/IdempotencyCacheEntry.cs new file mode 100644 index 0000000..b77a998 --- /dev/null +++ b/src/BuildingBlocks/Idempotency/Abstractions/IdempotencyCacheEntry.cs @@ -0,0 +1,28 @@ +namespace BuildingBlocks.Idempotency; + +/// +/// Wire format stored in . +/// Status values: Processing, Completed (and read-compat Failed). +/// +public sealed class IdempotencyCacheEntry +{ + /// Lifecycle status string. + public string Status { get; set; } = string.Empty; + + /// Serialized response body when completed (or read-compat Failed). + public string? Response { get; set; } + + /// HTTP status to replay. Null on legacy entries → treat as 200. + public int? StatusCode { get; set; } + + /// Content-Type to replay. Null on legacy entries → application/json. + public string? ContentType { get; set; } + + /// + /// SHA-256 hex of method + "\n" + path + "\n" + body when fingerprinting was enabled on first miss. + /// + public string? RequestFingerprint { get; set; } + + /// UTC instant after which a Processing entry is considered abandoned. + public DateTimeOffset? ProcessingExpiresAtUtc { get; set; } +} diff --git a/src/BuildingBlocks/Idempotency/Abstractions/IdempotencyOptions.cs b/src/BuildingBlocks/Idempotency/Abstractions/IdempotencyOptions.cs new file mode 100644 index 0000000..f36b6f4 --- /dev/null +++ b/src/BuildingBlocks/Idempotency/Abstractions/IdempotencyOptions.cs @@ -0,0 +1,111 @@ +namespace BuildingBlocks.Idempotency; + +/// +/// Configuration for HTTP idempotency. Hosts that need Lab-compatible anonymous +/// user scoping must set explicitly (for example "123"). +/// +/// +/// +/// TTL policy (publishable expiry): bounds in-flight leases; +/// bounds Completed replays. Keep longer than +/// the worst-case successful handler, or a second request may run while the first is still in flight +/// after lease expiry (lock covers GetOrCreate only). +/// +/// +/// Replay header default is X-Idempotent-Response. Override via if needed. +/// +/// +/// +/// +/// services.AddBuildingBlocksIdempotency(o => +/// { +/// o.UserIdFallback = "123"; +/// o.ProcessingTtl = TimeSpan.FromMinutes(2); +/// }); +/// +/// +public sealed class IdempotencyOptions +{ + /// HTTP header carrying the idempotency key. Default Idempotency-Key. + public string HeaderName { get; set; } = "Idempotency-Key"; + + /// Cache key prefix. Default Idempotency. + public string KeyPrefix { get; set; } = "Idempotency"; + + /// + /// Response header set on replay. Default X-Idempotent-Response. + /// + public string CachedResponseHeader { get; set; } = "X-Idempotent-Response"; + + /// TTL for Completed (and Failed read-compat) cache entries. Default 30 minutes. + public TimeSpan EntryTtl { get; set; } = TimeSpan.FromMinutes(30); + + /// + /// Lifetime of a Processing entry. After this, the same key may run again (abandoned in-flight). + /// Default 2 minutes. Must be shorter than in practice. + /// + public TimeSpan ProcessingTtl { get; set; } = TimeSpan.FromMinutes(2); + + /// Timeout for GetOrCreate / Mark Processing. Default 10 seconds. + public TimeSpan CacheOperationTimeout { get; set; } = TimeSpan.FromSeconds(10); + + /// Timeout for Complete / Remove. Default 5 seconds. + public TimeSpan CacheWriteTimeout { get; set; } = TimeSpan.FromSeconds(5); + + /// Redis lock expiry when UseLock is true. Default 10 seconds. + public TimeSpan LockExpiry { get; set; } = TimeSpan.FromSeconds(10); + + /// Claim type used for the user segment of the cache key. + public string UserIdClaimType { get; set; } = System.Security.Claims.ClaimTypes.NameIdentifier; + + /// + /// Optional extra claim types inserted into the cache key after the prefix and before the user id + /// (for example tenant id). Empty by default — key remains {prefix}_{user}_{key}. + /// + public IList KeyScopeClaimTypes { get; } = new List(); + + /// + /// Optional fallback when the user claim is missing. + /// Null/empty means missing claim → unauthorized (HTTP 401 ProblemDetails). + /// + public string? UserIdFallback { get; set; } + + /// When true (default), the header value must parse as a ULID. + public bool RequireUlid { get; set; } = true; + + /// + /// Maximum allowed length of the raw header value when validated. + /// Default 256. Applied even when is false. + /// + public int MaxKeyLength { get; set; } = 256; + + /// + /// When true, SHA-256 of method + "\n" + path + "\n" + body is stored on first miss and compared on reuse. + /// Default false (same key always replays regardless of body). + /// + public bool EnableRequestFingerprint { get; set; } + + /// + /// HTTP status when the same key is reused with a different fingerprint. + /// Default 422 Unprocessable Entity. + /// + public int FingerprintConflictStatusCode { get; set; } = 422; + + /// + /// HTTP status when another request holds a non-expired Processing lease. + /// Default 409 Conflict. + /// + public int ProcessingConflictStatusCode { get; set; } = 409; + + /// + /// Behavior when the same key hits a Completed (or Failed) entry. + /// Default . + /// + public DuplicateCompletedBehavior DuplicateCompletedBehavior { get; set; } = DuplicateCompletedBehavior.Replay; + + /// + /// HTTP status when is . + /// Default 409 Conflict. + /// + public int DuplicateConflictStatusCode { get; set; } = 409; +} diff --git a/src/BuildingBlocks/Idempotency/AspNetCore/IdempotentAttribute.cs b/src/BuildingBlocks/Idempotency/AspNetCore/IdempotentAttribute.cs new file mode 100644 index 0000000..8462646 --- /dev/null +++ b/src/BuildingBlocks/Idempotency/AspNetCore/IdempotentAttribute.cs @@ -0,0 +1,75 @@ +using Microsoft.AspNetCore.Mvc.Filters; +using Microsoft.Extensions.Caching.Distributed; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace BuildingBlocks.Idempotency.AspNetCore; + +/// +/// Applies Idempotency-Key handling via status tracking +/// and optional . +/// +/// +/// Defaults stay Exp 3 / 4 / 12 compatible: replay Completed, Processing → 409, fingerprint off, +/// optional Redis lock when is true. +/// Per-endpoint TTL: set / to a +/// positive value to override global (0 = use global). +/// +/// +/// +/// [HttpPost] +/// [Idempotent(useLock: true)] +/// public async Task<ActionResult> Create(...) { ... } +/// +/// +/// +/// +[AttributeUsage(AttributeTargets.Method)] +public sealed class IdempotentAttribute : Attribute, IFilterFactory +{ + /// When true, GetOrCreate is wrapped in . + public bool UseLock { get; set; } + + /// + /// Optional Processing TTL in seconds for this endpoint. 0 (default) uses global options. + /// + public int ProcessingTtlSeconds { get; set; } + + /// + /// Optional Completed entry TTL in seconds for this endpoint. 0 (default) uses global options. + /// + public int EntryTtlSeconds { get; set; } + + /// Creates the attribute. + public IdempotentAttribute(bool useLock = false) + { + UseLock = useLock; + } + + /// + public bool IsReusable => false; + + /// + public IFilterMetadata CreateInstance(IServiceProvider serviceProvider) + { + ArgumentNullException.ThrowIfNull(serviceProvider); + + var distributedCache = serviceProvider.GetRequiredService(); + var loggerFactory = serviceProvider.GetRequiredService(); + var options = serviceProvider.GetService>()?.Value + ?? new IdempotencyOptions(); + var idempotencyLock = serviceProvider.GetService(); + var telemetry = serviceProvider.GetService(); + + return new IdempotentAttributeFilter( + distributedCache, + loggerFactory, + idempotencyLock, + options, + useLock: UseLock, + telemetry, + ProcessingTtlSeconds, + EntryTtlSeconds); + } +} diff --git a/src/BuildingBlocks/Idempotency/AspNetCore/IdempotentAttributeFilter.cs b/src/BuildingBlocks/Idempotency/AspNetCore/IdempotentAttributeFilter.cs new file mode 100644 index 0000000..1cd12f1 --- /dev/null +++ b/src/BuildingBlocks/Idempotency/AspNetCore/IdempotentAttributeFilter.cs @@ -0,0 +1,214 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Filters; +using Microsoft.Extensions.Caching.Distributed; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace BuildingBlocks.Idempotency.AspNetCore; + +/// +/// Resource + action filter that implements Idempotency-Key handling via +/// and optional . +/// +/// +/// +public sealed class IdempotentAttributeFilter : IAsyncResourceFilter, IAsyncActionFilter +{ + private readonly IdempotencyGate _gate; + private readonly IdempotencyOptions _options; + private readonly IdempotencyTelemetry? _telemetry; + private readonly IdempotencyEndpointSettings _endpoint; + + /// Creates the filter. + public IdempotentAttributeFilter( + IDistributedCache distributedCache, + ILoggerFactory loggerFactory, + IIdempotencyLock? idempotencyLock, + IdempotencyOptions options, + bool useLock, + IdempotencyTelemetry? telemetry = null, + int processingTtlSeconds = 0, + int entryTtlSeconds = 0) + { + ArgumentNullException.ThrowIfNull(distributedCache); + ArgumentNullException.ThrowIfNull(loggerFactory); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _telemetry = telemetry; + _endpoint = IdempotencyEndpointSettings.Create(useLock, processingTtlSeconds, entryTtlSeconds); + _gate = new IdempotencyGate( + distributedCache, + loggerFactory.CreateLogger(), + _options, + idempotencyLock); + } + + /// + public async Task OnResourceExecutionAsync(ResourceExecutingContext context, ResourceExecutionDelegate next) + { + var headerName = _options.HeaderName; + string? rawKey = null; + if (context.HttpContext.Request.Headers.TryGetValue(headerName, out var headerValue)) + rawKey = headerValue.ToString(); + + if (!IdempotencyKeyValidator.TryValidate(rawKey, _options, out var error)) + { + var decision = IdempotencyDecision.BadKey(error!); + _telemetry?.RecordDecision(decision, _endpoint.UseLock, _options); + context.Result = ToProblemResult(decision); + return; + } + + if (_options.EnableRequestFingerprint) + context.HttpContext.Request.EnableBuffering(); + + await next().ConfigureAwait(false); + } + + /// + public async Task OnActionExecutionAsync(ActionExecutingContext context, ActionExecutionDelegate next) + { + var decision = await _gate + .BeginAsync(context.HttpContext, _endpoint, context.HttpContext.RequestAborted) + .ConfigureAwait(false); + + if (!decision.ShouldExecute) + { + _telemetry?.RecordDecision(decision, _endpoint.UseLock, _options); + + if (decision.Kind == IdempotencyDecisionKind.Replay && decision.Envelope is { } envelope) + { + context.Result = new ContentResult + { + Content = envelope.Body, + ContentType = envelope.ContentType, + StatusCode = envelope.StatusCode + }; + context.HttpContext.Response.Headers.Append(_options.CachedResponseHeader, "true"); + return; + } + + context.Result = ToProblemResult(decision); + return; + } + + var cacheKey = decision.CacheKey!; + var requestFingerprint = decision.RequestFingerprint; + + try + { + var executedContext = await next().ConfigureAwait(false); + + if (TryCaptureSuccessfulResponse(executedContext.Result, out var statusCode, out var contentType, out var body)) + { + var envelope = new IdempotencyEnvelope(statusCode, contentType, body); + await _gate + .CompleteAsync(cacheKey, envelope, requestFingerprint, _endpoint, context.HttpContext.RequestAborted) + .ConfigureAwait(false); + _telemetry?.Record(IdempotencyOutcomes.Executed, _endpoint.UseLock, cacheKey, statusCode); + } + else + { + await _gate.AbandonAsync(cacheKey, context.HttpContext.RequestAborted).ConfigureAwait(false); + } + } + catch (Exception) + { + await _gate.AbandonAsync(cacheKey, CancellationToken.None).ConfigureAwait(false); + throw; + } + } + + /// + /// Captures a successful action result for cache storage. + /// Keeps Newtonsoft serialization for ObjectResult values (same representation as historical Lab/Exp 3). + /// Caches all HTTP 2xx results (including null StatusCode meaning 200). + /// + internal static bool TryCaptureSuccessfulResponse( + IActionResult? result, + out int statusCode, + out string contentType, + out string body) + { + statusCode = StatusCodes.Status200OK; + contentType = "application/json"; + body = string.Empty; + + if (result is ObjectResult obj && IsSuccessStatusCode(obj.StatusCode)) + { + statusCode = obj.StatusCode ?? StatusCodes.Status200OK; + contentType = "application/json"; + body = JsonConvert.SerializeObject(obj.Value); + return true; + } + + if (result is ContentResult content && IsSuccessStatusCode(content.StatusCode)) + { + statusCode = content.StatusCode ?? StatusCodes.Status200OK; + contentType = string.IsNullOrEmpty(content.ContentType) ? "application/json" : content.ContentType; + body = content.Content ?? string.Empty; + return true; + } + + if (result is StatusCodeResult statusOnly && IsSuccessStatusCode(statusOnly.StatusCode)) + { + statusCode = statusOnly.StatusCode; + contentType = "application/json"; + body = string.Empty; + return true; + } + + if (result is EmptyResult) + { + statusCode = StatusCodes.Status200OK; + contentType = "application/json"; + body = string.Empty; + return true; + } + + return false; + } + + /// Validates and returns the raw idempotency header value. + /// Header is missing. + /// Header is empty, invalid, or not a ULID when required. + public string ExtractAndValidateIdempotencyKey(HttpRequest httpRequest) + { + ArgumentNullException.ThrowIfNull(httpRequest); + + var headerName = _options.HeaderName; + string? rawKey = null; + if (httpRequest.Headers.TryGetValue(headerName, out var value)) + rawKey = value.ToString(); + + if (rawKey is null) + throw new ArgumentNullException(headerName, $"The {headerName} header is missing."); + + if (!IdempotencyKeyValidator.TryValidate(rawKey, _options, out var error)) + throw new ArgumentException(error); + + return rawKey; + } + + /// Validates and parses the idempotency header as a ULID when is true. + public Ulid ExtractAndValidateIdempotencyKeyAsUlid(HttpRequest httpRequest) + { + var raw = ExtractAndValidateIdempotencyKey(httpRequest); + if (!Ulid.TryParse(raw, out var ulid)) + throw new ArgumentException($"Invalid {_options.HeaderName} format: {raw}"); + return ulid; + } + + private ObjectResult ToProblemResult(IdempotencyDecision decision) + { + var problem = IdempotencyProblemDetails.For(decision, _options); + return new ObjectResult(problem) + { + StatusCode = problem.Status, + ContentTypes = { "application/problem+json" } + }; + } + + private static bool IsSuccessStatusCode(int? statusCode) => + statusCode is null || (statusCode >= 200 && statusCode < 300); +} diff --git a/src/BuildingBlocks/Idempotency/AspNetCore/IdempotentEndpointFilter.cs b/src/BuildingBlocks/Idempotency/AspNetCore/IdempotentEndpointFilter.cs new file mode 100644 index 0000000..12bc3de --- /dev/null +++ b/src/BuildingBlocks/Idempotency/AspNetCore/IdempotentEndpointFilter.cs @@ -0,0 +1,208 @@ +using System.Text.Json; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.HttpResults; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Caching.Distributed; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace BuildingBlocks.Idempotency.AspNetCore; + +/// +/// Minimal API that applies the same semantics +/// as . +/// +/// +public sealed class IdempotentEndpointFilter : IEndpointFilter +{ + private readonly IdempotencyGate _gate; + private readonly IdempotencyOptions _options; + private readonly IdempotencyTelemetry? _telemetry; + private readonly IdempotencyEndpointSettings _endpoint; + + /// Creates the filter. + public IdempotentEndpointFilter( + IDistributedCache distributedCache, + ILoggerFactory loggerFactory, + IdempotencyOptions options, + IdempotencyEndpointSettings endpoint, + IIdempotencyLock? idempotencyLock = null, + IdempotencyTelemetry? telemetry = null) + { + ArgumentNullException.ThrowIfNull(distributedCache); + ArgumentNullException.ThrowIfNull(loggerFactory); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _endpoint = endpoint ?? throw new ArgumentNullException(nameof(endpoint)); + _telemetry = telemetry; + _gate = new IdempotencyGate( + distributedCache, + loggerFactory.CreateLogger(), + _options, + idempotencyLock); + } + + /// + public async ValueTask InvokeAsync(EndpointFilterInvocationContext context, EndpointFilterDelegate next) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(next); + + var httpContext = context.HttpContext; + + if (_options.EnableRequestFingerprint) + httpContext.Request.EnableBuffering(); + + var decision = await _gate + .BeginAsync(httpContext, _endpoint, httpContext.RequestAborted) + .ConfigureAwait(false); + + if (!decision.ShouldExecute) + { + _telemetry?.RecordDecision(decision, _endpoint.UseLock, _options); + + if (decision.Kind == IdempotencyDecisionKind.Replay && decision.Envelope is { } envelope) + { + httpContext.Response.Headers.Append(_options.CachedResponseHeader, "true"); + return Results.Content(envelope.Body, envelope.ContentType, statusCode: envelope.StatusCode); + } + + return ToProblemResult(decision); + } + + var cacheKey = decision.CacheKey!; + var requestFingerprint = decision.RequestFingerprint; + + try + { + var result = await next(context).ConfigureAwait(false); + + if (TryCaptureSuccessfulResult(result, out var statusCode, out var contentType, out var body)) + { + var envelope = new IdempotencyEnvelope(statusCode, contentType, body); + await _gate + .CompleteAsync(cacheKey, envelope, requestFingerprint, _endpoint, httpContext.RequestAborted) + .ConfigureAwait(false); + _telemetry?.Record(IdempotencyOutcomes.Executed, _endpoint.UseLock, cacheKey, statusCode); + } + else + { + await _gate.AbandonAsync(cacheKey, httpContext.RequestAborted).ConfigureAwait(false); + } + + return result; + } + catch + { + await _gate.AbandonAsync(cacheKey, CancellationToken.None).ConfigureAwait(false); + throw; + } + } + + internal static bool TryCaptureSuccessfulResult( + object? result, + out int statusCode, + out string contentType, + out string body) + { + statusCode = StatusCodes.Status200OK; + contentType = "application/json"; + body = string.Empty; + + if (result is null) + return false; + + if (result is IStatusCodeHttpResult statusResult) + { + var code = statusResult.StatusCode ?? StatusCodes.Status200OK; + if (code is < 200 or >= 300) + return false; + + statusCode = code; + + if (result is IValueHttpResult { Value: { } value }) + { + body = JsonSerializer.Serialize(value); + contentType = "application/json"; + return true; + } + + if (result is IContentTypeHttpResult { ContentType: { } ct } && !string.IsNullOrEmpty(ct)) + contentType = ct; + + if (result is ContentHttpResult content) + { + body = content.ResponseContent ?? string.Empty; + contentType = string.IsNullOrEmpty(content.ContentType) ? "application/json" : content.ContentType!; + return true; + } + + // Empty 2xx (e.g. 204 No Content, Results.Ok() without value) + body = string.Empty; + return true; + } + + if (result is ProblemDetails) + return false; + + // Untyped object return from handler — treat as 200 JSON + body = JsonSerializer.Serialize(result); + contentType = "application/json"; + statusCode = StatusCodes.Status200OK; + return true; + } + + private IResult ToProblemResult(IdempotencyDecision decision) + { + var problem = IdempotencyProblemDetails.For(decision, _options); + return Results.Problem( + detail: problem.Detail, + statusCode: problem.Status, + title: problem.Title, + type: problem.Type, + extensions: problem.Extensions); + } +} + +/// Minimal API registration helpers for BuildingBlocks.Idempotency. +public static class IdempotencyEndpointRouteBuilderExtensions +{ + /// + /// Adds to the endpoint. + /// + /// Route handler builder. + /// When true, GetOrCreate uses . + /// Optional Processing TTL override; 0 = global. + /// Optional Completed TTL override; 0 = global. + /// + /// + /// app.MapPost("/orders", CreateAsync).WithIdempotency(useLock: true); + /// + /// + /// + public static RouteHandlerBuilder WithIdempotency( + this RouteHandlerBuilder builder, + bool useLock = false, + int processingTtlSeconds = 0, + int entryTtlSeconds = 0) + { + ArgumentNullException.ThrowIfNull(builder); + + var endpoint = IdempotencyEndpointSettings.Create(useLock, processingTtlSeconds, entryTtlSeconds); + + return builder.AddEndpointFilter(async (context, next) => + { + var sp = context.HttpContext.RequestServices; + var filter = new IdempotentEndpointFilter( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetService>()?.Value ?? new IdempotencyOptions(), + endpoint, + sp.GetService(), + sp.GetService()); + + return await filter.InvokeAsync(context, next).ConfigureAwait(false); + }); + } +} diff --git a/src/BuildingBlocks/Idempotency/BuildingBlocks.Idempotency.csproj b/src/BuildingBlocks/Idempotency/BuildingBlocks.Idempotency.csproj new file mode 100644 index 0000000..6454cb4 --- /dev/null +++ b/src/BuildingBlocks/Idempotency/BuildingBlocks.Idempotency.csproj @@ -0,0 +1,54 @@ + + + + net8.0;net9.0;net10.0 + enable + enable + latest + true + CS1591 + BuildingBlocks.Idempotency + + true + BuildingBlocks.Idempotency + BuildingBlocks.Idempotency + 1.0.0 + Mohammad Hasan Hosseini + Mohammad Hasan Hosseini + Copyright (c) 2026 Mohammad Hasan Hosseini + ASP.NET Core HTTP Idempotency-Key filter (MVC + Minimal API): shared IdempotencyGate over IDistributedCache, 2xx response envelope replay, RFC 9457 ProblemDetails errors, optional Redis SET NX lock, ProcessingTtl abandoned recovery, opt-in method+path+body fingerprinting, per-endpoint TTL, and optional ActivitySource instrumentation. Host owns cache registration. No MCP or Telemetry package coupling. + idempotency;idempotency-key;aspnetcore;minimal-api;redis;distributed-lock;distributed-cache;fingerprint;problemdetails;opentelemetry;net8;net9;net10 + MIT + https://github.com/Maxofpower/FeatureFusion + https://github.com/Maxofpower/FeatureFusion + git + PACKAGE_README.md + 1.0.0: IdempotencyGate; MVC + Minimal API; 2xx envelope replay; ProblemDetails errors; ProcessingTtl / EntryTtl + per-endpoint overrides; optional Redis lock; opt-in method+path+body fingerprint; fluent UseRedisLock/UseTelemetry. Host owns IDistributedCache. https://github.com/Maxofpower/FeatureFusion/blob/main/CHANGELOG.md + true + true + true + snupkg + true + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/BuildingBlocks/Idempotency/Core/IdempotencyDecision.cs b/src/BuildingBlocks/Idempotency/Core/IdempotencyDecision.cs new file mode 100644 index 0000000..912c0c4 --- /dev/null +++ b/src/BuildingBlocks/Idempotency/Core/IdempotencyDecision.cs @@ -0,0 +1,148 @@ +namespace BuildingBlocks.Idempotency; + +/// Kind of outcome from . +public enum IdempotencyDecisionKind +{ + /// First owner — proceed to execute the handler. + Execute = 0, + + /// Replay a stored Completed (or Failed) envelope. + Replay = 1, + + /// Non-expired Processing lease held by another request. + ProcessingConflict = 2, + + /// Same key with a different request fingerprint. + FingerprintConflict = 3, + + /// Completed entry and . + DuplicateConflict = 4, + + /// Missing, empty, control-character, over-length, or invalid ULID key. + BadKey = 5, + + /// Caller identity missing and no . + Unauthorized = 6, + + /// Distributed lock or cache operation failure. + LockFailure = 7 +} + +/// +/// Gate decision for MVC / Minimal API hosts. Errors map via . +/// +public sealed class IdempotencyDecision +{ + private IdempotencyDecision( + IdempotencyDecisionKind kind, + string? cacheKey = null, + string? requestFingerprint = null, + IdempotencyEnvelope? envelope = null, + string? detail = null) + { + Kind = kind; + CacheKey = cacheKey; + RequestFingerprint = requestFingerprint; + Envelope = envelope; + Detail = detail; + } + + /// Decision kind. + public IdempotencyDecisionKind Kind { get; } + + /// Composite cache key when known. + public string? CacheKey { get; } + + /// Fingerprint used for this attempt (Execute path). + public string? RequestFingerprint { get; } + + /// Stored envelope when is . + public IdempotencyEnvelope? Envelope { get; } + + /// Human-readable detail for ProblemDetails. + public string? Detail { get; } + + /// Whether the host should invoke the action / endpoint. + public bool ShouldExecute => Kind == IdempotencyDecisionKind.Execute; + + /// Creates an Execute decision. + public static IdempotencyDecision Execute(string cacheKey, string? requestFingerprint) => + new(IdempotencyDecisionKind.Execute, cacheKey, requestFingerprint); + + /// Creates a Replay decision. + public static IdempotencyDecision Replay(string cacheKey, IdempotencyEnvelope envelope) => + new(IdempotencyDecisionKind.Replay, cacheKey, envelope: envelope); + + /// Creates a Processing conflict decision. + public static IdempotencyDecision ProcessingConflict(string cacheKey, string detail) => + new(IdempotencyDecisionKind.ProcessingConflict, cacheKey, detail: detail); + + /// Creates a fingerprint conflict decision. + public static IdempotencyDecision FingerprintConflict(string cacheKey, string detail) => + new(IdempotencyDecisionKind.FingerprintConflict, cacheKey, detail: detail); + + /// Creates a duplicate Completed conflict decision. + public static IdempotencyDecision DuplicateConflict(string cacheKey, string detail) => + new(IdempotencyDecisionKind.DuplicateConflict, cacheKey, detail: detail); + + /// Creates a bad-key decision. + public static IdempotencyDecision BadKey(string detail) => + new(IdempotencyDecisionKind.BadKey, detail: detail); + + /// Creates an unauthorized decision. + public static IdempotencyDecision Unauthorized(string detail) => + new(IdempotencyDecisionKind.Unauthorized, detail: detail); + + /// Creates a lock/infrastructure failure decision. + public static IdempotencyDecision LockFailure(string? cacheKey, string detail) => + new(IdempotencyDecisionKind.LockFailure, cacheKey, detail: detail); +} + +/// Cached HTTP response envelope for replay. +public sealed class IdempotencyEnvelope +{ + /// Creates an envelope. + public IdempotencyEnvelope(int statusCode, string contentType, string body) + { + StatusCode = statusCode; + ContentType = contentType; + Body = body; + } + + /// HTTP status to replay. + public int StatusCode { get; } + + /// Content-Type to replay. + public string ContentType { get; } + + /// Response body text. + public string Body { get; } +} + +/// +/// Per-endpoint overrides merged with . +/// Null TTL properties mean “use global options”. +/// +public sealed class IdempotencyEndpointSettings +{ + /// When true, GetOrCreate is wrapped in . + public bool UseLock { get; init; } + + /// Optional Processing TTL override. + public TimeSpan? ProcessingTtl { get; init; } + + /// Optional Completed Entry TTL override. + public TimeSpan? EntryTtl { get; init; } + + /// Creates settings with optional TTL seconds (0 = use global). + public static IdempotencyEndpointSettings Create( + bool useLock, + int processingTtlSeconds = 0, + int entryTtlSeconds = 0) => + new() + { + UseLock = useLock, + ProcessingTtl = processingTtlSeconds > 0 ? TimeSpan.FromSeconds(processingTtlSeconds) : null, + EntryTtl = entryTtlSeconds > 0 ? TimeSpan.FromSeconds(entryTtlSeconds) : null + }; +} diff --git a/src/BuildingBlocks/Idempotency/Core/IdempotencyGate.cs b/src/BuildingBlocks/Idempotency/Core/IdempotencyGate.cs new file mode 100644 index 0000000..4711028 --- /dev/null +++ b/src/BuildingBlocks/Idempotency/Core/IdempotencyGate.cs @@ -0,0 +1,356 @@ +using System.Text; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Caching.Distributed; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace BuildingBlocks.Idempotency; + +/// +/// Shared idempotency lifecycle over with optional . +/// +/// +/// MVC and Minimal API hosts call , then or +/// . This type does not produce HTTP results — map via +/// or replay . +/// +public sealed class IdempotencyGate +{ + private readonly IDistributedCache _distributedCache; + private readonly ILogger _logger; + private readonly IIdempotencyLock? _idempotencyLock; + private readonly IdempotencyOptions _options; + + /// Creates a gate. + /// Required dependency is null. + public IdempotencyGate( + IDistributedCache distributedCache, + ILogger logger, + IdempotencyOptions options, + IIdempotencyLock? idempotencyLock = null) + { + _distributedCache = distributedCache ?? throw new ArgumentNullException(nameof(distributedCache)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _idempotencyLock = idempotencyLock; + } + + /// + /// Validates the key, resolves identity, and GetOrCreates the cache entry. + /// Returns when the caller owns the lease. + /// + public async Task BeginAsync( + HttpContext httpContext, + IdempotencyEndpointSettings endpoint, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(httpContext); + ArgumentNullException.ThrowIfNull(endpoint); + + var headerName = _options.HeaderName; + string? rawKey = null; + if (httpContext.Request.Headers.TryGetValue(headerName, out var headerValue)) + rawKey = headerValue.ToString(); + + if (!IdempotencyKeyValidator.TryValidate(rawKey, _options, out var keyError)) + return IdempotencyDecision.BadKey(keyError!); + + if (!TryResolveUserId(httpContext, out var userId, out var authError)) + return IdempotencyDecision.Unauthorized(authError!); + + var cacheKey = BuildCacheKey(httpContext, userId!, rawKey!); + string? requestFingerprint = null; + + try + { + if (_options.EnableRequestFingerprint) + { + var request = httpContext.Request; + request.EnableBuffering(); + if (request.Body.CanSeek) + request.Body.Position = 0; + + requestFingerprint = await IdempotencyFingerprint + .ComputeAsync(request.Method, request.Path.Value ?? string.Empty, request.Body, cancellationToken) + .ConfigureAwait(false); + + if (request.Body.CanSeek) + request.Body.Position = 0; + } + + var (isNewlyCreated, cacheEntry) = await GetOrCreateCacheEntryAsync( + cacheKey, + requestFingerprint, + endpoint, + cancellationToken) + .ConfigureAwait(false); + + if (cacheEntry is not null && !isNewlyCreated) + { + if (_options.EnableRequestFingerprint + && !string.IsNullOrEmpty(cacheEntry.RequestFingerprint) + && requestFingerprint is not null + && !IdempotencyFingerprint.FixedTimeEquals(cacheEntry.RequestFingerprint, requestFingerprint)) + { + return IdempotencyDecision.FingerprintConflict( + cacheKey, + "Idempotency-Key was reused with a different request payload."); + } + + if (cacheEntry.Status == "Processing") + { + return IdempotencyDecision.ProcessingConflict( + cacheKey, + "Request is already being processed. Please wait."); + } + + if (cacheEntry.Status is "Completed" or "Failed") + { + if (_options.DuplicateCompletedBehavior == DuplicateCompletedBehavior.Conflict) + { + return IdempotencyDecision.DuplicateConflict( + cacheKey, + "Idempotency-Key was already used for a completed request."); + } + + var envelope = new IdempotencyEnvelope( + cacheEntry.StatusCode ?? StatusCodes.Status200OK, + string.IsNullOrEmpty(cacheEntry.ContentType) ? "application/json" : cacheEntry.ContentType, + cacheEntry.Response ?? string.Empty); + return IdempotencyDecision.Replay(cacheKey, envelope); + } + + return IdempotencyDecision.LockFailure( + cacheKey, + $"Unknown cache status: {cacheEntry.Status}"); + } + + await MarkRequestAsProcessingAsync(cacheKey, requestFingerprint, endpoint, cancellationToken) + .ConfigureAwait(false); + + return IdempotencyDecision.Execute(cacheKey, requestFingerprint); + } + catch (TimeoutException ex) + { + _logger.LogError(ex, "Idempotency cache operation timed out for key {CacheKey}.", cacheKey); + return IdempotencyDecision.LockFailure(cacheKey, ex.Message); + } + catch (InvalidOperationException ex) + { + _logger.LogError(ex, "Idempotency lock or cache failure for key {CacheKey}.", cacheKey); + return IdempotencyDecision.LockFailure(cacheKey, ex.Message); + } + } + + /// Stores a successful 2xx envelope as Completed. + public async Task CompleteAsync( + string cacheKey, + IdempotencyEnvelope envelope, + string? requestFingerprint, + IdempotencyEndpointSettings endpoint, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(cacheKey); + ArgumentNullException.ThrowIfNull(envelope); + ArgumentNullException.ThrowIfNull(endpoint); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(_options.CacheWriteTimeout); + + var cacheEntry = new IdempotencyCacheEntry + { + Status = "Completed", + Response = envelope.Body, + StatusCode = envelope.StatusCode, + ContentType = envelope.ContentType, + RequestFingerprint = requestFingerprint, + ProcessingExpiresAtUtc = null + }; + var cacheEntryData = JsonConvert.SerializeObject(cacheEntry); + var entryTtl = endpoint.EntryTtl ?? _options.EntryTtl; + + await _distributedCache.SetAsync( + cacheKey, + Encoding.UTF8.GetBytes(cacheEntryData), + new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = entryTtl }, + cts.Token).ConfigureAwait(false); + } + + /// Removes the cache entry so the key may be retried. + public async Task AbandonAsync( + string cacheKey, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(cacheKey); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(_options.CacheWriteTimeout); + await _distributedCache.RemoveAsync(cacheKey, cts.Token).ConfigureAwait(false); + } + + private bool TryResolveUserId(HttpContext httpContext, out string? userId, out string? error) + { + var claimValue = httpContext.User.FindFirst(_options.UserIdClaimType)?.Value; + userId = claimValue ?? _options.UserIdFallback; + + if (string.IsNullOrEmpty(userId)) + { + error = "User ID is missing in the request context."; + return false; + } + + error = null; + return true; + } + + private string BuildCacheKey(HttpContext httpContext, string userId, string idempotencyKey) + { + var segments = new List { _options.KeyPrefix }; + foreach (var claimType in _options.KeyScopeClaimTypes) + { + var value = httpContext.User.FindFirst(claimType)?.Value; + if (!string.IsNullOrEmpty(value)) + segments.Add(value); + } + + segments.Add(userId); + segments.Add(idempotencyKey); + return string.Join('_', segments); + } + + private async Task<(bool, IdempotencyCacheEntry?)> GetOrCreateCacheEntryAsync( + string cacheKey, + string? requestFingerprint, + IdempotencyEndpointSettings endpoint, + CancellationToken cancellationToken) + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(_options.CacheOperationTimeout); + + try + { + if (endpoint.UseLock) + { + if (_idempotencyLock is null) + throw new InvalidOperationException( + "UseLock=true requires a registered IIdempotencyLock."); + + var lockKey = $"{cacheKey}_lock"; + var lockValue = Guid.NewGuid().ToString(); + + var lockAcquired = await _idempotencyLock + .AcquireAsync(lockKey, lockValue, _options.LockExpiry, cts.Token) + .ConfigureAwait(false); + if (!lockAcquired) + throw new InvalidOperationException("Failed to acquire Redis lock."); + + try + { + return await GetOrCreateCacheEntryWithoutLockAsync( + cacheKey, + requestFingerprint, + endpoint, + cts.Token) + .ConfigureAwait(false); + } + finally + { + await _idempotencyLock.ReleaseAsync(lockKey, lockValue, CancellationToken.None) + .ConfigureAwait(false); + } + } + + return await GetOrCreateCacheEntryWithoutLockAsync( + cacheKey, + requestFingerprint, + endpoint, + cts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (!cancellationToken.IsCancellationRequested) + { + _logger.LogWarning("Cache operation timed out for key {CacheKey}: {Message}", cacheKey, ex.Message); + throw new TimeoutException($"Cache operation timed out for key: {cacheKey}", ex); + } + catch (Exception ex) when (ex is not TimeoutException and not InvalidOperationException) + { + _logger.LogError(ex, "Cache operation failed for key {CacheKey}: {Message}", cacheKey, ex.Message); + throw new InvalidOperationException($"Cache operation failed for key: {cacheKey}", ex); + } + } + + private async Task<(bool, IdempotencyCacheEntry?)> GetOrCreateCacheEntryWithoutLockAsync( + string cacheKey, + string? requestFingerprint, + IdempotencyEndpointSettings endpoint, + CancellationToken cancellationToken) + { + var processingTtl = endpoint.ProcessingTtl ?? _options.ProcessingTtl; + var cacheData = await _distributedCache.GetAsync(cacheKey, cancellationToken).ConfigureAwait(false); + if (cacheData != null) + { + var existing = JsonConvert.DeserializeObject(Encoding.UTF8.GetString(cacheData)); + if (existing is not null + && existing.Status == "Processing" + && IsProcessingExpired(existing)) + { + await _distributedCache.RemoveAsync(cacheKey, cancellationToken).ConfigureAwait(false); + } + else if (existing is not null) + { + return (false, existing); + } + } + + var expiresAt = DateTimeOffset.UtcNow.Add(processingTtl); + var cacheEntry = new IdempotencyCacheEntry + { + Status = "Processing", + Response = null, + RequestFingerprint = requestFingerprint, + ProcessingExpiresAtUtc = expiresAt + }; + var cacheEntryData = JsonConvert.SerializeObject(cacheEntry); + + await _distributedCache.SetAsync( + cacheKey, + Encoding.UTF8.GetBytes(cacheEntryData), + new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = processingTtl }, + cancellationToken).ConfigureAwait(false); + + return (true, cacheEntry); + } + + private static bool IsProcessingExpired(IdempotencyCacheEntry entry) + { + if (entry.ProcessingExpiresAtUtc is { } expires) + return expires <= DateTimeOffset.UtcNow; + + return false; + } + + private async Task MarkRequestAsProcessingAsync( + string cacheKey, + string? requestFingerprint, + IdempotencyEndpointSettings endpoint, + CancellationToken cancellationToken) + { + var processingTtl = endpoint.ProcessingTtl ?? _options.ProcessingTtl; + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(_options.CacheOperationTimeout); + + var cacheEntry = new IdempotencyCacheEntry + { + Status = "Processing", + RequestFingerprint = requestFingerprint, + ProcessingExpiresAtUtc = DateTimeOffset.UtcNow.Add(processingTtl) + }; + var cacheEntryData = JsonConvert.SerializeObject(cacheEntry); + + await _distributedCache.SetAsync( + cacheKey, + Encoding.UTF8.GetBytes(cacheEntryData), + new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = processingTtl }, + cts.Token).ConfigureAwait(false); + } +} diff --git a/src/BuildingBlocks/Idempotency/Core/IdempotencyKeyValidator.cs b/src/BuildingBlocks/Idempotency/Core/IdempotencyKeyValidator.cs new file mode 100644 index 0000000..0484634 --- /dev/null +++ b/src/BuildingBlocks/Idempotency/Core/IdempotencyKeyValidator.cs @@ -0,0 +1,122 @@ +using System.Security.Cryptography; +using System.Text; + +namespace BuildingBlocks.Idempotency; + +/// Validates raw Idempotency-Key header values. +public static class IdempotencyKeyValidator +{ + /// + /// Validates against . + /// + /// true when valid; otherwise false with a human-readable . + public static bool TryValidate(string? rawKey, IdempotencyOptions options, out string? error) + { + ArgumentNullException.ThrowIfNull(options); + + var headerName = options.HeaderName; + + if (rawKey is null) + { + error = $"The {headerName} header is missing."; + return false; + } + + if (string.IsNullOrWhiteSpace(rawKey)) + { + error = $"The {headerName} value cannot be empty."; + return false; + } + + if (ContainsControlCharacters(rawKey)) + { + error = $"The {headerName} value contains invalid control characters."; + return false; + } + + var max = options.MaxKeyLength > 0 ? options.MaxKeyLength : 256; + if (rawKey.Length > max) + { + error = $"The {headerName} value exceeds the maximum length of {max}."; + return false; + } + + if (options.RequireUlid && !Ulid.TryParse(rawKey, out _)) + { + error = $"Invalid {headerName} format: {rawKey}"; + return false; + } + + error = null; + return true; + } + + private static bool ContainsControlCharacters(string value) + { + foreach (var ch in value) + { + if (char.IsControl(ch)) + return true; + } + + return false; + } +} + +/// Computes request fingerprints for opt-in payload binding. +public static class IdempotencyFingerprint +{ + /// + /// SHA-256 hex of method + "\n" + path + "\n" + body. + /// + public static async Task ComputeAsync( + string method, + string path, + Stream body, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(method); + ArgumentNullException.ThrowIfNull(path); + ArgumentNullException.ThrowIfNull(body); + + using var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + + AppendUtf8(hasher, method); + hasher.AppendData("\n"u8); + AppendUtf8(hasher, path); + hasher.AppendData("\n"u8); + + var buffer = new byte[8192]; + int read; + while ((read = await body.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken).ConfigureAwait(false)) > 0) + hasher.AppendData(buffer.AsSpan(0, read)); + + Span hash = stackalloc byte[32]; + if (!hasher.TryGetHashAndReset(hash, out var written) || written != 32) + throw new CryptographicException("Failed to compute fingerprint hash."); + + return Convert.ToHexString(hash); + } + + /// Constant-time string compare for hex fingerprints. + public static bool FixedTimeEquals(string a, string b) + { + var ba = Encoding.UTF8.GetBytes(a); + var bb = Encoding.UTF8.GetBytes(b); + return ba.Length == bb.Length && CryptographicOperations.FixedTimeEquals(ba, bb); + } + + private static void AppendUtf8(IncrementalHash hasher, string value) + { + var byteCount = Encoding.UTF8.GetByteCount(value); + if (byteCount <= 256) + { + Span scratch = stackalloc byte[256]; + var written = Encoding.UTF8.GetBytes(value, scratch); + hasher.AppendData(scratch[..written]); + return; + } + + hasher.AppendData(Encoding.UTF8.GetBytes(value)); + } +} diff --git a/src/BuildingBlocks/Idempotency/Core/IdempotencyProblemDetails.cs b/src/BuildingBlocks/Idempotency/Core/IdempotencyProblemDetails.cs new file mode 100644 index 0000000..db0bbe8 --- /dev/null +++ b/src/BuildingBlocks/Idempotency/Core/IdempotencyProblemDetails.cs @@ -0,0 +1,93 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; + +namespace BuildingBlocks.Idempotency; + +/// +/// Builds RFC 9457 for idempotency filter/gate errors. +/// +/// +/// Stable type URIs use the https://buildingblocks.dev/errors/idempotency/ namespace. +/// Successful replays are not ProblemDetails — hosts return the cached envelope. +/// +public static class IdempotencyProblemDetails +{ + /// Base URI for machine-readable problem types. + public const string TypeBase = "https://buildingblocks.dev/errors/idempotency/"; + + /// Creates ProblemDetails for a non-success gate decision. + /// or is null. + /// is or . + public static ProblemDetails For(IdempotencyDecision decision, IdempotencyOptions options) + { + ArgumentNullException.ThrowIfNull(decision); + ArgumentNullException.ThrowIfNull(options); + + return decision.Kind switch + { + IdempotencyDecisionKind.BadKey => Create( + StatusCodes.Status400BadRequest, + "key-invalid", + "Idempotency-Key invalid", + decision.Detail ?? "The Idempotency-Key header is missing or invalid.", + IdempotencyOutcomes.BadKey), + + IdempotencyDecisionKind.Unauthorized => Create( + StatusCodes.Status401Unauthorized, + "unauthorized", + "Idempotency caller identity required", + decision.Detail ?? "User ID is missing in the request context.", + IdempotencyOutcomes.Unauthorized), + + IdempotencyDecisionKind.ProcessingConflict => Create( + options.ProcessingConflictStatusCode, + "processing", + "Request already in progress", + decision.Detail ?? "Request is already being processed. Please wait.", + IdempotencyOutcomes.ProcessingConflict), + + IdempotencyDecisionKind.FingerprintConflict => Create( + options.FingerprintConflictStatusCode, + "fingerprint-mismatch", + "Idempotency-Key payload mismatch", + decision.Detail ?? "Idempotency-Key was reused with a different request payload.", + IdempotencyOutcomes.FingerprintConflict), + + IdempotencyDecisionKind.DuplicateConflict => Create( + options.DuplicateConflictStatusCode, + "duplicate", + "Idempotency-Key already completed", + decision.Detail ?? "Idempotency-Key was already used for a completed request.", + IdempotencyOutcomes.DuplicateConflict), + + IdempotencyDecisionKind.LockFailure => Create( + StatusCodes.Status500InternalServerError, + "lock-failure", + "Idempotency infrastructure failure", + decision.Detail ?? "Idempotency infrastructure failure.", + IdempotencyOutcomes.LockFailure), + + _ => throw new ArgumentException( + $"Decision kind {decision.Kind} does not map to ProblemDetails.", + nameof(decision)) + }; + } + + private static ProblemDetails Create( + int status, + string typeSuffix, + string title, + string detail, + string outcome) + { + var problem = new ProblemDetails + { + Status = status, + Type = TypeBase + typeSuffix, + Title = title, + Detail = detail + }; + problem.Extensions["idempotency.outcome"] = outcome; + return problem; + } +} diff --git a/src/BuildingBlocks/Idempotency/DependencyInjection/IdempotencyServiceCollectionExtensions.cs b/src/BuildingBlocks/Idempotency/DependencyInjection/IdempotencyServiceCollectionExtensions.cs new file mode 100644 index 0000000..fd46d22 --- /dev/null +++ b/src/BuildingBlocks/Idempotency/DependencyInjection/IdempotencyServiceCollectionExtensions.cs @@ -0,0 +1,104 @@ +using BuildingBlocks.Idempotency.Redis; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StackExchange.Redis; + +namespace BuildingBlocks.Idempotency.DependencyInjection; + +/// +/// Fluent registration returned by . +/// +public sealed class IdempotencyBuilder +{ + /// Creates a builder over . + public IdempotencyBuilder(IServiceCollection services) + { + Services = services ?? throw new ArgumentNullException(nameof(services)); + } + + /// Underlying service collection. + public IServiceCollection Services { get; } + + /// + /// Registers as using the host's + /// . + /// + public IdempotencyBuilder UseRedisLock() + { + Services.AddRedisIdempotencyLock(); + return this; + } + + /// + /// Registers optional (ActivitySource). + /// Hosts that export OTLP should AddSource("BuildingBlocks.Idempotency"). + /// + public IdempotencyBuilder UseTelemetry(Action? configure = null) + { + Services.AddIdempotencyTelemetry(configure); + return this; + } +} + +/// DI registration for BuildingBlocks.Idempotency. +public static class IdempotencyServiceCollectionExtensions +{ + /// + /// Registers . Host must already register + /// . + /// Chain when endpoints use UseLock, + /// and for ActivitySource outcomes. + /// + /// + /// + /// services.AddBuildingBlocksIdempotency(o => o.UserIdFallback = "123") + /// .UseRedisLock() + /// .UseTelemetry(); + /// + /// + public static IdempotencyBuilder AddBuildingBlocksIdempotency( + this IServiceCollection services, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + if (configure is not null) + services.Configure(configure); + else + services.AddOptions(); + + return new IdempotencyBuilder(services); + } + + /// + /// Registers as using the host's + /// . Does not register Redis cache or connection hosting. + /// Prefer when chaining. + /// + public static IServiceCollection AddRedisIdempotencyLock(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(sp => + new RedisIdempotencyLock(sp.GetRequiredService())); + + return services; + } + + /// + /// Registers optional (ActivitySource). + /// Hosts that export OTLP should AddSource("BuildingBlocks.Idempotency") — no Telemetry package dependency. + /// Prefer when chaining. + /// + public static IServiceCollection AddIdempotencyTelemetry( + this IServiceCollection services, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + var options = new IdempotencyTelemetryOptions(); + configure?.Invoke(options); + services.TryAddSingleton(new IdempotencyTelemetry(options)); + return services; + } +} diff --git a/src/BuildingBlocks/Idempotency/PACKAGE_README.md b/src/BuildingBlocks/Idempotency/PACKAGE_README.md new file mode 100644 index 0000000..4726a06 --- /dev/null +++ b/src/BuildingBlocks/Idempotency/PACKAGE_README.md @@ -0,0 +1,127 @@ +# BuildingBlocks.Idempotency + +ASP.NET Core **Idempotency-Key** for MVC and Minimal API. Host-owned `IDistributedCache`, optional Redis lock, opt-in request fingerprint, RFC 9457 ProblemDetails on conflicts, optional ActivitySource. + +Requires **.NET 8**, **.NET 9**, or **.NET 10**. + +## Install + +```bash +dotnet add package BuildingBlocks.Idempotency +``` + +## Features (1.0.0) + +- Shared gate for **MVC** (`[Idempotent]`) and **Minimal API** (`WithIdempotency`) +- Cache **HTTP 2xx** envelopes (status, content-type, body) and replay them +- Processing lease (`ProcessingTtl`) with abandoned recovery; Completed window (`EntryTtl`) +- Per-endpoint TTL overrides (`ProcessingTtlSeconds` / `EntryTtlSeconds`) +- Optional Redis **SET NX** lock around GetOrCreate (`.UseRedisLock()`) +- Opt-in fingerprint: `SHA-256(method + "\n" + path + "\n" + body)` +- Key validation (ULID by default, `MaxKeyLength`, reject empty / control characters) +- `DuplicateCompletedBehavior`: Replay (default) or Conflict +- ProblemDetails errors with stable `type` URIs +- Optional telemetry (`.UseTelemetry()`); host `AddSource("BuildingBlocks.Idempotency")` + +## Quick start + +```csharp +// Host must already register IDistributedCache. +// For UseLock, also register IConnectionMultiplexer. +builder.Services.AddBuildingBlocksIdempotency(o => +{ + o.ProcessingTtl = TimeSpan.FromMinutes(2); // keep longer than worst-case handler + // o.EnableRequestFingerprint = true; + // o.UserIdFallback = "anonymous"; // when NameIdentifier is missing +}) +.UseRedisLock() // optional +.UseTelemetry(); // optional + +// MVC +[HttpPost] +[Idempotent(useLock: true)] +public async Task> Create([FromBody] CreateOrder request) { ... } + +// Minimal API +app.MapPost("/orders", CreateAsync).WithIdempotency(useLock: true); +``` + +Aliases still work: `AddRedisIdempotencyLock()`, `AddIdempotencyTelemetry()`. + +## Behavior + +| Situation | Result | +|-----------|--------| +| New key | Execute; on **2xx** store Completed envelope | +| Same key, Completed (default) | Replay envelope + `X-Idempotent-Response` | +| Same key, Completed + `DuplicateCompletedBehavior.Conflict` | ProblemDetails (default 409) | +| Same key, Processing (lease active) | ProblemDetails (default 409) | +| Same key, Processing expired | Miss — may execute again | +| Fingerprint enabled + method/path/body mismatch | ProblemDetails (default 422) | +| Missing / invalid key | ProblemDetails 400 | +| Missing caller identity (no `UserIdFallback`) | ProblemDetails 401 | +| Non-2xx or exception | Remove cache entry (retry allowed) | +| `UseLock` and lock not acquired | ProblemDetails 500 | + +Cache key: `{KeyPrefix}_{optionalScopeClaims}_{userId}_{key}` (default prefix `Idempotency`). + +Replay header name is configurable (`CachedResponseHeader`, default `X-Idempotent-Response`). + +### ProblemDetails + +Conflicts and validation failures return `application/problem+json`. Stable `type` values: + +| Suffix | Typical status | +|--------|----------------| +| `key-invalid` | 400 | +| `unauthorized` | 401 | +| `processing` | 409 | +| `fingerprint-mismatch` | 422 | +| `duplicate` | 409 | +| `lock-failure` | 500 | + +Base: `https://buildingblocks.dev/errors/idempotency/`. + +### TTL + +| Setting | Default | Role | +|---------|---------|------| +| `ProcessingTtl` | 2 min | In-flight lease | +| `EntryTtl` | 30 min | Completed replay window | +| Attribute / `WithIdempotency` seconds | `0` | Use global; `> 0` overrides | + +After `ProcessingTtl` expires, another request with the same key may run while a slow first request is still in flight. The optional lock covers GetOrCreate only. + +### Fingerprint + +Off by default (same key replays even if the body changes). When `EnableRequestFingerprint` is true, the hash includes HTTP method, path, and raw body bytes (not canonical JSON). + +## Options (common) + +| Option | Default | Notes | +|--------|---------|-------| +| `HeaderName` | `Idempotency-Key` | Request header | +| `RequireUlid` | `true` | Set `false` for opaque strings (still length/control checks) | +| `MaxKeyLength` | `256` | Always enforced | +| `KeyScopeClaimTypes` | empty | Extra claims in the cache key (e.g. tenant) | +| `UserIdFallback` | `null` | Missing user claim → 401 unless set | +| `DuplicateCompletedBehavior` | `Replay` | Or `Conflict` | +| `FingerprintConflictStatusCode` | `422` | Configurable | +| `ProcessingConflictStatusCode` | `409` | Configurable | +| `DuplicateConflictStatusCode` | `409` | When Conflict strategy | + +## Telemetry + +`.UseTelemetry()` registers ActivitySource `BuildingBlocks.Idempotency`. Outcomes: `executed`, `replayed`, `processing_conflict`, `fingerprint_conflict`, `duplicate_conflict`, `bad_key`, `unauthorized`, `lock_failure`. Cache keys are not tagged unless `IncludeCacheKeyInTelemetry` is true. + +This package does **not** depend on BuildingBlocks.Telemetry — add `AddSource("BuildingBlocks.Idempotency")` on the host. + +## Host requirements + +- `IDistributedCache` (required) +- `IConnectionMultiplexer` when using `.UseRedisLock()` / `UseLock: true` +- No references to BuildingBlocks.Mcp, Mediator, Telemetry, or EventBus + +## Not this package + +MCP tool write idempotency (`UseMemoryIdempotency` / `IMcpIdempotencyStore`) is part of **BuildingBlocks.Mcp**. diff --git a/src/BuildingBlocks/Idempotency/Redis/RedisIdempotencyLock.cs b/src/BuildingBlocks/Idempotency/Redis/RedisIdempotencyLock.cs new file mode 100644 index 0000000..061af8e --- /dev/null +++ b/src/BuildingBlocks/Idempotency/Redis/RedisIdempotencyLock.cs @@ -0,0 +1,68 @@ +using StackExchange.Redis; + +namespace BuildingBlocks.Idempotency.Redis; + +/// +/// Redis SET NX / compare-and-delete lock matching the Lab EventBus-era lock scripts. +/// +public sealed class RedisIdempotencyLock : IIdempotencyLock +{ + private readonly IConnectionMultiplexer _multiplexer; + + /// Creates a lock that uses database 0 of . + public RedisIdempotencyLock(IConnectionMultiplexer multiplexer) + { + _multiplexer = multiplexer ?? throw new ArgumentNullException(nameof(multiplexer)); + } + + /// + public async Task AcquireAsync(string key, string value, TimeSpan expiry, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var database = _multiplexer.GetDatabase(); + var expiryMilliseconds = (int)expiry.TotalMilliseconds; + + const string script = """ + local key = KEYS[1] + local value = ARGV[1] + local expiry = ARGV[2] + + local result = redis.call('SET', key, value, 'NX', 'PX', expiry) + + if result then + return true + else + return false + end + """; + + var result = (bool)await database.ScriptEvaluateAsync( + script, + [key], + [value, expiryMilliseconds]).ConfigureAwait(false); + + return result; + } + + /// + public async Task ReleaseAsync(string key, string value, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var database = _multiplexer.GetDatabase(); + + const string script = """ + if redis.call('get', KEYS[1]) == ARGV[1] then + return redis.call('del', KEYS[1]) + else + return 0 + end + """; + + var result = (int)await database.ScriptEvaluateAsync( + script, + [key], + [value]).ConfigureAwait(false); + + return result == 1; + } +} diff --git a/src/BuildingBlocks/Idempotency/Telemetry/IdempotencyOutcomes.cs b/src/BuildingBlocks/Idempotency/Telemetry/IdempotencyOutcomes.cs new file mode 100644 index 0000000..03b8e62 --- /dev/null +++ b/src/BuildingBlocks/Idempotency/Telemetry/IdempotencyOutcomes.cs @@ -0,0 +1,29 @@ +namespace BuildingBlocks.Idempotency; + +/// Well-known idempotency.outcome Activity tag values. +public static class IdempotencyOutcomes +{ + /// First successful execution cached as Completed. + public const string Executed = "executed"; + + /// Cached Completed (or Failed) response replayed. + public const string Replayed = "replayed"; + + /// In-flight Processing conflict. + public const string ProcessingConflict = "processing_conflict"; + + /// Same key, different request fingerprint. + public const string FingerprintConflict = "fingerprint_conflict"; + + /// Completed entry rejected under Conflict strategy. + public const string DuplicateConflict = "duplicate_conflict"; + + /// Missing or invalid Idempotency-Key. + public const string BadKey = "bad_key"; + + /// Caller identity missing. + public const string Unauthorized = "unauthorized"; + + /// Distributed lock not acquired or cache operation timed out. + public const string LockFailure = "lock_failure"; +} diff --git a/src/BuildingBlocks/Idempotency/Telemetry/IdempotencyTelemetry.cs b/src/BuildingBlocks/Idempotency/Telemetry/IdempotencyTelemetry.cs new file mode 100644 index 0000000..ed3c060 --- /dev/null +++ b/src/BuildingBlocks/Idempotency/Telemetry/IdempotencyTelemetry.cs @@ -0,0 +1,89 @@ +using System.Diagnostics; +using Microsoft.AspNetCore.Http; + +namespace BuildingBlocks.Idempotency; + +/// Options for optional idempotency ActivitySource instrumentation. +public sealed class IdempotencyTelemetryOptions +{ + /// ActivitySource name. Default BuildingBlocks.Idempotency. + public string ActivitySourceName { get; set; } = "BuildingBlocks.Idempotency"; + + /// + /// When true, tags activities with the full composite cache key (high cardinality / possible PII). + /// Default false. + /// + public bool IncludeCacheKeyInTelemetry { get; set; } +} + +/// +/// Optional ActivitySource for idempotency outcomes. Register via +/// UseTelemetry / AddIdempotencyTelemetry; omit for zero library telemetry overhead. +/// Hosts export with AddSource("BuildingBlocks.Idempotency") — no Telemetry package dependency. +/// +public sealed class IdempotencyTelemetry +{ + private readonly IdempotencyTelemetryOptions _options; + + /// Creates telemetry from options. + public IdempotencyTelemetry(IdempotencyTelemetryOptions options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + var name = string.IsNullOrWhiteSpace(options.ActivitySourceName) + ? "BuildingBlocks.Idempotency" + : options.ActivitySourceName; + Source = new ActivitySource(name); + } + + /// Activity source for short-lived idempotency outcome activities. + public ActivitySource Source { get; } + + /// Starts a completed activity tagged with the outcome (no-op when no listeners). + /// Value from . + /// Whether the endpoint requested a distributed lock. + /// Composite cache key; exported only when is true. + /// Optional HTTP status associated with the decision. + public void Record( + string outcome, + bool useLock = false, + string? cacheKey = null, + int? statusCode = null) + { + using var activity = Source.StartActivity("idempotency", ActivityKind.Internal); + if (activity is null) + return; + + activity.SetTag("idempotency.outcome", outcome); + activity.SetTag("idempotency.use_lock", useLock); + if (statusCode is { } code) + activity.SetTag("http.response.status_code", code); + if (_options.IncludeCacheKeyInTelemetry && cacheKey is not null) + activity.SetTag("idempotency.cache_key", cacheKey); + } + + /// Records telemetry for a gate decision. + public void RecordDecision(IdempotencyDecision decision, bool useLock, IdempotencyOptions options) + { + ArgumentNullException.ThrowIfNull(decision); + ArgumentNullException.ThrowIfNull(options); + + var (outcome, status) = decision.Kind switch + { + IdempotencyDecisionKind.Execute => (IdempotencyOutcomes.Executed, (int?)null), + IdempotencyDecisionKind.Replay => (IdempotencyOutcomes.Replayed, decision.Envelope?.StatusCode), + IdempotencyDecisionKind.ProcessingConflict => (IdempotencyOutcomes.ProcessingConflict, options.ProcessingConflictStatusCode), + IdempotencyDecisionKind.FingerprintConflict => (IdempotencyOutcomes.FingerprintConflict, options.FingerprintConflictStatusCode), + IdempotencyDecisionKind.DuplicateConflict => (IdempotencyOutcomes.DuplicateConflict, options.DuplicateConflictStatusCode), + IdempotencyDecisionKind.BadKey => (IdempotencyOutcomes.BadKey, StatusCodes.Status400BadRequest), + IdempotencyDecisionKind.Unauthorized => (IdempotencyOutcomes.Unauthorized, StatusCodes.Status401Unauthorized), + IdempotencyDecisionKind.LockFailure => (IdempotencyOutcomes.LockFailure, StatusCodes.Status500InternalServerError), + _ => (decision.Kind.ToString().ToLowerInvariant(), null) + }; + + // Execute is recorded after successful Complete by the host. + if (decision.Kind == IdempotencyDecisionKind.Execute) + return; + + Record(outcome, useLock, decision.CacheKey, status); + } +} diff --git a/src/BuildingBlocks/Mcp/AGENTS.md b/src/BuildingBlocks/Mcp/AGENTS.md index dfebb75..70030fd 100644 --- a/src/BuildingBlocks/Mcp/AGENTS.md +++ b/src/BuildingBlocks/Mcp/AGENTS.md @@ -59,4 +59,4 @@ Writes (Command / POST / PUT) are never retried. Commands default to requiring ` Agents: send a **new UUID** for a new write; **reuse** the key only on retry of that write. Do not invent keys for query tools. -Optional `IMcpToolFilter`, `IMcpRateLimiter`. +Optional `IMcpToolFilter`, `IMcpRateLimiter`. `UseTelemetry()` emits `mcp.tool` spans. `IncludeExceptionDetails` is explicit opt-in (not auto-Development). Host export needs `IntegrateMcp`. diff --git a/src/BuildingBlocks/Mcp/Abstractions/McpToolAttribute.cs b/src/BuildingBlocks/Mcp/Abstractions/McpToolAttribute.cs index 039c001..99f2174 100644 --- a/src/BuildingBlocks/Mcp/Abstractions/McpToolAttribute.cs +++ b/src/BuildingBlocks/Mcp/Abstractions/McpToolAttribute.cs @@ -9,7 +9,6 @@ namespace BuildingBlocks.Mcp; /// On a public static Minimal API method: JSON binds to the request parameter; other parameters resolve from DI. /// MVC controllers are unsupported for now (actions, [FromHeader], ActionResult). /// Use a public static Minimal API method or a message type. Do not use for OpenAPI generation. -/// Catalog: docs/linkedin-posts.md (mcp-message-tools). /// /// /// diff --git a/src/BuildingBlocks/Mcp/PACKAGE_README.md b/src/BuildingBlocks/Mcp/PACKAGE_README.md index 34b37b6..ecf800c 100644 --- a/src/BuildingBlocks/Mcp/PACKAGE_README.md +++ b/src/BuildingBlocks/Mcp/PACKAGE_README.md @@ -17,7 +17,7 @@ Map **application message types** (commands, queries, DTOs) and **public static - Successful calls return JSON text **and** `structuredContent` - Safe writes: `idempotencyKey`, confirmation, timeout, `IMcpToolFilter`, `IMcpRateLimiter`, `catalog://tools` - Analyzers BBMCP001–005 packed in the NuGet -- Opt-in ActivitySource `BuildingBlocks.Mcp` +- Opt-in ActivitySource `BuildingBlocks.Mcp` (`UseTelemetry()`). `IncludeExceptionDetails` is explicit opt-in (Development only recommended); it is not auto-enabled by environment. ## Install @@ -177,7 +177,7 @@ Cursor HTTP (API must already be running): - **`structuredContent`** on success - **Catalog resource:** `catalog://tools` - **Roslyn analyzers** BBMCP001–005 -- **`UseTelemetry()`:** ActivitySource `BuildingBlocks.Mcp` +- **`UseTelemetry()`:** ActivitySource `BuildingBlocks.Mcp`. Optional `t => t.IncludeExceptionDetails = true` includes exception messages on `McpErrorCode.Internal` (keep off in production). Host export still requires `IntegrateMcp = true` on `AddTelemetry` / `AddServiceDefaults`. ## What it is not (v1) diff --git a/src/BuildingBlocks/Mediator/Abstractions/Contracts.cs b/src/BuildingBlocks/Mediator/Abstractions/Contracts.cs index 6758653..fdafe37 100644 --- a/src/BuildingBlocks/Mediator/Abstractions/Contracts.cs +++ b/src/BuildingBlocks/Mediator/Abstractions/Contracts.cs @@ -7,9 +7,6 @@ namespace BuildingBlocks.Mediator; /// Prefer this over at call sites for a narrower dependency. /// CQRS-first: / / — /// there is no public IRequest. -/// LinkedIn: https://lnkd.in/p/eU5TsuR4 -/// LinkedIn (prior): https://www.linkedin.com/feed/update/urn:li:activity:7311311587372367873/ -/// Catalog: docs/linkedin-posts.md (mediator-building-blocks). /// /// /// @@ -57,11 +54,6 @@ Task Send(TCommand command, CancellationToken cancellationToken = defa /// /// Extends with the familiar Mediator name. No Publish/notifications in v1. /// -/// -/// LinkedIn: https://lnkd.in/p/eU5TsuR4 -/// LinkedIn (prior): https://www.linkedin.com/feed/update/urn:li:activity:7311311587372367873/ -/// Catalog: docs/linkedin-posts.md (mediator-building-blocks). -/// public interface IMediator : ISender { } diff --git a/src/BuildingBlocks/Mediator/PACKAGE_README.md b/src/BuildingBlocks/Mediator/PACKAGE_README.md index 8adef7b..e8e8cad 100644 --- a/src/BuildingBlocks/Mediator/PACKAGE_README.md +++ b/src/BuildingBlocks/Mediator/PACKAGE_README.md @@ -143,7 +143,6 @@ Scanner is built-in (no Scrutor). Analyzers **BBM001 / BBM002** ship in the NuGe - [Cookbook](https://github.com/Maxofpower/FeatureFusion/blob/main/docs/building-blocks/cookbook.md) - [Test matrix](https://github.com/Maxofpower/FeatureFusion/blob/main/docs/building-blocks/TEST_MATRIX.md) - [CHANGELOG](https://github.com/Maxofpower/FeatureFusion/blob/main/CHANGELOG.md) -- LinkedIn: [BuildingBlocks.Mediator v1.0.1](https://lnkd.in/p/eU5TsuR4) · [Mediator Pattern + Pipeline Behavior](https://www.linkedin.com/feed/update/urn:li:activity:7311311587372367873/) (prior) Demo host: **FeatureFusion** in the same repository. diff --git a/src/BuildingBlocks/Pagination.EntityFrameworkCore/PACKAGE_README.md b/src/BuildingBlocks/Pagination.EntityFrameworkCore/PACKAGE_README.md index 2a8d5f6..2e28151 100644 --- a/src/BuildingBlocks/Pagination.EntityFrameworkCore/PACKAGE_README.md +++ b/src/BuildingBlocks/Pagination.EntityFrameworkCore/PACKAGE_README.md @@ -8,6 +8,8 @@ Typed **keyset (cursor)** pagination for EF Core: `SortKey`, opaque versioned cu **When to use:** Stable “next page” over large tables without `OFFSET`. Map a host sort enum to a **prebuilt** `SortKey` — the library never turns `"Price"` into a property name. +![OFFSET skip growing with page number versus keyset seeking from the last Price and Id.](https://raw.githubusercontent.com/Maxofpower/FeatureFusion/main/docs/medium/images/04-offset-vs-keyset.png) + Dapper is an **in-repo project** (not a NuGet package). A LinqToDB adapter is not shipped. ## What you get @@ -55,6 +57,8 @@ builder.HasKeysetIndex(priceDescKey).HasDatabaseName("IX_products_price_id_desc" Empty cursor + `PageDirection.Backward` is the last page. +![First page, next cursor, previous cursor, last page via pageDirection=Backward.](https://raw.githubusercontent.com/Maxofpower/FeatureFusion/main/docs/medium/images/04b-cursor-flow.png) + There is **no** `IEnumerable` / in-memory adapter. Relational providers execute seek SQL. EF Core InMemory is the same API in-process (tests only). ## Caveats diff --git a/src/BuildingBlocks/Pagination/README.md b/src/BuildingBlocks/Pagination/README.md index bd80e8e..9525f88 100644 --- a/src/BuildingBlocks/Pagination/README.md +++ b/src/BuildingBlocks/Pagination/README.md @@ -3,3 +3,5 @@ Non-packable project: `SortKey`, opaque cursors, `CursorPage` / `CursorRequest`. **Not a NuGet.** Shipped inside **BuildingBlocks.Pagination.EntityFrameworkCore** (`BuildingBlocks.Pagination.dll` is bundled in that nupkg). Optional `PaginationOptions.Hint` and the `--probe` benchmark live in that package — see [`../Pagination.EntityFrameworkCore/PACKAGE_README.md`](../Pagination.EntityFrameworkCore/PACKAGE_README.md). + +![OFFSET skip growing with page number versus keyset seeking from the last Price and Id.](../../../docs/medium/images/04-offset-vs-keyset.png) diff --git a/src/BuildingBlocks/Telemetry/PACKAGE_README.md b/src/BuildingBlocks/Telemetry/PACKAGE_README.md index 3c4d72f..ae034a3 100644 --- a/src/BuildingBlocks/Telemetry/PACKAGE_README.md +++ b/src/BuildingBlocks/Telemetry/PACKAGE_README.md @@ -22,6 +22,10 @@ Config-driven OpenTelemetry traces, metrics, and logs for ASP.NET Core from one This package is **not** a SigNoz SDK. Local Aspire SigNoz: `BuildingBlocks.Aspire.Hosting.SigNoz`. Libraries **emit** with `UseTelemetry()`; this host **exports** when `IntegrateMediator` / `IntegrateMcp` is on. +![SigNoz metrics explorer: HTTP server duration, mediator.send.duration, Npgsql connection create time.](https://raw.githubusercontent.com/Maxofpower/FeatureFusion/main/docs/medium/images/07b-signoz-metrics-explorer.png) + +*`mediator.send.duration` is Mediator telemetry leaving the FeatureFusion lab process. This package exports OTLP; it is not a SigNoz SDK.* + ## Install ```bash diff --git a/src/Lab/EventBus/Infrastructure/Messaging/IEventBusLabHook.cs b/src/Lab/EventBus/Infrastructure/Messaging/IEventBusLabHook.cs new file mode 100644 index 0000000..9ea5f4e --- /dev/null +++ b/src/Lab/EventBus/Infrastructure/Messaging/IEventBusLabHook.cs @@ -0,0 +1,37 @@ +using EventBusRabbitMQ.Events; + +namespace EventBusRabbitMQ.Infrastructure.Messaging; + +/// +/// Optional Lab-only hook invoked by OutboxWorker after a successful +/// PublishDirect and before marking the outbox row processed. +/// Not registered in normal hosts — absence means no-op (production semantics unchanged). +/// +public interface IEventBusLabHook +{ + /// + /// Called after the broker publish succeeds for . + /// Throw to leave the outbox row pending + /// (simulate process crash before MarkProcessed). Any other exception uses the worker's + /// normal failure path. + /// + Task OnAfterPublishBeforeOutboxMarkAsync( + Guid messageId, + string eventType, + IntegrationEvent @event, + CancellationToken cancellationToken = default); +} + +/// +/// Lab-only signal: outbox worker skips MarkProcessed/MarkFailed and leaves the row pending. +/// +public sealed class EventBusLabSimulatedCrashException : Exception +{ + public EventBusLabSimulatedCrashException(Guid messageId) + : base($"Lab simulated crash after publish for outbox message {messageId}.") + { + MessageId = messageId; + } + + public Guid MessageId { get; } +} diff --git a/src/Lab/EventBus/Infrastructure/Messaging/ITransactionalOutbox.cs b/src/Lab/EventBus/Infrastructure/Messaging/ITransactionalOutbox.cs index a59628b..c9805cb 100644 --- a/src/Lab/EventBus/Infrastructure/Messaging/ITransactionalOutbox.cs +++ b/src/Lab/EventBus/Infrastructure/Messaging/ITransactionalOutbox.cs @@ -1,4 +1,4 @@ -using EventBusRabbitMQ.Domain; +using EventBusRabbitMQ.Domain; using EventBusRabbitMQ.Events; using EventBusRabbitMQ.Infrastructure.EventBus; using Microsoft.EntityFrameworkCore.Storage; @@ -19,6 +19,7 @@ Task StoreIncomingMessageAsync( byte[] payload, string serviceName); Task MarkMessageAsProcessedAsync(Guid messageId); - Task UpdateHandlerStatuses(List<(string handlerType, ProcessingResult result, Guid messageID)> resultStatuses); + Task> UpdateHandlerStatuses( + List<(string handlerType, ProcessingResult result, Guid messageID)> resultStatuses); } } \ No newline at end of file diff --git a/src/Lab/EventBus/Infrastructure/Messaging/MessageProcessor.cs b/src/Lab/EventBus/Infrastructure/Messaging/MessageProcessor.cs index be13388..a1889ff 100644 --- a/src/Lab/EventBus/Infrastructure/Messaging/MessageProcessor.cs +++ b/src/Lab/EventBus/Infrastructure/Messaging/MessageProcessor.cs @@ -1,4 +1,4 @@ -using EventBusRabbitMQ.Domain; +using EventBusRabbitMQ.Domain; using EventBusRabbitMQ.Events; using EventBusRabbitMQ.Infrastructure.EventBus; using EventBusRabbitMQ.Utilities; @@ -183,9 +183,59 @@ private async Task DispatchToHandlers( } } - await inbox.UpdateHandlerStatuses(resultStatuses); + var attemptRecords = await inbox.UpdateHandlerStatuses(resultStatuses); + // Per-subscriber Attempts is the budget. Exhausted RetryLater becomes PermanentFailure + // before aggregation so Phase 3 order (RetryLater > PermanentFailure > Success) is unchanged. + var budgetedResults = ApplyRetryBudget(resultStatuses, attemptRecords); - return ProcessingResult.Success; + return AggregateHandlerResults(budgetedResults); + } + + private List<(string handlerType, ProcessingResult result, Guid messageId)> ApplyRetryBudget( + IReadOnlyList<(string handlerType, ProcessingResult result, Guid messageId)> resultStatuses, + IReadOnlyList<(string HandlerType, int Attempts)> attemptRecords) + { + var attemptsByHandler = attemptRecords.ToDictionary( + r => r.HandlerType, + r => r.Attempts, + StringComparer.Ordinal); + + var budgeted = new List<(string handlerType, ProcessingResult result, Guid messageId)>(resultStatuses.Count); + foreach (var (handlerType, result, messageId) in resultStatuses) + { + if (result == ProcessingResult.RetryLater + && attemptsByHandler.TryGetValue(handlerType, out var attempts) + && attempts >= _options.RetryCount) + { + budgeted.Add((handlerType, ProcessingResult.PermanentFailure, messageId)); + continue; + } + + budgeted.Add((handlerType, result, messageId)); + } + + return budgeted; + } + + private static ProcessingResult AggregateHandlerResults( + IReadOnlyList<(string handlerType, ProcessingResult result, Guid messageId)> resultStatuses) + { + var aggregate = ProcessingResult.Success; + + foreach (var (_, result, _) in resultStatuses) + { + if (result == ProcessingResult.RetryLater) + { + return ProcessingResult.RetryLater; + } + + if (result == ProcessingResult.PermanentFailure) + { + aggregate = ProcessingResult.PermanentFailure; + } + } + + return aggregate; } } diff --git a/src/Lab/EventBus/Infrastructure/Messaging/OutBoxWorker.cs b/src/Lab/EventBus/Infrastructure/Messaging/OutBoxWorker.cs index 68f768d..e55e35a 100644 --- a/src/Lab/EventBus/Infrastructure/Messaging/OutBoxWorker.cs +++ b/src/Lab/EventBus/Infrastructure/Messaging/OutBoxWorker.cs @@ -2,6 +2,7 @@ using EventBusRabbitMQ.Events; using EventBusRabbitMQ.Infrastructure; using EventBusRabbitMQ.Infrastructure.EventBus; +using EventBusRabbitMQ.Infrastructure.Messaging; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -86,6 +87,29 @@ await MarkMessageAsFailedAsync(dbContext, message, } await eventBus.PublishDirect((dynamic)@event, ct: stoppingToken); + + // Optional Lab seam (IEventBusLabHook). Unregistered → no-op. + var labHook = scope.ServiceProvider.GetService(); + if (labHook is not null) + { + try + { + await labHook.OnAfterPublishBeforeOutboxMarkAsync( + message.Id, + message.EventType, + @event, + stoppingToken); + } + catch (EventBusLabSimulatedCrashException) + { + // Leave ProcessedAt null (pending) — do not MarkProcessed or MarkFailed. + _logger.LogWarning( + "Lab simulated crash after publish for outbox message {MessageId}; leaving pending", + message.Id); + continue; + } + } + await MarkMessageAsProcessedAsync(dbContext, message, stoppingToken); } catch (JsonException jsonEx) diff --git a/src/Lab/EventBus/Infrastructure/Messaging/TransactionalOutbox.cs b/src/Lab/EventBus/Infrastructure/Messaging/TransactionalOutbox.cs index 58bfcc9..81799ff 100644 --- a/src/Lab/EventBus/Infrastructure/Messaging/TransactionalOutbox.cs +++ b/src/Lab/EventBus/Infrastructure/Messaging/TransactionalOutbox.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Data; using System.Reflection.Emit; using System.Text.Json; @@ -174,11 +174,18 @@ public async Task StoreIncomingMessageAsync( return MessageStoreResult.StorageFailed; } - if (await _dbContext.InboxMessages + var existing = await _dbContext.InboxMessages .AsNoTracking() - .AnyAsync(m => m.Id == messageId)) + .FirstOrDefaultAsync(m => m.Id == messageId); + + if (existing is not null) { - return MessageStoreResult.Duplicate; + if (IsSuccessfullyProcessed(existing)) + { + return MessageStoreResult.Duplicate; + } + + return MessageStoreResult.Success; } var message = new InboxMessage @@ -240,15 +247,21 @@ public async Task StoreIncomingMessageAsync( public async Task IsDuplicateAsync(Guid messageId) { - //var cutoff = DateTime.UtcNow - _deduplicationWindow; //if needed - - if (await _dbContext.InboxMessages + var message = await _dbContext.InboxMessages .AsNoTracking() - .AnyAsync(x => x.Id == messageId)) - return MessageStoreResult.Duplicate; - return MessageStoreResult.Success; + .FirstOrDefaultAsync(x => x.Id == messageId); + + if (message is null || !IsSuccessfullyProcessed(message)) + { + return MessageStoreResult.Success; + } + + return MessageStoreResult.Duplicate; } + private static bool IsSuccessfullyProcessed(InboxMessage message) => + message.IsProcessed || message.Status == MessageStatus.Processed; + public async Task MarkMessageAsProcessedAsync(Guid messageId) { @@ -264,36 +277,32 @@ public async Task MarkMessageAsProcessedAsync(Guid messageId) } } - public async Task UpdateHandlerStatuses(List<(string handlerType, ProcessingResult result,Guid messageID)> resultStatuses) + public async Task> UpdateHandlerStatuses( + List<(string handlerType, ProcessingResult result, Guid messageID)> resultStatuses) { - try + var attempts = new List<(string HandlerType, int Attempts)>(resultStatuses.Count); + var attemptedAt = DateTime.UtcNow; + + foreach (var (handlerType, result, messageId) in resultStatuses) { - - foreach (var (handlerType, result,messageId) in resultStatuses) + var handlerRecord = await _dbContext.InboxSubscriber + .FirstOrDefaultAsync(h => h.SubscriberName == handlerType && h.MessageId == messageId); + if (handlerRecord is null) { - - var handlerRecord = await _dbContext.InboxSubscriber - .FirstOrDefaultAsync(h => h.SubscriberName == handlerType && h.MessageId==messageId); - if (handlerRecord != null) - { - if (result == ProcessingResult.Success) - { - handlerRecord.Status = MessageStatus.Processed; - } - else - { - handlerRecord.Status = MessageStatus.Failed; - } - } - + continue; } - await _dbContext.SaveChangesAsync(); - } - catch - { - // we cant re throw here + + handlerRecord.Attempts++; + handlerRecord.LastAttemptedAt = attemptedAt; + handlerRecord.Status = result == ProcessingResult.Success + ? MessageStatus.Processed + : MessageStatus.Failed; + + attempts.Add((handlerType, handlerRecord.Attempts)); } - + + await _dbContext.SaveChangesAsync(); + return attempts; } private List GetSubscribersForEventType(EventBusSubscriptionInfo subscriptionInfo, string eventType) { diff --git a/src/Lab/FeatureFusion/Apis/MinimalApiGreeting.cs b/src/Lab/FeatureFusion/Apis/MinimalApiGreeting.cs index 43b3136..ac67eed 100644 --- a/src/Lab/FeatureFusion/Apis/MinimalApiGreeting.cs +++ b/src/Lab/FeatureFusion/Apis/MinimalApiGreeting.cs @@ -1,3 +1,4 @@ +using BuildingBlocks.Idempotency.AspNetCore; using BuildingBlocks.Mcp; using BuildingBlocks.Mcp.Hosting; using Asp.Versioning.Conventions; @@ -57,6 +58,12 @@ public static RouteGroupBuilder MapGreetingApiV2(this IEndpointRouteBuilder app) .WithSummary("Minimal API ping (not a Mediator command). Same method is an MCP tool lab.ping.") .WithMcp(app); + // Smoke: BuildingBlocks.Idempotency Minimal API surface (not an Exp gate). + api.MapPost("/idempotency-smoke", () => Results.Ok(new { ok = true })) + .WithName("IdempotencySmoke") + .WithSummary("Minimal API Idempotency-Key smoke (WithIdempotency).") + .WithIdempotency(useLock: true); + return api; } diff --git a/src/Lab/FeatureFusion/Controllers/V2/OrderController.cs b/src/Lab/FeatureFusion/Controllers/V2/OrderController.cs index 42ef399..c2511d3 100644 --- a/src/Lab/FeatureFusion/Controllers/V2/OrderController.cs +++ b/src/Lab/FeatureFusion/Controllers/V2/OrderController.cs @@ -1,4 +1,4 @@ -using FeatureFusion.Infrastructure.Filters; +using BuildingBlocks.Idempotency.AspNetCore; using BuildingBlocks.Mediator; using FeatureFusion.Models; using FeatureFusion.Services.FeatureToggleService; diff --git a/src/Lab/FeatureFusion/Dtos/Validator/GetProductsCommandValidator.cs b/src/Lab/FeatureFusion/Dtos/Validator/GetProductsCommandValidator.cs index 0de7e2f..96915bb 100644 --- a/src/Lab/FeatureFusion/Dtos/Validator/GetProductsCommandValidator.cs +++ b/src/Lab/FeatureFusion/Dtos/Validator/GetProductsCommandValidator.cs @@ -1,4 +1,4 @@ -using BuildingBlocks.Pagination; +using BuildingBlocks.Pagination; using FeatureFusion.Features.Products.Queries; using FeatureFusion.Infrastructure.Pagination; using FluentValidation; diff --git a/src/Lab/FeatureFusion/FeatureFusion.csproj b/src/Lab/FeatureFusion/FeatureFusion.csproj index 5d379a4..838a2c0 100644 --- a/src/Lab/FeatureFusion/FeatureFusion.csproj +++ b/src/Lab/FeatureFusion/FeatureFusion.csproj @@ -1,4 +1,4 @@ - + net10.0 @@ -44,6 +44,7 @@ + diff --git a/src/Lab/FeatureFusion/Infrastructure/CursorPagination/PagedResult.cs b/src/Lab/FeatureFusion/Infrastructure/CursorPagination/PagedResult.cs index 22dd686..3e25c3a 100644 --- a/src/Lab/FeatureFusion/Infrastructure/CursorPagination/PagedResult.cs +++ b/src/Lab/FeatureFusion/Infrastructure/CursorPagination/PagedResult.cs @@ -1,4 +1,4 @@ -namespace FeatureFusion.Infrastructure.CursorPagination +namespace FeatureFusion.Infrastructure.CursorPagination { public record PagedResult( IReadOnlyList Items, diff --git a/src/Lab/FeatureFusion/Infrastructure/EntitiyConfiguration/ProductEntityTypeConfiguration.cs b/src/Lab/FeatureFusion/Infrastructure/EntitiyConfiguration/ProductEntityTypeConfiguration.cs index e402479..7b0a16c 100644 --- a/src/Lab/FeatureFusion/Infrastructure/EntitiyConfiguration/ProductEntityTypeConfiguration.cs +++ b/src/Lab/FeatureFusion/Infrastructure/EntitiyConfiguration/ProductEntityTypeConfiguration.cs @@ -1,4 +1,4 @@ -using BuildingBlocks.Pagination.EntityFrameworkCore; +using BuildingBlocks.Pagination.EntityFrameworkCore; using FeatureFusion.Domain.Entities; using FeatureFusion.Infrastructure.Pagination; using Microsoft.EntityFrameworkCore; diff --git a/src/Lab/FeatureFusion/Infrastructure/Extensions/BuilderExtensions.cs b/src/Lab/FeatureFusion/Infrastructure/Extensions/BuilderExtensions.cs index 6a0a88d..6bbe542 100644 --- a/src/Lab/FeatureFusion/Infrastructure/Extensions/BuilderExtensions.cs +++ b/src/Lab/FeatureFusion/Infrastructure/Extensions/BuilderExtensions.cs @@ -1,6 +1,7 @@ using Asp.Versioning; using Asp.Versioning.ApiExplorer; using Asp.Versioning.Conventions; +using BuildingBlocks.Idempotency.DependencyInjection; using FeatureFusion.Features.Order.IntegrationEvents; using FeatureFusion.Features.Order.IntegrationEvents.EventHandling; using FeatureFusion.Features.Order.IntegrationEvents.Events; @@ -214,6 +215,14 @@ public static void RegisterServices(this IServiceCollection services) services.AddSingleton(); + // BuildingBlocks.Idempotency — Lab-compatible UserIdFallback; host owns IDistributedCache + multiplexer. + services.AddBuildingBlocksIdempotency(o => + { + o.UserIdFallback = "123"; + }) + .UseRedisLock() + .UseTelemetry(); + services.AddSingleton(); // AppInitializer is registered in AddApplicationServices after DB migrations. diff --git a/src/Lab/FeatureFusion/Infrastructure/Filters/IdempotentAttribute.cs b/src/Lab/FeatureFusion/Infrastructure/Filters/IdempotentAttribute.cs deleted file mode 100644 index d0a6468..0000000 --- a/src/Lab/FeatureFusion/Infrastructure/Filters/IdempotentAttribute.cs +++ /dev/null @@ -1,42 +0,0 @@ -namespace FeatureFusion.Infrastructure.Filters -{ - using FeatureFusion.Infrastructure.Caching; - using Microsoft.AspNetCore.Mvc.Filters; - using Microsoft.Extensions.Caching.Distributed; - using Microsoft.Extensions.Caching.Hybrid; - - /// - /// IdempotentFusion: applies ULID Idempotency-Key handling via Redis-backed status tracking (optional lock). - /// - /// - /// LinkedIn (IdentifiedCommand / CQRS idempotency pattern): - /// https://www.linkedin.com/feed/update/urn:li:activity:7303686809891356676/ - /// LinkedIn (IdempotentFusion in this repo): - /// https://www.linkedin.com/feed/update/urn:li:activity:7309149985307029504/ - /// Catalog: docs/linkedin-posts.md (idempotency-cqrs, idempotentfusion). - /// - [AttributeUsage(AttributeTargets.Method)] - public class IdempotentAttribute : Attribute, IFilterFactory - { - public bool UseLock { get; set; } - - public IdempotentAttribute(bool useLock = false) - { - UseLock = useLock; - } - public bool IsReusable => false; // Filters are not reusable - - public IFilterMetadata CreateInstance(IServiceProvider serviceProvider) - { - var distributedCache = serviceProvider.GetService(); - var redisWrapper = serviceProvider.GetService(); - var loggerFactory = (ILoggerFactory)serviceProvider.GetService(typeof(ILoggerFactory)); - - return new IdempotentAttributeFilter( - distributedCache, - loggerFactory, - redisWrapper, - useLock: UseLock); - } - } -} diff --git a/src/Lab/FeatureFusion/Infrastructure/Filters/IdempotentAttributeFilter.cs b/src/Lab/FeatureFusion/Infrastructure/Filters/IdempotentAttributeFilter.cs deleted file mode 100644 index 3e89944..0000000 --- a/src/Lab/FeatureFusion/Infrastructure/Filters/IdempotentAttributeFilter.cs +++ /dev/null @@ -1,228 +0,0 @@ -using Microsoft.AspNetCore.Mvc.Filters; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Caching.Distributed; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json; -using System; -using System.Security.Claims; -using System.Threading; -using System.Threading.Tasks; -using static System.Ulid; -using FeatureFusion.Infrastructure.Caching; -using System.Text; - -public sealed class IdempotentAttributeFilter : IAsyncResourceFilter, IAsyncActionFilter -{ - private readonly IDistributedCache _distributedCache; - private readonly ILogger _logger; - private readonly IRedisConnectionWrapper _redisConnectionWrapper; - private readonly bool _useLock; - - public IdempotentAttributeFilter( - IDistributedCache distributedCache, - ILoggerFactory loggerFactory, - IRedisConnectionWrapper redisConnectionWrapper, - bool useLock) - { - _distributedCache = distributedCache; - _logger = loggerFactory.CreateLogger(); - _redisConnectionWrapper = redisConnectionWrapper; - _useLock = useLock; - } - - // Early validation (before model binding) - public async Task OnResourceExecutionAsync(ResourceExecutingContext context, ResourceExecutionDelegate next) - { - try - { - ExtractAndValidateIdempotencyKey(context.HttpContext.Request); - await next(); - } - catch (ArgumentNullException ex) - { - context.Result = new BadRequestObjectResult(ex.Message); - } - catch (ArgumentException ex) - { - context.Result = new BadRequestObjectResult(ex.Message); - } - } - // Main idempotency logic (after model binding) - public async Task OnActionExecutionAsync(ActionExecutingContext context, ActionExecutionDelegate next) - { - string cacheKey = null; - - try - { - // Extract and validate the Idempotency-Key header as a ULID - var idempotencyKey = ExtractAndValidateIdempotencyKey(context.HttpContext.Request); - var userId = context.HttpContext.User.FindFirst(ClaimTypes.NameIdentifier)?.Value ?? "123"; - - if (string.IsNullOrEmpty(userId)) - throw new UnauthorizedAccessException("User ID is missing in the request context."); - - cacheKey = $"Idempotency_{userId}_{idempotencyKey}"; - - // request status tracking - var (isNewlyCreated, cacheEntry) = await GetOrCreateCacheEntryAsync(cacheKey); - if (cacheEntry != null && !isNewlyCreated) - { - context.Result = cacheEntry.Status switch - { - "Processing" => new ConflictObjectResult("Request is already being processed. Please wait."), - "Completed" or "Failed" => new ContentResult - { - Content = cacheEntry.Response, - ContentType = "application/json", - StatusCode = 200 - }, - _ => throw new InvalidOperationException($"Unknown cache status: {cacheEntry.Status}") - }; - - if (context.Result is ContentResult contentResult) - context.HttpContext.Response.Headers.Append("X-Idempotent-Response", "true"); - - return; - } - - await MarkRequestAsProcessingAsync(cacheKey); - var executedContext = await next(); - - if (executedContext.Result is ObjectResult objResult && objResult.StatusCode == 200) - { - var responseJson = JsonConvert.SerializeObject(objResult.Value); - await CacheSuccessfulResponseAsync(cacheKey, responseJson); - } - else - { - await RemoveCacheEntryAsync(cacheKey); - } - } - catch (Exception ex) - { - _logger.LogError(ex, "An error occurred while processing the request."); - if (cacheKey != null) await RemoveCacheEntryAsync(cacheKey); - context.Result = new StatusCodeResult(500); - } - } - - private async Task<(bool, IdempotencyCacheEntry)> GetOrCreateCacheEntryAsync(string cacheKey) - { - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); - - try - { - if (_useLock) - { - string lockKey = $"{cacheKey}_lock"; - string lockValue = Guid.NewGuid().ToString(); - TimeSpan lockExpiry = TimeSpan.FromSeconds(10); - - bool lockAcquired = await _redisConnectionWrapper.AcquireLockAsync(lockKey, lockValue, lockExpiry); - if (!lockAcquired) throw new InvalidOperationException("Failed to acquire Redis lock."); - - try - { - return await GetOrCreateCacheEntryWithoutLockAsync(cacheKey, cts.Token); - } - finally - { - await _redisConnectionWrapper.ReleaseLockAsync(lockKey, lockValue); - } - } - else - { - return await GetOrCreateCacheEntryWithoutLockAsync(cacheKey, cts.Token); - } - } - catch (OperationCanceledException ex) - { - _logger.LogWarning($"Cache operation timed out for key {cacheKey}: {ex.Message}"); - throw new TimeoutException($"Cache operation timed out for key: {cacheKey}", ex); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Cache operation failed for key {cacheKey}: {ex.Message}"); - throw new InvalidOperationException($"Cache operation failed for key: {cacheKey}", ex); - } - } - - private async Task<(bool, IdempotencyCacheEntry)> GetOrCreateCacheEntryWithoutLockAsync(string cacheKey, CancellationToken cancellationToken) - { - var cacheData = await _distributedCache.GetAsync(cacheKey, cancellationToken); - if (cacheData != null) - { - return (false, JsonConvert.DeserializeObject(Encoding.UTF8.GetString(cacheData))); - } - - var cacheEntry = new IdempotencyCacheEntry { Status = "Processing", Response = null }; - var cacheEntryData = JsonConvert.SerializeObject(cacheEntry); - - await _distributedCache.SetAsync( - cacheKey, - Encoding.UTF8.GetBytes(cacheEntryData), - new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(30) }, - cancellationToken - ); - - return (true, cacheEntry); - } - - private async Task MarkRequestAsProcessingAsync(string cacheKey) - { - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); - var cacheEntry = new IdempotencyCacheEntry { Status = "Processing" }; - var cacheEntryData = JsonConvert.SerializeObject(cacheEntry); - - await _distributedCache.SetAsync( - cacheKey, - Encoding.UTF8.GetBytes(cacheEntryData), - new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(30) }, - cts.Token - ); - } - - private async Task CacheSuccessfulResponseAsync(string cacheKey, string responseJson) - { - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - var cacheEntry = new IdempotencyCacheEntry { Status = "Completed", Response = responseJson }; - var cacheEntryData = JsonConvert.SerializeObject(cacheEntry); - - await _distributedCache.SetAsync( - cacheKey, - Encoding.UTF8.GetBytes(cacheEntryData), - new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(30) }, - cts.Token - ); - } - - private async Task RemoveCacheEntryAsync(string cacheKey) - { - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - await _distributedCache.RemoveAsync(cacheKey, cts.Token); - } - - public Ulid ExtractAndValidateIdempotencyKey(HttpRequest httpRequest) - { - const string idempotencyHeaderName = "Idempotency-Key"; - - if (!httpRequest.Headers.TryGetValue(idempotencyHeaderName, out var value)) - throw new ArgumentNullException(idempotencyHeaderName, "The Idempotency-Key header is missing."); - - var idempotencyKeyValue = value.ToString(); - - if (string.IsNullOrWhiteSpace(idempotencyKeyValue)) - throw new ArgumentException("The Idempotency-Key value cannot be empty."); - - if (!Ulid.TryParse(idempotencyKeyValue, out var idempotencyKey)) - throw new ArgumentException($"Invalid Idempotency-Key format: {idempotencyKeyValue}"); - - return idempotencyKey; - } -} - -public class IdempotencyCacheEntry -{ - public string Status { get; set; } - public string Response { get; set; } -} \ No newline at end of file diff --git a/src/Lab/FeatureFusion/Infrastructure/Migrations/CatalogDbContextModelSnapshot.cs b/src/Lab/FeatureFusion/Infrastructure/Migrations/CatalogDbContextModelSnapshot.cs index a0e6b08..9e147e1 100644 --- a/src/Lab/FeatureFusion/Infrastructure/Migrations/CatalogDbContextModelSnapshot.cs +++ b/src/Lab/FeatureFusion/Infrastructure/Migrations/CatalogDbContextModelSnapshot.cs @@ -1,4 +1,4 @@ -// +// using System; using FeatureFusion.Infrastructure.Context; using Microsoft.EntityFrameworkCore; diff --git a/src/Lab/FeatureFusion/Program.cs b/src/Lab/FeatureFusion/Program.cs index 9694ec0..9807cfe 100644 --- a/src/Lab/FeatureFusion/Program.cs +++ b/src/Lab/FeatureFusion/Program.cs @@ -38,6 +38,7 @@ configureTelemetry: telemetry => { telemetry.AddSource("DbMigrations"); + telemetry.AddSource("BuildingBlocks.Idempotency"); telemetry.ConfigureTracing(t => t .AddEntityFrameworkCoreInstrumentation() .AddRedisInstrumentation()); @@ -70,7 +71,7 @@ builder.Services.AddBuildingBlocksMcp(o => { o.ScanAssembly(Assembly.GetExecutingAssembly()); - o.UseTelemetry(); + o.UseTelemetry(t => t.IncludeExceptionDetails = true); o.UseMemoryIdempotency(TimeSpan.FromHours(1)); }).UseDispatcher(async (sp, msg, ct) => { diff --git a/tests/BuildingBlocks/Idempotency.Tests/BuildingBlocks.Idempotency.Tests.csproj b/tests/BuildingBlocks/Idempotency.Tests/BuildingBlocks.Idempotency.Tests.csproj new file mode 100644 index 0000000..5c092ee --- /dev/null +++ b/tests/BuildingBlocks/Idempotency.Tests/BuildingBlocks.Idempotency.Tests.csproj @@ -0,0 +1,29 @@ + + + + net8.0;net9.0;net10.0 + latest + enable + enable + false + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + diff --git a/tests/BuildingBlocks/Idempotency.Tests/IdempotentAttributeFilterTests.cs b/tests/BuildingBlocks/Idempotency.Tests/IdempotentAttributeFilterTests.cs new file mode 100644 index 0000000..dd27423 --- /dev/null +++ b/tests/BuildingBlocks/Idempotency.Tests/IdempotentAttributeFilterTests.cs @@ -0,0 +1,623 @@ +using System.Diagnostics; +using System.Security.Claims; +using System.Text; +using BuildingBlocks.Idempotency; +using BuildingBlocks.Idempotency.AspNetCore; +using BuildingBlocks.Idempotency.DependencyInjection; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Abstractions; +using Microsoft.AspNetCore.Mvc.Filters; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.Caching.Distributed; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; +using Newtonsoft.Json; +using Xunit; + +namespace BuildingBlocks.Idempotency.Tests; + +public sealed class IdempotentAttributeFilterTests +{ + private readonly IDistributedCache _distributedCache = + new MemoryDistributedCache(Options.Create(new MemoryDistributedCacheOptions())); + + private readonly Mock _lockMock = new(); + private readonly IdempotencyOptions _options = new() + { + UserIdFallback = "123" + }; + + public IdempotentAttributeFilterTests() + { + _lockMock + .Setup(l => l.AcquireAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(true); + _lockMock + .Setup(l => l.ReleaseAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(true); + } + + private HttpContext CreateHttpContext( + string idempotencyKey, + string? userId = "456", + string? body = null, + string method = "POST", + string path = "/api/orders") + { + var httpContext = new DefaultHttpContext(); + httpContext.Request.Method = method; + httpContext.Request.Path = path; + httpContext.Request.Headers["Idempotency-Key"] = idempotencyKey; + if (userId is not null) + { + httpContext.User = new ClaimsPrincipal(new ClaimsIdentity( + [ + new Claim(ClaimTypes.NameIdentifier, userId) + ])); + } + + if (body is not null) + { + var bytes = Encoding.UTF8.GetBytes(body); + httpContext.Request.Body = new MemoryStream(bytes); + httpContext.Request.ContentLength = bytes.Length; + httpContext.Request.ContentType = "application/json"; + httpContext.Request.EnableBuffering(); + } + + return httpContext; + } + + private IdempotentAttributeFilter CreateFilter( + bool useLock = false, + IdempotencyOptions? options = null, + IdempotencyTelemetry? telemetry = null, + int processingTtlSeconds = 0, + int entryTtlSeconds = 0) + { + return new IdempotentAttributeFilter( + _distributedCache, + NullLoggerFactory.Instance, + useLock ? _lockMock.Object : null, + options ?? _options, + useLock, + telemetry, + processingTtlSeconds, + entryTtlSeconds); + } + + private static ActionExecutingContext CreateActionContext(HttpContext httpContext) + { + return new ActionExecutingContext( + new ActionContext(httpContext, new RouteData(), new ActionDescriptor()), + new List(), + new Dictionary(), + controller: new object()); + } + + private static ActionExecutedContext CreateSuccessfulActionExecutedContext( + ActionExecutingContext context, + int statusCode = 200) + { + return new ActionExecutedContext(context, new List(), null!) + { + Result = new ObjectResult(new { message = "Success" }) { StatusCode = statusCode } + }; + } + + private static ProblemDetails AssertProblem(IActionResult? result, int status, string typeSuffix) + { + var objectResult = Assert.IsType(result); + Assert.Equal(status, objectResult.StatusCode); + var problem = Assert.IsType(objectResult.Value); + Assert.Equal(status, problem.Status); + Assert.Equal(IdempotencyProblemDetails.TypeBase + typeSuffix, problem.Type); + return problem; + } + + [Fact] + public void ExtractAndValidateIdempotencyKey_ValidUlid_ReturnsKey() + { + var httpContext = CreateHttpContext(Ulid.NewUlid().ToString()); + var filter = CreateFilter(); + var result = filter.ExtractAndValidateIdempotencyKey(httpContext.Request); + Assert.Equal(httpContext.Request.Headers["Idempotency-Key"], result); + } + + [Fact] + public void ExtractAndValidateIdempotencyKey_MissingHeader_ThrowsArgumentNullException() + { + var httpContext = new DefaultHttpContext(); + var filter = CreateFilter(); + var exception = Assert.Throws(() => filter.ExtractAndValidateIdempotencyKey(httpContext.Request)); + Assert.Equal("Idempotency-Key", exception.ParamName); + } + + [Fact] + public void ExtractAndValidateIdempotencyKey_EmptyHeader_ThrowsArgumentException() + { + var httpContext = CreateHttpContext(string.Empty); + var filter = CreateFilter(); + var exception = Assert.Throws(() => filter.ExtractAndValidateIdempotencyKey(httpContext.Request)); + Assert.Equal("The Idempotency-Key value cannot be empty.", exception.Message); + } + + [Fact] + public void ExtractAndValidateIdempotencyKey_InvalidUlid_ThrowsArgumentException() + { + var httpContext = CreateHttpContext("invalid-ulid"); + var filter = CreateFilter(); + var exception = Assert.Throws(() => filter.ExtractAndValidateIdempotencyKey(httpContext.Request)); + Assert.Equal("Invalid Idempotency-Key format: invalid-ulid", exception.Message); + } + + [Fact] + public void KeyValidator_RejectsControlCharactersAndOverLength() + { + var options = new IdempotencyOptions { RequireUlid = false, MaxKeyLength = 8 }; + Assert.False(IdempotencyKeyValidator.TryValidate("ab\nc", options, out _)); + Assert.False(IdempotencyKeyValidator.TryValidate("123456789", options, out _)); + Assert.True(IdempotencyKeyValidator.TryValidate("ok-key", options, out _)); + } + + [Fact] + public async Task FirstRequest_CachesEnvelope() + { + var idempotencyKey = Ulid.NewUlid().ToString(); + var httpContext = CreateHttpContext(idempotencyKey); + var filter = CreateFilter(); + var context = CreateActionContext(httpContext); + var executedContext = CreateSuccessfulActionExecutedContext(context); + + await filter.OnActionExecutionAsync(context, () => Task.FromResult(executedContext)); + + var cacheKey = $"Idempotency_456_{idempotencyKey}"; + var cachedData = await _distributedCache.GetAsync(cacheKey); + Assert.NotNull(cachedData); + + var cachedEntry = JsonConvert.DeserializeObject(Encoding.UTF8.GetString(cachedData)); + Assert.Equal("Completed", cachedEntry?.Status); + Assert.Equal(200, cachedEntry?.StatusCode); + Assert.Equal("application/json", cachedEntry?.ContentType); + Assert.Equal("{\"message\":\"Success\"}", cachedEntry?.Response); + } + + [Fact] + public async Task Created201_IsCachedAndReplayed() + { + var idempotencyKey = Ulid.NewUlid().ToString(); + var filter = CreateFilter(); + var first = CreateActionContext(CreateHttpContext(idempotencyKey, "1")); + await filter.OnActionExecutionAsync(first, () => Task.FromResult(CreateSuccessfulActionExecutedContext(first, 201))); + + var second = CreateActionContext(CreateHttpContext(idempotencyKey, "1")); + var ran = false; + await filter.OnActionExecutionAsync(second, () => + { + ran = true; + return Task.FromResult(CreateSuccessfulActionExecutedContext(second, 201)); + }); + + Assert.False(ran); + var content = Assert.IsType(second.Result); + Assert.Equal(201, content.StatusCode); + } + + [Fact] + public async Task DuplicateRequest_ReplaysStoredStatusAndContentType() + { + var idempotencyKey = Ulid.NewUlid().ToString(); + var httpContext = CreateHttpContext(idempotencyKey, "1234"); + var cacheKey = $"Idempotency_1234_{idempotencyKey}"; + var cacheEntry = new IdempotencyCacheEntry + { + Status = "Completed", + Response = "{\"message\":\"Cached\"}", + StatusCode = 200, + ContentType = "application/json" + }; + await _distributedCache.SetAsync(cacheKey, Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(cacheEntry))); + + var filter = CreateFilter(); + var context = CreateActionContext(httpContext); + await filter.OnActionExecutionAsync(context, () => Task.FromResult(new ActionExecutedContext(context, new List(), null!))); + + var contentResult = Assert.IsType(context.Result); + Assert.Equal("{\"message\":\"Cached\"}", contentResult.Content); + Assert.Equal("application/json", contentResult.ContentType); + Assert.Equal(200, contentResult.StatusCode); + Assert.True(httpContext.Response.Headers.ContainsKey("X-Idempotent-Response")); + } + + [Fact] + public async Task DuplicateCompleted_ConflictStrategy_Returns409Problem() + { + var options = new IdempotencyOptions + { + UserIdFallback = "123", + DuplicateCompletedBehavior = DuplicateCompletedBehavior.Conflict + }; + var idempotencyKey = Ulid.NewUlid().ToString(); + var cacheKey = $"Idempotency_1_{idempotencyKey}"; + await _distributedCache.SetAsync(cacheKey, Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new IdempotencyCacheEntry + { + Status = "Completed", + Response = "{}", + StatusCode = 200 + }))); + + var filter = CreateFilter(options: options); + var context = CreateActionContext(CreateHttpContext(idempotencyKey, "1")); + await filter.OnActionExecutionAsync(context, () => Task.FromResult(CreateSuccessfulActionExecutedContext(context))); + + AssertProblem(context.Result, 409, "duplicate"); + } + + [Fact] + public async Task ProcessingRequest_ReturnsConflictProblem() + { + var idempotencyKey = Ulid.NewUlid().ToString(); + var httpContext = CreateHttpContext(idempotencyKey, "1234"); + var cacheKey = $"Idempotency_1234_{idempotencyKey}"; + var cacheEntry = new IdempotencyCacheEntry + { + Status = "Processing", + ProcessingExpiresAtUtc = DateTimeOffset.UtcNow.AddMinutes(1) + }; + await _distributedCache.SetAsync(cacheKey, Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(cacheEntry))); + + var filter = CreateFilter(); + var context = CreateActionContext(httpContext); + await filter.OnActionExecutionAsync(context, () => Task.FromResult(new ActionExecutedContext(context, new List(), null!))); + + var problem = AssertProblem(context.Result, 409, "processing"); + Assert.Equal("Request is already being processed. Please wait.", problem.Detail); + } + + [Fact] + public async Task ExpiredProcessing_AllowsNewExecution() + { + var idempotencyKey = Ulid.NewUlid().ToString(); + var httpContext = CreateHttpContext(idempotencyKey, "999"); + var cacheKey = $"Idempotency_999_{idempotencyKey}"; + var stale = new IdempotencyCacheEntry + { + Status = "Processing", + ProcessingExpiresAtUtc = DateTimeOffset.UtcNow.AddMinutes(-1) + }; + await _distributedCache.SetAsync(cacheKey, Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(stale))); + + var filter = CreateFilter(); + var context = CreateActionContext(httpContext); + var executed = CreateSuccessfulActionExecutedContext(context); + var ran = false; + + await filter.OnActionExecutionAsync(context, () => + { + ran = true; + return Task.FromResult(executed); + }); + + Assert.True(ran); + var cachedData = await _distributedCache.GetAsync(cacheKey); + var entry = JsonConvert.DeserializeObject(Encoding.UTF8.GetString(cachedData!)); + Assert.Equal("Completed", entry?.Status); + } + + [Fact] + public async Task OnActionExecutionAsync_FailedRequest_RemovesCacheEntry() + { + var idempotencyKey = Ulid.NewUlid().ToString(); + var httpContext = CreateHttpContext(idempotencyKey, "2565"); + var filter = CreateFilter(); + var context = CreateActionContext(httpContext); + + async Task Next() + { + await Task.Yield(); + throw new InvalidOperationException("Test exception"); + } + + await Assert.ThrowsAsync(() => filter.OnActionExecutionAsync(context, Next)); + Assert.Null(await _distributedCache.GetAsync($"Idempotency_2565_{idempotencyKey}")); + } + + [Fact] + public async Task FingerprintEnabled_SameBody_Replays() + { + var options = new IdempotencyOptions { UserIdFallback = "123", EnableRequestFingerprint = true }; + var body = """{"quantity":2}"""; + var idempotencyKey = Ulid.NewUlid().ToString(); + var httpContext = CreateHttpContext(idempotencyKey, "1", body); + var filter = CreateFilter(options: options); + var context = CreateActionContext(httpContext); + await filter.OnActionExecutionAsync(context, () => Task.FromResult(CreateSuccessfulActionExecutedContext(context))); + + var httpContext2 = CreateHttpContext(idempotencyKey, "1", body); + var context2 = CreateActionContext(httpContext2); + var ran = false; + await filter.OnActionExecutionAsync(context2, () => + { + ran = true; + return Task.FromResult(CreateSuccessfulActionExecutedContext(context2)); + }); + + Assert.False(ran); + Assert.IsType(context2.Result); + Assert.True(httpContext2.Response.Headers.ContainsKey("X-Idempotent-Response")); + } + + [Fact] + public async Task FingerprintEnabled_DifferentBody_Returns422Problem() + { + var options = new IdempotencyOptions { UserIdFallback = "123", EnableRequestFingerprint = true }; + var idempotencyKey = Ulid.NewUlid().ToString(); + var filter = CreateFilter(options: options); + + var first = CreateActionContext(CreateHttpContext(idempotencyKey, "1", """{"quantity":2}""")); + await filter.OnActionExecutionAsync(first, () => Task.FromResult(CreateSuccessfulActionExecutedContext(first))); + + var second = CreateActionContext(CreateHttpContext(idempotencyKey, "1", """{"quantity":5}""")); + await filter.OnActionExecutionAsync(second, () => Task.FromResult(CreateSuccessfulActionExecutedContext(second))); + + AssertProblem(second.Result, 422, "fingerprint-mismatch"); + } + + [Fact] + public async Task FingerprintEnabled_DifferentPath_Returns422Problem() + { + var options = new IdempotencyOptions { UserIdFallback = "123", EnableRequestFingerprint = true }; + var idempotencyKey = Ulid.NewUlid().ToString(); + var body = """{"quantity":2}"""; + var filter = CreateFilter(options: options); + + var first = CreateActionContext(CreateHttpContext(idempotencyKey, "1", body, path: "/api/a")); + await filter.OnActionExecutionAsync(first, () => Task.FromResult(CreateSuccessfulActionExecutedContext(first))); + + var second = CreateActionContext(CreateHttpContext(idempotencyKey, "1", body, path: "/api/b")); + await filter.OnActionExecutionAsync(second, () => Task.FromResult(CreateSuccessfulActionExecutedContext(second))); + + AssertProblem(second.Result, 422, "fingerprint-mismatch"); + } + + [Fact] + public async Task FingerprintDisabled_DifferentBody_ReplaysOriginal() + { + var options = new IdempotencyOptions { UserIdFallback = "123", EnableRequestFingerprint = false }; + var idempotencyKey = Ulid.NewUlid().ToString(); + var filter = CreateFilter(options: options); + + var first = CreateActionContext(CreateHttpContext(idempotencyKey, "1", """{"quantity":2}""")); + await filter.OnActionExecutionAsync(first, () => Task.FromResult(CreateSuccessfulActionExecutedContext(first))); + + var second = CreateActionContext(CreateHttpContext(idempotencyKey, "1", """{"quantity":5}""")); + var ran = false; + await filter.OnActionExecutionAsync(second, () => + { + ran = true; + return Task.FromResult(CreateSuccessfulActionExecutedContext(second)); + }); + + Assert.False(ran); + Assert.IsType(second.Result); + } + + [Fact] + public async Task KeyScopeClaim_IsIncludedInCacheKey() + { + var options = new IdempotencyOptions { UserIdFallback = "123" }; + options.KeyScopeClaimTypes.Add("tenant_id"); + + var idempotencyKey = Ulid.NewUlid().ToString(); + var httpContext = CreateHttpContext(idempotencyKey, "456"); + httpContext.User = new ClaimsPrincipal(new ClaimsIdentity( + [ + new Claim(ClaimTypes.NameIdentifier, "456"), + new Claim("tenant_id", "acme") + ])); + + var filter = CreateFilter(options: options); + var context = CreateActionContext(httpContext); + await filter.OnActionExecutionAsync(context, () => Task.FromResult(CreateSuccessfulActionExecutedContext(context))); + + Assert.NotNull(await _distributedCache.GetAsync($"Idempotency_acme_456_{idempotencyKey}")); + Assert.Null(await _distributedCache.GetAsync($"Idempotency_456_{idempotencyKey}")); + } + + [Fact] + public async Task PerEndpointProcessingTtl_IsApplied() + { + var idempotencyKey = Ulid.NewUlid().ToString(); + var filter = CreateFilter(processingTtlSeconds: 30); + var context = CreateActionContext(CreateHttpContext(idempotencyKey, "7")); + + var gateEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var allowComplete = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var run = filter.OnActionExecutionAsync(context, async () => + { + gateEntered.SetResult(); + await allowComplete.Task; + return CreateSuccessfulActionExecutedContext(context); + }); + + await gateEntered.Task; + var cacheKey = $"Idempotency_7_{idempotencyKey}"; + var raw = await _distributedCache.GetAsync(cacheKey); + Assert.NotNull(raw); + var entry = JsonConvert.DeserializeObject(Encoding.UTF8.GetString(raw)); + Assert.NotNull(entry?.ProcessingExpiresAtUtc); + var remaining = entry!.ProcessingExpiresAtUtc!.Value - DateTimeOffset.UtcNow; + Assert.True(remaining <= TimeSpan.FromSeconds(30) + TimeSpan.FromSeconds(2)); + Assert.True(remaining > TimeSpan.FromSeconds(20)); + + allowComplete.SetResult(); + await run; + } + + [Fact] + public async Task Telemetry_RecordsExecutedAndReplayed_WithoutCacheKeyByDefault() + { + var outcomes = new List(); + var sawCacheKey = false; + using var listener = new ActivityListener + { + ShouldListenTo = s => s.Name == "BuildingBlocks.Idempotency", + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllData, + ActivityStopped = a => + { + var o = a.GetTagItem("idempotency.outcome")?.ToString(); + if (o is not null) + outcomes.Add(o); + if (a.GetTagItem("idempotency.cache_key") is not null) + sawCacheKey = true; + } + }; + ActivitySource.AddActivityListener(listener); + + var telemetry = new IdempotencyTelemetry(new IdempotencyTelemetryOptions()); + var idempotencyKey = Ulid.NewUlid().ToString(); + var filter = CreateFilter(telemetry: telemetry); + + var first = CreateActionContext(CreateHttpContext(idempotencyKey)); + await filter.OnActionExecutionAsync(first, () => Task.FromResult(CreateSuccessfulActionExecutedContext(first))); + + var second = CreateActionContext(CreateHttpContext(idempotencyKey)); + await filter.OnActionExecutionAsync(second, () => Task.FromResult(CreateSuccessfulActionExecutedContext(second))); + + Assert.Contains(IdempotencyOutcomes.Executed, outcomes); + Assert.Contains(IdempotencyOutcomes.Replayed, outcomes); + Assert.False(sawCacheKey); + } + + [Fact] + public async Task UseLockTrue_AcquiresAndReleasesLock() + { + var idempotencyKey = Ulid.NewUlid().ToString(); + var httpContext = CreateHttpContext(idempotencyKey); + var filter = CreateFilter(useLock: true); + var context = CreateActionContext(httpContext); + var lockKey = $"Idempotency_456_{idempotencyKey}_lock"; + + await filter.OnActionExecutionAsync(context, () => Task.FromResult(CreateSuccessfulActionExecutedContext(context))); + + _lockMock.Verify( + l => l.AcquireAsync(lockKey, It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + _lockMock.Verify( + l => l.ReleaseAsync(lockKey, It.IsAny(), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task UseLockTrue_FailsToAcquireLock_Returns500Problem() + { + var idempotencyKey = Ulid.NewUlid().ToString(); + var httpContext = CreateHttpContext(idempotencyKey, "466"); + _lockMock + .Setup(l => l.AcquireAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(false); + + var filter = CreateFilter(useLock: true); + var context = CreateActionContext(httpContext); + await filter.OnActionExecutionAsync(context, () => Task.FromResult(new ActionExecutedContext(context, new List(), null!))); + + AssertProblem(context.Result, 500, "lock-failure"); + } + + [Fact] + public async Task MissingUserClaim_WithoutFallback_Returns401Problem() + { + var options = new IdempotencyOptions { UserIdFallback = null }; + var httpContext = CreateHttpContext(Ulid.NewUlid().ToString(), userId: null); + var filter = CreateFilter(options: options); + var context = CreateActionContext(httpContext); + await filter.OnActionExecutionAsync(context, () => Task.FromResult(CreateSuccessfulActionExecutedContext(context))); + AssertProblem(context.Result, 401, "unauthorized"); + } + + [Fact] + public async Task MinimalApiFilter_MissThenReplay() + { + var options = new IdempotencyOptions { UserIdFallback = "123" }; + var endpoint = IdempotencyEndpointSettings.Create(useLock: false); + var filter = new IdempotentEndpointFilter( + _distributedCache, + NullLoggerFactory.Instance, + options, + endpoint); + + var key = Ulid.NewUlid().ToString(); + var http1 = CreateHttpContext(key, "9"); + var ctx1 = new DefaultEndpointFilterInvocationContext(http1); + var result1 = await filter.InvokeAsync(ctx1, _ => new ValueTask(Results.Ok(new { message = "ok" }))); + Assert.NotNull(result1); + + var http2 = CreateHttpContext(key, "9"); + var ctx2 = new DefaultEndpointFilterInvocationContext(http2); + var ran = false; + var result2 = await filter.InvokeAsync(ctx2, _ => + { + ran = true; + return new ValueTask(Results.Ok(new { message = "again" })); + }); + + Assert.False(ran); + Assert.True(http2.Response.Headers.ContainsKey("X-Idempotent-Response")); + Assert.NotNull(result2); + } + + [Fact] + public void FluentDi_RegistersOptionsLockAndTelemetry() + { + var services = new ServiceCollection(); + services.AddBuildingBlocksIdempotency(o => o.UserIdFallback = "x") + .UseTelemetry(t => t.IncludeCacheKeyInTelemetry = true); + + var sp = services.BuildServiceProvider(); + Assert.Equal("x", sp.GetRequiredService>().Value.UserIdFallback); + Assert.NotNull(sp.GetService()); + } + + [Fact] + public void TryCaptureSuccessfulResponse_Accepts2xx() + { + Assert.True(IdempotentAttributeFilter.TryCaptureSuccessfulResponse( + new ObjectResult(new { a = 1 }) { StatusCode = 201 }, + out var code, + out _, + out _)); + Assert.Equal(201, code); + + Assert.True(IdempotentAttributeFilter.TryCaptureSuccessfulResponse( + new StatusCodeResult(204), + out code, + out _, + out _)); + Assert.Equal(204, code); + + Assert.False(IdempotentAttributeFilter.TryCaptureSuccessfulResponse( + new ObjectResult("err") { StatusCode = 400 }, + out _, + out _, + out _)); + } +} + +internal sealed class DefaultEndpointFilterInvocationContext : EndpointFilterInvocationContext +{ + public DefaultEndpointFilterInvocationContext(HttpContext httpContext) + { + HttpContext = httpContext; + } + + public override HttpContext HttpContext { get; } + + public override IList Arguments { get; } = new List(); + + public override T GetArgument(int index) => (T)Arguments[index]!; +} diff --git a/tests/BuildingBlocks/Mcp.Tests/CatalogAndInvokerTests.cs b/tests/BuildingBlocks/Mcp.Tests/CatalogAndInvokerTests.cs index 5c11fe4..6943211 100644 --- a/tests/BuildingBlocks/Mcp.Tests/CatalogAndInvokerTests.cs +++ b/tests/BuildingBlocks/Mcp.Tests/CatalogAndInvokerTests.cs @@ -261,6 +261,35 @@ public async Task Handler_Throw_Is_Internal_Without_Stack() Assert.DoesNotContain("secret-stack", result.Error.Message); } + [Fact] + public async Task Handler_Throw_Includes_Exception_Message_When_Details_Enabled() + { + const string knownMessage = "known-exception-detail"; + var descriptor = McpToolScanner.FromType( + typeof(ListedOrder), + (_, _, _, _) => throw new InvalidOperationException(knownMessage)); + var invoker = new McpInvoker( + [descriptor], + new ServiceCollection().BuildServiceProvider(), + [], + new NoOpRateLimiter(), + new DefaultMcpResultMapper(), + dispatcher: null, + idempotency: null, + resilience: null, + telemetry: null, + includeExceptionDetails: true); + + var result = await invoker.InvokeAsync( + "tests.list", + JsonDocument.Parse("""{"sku":"x"}""").RootElement, + McpInvokeContext.None, + CancellationToken.None); + + Assert.Equal(McpErrorCode.Internal, result.Error!.Code); + Assert.Contains(knownMessage, result.Error.Message); + } + [Fact] public async Task Writes_Are_Not_Retried_By_Invoker() { diff --git a/tests/BuildingBlocks/Telemetry.Tests/AddTelemetry/AddTelemetryRegistrationTests.cs b/tests/BuildingBlocks/Telemetry.Tests/AddTelemetry/AddTelemetryRegistrationTests.cs index 4cb0267..6b136b2 100644 --- a/tests/BuildingBlocks/Telemetry.Tests/AddTelemetry/AddTelemetryRegistrationTests.cs +++ b/tests/BuildingBlocks/Telemetry.Tests/AddTelemetry/AddTelemetryRegistrationTests.cs @@ -79,6 +79,7 @@ public void Opt_in_extras_can_be_enabled() o.Instrumentation.EventBus = true; o.Instrumentation.SqlClient = true; o.Instrumentation.MassTransit = true; + o.IntegrateMcp = true; }); using var app = host.Build(); @@ -87,6 +88,7 @@ public void Opt_in_extras_can_be_enabled() Assert.True(options.Instrumentation.EventBus); Assert.True(options.Instrumentation.SqlClient); Assert.True(options.Instrumentation.MassTransit); + Assert.True(options.IntegrateMcp); } [Fact] diff --git a/tests/BuildingBlocks/Telemetry.Tests/Instrumentations/TelemetryComponentEnrichmentTests.cs b/tests/BuildingBlocks/Telemetry.Tests/Instrumentations/TelemetryComponentEnrichmentTests.cs index 09863ef..9f8cac6 100644 --- a/tests/BuildingBlocks/Telemetry.Tests/Instrumentations/TelemetryComponentEnrichmentTests.cs +++ b/tests/BuildingBlocks/Telemetry.Tests/Instrumentations/TelemetryComponentEnrichmentTests.cs @@ -64,6 +64,7 @@ public void ApplySqlClientDefaults_sets_component_before_user_enricher() [InlineData("Npgsql", TelemetryComponentTags.Npgsql)] [InlineData("EventBus", TelemetryComponentTags.EventBus)] [InlineData("BuildingBlocks.Mediator", TelemetryComponentTags.Mediator)] + [InlineData("BuildingBlocks.Mcp", TelemetryComponentTags.Mcp)] [InlineData("MassTransit", TelemetryComponentTags.MassTransit)] [InlineData("Microsoft.AspNetCore", TelemetryComponentTags.AspNetCore)] [InlineData("System.Net.Http", TelemetryComponentTags.HttpClient)] diff --git a/tests/Lab/FeatureFusion.Tests/IdempotencyTest.cs b/tests/Lab/FeatureFusion.Tests/IdempotencyTest.cs deleted file mode 100644 index cb0718b..0000000 --- a/tests/Lab/FeatureFusion.Tests/IdempotencyTest.cs +++ /dev/null @@ -1,322 +0,0 @@ -using Castle.Core.Logging; -using FeatureFusion.Controllers.V2; -using FeatureFusion.Features.Orders.Commands; -using FeatureFusion.Infrastructure.Caching; -using BuildingBlocks.Mediator; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.Mvc.Abstractions; -using Microsoft.AspNetCore.Mvc.Filters; -using Microsoft.AspNetCore.Routing; -using Microsoft.Extensions.Caching.Distributed; -using Microsoft.Extensions.Caching.StackExchangeRedis; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using Moq; -using Newtonsoft.Json; -using System; -using System.Collections.Generic; -using System.Security.Claims; -using System.Text; -using System.Threading.Tasks; -using Testcontainers.Redis; -using Xunit; - -namespace Tests.FeatureFusion.IdempotentAttributeFilterTests -{ - public class IdempotentAttributeFilterTests : IAsyncLifetime - { - private readonly RedisContainer _redisContainer = new RedisBuilder().Build(); - private IDistributedCache? _distributedCache; - private Mock? _redisConnectionWrapperMock; - - public async Task InitializeAsync() - { - await _redisContainer.StartAsync(); - - var services = new ServiceCollection(); - services.AddStackExchangeRedisCache(options => - { - options.Configuration = _redisContainer.GetConnectionString(); - }); - - var serviceProvider = services.BuildServiceProvider(); - _distributedCache = serviceProvider.GetRequiredService(); - - _redisConnectionWrapperMock = new Mock(); - } - - public async Task DisposeAsync() - { - await _redisContainer.DisposeAsync(); - } - - // Helper method to create an HttpContext with an Idempotency-Key header and user claims - private HttpContext CreateHttpContext(string idempotencyKey, string userId = "456") - { - var httpContext = new DefaultHttpContext(); - httpContext.Request.Headers["Idempotency-Key"] = idempotencyKey; - httpContext.User = new ClaimsPrincipal(new ClaimsIdentity(new[] { new Claim(ClaimTypes.NameIdentifier, userId) })); - return httpContext; - } - - // Helper method to create an IdempotentAttributeFilter instance - private IdempotentAttributeFilter CreateFilter(bool useLock = false) - { - return new IdempotentAttributeFilter(_distributedCache, new NullLoggerFactory(), _redisConnectionWrapperMock?.Object, useLock); - } - - // Helper method to create an ActionExecutingContext - private ActionExecutingContext CreateActionContext(HttpContext httpContext, object controller) - { - return new ActionExecutingContext( - new ActionContext(httpContext, new RouteData(), new ActionDescriptor()), - new List(), - new Dictionary(), - controller); - } - - // Helper method to create a mock OrderController - private OrderController CreateMockOrderController() - { - var mockLogger = new Mock>(); - var validator = new OrderRequestValidator(mockLogger.Object); - var mockMediator = new Mock(); - return new OrderController(validator, mockMediator.Object); - } - - // Helper method to create a successful ActionExecutedContext - private ActionExecutedContext CreateSuccessfulActionExecutedContext(ActionExecutingContext context) - { - return new ActionExecutedContext(context, new List(), new Exception("No exception occurred")) - { - Result = new ObjectResult(new { message = "Success" }) { StatusCode = 200 } - }; - } - - [Fact] - public void ExtractAndValidateIdempotencyKey_ValidUlid_ReturnsUlid() - { - // Arrange - var httpContext = CreateHttpContext(Ulid.NewUlid().ToString()); - var filter = CreateFilter(); - - // Act - var result = filter.ExtractAndValidateIdempotencyKey(httpContext.Request); - - // Assert - Assert.Equal(httpContext.Request.Headers["Idempotency-Key"], result.ToString()); - } - - [Fact] - public void ExtractAndValidateIdempotencyKey_MissingHeader_ThrowsArgumentNullException() - { - // Arrange - var httpContext = new DefaultHttpContext(); - var filter = CreateFilter(); - - // Act & Assert - var exception = Assert.Throws(() => filter.ExtractAndValidateIdempotencyKey(httpContext.Request)); - Assert.Equal("Idempotency-Key", exception.ParamName); - Assert.Contains("The Idempotency-Key header is missing.", exception.Message); - } - - [Fact] - public void ExtractAndValidateIdempotencyKey_EmptyHeader_ThrowsArgumentException() - { - // Arrange - var httpContext = CreateHttpContext(string.Empty); - var filter = CreateFilter(); - - // Act & Assert - var exception = Assert.Throws(() => filter.ExtractAndValidateIdempotencyKey(httpContext.Request)); - Assert.Equal("The Idempotency-Key value cannot be empty.", exception.Message); - } - - [Fact] - public void ExtractAndValidateIdempotencyKey_InvalidUlid_ThrowsArgumentException() - { - // Arrange - var httpContext = CreateHttpContext("invalid-ulid"); - var filter = CreateFilter(); - - // Act & Assert - var exception = Assert.Throws(() => filter.ExtractAndValidateIdempotencyKey(httpContext.Request)); - Assert.Equal("Invalid Idempotency-Key format: invalid-ulid", exception.Message); - } - - [Fact] - public async Task FirstRequest_CachesResponse() - { - // Arrange - var idempotencyKey = Ulid.NewUlid().ToString(); - var httpContext = CreateHttpContext(idempotencyKey); - var filter = CreateFilter(); - var context = CreateActionContext(httpContext, CreateMockOrderController()); - var executedContext = CreateSuccessfulActionExecutedContext(context); - - // Act - await filter.OnActionExecutionAsync(context, () => Task.FromResult(executedContext)); - - // Assert - var cacheKey = $"Idempotency_456_{idempotencyKey}"; - var cachedData = await _distributedCache!.GetAsync(cacheKey); - Assert.NotNull(cachedData); - - var cachedEntry = JsonConvert.DeserializeObject(Encoding.UTF8.GetString(cachedData)); - Assert.Equal("Completed", cachedEntry?.Status); - Assert.Equal("{\"message\":\"Success\"}", cachedEntry?.Response); - } - - [Fact] - public async Task DuplicateRequest_ReturnsCachedResponse() - { - // Arrange - var idempotencyKey = Ulid.NewUlid().ToString(); - var httpContext = CreateHttpContext(idempotencyKey, "1234"); - var cacheKey = $"Idempotency_1234_{idempotencyKey}"; - var cacheEntry = new IdempotencyCacheEntry { Status = "Completed", Response = "{\"message\":\"Cached\"}" }; - await _distributedCache!.SetAsync(cacheKey, Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(cacheEntry))); - - var filter = CreateFilter(); - var context = CreateActionContext(httpContext, CreateMockOrderController()); - - // Act - await filter.OnActionExecutionAsync(context, () => Task.FromResult(new ActionExecutedContext(context, new List(), null!))); - - // Assert - Assert.IsType(context.Result); - var contentResult = context.Result as ContentResult; - Assert.Equal("{\"message\":\"Cached\"}", contentResult?.Content); - Assert.Equal("application/json", contentResult?.ContentType); - Assert.Equal(200, contentResult?.StatusCode); - Assert.True(httpContext.Response.Headers.ContainsKey("X-Idempotent-Response")); - } - - [Fact] - public async Task ProcessingRequest_ReturnsConflict() - { - // Arrange - var idempotencyKey = Ulid.NewUlid().ToString(); - var httpContext = CreateHttpContext(idempotencyKey, "1234"); - var cacheKey = $"Idempotency_1234_{idempotencyKey}"; - var cacheEntry = new IdempotencyCacheEntry { Status = "Processing" }; - await _distributedCache!.SetAsync(cacheKey, Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(cacheEntry))); - - var filter = CreateFilter(); - var context = CreateActionContext(httpContext, CreateMockOrderController()); - - // Act - await filter.OnActionExecutionAsync(context, () => Task.FromResult(new ActionExecutedContext(context, new List(), null!))); - - // Assert - Assert.IsType(context.Result); - var conflictResult = context.Result as ConflictObjectResult; - Assert.Equal("Request is already being processed. Please wait.", conflictResult?.Value); - } - - [Fact] - public async Task OnActionExecutionAsync_FailedRequest_RemovesCacheEntry() - { - // Arrange - var idempotencyKey = Ulid.NewUlid().ToString(); - var httpContext = CreateHttpContext(idempotencyKey, "2565"); - var filter = CreateFilter(); - var context = CreateActionContext(httpContext, CreateMockOrderController()); - - // Simulate a failed action execution - async Task Next() - { - await Task.Yield(); - throw new Exception("Test exception"); - - } - - // Act - await filter.OnActionExecutionAsync(context, Next); - - // Assert - var cacheKey = $"Idempotency_2565_{idempotencyKey}"; - var cachedData = await _distributedCache!.GetAsync(cacheKey); - Assert.Null(cachedData); - } - - [Fact] - public async Task UseLockTrue_AcquiresLock() - { - // Arrange - var idempotencyKey = Ulid.NewUlid().ToString(); - var httpContext = CreateHttpContext(idempotencyKey); - var filter = CreateFilter(useLock: true); - var context = CreateActionContext(httpContext, CreateMockOrderController()); - var executedContext = CreateSuccessfulActionExecutedContext(context); - - // Act - await filter.OnActionExecutionAsync(context, () => Task.FromResult(executedContext)); - - // Assert - _redisConnectionWrapperMock?.Verify(r => r.AcquireLockAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); - } - - [Fact] - public async Task UseLockTrue_ReleasesLock() - { - // Arrange - var idempotencyKey = Ulid.NewUlid().ToString(); - var httpContext = CreateHttpContext(idempotencyKey); - var filter = CreateFilter(useLock: true); - var context = CreateActionContext(httpContext, CreateMockOrderController()); - var executedContext = CreateSuccessfulActionExecutedContext(context); - - - var lockKey = $"Idempotency_456_{idempotencyKey}_lock"; - var lockValue = Guid.NewGuid().ToString(); - - // Mock the Redis connection wrapper to simulate successful lock acquisition and release - _redisConnectionWrapperMock? - .Setup(r => r.AcquireLockAsync(lockKey, It.IsAny(), It.IsAny())) - .ReturnsAsync(true); - - _redisConnectionWrapperMock? - .Setup(r => r.ReleaseLockAsync(lockKey, It.IsAny())) - .Returns(Task.FromResult(true)); - - // Act - await filter.OnActionExecutionAsync(context, () => Task.FromResult(executedContext)); - - // Assert - _redisConnectionWrapperMock?.Verify(r => r.ReleaseLockAsync(lockKey, It.IsAny()), Times.Once); - } - - [Fact] - public async Task UseLockTrue_FailsToAcquireLock_Returns500() - { - // Arrange - var idempotencyKey = Ulid.NewUlid().ToString(); - var httpContext = CreateHttpContext(idempotencyKey, "466"); - var redisConnectionWrapper = new RedisConnectionWrapper(Options.Create(new RedisCacheOptions - { - Configuration = _redisContainer.GetConnectionString() - })); - - var lockKey = $"Idempotency_466_{idempotencyKey}_lock"; - await redisConnectionWrapper.AcquireLockAsync(lockKey, "another-process-lock-value", TimeSpan.FromSeconds(10)); - - var filter = new IdempotentAttributeFilter(_distributedCache, new NullLoggerFactory(), redisConnectionWrapper, true); - var context = CreateActionContext(httpContext, CreateMockOrderController()); - - // Act - await filter.OnActionExecutionAsync(context, () => Task.FromResult(new ActionExecutedContext(context, new List(), null!))); - - // Assert - Assert.IsType(context.Result); - var statusCodeResult = context.Result as StatusCodeResult; - Assert.Equal(StatusCodes.Status500InternalServerError, statusCodeResult?.StatusCode); - - // Clean up - await redisConnectionWrapper.ReleaseLockAsync(lockKey, "another-process-lock-value"); - } - } -} \ No newline at end of file diff --git a/tests/Lab/IntegrationTests/Api/FeatureFusionMcpTests.cs b/tests/Lab/IntegrationTests/Api/FeatureFusionMcpTests.cs index 505db0b..3eb671c 100644 --- a/tests/Lab/IntegrationTests/Api/FeatureFusionMcpTests.cs +++ b/tests/Lab/IntegrationTests/Api/FeatureFusionMcpTests.cs @@ -2,6 +2,7 @@ using BuildingBlocks.Mcp; using FluentAssertions; using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Mcp; using Microsoft.AspNetCore.Mvc.Testing; using ModelContextProtocol.Client; using ModelContextProtocol.Protocol; @@ -27,7 +28,7 @@ public FeatureFusionMcpTests(AspireFixture fixture) [Fact] public async Task Tools_List_Contains_Opt_In_Tools_Only() { - await using var mcp = await CreateClientAsync(); + await using var mcp = await LabMcpClient.CreateAsync(_http); var tools = await mcp.ListToolsAsync(); var names = tools.Select(t => t.Name).ToArray(); @@ -38,7 +39,7 @@ public async Task Tools_List_Contains_Opt_In_Tools_Only() [Fact] public async Task Call_Demo_Echo_Succeeds() { - await using var mcp = await CreateClientAsync(); + await using var mcp = await LabMcpClient.CreateAsync(_http); var result = await mcp.CallToolAsync( "demo.echo", new Dictionary { ["message"] = "hello-mcp" }); @@ -51,7 +52,7 @@ public async Task Call_Demo_Echo_Succeeds() [Fact] public async Task Call_Orders_Create_Without_Confirm_And_Key_Is_Error() { - await using var mcp = await CreateClientAsync(); + await using var mcp = await LabMcpClient.CreateAsync(_http); var result = await mcp.CallToolAsync( "orders.create", new Dictionary @@ -73,7 +74,7 @@ public async Task Call_Orders_Create_Without_Confirm_And_Key_Is_Error() [Fact] public async Task Call_Orders_Create_With_Confirm_And_Key_Succeeds() { - await using var mcp = await CreateClientAsync(); + await using var mcp = await LabMcpClient.CreateAsync(_http); var result = await mcp.CallToolAsync( "orders.create", new Dictionary @@ -96,7 +97,7 @@ public async Task Call_Orders_Create_With_Confirm_And_Key_Succeeds() [Fact] public async Task Products_List_Schema_Has_Enums_And_Optional_Cursor() { - await using var mcp = await CreateClientAsync(); + await using var mcp = await LabMcpClient.CreateAsync(_http); var tools = await mcp.ListToolsAsync(); var products = tools.Should().ContainSingle(t => t.Name == "products.list").Subject; var schema = products.ProtocolTool.InputSchema; @@ -122,7 +123,7 @@ public async Task Products_List_Schema_Has_Enums_And_Optional_Cursor() [Fact] public async Task Call_Demo_Echo_Includes_StructuredContent() { - await using var mcp = await CreateClientAsync(); + await using var mcp = await LabMcpClient.CreateAsync(_http); var result = await mcp.CallToolAsync( "demo.echo", new Dictionary { ["message"] = "hello-mcp" }); @@ -136,7 +137,7 @@ public async Task Call_Demo_Echo_Includes_StructuredContent() [Fact] public async Task Catalog_Resource_Lists_Lab_Tools() { - await using var mcp = await CreateClientAsync(); + await using var mcp = await LabMcpClient.CreateAsync(_http); var read = await mcp.ReadResourceAsync(new Uri(McpDefaults.CatalogResourceUri)); var markdown = string.Join("\n", read.Contents.OfType().Select(c => c.Text)); markdown.Should().Contain("demo.echo"); @@ -148,7 +149,7 @@ public async Task Catalog_Resource_Lists_Lab_Tools() [Fact] public async Task Call_Lab_Ping_From_Minimal_Api_Method_Succeeds() { - await using var mcp = await CreateClientAsync(); + await using var mcp = await LabMcpClient.CreateAsync(_http); var result = await mcp.CallToolAsync( "lab.ping", new Dictionary { ["name"] = "Ada" }); @@ -158,16 +159,6 @@ public async Task Call_Lab_Ping_From_Minimal_Api_Method_Succeeds() result.StructuredContent.Should().NotBeNull(); } - private async Task CreateClientAsync() - { - var endpoint = new Uri(_http.BaseAddress ?? new Uri("http://localhost"), "mcp"); - var transport = new HttpClientTransport( - new HttpClientTransportOptions { Endpoint = endpoint }, - _http, - ownsHttpClient: false); - return await McpClient.CreateAsync(transport); - } - private static string GetText(CallToolResult result) { return string.Join( diff --git a/tests/Lab/IntegrationTests/Api/FeatureFusionTraceEvidenceTests.cs b/tests/Lab/IntegrationTests/Api/FeatureFusionTraceEvidenceTests.cs new file mode 100644 index 0000000..2a22be5 --- /dev/null +++ b/tests/Lab/IntegrationTests/Api/FeatureFusionTraceEvidenceTests.cs @@ -0,0 +1,408 @@ +using System.Diagnostics; + +using System.Net.Http.Json; + +using System.Text.Json; + +using BuildingBlocks.Telemetry; + +using FluentAssertions; + +using IntegrationTests.Aspire; + +using IntegrationTests.Infrastructure.Mcp; +using IntegrationTests.Infrastructure.Telemetry; + +using Microsoft.AspNetCore.Mvc.Testing; + +using ModelContextProtocol.Client; + +using ModelContextProtocol.Protocol; + +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + + + +namespace IntegrationTests.Api; + + + +/// + +/// Phase 1 evidence-path proof: the Aspire fixture hosts FeatureFusion in-process + +/// ( is a ). + +/// Postgres/Redis/RabbitMQ/Memcached run in Aspire containers. An + +/// in the test process therefore sees application + +/// Activities — no OTLP collector required for this proof. + +/// + +[Collection(AspireCollection.Name)] + +public sealed class FeatureFusionTraceEvidenceTests + +{ + + private static readonly JsonSerializerOptions JsonOptions = new() + + { + + PropertyNameCaseInsensitive = true + + }; + + + + private readonly HttpClient _http; + + + + public FeatureFusionTraceEvidenceTests(AspireFixture fixture) + + { + + _http = fixture.CreateClient(new WebApplicationFactoryClientOptions + + { + + AllowAutoRedirect = false + + }); + + } + + + + [Fact] + + public async Task Http_products_page_produces_trace_evidence() + + { + + using var capture = new InProcessActivityCapture(); + + var (traceId, spanId) = NewTraceParent(); + + const int pageSize = 5; + + + + using var request = new HttpRequestMessage( + + HttpMethod.Get, + + $"/api/v2/products-page?limit={pageSize}&sortBy=Id&sortDirection=Ascending"); + + request.Headers.TryAddWithoutValidation("traceparent", FormatTraceParent(traceId, spanId)); + + + + var started = Stopwatch.StartNew(); + + var response = await _http.SendAsync(request); + + started.Stop(); + + + + response.StatusCode.Should().Be(HttpStatusCode.OK, await response.Content.ReadAsStringAsync()); + + var page = await response.Content.ReadFromJsonAsync(JsonOptions); + + page.Should().NotBeNull(); + + page!.Items.Should().HaveCount(pageSize); + + page.NextCursor.Should().NotBeNullOrWhiteSpace(); + + page.Items.Select(i => i.Id).Should().OnlyHaveUniqueItems(); + + + + var spans = capture.ForTrace(traceId); + + spans.Should().NotBeEmpty( + + "injected traceparent {0} should correlate ASP.NET Core → Mediator → Npgsql in the same process. Captured traces: {1}", + + traceId, + + DescribeTraces(capture)); + + + + var httpSpans = spans.Where(IsAspNetCore).ToList(); + + var mediatorSpans = spans.Where(IsMediator).ToList(); + + var dbSpans = spans.Where(IsDatabase).ToList(); + + + + httpSpans.Should().NotBeEmpty("ASP.NET Core instrumentation should emit the incoming products-page span"); + + httpSpans.Should().Contain( + + s => s.DisplayName.Contains("products-page", StringComparison.OrdinalIgnoreCase) + + || HasTag(s, "url.path", "/api/v2/products-page") + + || HasTagContaining(s, "http.route", "products-page"), + + "incoming HTTP span should identify /api/v2/products-page. Spans: {0}", + + Describe(spans)); + + + + mediatorSpans.Should().Contain( + + s => s.DisplayName.Contains("GetProductsQuery", StringComparison.Ordinal), + + "Mediator UseTelemetry should emit GetProductsQuery Handling. Spans: {0}", + + Describe(spans)); + + mediatorSpans.Should().Contain(s => HasTag(s, "mediator.request_name", "GetProductsQuery")); + + mediatorSpans.Should().Contain(s => HasTag(s, TelemetryComponentTags.TagName, TelemetryComponentTags.Mediator)); + + + + dbSpans.Should().NotBeEmpty( + + "Npgsql and/or EF Core instrumentation should emit at least one database span. Spans: {0}", + + Describe(spans)); + + + + started.Elapsed.Should().BeGreaterThan(TimeSpan.Zero); + + httpSpans.Concat(mediatorSpans).Should().Contain(s => s.Duration > TimeSpan.Zero); + + + + // Pagination evidence lives on the HTTP payload today (not Activity tags). + + page.Items.Count.Should().Be(pageSize); + + page.HasMore.Should().BeTrue(); + + traceId.ToHexString().Should().HaveLength(32); + + } + + + + [Fact] + + public async Task Mcp_products_list_produces_trace_evidence() + + { + + using var capture = new InProcessActivityCapture(); + + var (traceId, spanId) = NewTraceParent(); + + _http.DefaultRequestHeaders.Remove("traceparent"); + + _http.DefaultRequestHeaders.TryAddWithoutValidation("traceparent", FormatTraceParent(traceId, spanId)); + + + + await using var mcp = await LabMcpClient.CreateAsync(_http); + + var result = await mcp.CallToolAsync( + + "products.list", + + new Dictionary + + { + + ["limit"] = 5, + + ["sortBy"] = "Id", + + ["sortDirection"] = "Ascending" + + }); + + + + (result.IsError ?? false).Should().BeFalse(GetText(result)); + + result.StructuredContent.Should().NotBeNull(); + + var page = ParseProductsPage(result.StructuredContent!.Value); + + page.Items.Should().HaveCount(5); + + page.NextCursor.Should().NotBeNullOrWhiteSpace(); + + page.Items.Select(i => i.Id).Should().OnlyHaveUniqueItems(); + + + + // Incoming /mcp continues the injected traceparent. Tool invoke may run + + // without Activity.Current (Streamable HTTP session), so also search the + + // in-process sink by source rather than assuming one TraceId. + + var httpSpans = capture.ForTrace(traceId); + + httpSpans.Where(IsAspNetCore).Should().NotBeEmpty( + + "MCP Streamable HTTP is still an incoming ASP.NET Core request on trace {0}. Captured: {1}", + + traceId, + + DescribeTraces(capture)); + + + + var mcpSpan = capture.All.Should().ContainSingle( + + s => s.DisplayName == "mcp.tool" && HasTag(s, "mcp.tool.name", "products.list"), + + "MCP UseTelemetry should emit mcp.tool with mcp.tool.name=products.list. Captured: {0}", + + Describe(capture.All)).Which; + + mcpSpan.Tags.Should().ContainKey(TelemetryComponentTags.TagName) + + .WhoseValue.Should().Be(TelemetryComponentTags.Mcp); + + + + var mediatorSpan = capture.All.Should().ContainSingle( + + s => IsMediator(s) && s.DisplayName.Contains("GetProductsQuery", StringComparison.Ordinal), + + "products.list should dispatch GetProductsQuery through Mediator. Captured: {0}", + + Describe(capture.All)).Which; + + + + var dbSpans = capture.All.Where(s => s.TraceId == mediatorSpan.TraceId).Where(IsDatabase).ToList(); + + dbSpans.Should().NotBeEmpty( + + "GetProductsQuery should hit PostgreSQL on the same trace as Mediator ({0}). Captured: {1}", + + mediatorSpan.TraceId, + + Describe(capture.All.Where(s => s.TraceId == mediatorSpan.TraceId).ToList())); + + + + mcpSpan.TraceId.Should().Be( + + mediatorSpan.TraceId, + + "McpInvoker starts mcp.tool then ISender.Send; those Activities share Activity.Current"); + + } + + + + private static bool IsDatabase(CapturedActivity span) + + => span.Source is "Npgsql" or "OpenTelemetry.Instrumentation.EntityFrameworkCore" + + || HasTag(span, TelemetryComponentTags.TagName, TelemetryComponentTags.Npgsql) + + || HasTag(span, TelemetryComponentTags.TagName, TelemetryComponentTags.EntityFrameworkCore); + + + + private static string Describe(IReadOnlyList spans) + + => string.Join(" | ", spans.Select(s => $"{s.Source}:{s.DisplayName}")); + + + + private static string DescribeTraces(InProcessActivityCapture capture) + + { + + var groups = capture.All + + .GroupBy(s => s.TraceId) + + .Select(g => $"{g.Key}({string.Join(",", g.Select(s => s.Source).Distinct())})"); + + return string.Join("; ", groups); + + } + + + + private static string GetText(CallToolResult result) + + => string.Join("\n", result.Content.OfType().Select(b => b.Text)); + + + + private static ProductsPage ParseProductsPage(JsonElement structured) + + { + + var json = structured.GetRawText(); + + var page = JsonSerializer.Deserialize(json, JsonOptions); + + if (page?.Items is { Count: > 0 }) + + return page; + + + + if (structured.ValueKind == JsonValueKind.Object && structured.TryGetProperty("value", out var value)) + + { + + page = JsonSerializer.Deserialize(value.GetRawText(), JsonOptions); + + if (page?.Items is { Count: > 0 }) + + return page; + + } + + + + throw new InvalidOperationException($"Could not parse products page from MCP structured content: {json}"); + + } + + + + private sealed record ProductsPage( + + IReadOnlyList Items, + + string NextCursor, + + string PreviousCursor, + + bool HasMore, + + bool HasPrevious, + + int TotalCount); + + + + private sealed record ProductItem(int Id, string Name, decimal Price, DateTime CreatedAt); + +} + + diff --git a/tests/Lab/IntegrationTests/Aspire/AspireFixture.cs b/tests/Lab/IntegrationTests/Aspire/AspireFixture.cs index 70a625d..6490d02 100644 --- a/tests/Lab/IntegrationTests/Aspire/AspireFixture.cs +++ b/tests/Lab/IntegrationTests/Aspire/AspireFixture.cs @@ -5,13 +5,16 @@ using EventBusRabbitMQ.Infrastructure; using EventBusRabbitMQ.Infrastructure.EventBus; using EventBusRabbitMQ.Infrastructure.Context; +using EventBusRabbitMQ.Infrastructure.Messaging; using FeatureFusion.Features.Order.IntegrationEvents.EventHandling; using FeatureFusion.Features.Order.IntegrationEvents.Events; using IntegrationTests.EventBus; +using IntegrationTests.Infrastructure.EventBusLab; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -43,6 +46,12 @@ public sealed class AspireFixture : WebApplicationFactory, IAsyncLifeti public List ProcessedEvents { get; } = new(); + /// Lab-only EventBus stage journal (Exp 19/20). Cleared by experiments as needed. + public EventBusLabJournal EventBusJournal { get; } = new(); + + /// Lab-only fault arming for point B (publish-then-crash). Default off. + public EventBusLabFaultController EventBusFaults { get; } = new(); + public AspireFixture() { EnsureDockerContainerRuntime(); @@ -129,9 +138,20 @@ protected override IHost CreateHost(IHostBuilder builder) services.AddKeyedScoped, TestIntegrationEventHandler>(typeof(TestIntegrationEvent)); + services.AddKeyedScoped( + typeof(TestIntegrationEvent)); services.AddKeyedScoped, FailingIntegrationEventHandler>(typeof(FailingIntegrationEvent)); + services.AddKeyedScoped( + typeof(FailingIntegrationEvent)); + + services.AddKeyedScoped( + typeof(TransientThrowingIntegrationEvent)); + services.AddKeyedScoped( + typeof(BusinessFailureIntegrationEvent)); + services.AddKeyedScoped( + typeof(OnceTransientThenSucceedIntegrationEvent)); services.AddKeyedScoped( typeof(OrderCreatedIntegrationEvent), @@ -139,11 +159,30 @@ protected override IHost CreateHost(IHostBuilder builder) sp.GetRequiredService(), ProcessedEvents)); + // Lab EventBus observation seam (no-op when faults disarmed). + services.AddSingleton(EventBusJournal); + services.AddSingleton(EventBusFaults); + services.AddSingleton(sp => + new EventBusLabHook(EventBusJournal, EventBusFaults)); + + services.RemoveAll(); + services.AddScoped(); + services.AddScoped(sp => + new LabMessageProcessorDecorator( + sp.GetRequiredService(), + EventBusJournal)); + services.Configure(o => { o.EventTypes[typeof(OrderCreatedIntegrationEvent).Name] = typeof(OrderCreatedIntegrationEvent); o.EventTypes[typeof(TestIntegrationEvent).Name] = typeof(TestIntegrationEvent); o.EventTypes[typeof(FailingIntegrationEvent).Name] = typeof(FailingIntegrationEvent); + o.EventTypes[typeof(TransientThrowingIntegrationEvent).Name] = + typeof(TransientThrowingIntegrationEvent); + o.EventTypes[typeof(BusinessFailureIntegrationEvent).Name] = + typeof(BusinessFailureIntegrationEvent); + o.EventTypes[typeof(OnceTransientThenSucceedIntegrationEvent).Name] = + typeof(OnceTransientThenSucceedIntegrationEvent); }); services.AddHostedService(provider => diff --git a/tests/Lab/IntegrationTests/EventBus/RabbitMQEventBusTests.cs b/tests/Lab/IntegrationTests/EventBus/RabbitMQEventBusTests.cs index 3b0a12e..7e56532 100644 --- a/tests/Lab/IntegrationTests/EventBus/RabbitMQEventBusTests.cs +++ b/tests/Lab/IntegrationTests/EventBus/RabbitMQEventBusTests.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Diagnostics; using System.Text; using System.Text.Json; @@ -5,9 +6,11 @@ using EventBusRabbitMQ.Events; using EventBusRabbitMQ.Infrastructure; using EventBusRabbitMQ.Infrastructure.EventBus; +using EventBusRabbitMQ.Utilities; using FeatureFusion.Features.Order.IntegrationEvents.Events; using FluentAssertions; using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Async; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using RabbitMQ.Client; @@ -38,12 +41,77 @@ await TestEventProcessing( () => new OrderCreatedIntegrationEvent(Guid.NewGuid(), 99.0m)); } + /// + /// Smoke: handler executes once, then permanent failure dead-letters. + /// See Experiment 11. + /// [Fact] public async Task Publishes_And_Processes_Failed_Events() { await _fixture.ResetRabbitMQ(); + FailingIntegrationEventHandler.ResetInvocationCount(); var testEvent = new FailingIntegrationEvent(Guid.NewGuid(), 110.0m); - await PublishAndVerifyDlq(testEvent); + var eventBus = GetRequiredService(); + + await eventBus.PublishDirect(testEvent); + + await Wait.UntilAsync( + () => FailingIntegrationEventHandler.InvocationCount >= 1, + TimeSpan.FromSeconds(20)); + + await using var channel = await CreateChannelAsync(); + var dlqName = GetDlqName(); + await WaitForMessageCount(channel, dlqName, 1, TimeSpan.FromSeconds(20)); + + FailingIntegrationEventHandler.InvocationCount.Should().Be(1); + } + + /// + /// Smoke: is retried until RetryCount handler + /// executions, then dead-letters. See Experiment 11. + /// + [Fact] + public async Task Transient_handler_failure_is_retried_on_redelivery() + { + await _fixture.ResetRabbitMQ(); + TransientThrowingIntegrationEventHandler.ResetInvocationCount(); + var retryCount = GetRequiredService>().Value.RetryCount; + var testEvent = new TransientThrowingIntegrationEvent(Guid.NewGuid(), 120.0m); + var eventBus = GetRequiredService(); + + await eventBus.PublishDirect(testEvent); + + await Wait.UntilAsync( + () => TransientThrowingIntegrationEventHandler.InvocationCount >= retryCount, + TimeSpan.FromSeconds(30)); + + await using var channel = await CreateChannelAsync(); + var dlqName = GetDlqName(); + await WaitForMessageCount(channel, dlqName, 1, TimeSpan.FromSeconds(20)); + + TransientThrowingIntegrationEventHandler.InvocationCount.Should().Be(retryCount); + } + + /// + /// Smoke: first requeues; the next handler execution succeeds. + /// + [Fact] + public async Task Transient_handler_failure_can_succeed_on_retry() + { + await _fixture.ResetRabbitMQ(); + OnceTransientThenSucceedIntegrationEventHandler.Reset(); + var testEvent = new OnceTransientThenSucceedIntegrationEvent(Guid.NewGuid(), 140.0m); + var eventBus = GetRequiredService(); + + await eventBus.PublishDirect(testEvent); + + await Wait.UntilAsync( + () => OnceTransientThenSucceedIntegrationEventHandler.InvocationCountFor(testEvent.Id) >= 2, + TimeSpan.FromSeconds(30)); + + await Task.Delay(1000); + + OnceTransientThenSucceedIntegrationEventHandler.InvocationCountFor(testEvent.Id).Should().Be(2); } [Fact] @@ -236,22 +304,107 @@ public FailingIntegrationEvent(Guid id, decimal total) public class FailingIntegrationEventHandler : IIntegrationEventHandler { + private static int _invocationCount; + + /// Test-only observation counter for behavioral experiments. + public static int InvocationCount => _invocationCount; + + public static void ResetInvocationCount() => + Interlocked.Exchange(ref _invocationCount, 0); + public Task Handle(FailingIntegrationEvent @event) { + Interlocked.Increment(ref _invocationCount); throw new InvalidOperationException("Simulated handler failure"); } } -public static class Wait +public record TransientThrowingIntegrationEvent : IntegrationEvent +{ + public decimal Total { get; } + + public TransientThrowingIntegrationEvent(Guid id, decimal total) + { + Id = id; + Total = total; + } +} + +public sealed class TransientThrowingIntegrationEventHandler + : IIntegrationEventHandler +{ + private static int _invocationCount; + + public static int InvocationCount => _invocationCount; + + public static void ResetInvocationCount() => + Interlocked.Exchange(ref _invocationCount, 0); + + public Task Handle(TransientThrowingIntegrationEvent @event) + { + Interlocked.Increment(ref _invocationCount); + throw new TransientException("Simulated transient handler failure"); + } +} + +public record BusinessFailureIntegrationEvent : IntegrationEvent +{ + public decimal Total { get; } + + public BusinessFailureIntegrationEvent(Guid id, decimal total) + { + Id = id; + Total = total; + } +} + +public sealed class BusinessFailureIntegrationEventHandler + : IIntegrationEventHandler +{ + private static int _invocationCount; + + public static int InvocationCount => _invocationCount; + + public static void ResetInvocationCount() => + Interlocked.Exchange(ref _invocationCount, 0); + + public Task Handle(BusinessFailureIntegrationEvent @event) + { + Interlocked.Increment(ref _invocationCount); + throw new BusinessException("Simulated business handler failure"); + } +} + +public record OnceTransientThenSucceedIntegrationEvent : IntegrationEvent { - public static async Task UntilAsync(Func condition, TimeSpan timeout) + public decimal Total { get; } + + public OnceTransientThenSucceedIntegrationEvent(Guid id, decimal total) { - var stopwatch = Stopwatch.StartNew(); - while (stopwatch.Elapsed < timeout) + Id = id; + Total = total; + } +} + +public sealed class OnceTransientThenSucceedIntegrationEventHandler + : IIntegrationEventHandler +{ + private static readonly ConcurrentDictionary AttemptsByMessageId = new(); + + public static int InvocationCountFor(Guid messageId) => + AttemptsByMessageId.TryGetValue(messageId, out var count) ? count : 0; + + public static void Reset() => AttemptsByMessageId.Clear(); + + public Task Handle(OnceTransientThenSucceedIntegrationEvent @event) + { + var attempt = AttemptsByMessageId.AddOrUpdate(@event.Id, 1, (_, previous) => previous + 1); + if (attempt == 1) { - if (condition()) return; - await Task.Delay(100); + throw new TransientException("Simulated first-attempt transient failure"); } - throw new TimeoutException("Condition not met within timeout"); + + return Task.CompletedTask; } } + diff --git a/tests/Lab/IntegrationTests/Experiments/AsyncTraceCorrelation/AsyncTraceCorrelationExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/AsyncTraceCorrelation/AsyncTraceCorrelationExperimentTests.cs new file mode 100644 index 0000000..bdada2d --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/AsyncTraceCorrelation/AsyncTraceCorrelationExperimentTests.cs @@ -0,0 +1,295 @@ +using System.Diagnostics; +using System.Text.Json; +using FeatureFusion.Infrastructure.Context; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Async; +using IntegrationTests.Infrastructure.Orders; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Xunit.Abstractions; +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Experiments.AsyncTraceCorrelation; + +/// +/// Experiment 18 — async trace correlation across HTTP → outbox → RabbitMQ → consumer. +/// Characterizes current production telemetry: whether the originating HTTP/Mediator TraceId +/// is preserved into EventBus ProcessMessage, or whether the consumer starts a +/// separate trace and only business ids (IntegrationEvent.Id / OrderId) remain +/// correlatable. Does not modify production EventBus/Telemetry. Does not inject RabbitMQ +/// traceparent. HTTP traceparent uses the same Lab client convention as Exp 8 +/// so the originating operation TraceId is known. OutBoxWorker and PublishDirect currently +/// emit no Activities; ProcessMessage starts a root Activity with message.id tags only. +/// +[Collection(AspireCollection.Name)] +public sealed class AsyncTraceCorrelationExperimentTests +{ + private const int Quantity = 2; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly AspireFixture _fixture; + private readonly IServiceProvider _services; + private readonly HttpClient _http; + private readonly ITestOutputHelper _output; + + public AsyncTraceCorrelationExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + _services = fixture.Services; + _http = fixture.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + } + + [Fact] + public async Task Http_order_outbox_consumer_trace_correlation_is_characterized() + { + _fixture.ProcessedEvents.Clear(); + + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var key = System.Ulid.NewUlid().ToString(); + + var (originTraceId, originSpanId) = NewTraceParent(); + var originTraceHex = originTraceId.ToHexString(); + + var httpResult = await HttpOrderCreate.PostAsync( + _http, + capture, + key, + Quantity, + traceId: originTraceId, + parentSpanId: originSpanId, + jsonOptions: JsonOptions); + + httpResult.HttpStatus.Should().Be(200, httpResult.Body); + httpResult.OrderId.Should().NotBeEmpty(); + + var originSpans = capture.ForTrace(originTraceId); + var originHasAspNet = originSpans.Any(IsAspNetCore); + var originHasMediator = originSpans.Any(IsMediator); + var originHasNpgsql = originSpans.Any(IsNpgsql); + var originEventBusSpans = originSpans.Where(IsEventBusProcessMessage).ToList(); + + var outboxRow = await OrderOutboxObserver.WaitUntilProcessedAsync(_services, httpResult.OrderId); + var outbox = ToSnapshot(outboxRow); + await Wait.UntilAsync( + () => _fixture.ProcessedEvents.Any(e => e.OrderId == httpResult.OrderId), + TimeSpan.FromSeconds(20)); + + var delivered = _fixture.ProcessedEvents.Single(e => e.OrderId == httpResult.OrderId); + delivered.Id.Should().Be(outbox.IntegrationEventId, + "handler IntegrationEvent.Id must match outbox_messages.Id"); + + await WaitForInboxProcessedAsync(delivered.Id); + var inbox = await QueryInboxAsync(delivered.Id); + + await Wait.UntilAsync( + () => capture.All.Any(s => IsEventBusProcessMessage(s) && HasMessageId(s, delivered.Id)), + TimeSpan.FromSeconds(10)); + + var processSpansForMessage = capture.All + .Where(s => IsEventBusProcessMessage(s) && HasMessageId(s, delivered.Id)) + .ToList(); + + var consumerTraceIds = processSpansForMessage.Select(s => s.TraceId).Distinct().ToList(); + var consumerSharesOriginTrace = consumerTraceIds.Contains(originTraceHex, StringComparer.Ordinal); + var consumerHasParentOnOrigin = processSpansForMessage.Any(s => + s.ParentSpanId is not null + && originSpans.Any(o => o.SpanId == s.ParentSpanId)); + var consumerLinksToOrigin = processSpansForMessage.Any(s => + s.Links.Any(l => string.Equals(l.TraceId, originTraceHex, StringComparison.Ordinal))); + var outboxWorkerSpans = capture.All.Where(s => + s.DisplayName.Contains("Outbox", StringComparison.OrdinalIgnoreCase) + || s.Source.Contains("Outbox", StringComparison.OrdinalIgnoreCase)).ToList(); + var rabbitPublishSpans = capture.All.Where(s => + s.DisplayName.Contains("publish", StringComparison.OrdinalIgnoreCase) + && (s.Source.Contains("Rabbit", StringComparison.OrdinalIgnoreCase) + || s.Source.Contains("EventBus", StringComparison.OrdinalIgnoreCase))).ToList(); + + var w3cCrossesRabbitBoundary = consumerSharesOriginTrace + || consumerHasParentOnOrigin + || consumerLinksToOrigin; + + var characterization = w3cCrossesRabbitBoundary + ? "W3C_TRACE_CONTEXT_CORRELATED" + : "SEPARATE_CONSUMER_TRACE_BUSINESS_ID_CORRELATION_ONLY"; + + var result = new + { + name = "http-order-async-trace-correlation-v1", + startedUtc, + gitSha = LabRunInfo.ReadGitSha(), + environment = "Development", + configuration = new + { + path = HttpOrderCreate.Path, + asyncPath = "POST order → CreateOrderCommandHandler → IntegrationEventService → outbox_messages → OutBoxWorker → PublishDirect → RabbitMQ → EventBus.ProcessMessage → handler", + instrumentationNotes = new[] + { + "HTTP client injects LabTrace traceparent (same convention as Exp 8) — not RabbitMQ propagation.", + "EventBus.StartActivity creates ActivitySource('EventBus').StartActivity('ProcessMessage') with no parent ActivityContext from headers.", + "RabbitMQ BasicProperties headers carry event type / occurred-on / source / x-retry-count only — no traceparent.", + "OutBoxWorker and PublishDirect emit no ActivitySource spans today." + } + }, + origin = new + { + httpStatus = httpResult.HttpStatus, + orderId = httpResult.OrderId, + clientDurationMs = httpResult.ClientDurationMs, + traceId = originTraceHex, + spanCount = originSpans.Count, + hasAspNetCore = originHasAspNet, + hasMediator = originHasMediator, + hasNpgsql = originHasNpgsql, + eventBusProcessMessageOnOriginTrace = originEventBusSpans.Count, + sources = originSpans.Select(s => s.Source).Distinct().OrderBy(x => x).ToArray() + }, + asyncBoundary = new + { + integrationEventId = delivered.Id, + orderId = delivered.OrderId, + outboxMessageId = outbox.OutboxMessageId, + outboxStatus = outbox.Status, + inboxRowCount = inbox.InboxRowCount, + inboxIsProcessed = inbox.IsProcessed, + handlerObservationsForOrder = _fixture.ProcessedEvents.Count(e => e.OrderId == httpResult.OrderId), + outboxWorkerSpanCount = outboxWorkerSpans.Count, + rabbitPublishSpanCount = rabbitPublishSpans.Count, + processMessageSpans = processSpansForMessage.Select(s => new + { + s.TraceId, + s.SpanId, + s.ParentSpanId, + linkCount = s.Links.Count, + links = s.Links, + messageId = GetTag(s, "message.id"), + messageStatus = GetTag(s, "message.status") + }).ToList() + }, + correlation = new + { + characterization, + w3cTraceContextCrossesRabbitBoundary = w3cCrossesRabbitBoundary, + consumerSharesOriginTraceId = consumerSharesOriginTrace, + consumerParentSpanOnOriginTrace = consumerHasParentOnOrigin, + consumerActivityLinksToOrigin = consumerLinksToOrigin, + distinctConsumerTraceIds = consumerTraceIds, + businessIdCorrelation = new + { + outboxIdEqualsIntegrationEventId = outbox.OutboxMessageId == delivered.Id, + processMessageTaggedWithIntegrationEventId = processSpansForMessage.All(s => HasMessageId(s, delivered.Id)), + handlerOrderIdMatchesHttpOrderId = delivered.OrderId == httpResult.OrderId + } + }, + notes = new[] + { + "If characterization is SEPARATE_CONSUMER_TRACE_BUSINESS_ID_CORRELATION_ONLY, Exp 18 documents the gap — it does not fix propagation.", + "Correlate async path via IntegrationEvent.Id (== outbox Id == RabbitMQ MessageId == ProcessMessage message.id tag) and OrderId.", + $"Origin TraceId={originTraceHex}; consumer TraceId(s)=[{string.Join(",", consumerTraceIds)}]." + } + }; + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + // Originating operation + httpResult.HttpStatus.Should().Be(200); + originHasMediator.Should().BeTrue("HTTP cache-miss order create should run Mediator on the originating TraceId"); + originHasNpgsql.Should().BeTrue("HTTP cache-miss should touch Npgsql on the originating TraceId"); + httpResult.CachedResponseHeader.Should().BeFalse(); + + // Outbox + handler once + outbox.OutboxMessageId.Should().Be(delivered.Id); + outbox.OrderId.Should().Be(httpResult.OrderId); + _fixture.ProcessedEvents.Count(e => e.OrderId == httpResult.OrderId).Should().Be(1); + inbox.InboxRowCount.Should().BeGreaterThanOrEqualTo(1); + inbox.IsProcessed.Should().BeTrue(); + + // Consumer processing occurred + processSpansForMessage.Should().NotBeEmpty( + "EventBus ProcessMessage Activity tagged with message.id={0} should be captured", delivered.Id); + processSpansForMessage.Should().OnlyContain(s => + string.Equals(GetTag(s, "message.status"), "processed", StringComparison.Ordinal)); + + // Business-id correlation across the async boundary + processSpansForMessage.Should().OnlyContain(s => HasMessageId(s, delivered.Id)); + delivered.OrderId.Should().Be(httpResult.OrderId); + + // Characterize W3C vs separate consumer trace (do not force propagation) + originEventBusSpans.Should().BeEmpty( + "ProcessMessage should not appear on the HTTP originating TraceId when the consumer starts its own Activity without parent context from RabbitMQ headers"); + consumerSharesOriginTrace.Should().BeFalse( + "current EventBus.StartActivity does not continue the HTTP TraceId across RabbitMQ"); + consumerHasParentOnOrigin.Should().BeFalse(); + consumerLinksToOrigin.Should().BeFalse(); + w3cCrossesRabbitBoundary.Should().BeFalse( + "W3C trace context does not cross the RabbitMQ boundary in current production instrumentation"); + consumerTraceIds.Should().NotBeEmpty(); + consumerTraceIds.Should().NotContain(originTraceHex); + + characterization.Should().Be("SEPARATE_CONSUMER_TRACE_BUSINESS_ID_CORRELATION_ONLY"); + } + + private async Task WaitForInboxProcessedAsync(Guid messageId) + { + var stopwatch = Stopwatch.StartNew(); + while (stopwatch.Elapsed < TimeSpan.FromSeconds(20)) + { + var state = await QueryInboxAsync(messageId); + if (state.InboxRowCount >= 1 && state.IsProcessed) + return; + await Task.Delay(100); + } + + throw new TimeoutException($"Inbox message {messageId} was not marked processed within timeout"); + } + + private static OutboxSnapshot ToSnapshot(OrderOutboxRow row) + => new( + row.OutboxMessageId, + row.IntegrationEventId, + row.OrderId, + row.Status, + row.WorkerPending, + row.WorkerProcessed); + + private async Task QueryInboxAsync(Guid messageId) + { + await using var scope = _services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var rows = await db.InboxMessages.AsNoTracking().Where(m => m.Id == messageId).ToListAsync(); + var row = rows.SingleOrDefault(); + return new InboxSnapshot(rows.Count, row?.IsProcessed ?? false, row?.Status.ToString()); + } + + private static bool IsEventBusProcessMessage(CapturedActivity span) + => span.Source == "EventBus" && span.DisplayName == "ProcessMessage"; + + private static bool HasMessageId(CapturedActivity span, Guid messageId) + => span.Tags.TryGetValue("message.id", out var mid) + && string.Equals(mid, messageId.ToString(), StringComparison.Ordinal); + + private static string? GetTag(CapturedActivity span, string key) + => span.Tags.TryGetValue(key, out var value) ? value : null; + + private sealed record OutboxSnapshot( + Guid OutboxMessageId, + Guid IntegrationEventId, + Guid OrderId, + string Status, + bool WorkerPending, + bool WorkerProcessed); + + private sealed record InboxSnapshot(int InboxRowCount, bool IsProcessed, string? Status); +} diff --git a/tests/Lab/IntegrationTests/Experiments/CacheVsProduction/CacheVsProductionExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/CacheVsProduction/CacheVsProductionExperimentTests.cs new file mode 100644 index 0000000..0d4b4f2 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/CacheVsProduction/CacheVsProductionExperimentTests.cs @@ -0,0 +1,280 @@ +using System.Diagnostics; +using System.Text.Json; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Orders; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using Xunit.Abstractions; +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Experiments.CacheVsProduction; + +/// +/// Experiment 3 — Cache vs Production (HTTP Redis idempotency). +/// Originally characterized Lab order-create idempotency; now a regression/provenance +/// gate that the extracted BuildingBlocks.Idempotency filter preserves miss → +/// production vs hit → replay (Redis IDistributedCache, fingerprint off). +/// Not a cache-library unit test. Does not cover MCP orders.create +/// (in-memory IMcpIdempotencyStore — Exp 6). Does not prove fingerprinting (Exp 12) +/// or concurrency lock races (Exp 4). +/// +[Collection(AspireCollection.Name)] +public sealed class CacheVsProductionExperimentTests +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly HttpClient _http; + private readonly ITestOutputHelper _output; + + public CacheVsProductionExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _output = output; + _http = fixture.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + } + + [Fact] + public async Task Order_create_redis_idempotency_cache_vs_production_is_observed() + { + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var cachedKey = System.Ulid.NewUlid().ToString(); + var controlKey = System.Ulid.NewUlid().ToString(); + var calls = new List(); + + var miss = await SendAsync( + capture, + calls, + behavior: "MissPopulate", + idempotencyKey: cachedKey, + quantity: 2); + + var hit = await SendAsync( + capture, + calls, + behavior: "HitReplay", + idempotencyKey: cachedKey, + quantity: 2); + + var mutatedBody = await SendAsync( + capture, + calls, + behavior: "SameKeyDifferentQuantity", + idempotencyKey: cachedKey, + quantity: 5); + + var freshProduction = await SendAsync( + capture, + calls, + behavior: "NewKeyDifferentQuantity", + idempotencyKey: controlKey, + quantity: 5); + + var result = new CacheVsProductionExperimentResult( + Name: "order-create-redis-idempotency-v1", + StartedUtc: startedUtc, + GitSha: LabRunInfo.ReadGitSha(), + Environment: "Development", + Configuration: new CacheVsProductionConfiguration( + HttpOrderCreate.Path, + HttpOrderCreate.IdempotencyHeader, + HttpOrderCreate.CachedResponseHeader, + Cache: "Redis IDistributedCache (BuildingBlocks.Idempotency IdempotentAttributeFilter, useLock: true)", + Production: "OrderController.CreateOrder → ISender.Send(CreateOrderCommand) → catalog SaveChanges + outbox"), + Calls: calls, + Observations: new CacheVsProductionObservations( + MissHttpStatus: miss.HttpStatus, + MissOrderId: miss.OrderId, + MissQuantity: miss.Quantity, + MissCachedHeader: miss.CachedResponseHeader, + MissMediatorSpans: miss.MediatorSpanCount, + MissNpgsqlSpans: miss.NpgsqlSpanCount, + MissRedisSpans: miss.RedisSpanCount, + HitHttpStatus: hit.HttpStatus, + HitOrderId: hit.OrderId, + HitQuantity: hit.Quantity, + HitCachedHeader: hit.CachedResponseHeader, + HitMediatorSpans: hit.MediatorSpanCount, + HitNpgsqlSpans: hit.NpgsqlSpanCount, + HitRedisSpans: hit.RedisSpanCount, + HitSameOrderIdAsMiss: hit.OrderId == miss.OrderId && miss.OrderId != Guid.Empty, + MutatedBodyHttpStatus: mutatedBody.HttpStatus, + MutatedBodyOrderId: mutatedBody.OrderId, + MutatedBodyQuantity: mutatedBody.Quantity, + MutatedBodyCachedHeader: mutatedBody.CachedResponseHeader, + MutatedBodyKeptOriginalQuantity: mutatedBody.Quantity == miss.Quantity, + FreshProductionHttpStatus: freshProduction.HttpStatus, + FreshProductionOrderId: freshProduction.OrderId, + FreshProductionQuantity: freshProduction.Quantity, + FreshProductionCachedHeader: freshProduction.CachedResponseHeader, + FreshProductionDifferentOrderId: freshProduction.OrderId != miss.OrderId && freshProduction.OrderId != Guid.Empty, + Notes: + [ + $"Miss TraceId={miss.TraceId}; sources={DescribeSources(capture.ForTraceHex(miss.TraceId))}.", + $"Hit TraceId={hit.TraceId}; sources={DescribeSources(capture.ForTraceHex(hit.TraceId))}.", + $"Same-key quantity=5 returned HTTP {mutatedBody.HttpStatus}, quantity={mutatedBody.Quantity}, cachedHeader={mutatedBody.CachedResponseHeader}.", + $"New-key quantity=5 returned HTTP {freshProduction.HttpStatus}, orderId={freshProduction.OrderId}, quantity={freshProduction.Quantity}." + ])); + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + miss.HttpStatus.Should().Be(200); + miss.OrderId.Should().NotBeEmpty(); + miss.Quantity.Should().Be(2); + miss.CachedResponseHeader.Should().BeFalse( + "the first request with a new Idempotency-Key should execute production, not replay a cached body"); + miss.MediatorSpanCount.Should().BeGreaterThan(0, + "cache miss should reach ISender.Send(CreateOrderCommand). Spans: {0}", + Describe(capture.ForTraceHex(miss.TraceId))); + miss.NpgsqlSpanCount.Should().BeGreaterThan(0, + "cache miss should persist catalog/outbox via SaveChanges. Spans: {0}", + Describe(capture.ForTraceHex(miss.TraceId))); + + hit.HttpStatus.Should().Be(200); + hit.OrderId.Should().Be(miss.OrderId); + hit.Quantity.Should().Be(2); + hit.CachedResponseHeader.Should().BeTrue( + "replay of a completed Idempotency-Key should set {0}", HttpOrderCreate.CachedResponseHeader); + hit.MediatorSpanCount.Should().Be(0, + "cache hit short-circuits before the controller action. Spans: {0}", + Describe(capture.ForTraceHex(hit.TraceId))); + hit.NpgsqlSpanCount.Should().Be(0, + "cache hit should not re-run catalog SaveChanges. Spans: {0}", + Describe(capture.ForTraceHex(hit.TraceId))); + + miss.Body.Should().Contain("\"orderId\"", + "production MVC serialization is camelCase System.Text.Json"); + hit.Body.Should().Contain("\"OrderId\"", + "the cached replay is Newtonsoft.Json from BuildingBlocks.Idempotency, which defaults to PascalCase"); + + mutatedBody.HttpStatus.Should().Be(200); + mutatedBody.OrderId.Should().Be(miss.OrderId); + mutatedBody.Quantity.Should().Be(2, + "the Redis entry is keyed by user + Idempotency-Key, not by request body"); + mutatedBody.CachedResponseHeader.Should().BeTrue(); + mutatedBody.MediatorSpanCount.Should().Be(0); + mutatedBody.NpgsqlSpanCount.Should().Be(0); + + freshProduction.HttpStatus.Should().Be(200); + freshProduction.OrderId.Should().NotBe(miss.OrderId); + freshProduction.Quantity.Should().Be(5); + freshProduction.CachedResponseHeader.Should().BeFalse(); + freshProduction.MediatorSpanCount.Should().BeGreaterThan(0, + "a new Idempotency-Key must run production again. Spans: {0}", + Describe(capture.ForTraceHex(freshProduction.TraceId))); + freshProduction.NpgsqlSpanCount.Should().BeGreaterThan(0); + } + + private async Task SendAsync( + InProcessActivityCapture capture, + List calls, + string behavior, + string idempotencyKey, + int quantity) + { + var result = await HttpOrderCreate.PostAsync( + _http, + capture, + idempotencyKey, + quantity, + jsonOptions: JsonOptions); + + var call = new CacheVsProductionCall( + calls.Count + 1, + behavior, + idempotencyKey, + quantity, + result.HttpStatus, + result.OrderId, + result.Quantity, + result.TotalAmount, + result.CachedResponseHeader, + result.ClientDurationMs, + result.TraceIdHex, + result.Spans.Count(IsMediator), + result.Spans.Count(IsNpgsql), + result.Spans.Count(IsRedis), + result.Body); + + calls.Add(call); + return call; + } + + private static bool IsRedis(CapturedActivity span) + => span.Source.Contains("Redis", StringComparison.OrdinalIgnoreCase) + || span.Source.Contains("StackExchange", StringComparison.OrdinalIgnoreCase); + + private static string Describe(IReadOnlyList spans) + => string.Join("; ", spans.Select(s => $"{s.Source}:{s.DisplayName}")); + + private static string DescribeSources(IReadOnlyList spans) + => string.Join(",", spans.Select(s => s.Source).Distinct().OrderBy(s => s, StringComparer.Ordinal)); + + private sealed record CacheVsProductionCall( + int RequestNumber, + string Behavior, + string IdempotencyKey, + int RequestedQuantity, + int HttpStatus, + Guid OrderId, + int? Quantity, + decimal? TotalAmount, + bool CachedResponseHeader, + double ClientDurationMs, + string TraceId, + int MediatorSpanCount, + int NpgsqlSpanCount, + int RedisSpanCount, + string Body); + + private sealed record CacheVsProductionConfiguration( + string Path, + string IdempotencyHeader, + string CachedResponseHeader, + string Cache, + string Production); + + private sealed record CacheVsProductionObservations( + int MissHttpStatus, + Guid MissOrderId, + int? MissQuantity, + bool MissCachedHeader, + int MissMediatorSpans, + int MissNpgsqlSpans, + int MissRedisSpans, + int HitHttpStatus, + Guid HitOrderId, + int? HitQuantity, + bool HitCachedHeader, + int HitMediatorSpans, + int HitNpgsqlSpans, + int HitRedisSpans, + bool HitSameOrderIdAsMiss, + int MutatedBodyHttpStatus, + Guid MutatedBodyOrderId, + int? MutatedBodyQuantity, + bool MutatedBodyCachedHeader, + bool MutatedBodyKeptOriginalQuantity, + int FreshProductionHttpStatus, + Guid FreshProductionOrderId, + int? FreshProductionQuantity, + bool FreshProductionCachedHeader, + bool FreshProductionDifferentOrderId, + IReadOnlyList Notes); + + private sealed record CacheVsProductionExperimentResult( + string Name, + DateTimeOffset StartedUtc, + string GitSha, + string Environment, + CacheVsProductionConfiguration Configuration, + IReadOnlyList Calls, + CacheVsProductionObservations Observations); +} diff --git a/tests/Lab/IntegrationTests/Experiments/Concurrency/ConcurrencyRaceExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/Concurrency/ConcurrencyRaceExperimentTests.cs new file mode 100644 index 0000000..2489633 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/Concurrency/ConcurrencyRaceExperimentTests.cs @@ -0,0 +1,258 @@ +using System.Diagnostics; +using System.Text.Json; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Orders; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using Xunit.Abstractions; +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Experiments.Concurrency; + +/// +/// Experiment 4 — Concurrency Race (HTTP Redis idempotency lock). +/// Originally proved the concurrent same-key race and Redis SET NX coordination; +/// now a regression/provenance gate for RedisIdempotencyLock + +/// [Idempotent(useLock: true)] on order create. Same key: one production +/// winner; siblings are replay / Processing 409 / lock-failure 500. Different keys +/// do not share a lock. ProcessingTtl lease overlap is Exp 15. Does not prove +/// fingerprinting (Exp 12), or MCP idempotency (Exp 6). +/// +[Collection(AspireCollection.Name)] +public sealed class ConcurrencyRaceExperimentTests +{ + private const int SameKeyConcurrency = 3; + private const int IndependentConcurrency = 2; + private const int Quantity = 1; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly HttpClient _http; + private readonly ITestOutputHelper _output; + + public ConcurrencyRaceExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _output = output; + _http = fixture.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + } + + [Fact] + public async Task Concurrent_order_creates_coordinate_same_key_and_run_independent_keys() + { + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var sameKey = System.Ulid.NewUlid().ToString(); + var independentKeys = Enumerable.Range(0, IndependentConcurrency) + .Select(_ => System.Ulid.NewUlid().ToString()) + .ToArray(); + + var coordinated = await StartTogether( + SameKeyConcurrency, + i => SendAsync(capture, "SameKeyConcurrent", sameKey, Quantity)); + NumberCalls(coordinated, start: 1); + + var independent = await StartTogether( + IndependentConcurrency, + i => SendAsync(capture, "DifferentKeyConcurrent", independentKeys[i], Quantity)); + NumberCalls(independent, start: SameKeyConcurrency + 1); + + var coordinatedProduction = coordinated.Where(IsProductionWinner).ToList(); + var coordinatedCached = coordinated.Where(IsCachedReplay).ToList(); + var coordinatedConflict = coordinated.Where(c => c.HttpStatus == 409).ToList(); + var coordinatedLockFail = coordinated.Where(c => c.HttpStatus == 500).ToList(); + var coordinatedOrderIds = coordinated + .Where(c => c.HttpStatus == 200 && c.OrderId != Guid.Empty) + .Select(c => c.OrderId) + .Distinct() + .ToList(); + var coordinatedMediatorCallers = coordinated.Count(c => c.MediatorSpanCount > 0); + var coordinatedNpgsqlCallers = coordinated.Count(c => c.NpgsqlSpanCount > 0); + + var independentOrderIds = independent.Select(c => c.OrderId).Distinct().ToList(); + var independentMediatorCallers = independent.Count(c => c.MediatorSpanCount > 0); + + var result = new ConcurrencyRaceExperimentResult( + Name: "order-create-idempotency-concurrency-v1", + StartedUtc: startedUtc, + GitSha: LabRunInfo.ReadGitSha(), + Environment: "Development", + Configuration: new ConcurrencyRaceConfiguration( + HttpOrderCreate.Path, + SameKeyConcurrency, + IndependentConcurrency, + Mechanism: "Redis SET NX lock around GetOrCreate of Idempotency_{user}_{key}, then Processing/Completed status in IDistributedCache"), + Calls: [.. coordinated, .. independent], + Observations: new ConcurrencyRaceObservations( + CoordinatedStatuses: coordinated.Select(c => c.HttpStatus).ToArray(), + CoordinatedProductionWinners: coordinatedProduction.Count, + CoordinatedCachedReplays: coordinatedCached.Count, + CoordinatedConflicts: coordinatedConflict.Count, + CoordinatedLockFailures: coordinatedLockFail.Count, + CoordinatedUniqueOrderIds: coordinatedOrderIds.Count, + CoordinatedMediatorCallers: coordinatedMediatorCallers, + CoordinatedNpgsqlCallers: coordinatedNpgsqlCallers, + IndependentStatuses: independent.Select(c => c.HttpStatus).ToArray(), + IndependentUniqueOrderIds: independentOrderIds.Count, + IndependentMediatorCallers: independentMediatorCallers, + Notes: + [ + $"Same-key statuses [{string.Join(",", coordinated.Select(c => c.HttpStatus))}] roles [{string.Join(",", coordinated.Select(DescribeRole))}].", + $"Same-key mediator callers={coordinatedMediatorCallers}; npgsql callers={coordinatedNpgsqlCallers}; unique orderIds={coordinatedOrderIds.Count}.", + $"Independent statuses [{string.Join(",", independent.Select(c => c.HttpStatus))}] orderIds [{string.Join(",", independent.Select(c => c.OrderId))}].", + .. coordinated.Select(c => $"Same-key #{c.RequestNumber} {DescribeRole(c)} TraceId={c.TraceId} sources={DescribeSources(capture.ForTraceHex(c.TraceId))}.") + ])); + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + coordinated.Should().HaveCount(SameKeyConcurrency); + coordinatedProduction.Should().ContainSingle( + "exactly one same-key request should execute production (lock winner). Roles: {0}", + string.Join(",", coordinated.Select(DescribeRole))); + coordinatedOrderIds.Should().ContainSingle( + "coordinated 200s must replay one order, not create two. Ids: {0}", + string.Join(",", coordinatedOrderIds)); + coordinatedMediatorCallers.Should().Be(1, + "Redis lock + Processing status should admit one CreateOrderCommand. Calls: {0}", + DescribeCalls(coordinated)); + coordinatedNpgsqlCallers.Should().Be(1, + "only the production winner should persist outbox/catalog. Calls: {0}", + DescribeCalls(coordinated)); + coordinated.Where(c => !IsProductionWinner(c)).Should().OnlyContain( + c => IsCachedReplay(c) || c.HttpStatus == 409 || c.HttpStatus == 500, + "losers should be cached replay, in-flight 409, or lock-failure 500 — not a second production winner"); + + independent.Should().OnlyContain(c => c.HttpStatus == 200); + independent.Should().OnlyContain(c => !c.CachedResponseHeader); + independentOrderIds.Should().HaveCount(IndependentConcurrency, + "different Idempotency-Keys must not share a lock. Ids: {0}", + string.Join(",", independentOrderIds)); + independentMediatorCallers.Should().Be(IndependentConcurrency, + "independent keys should each run production. Calls: {0}", + DescribeCalls(independent)); + independent.Should().OnlyContain(c => c.NpgsqlSpanCount > 0); + } + + private static async Task StartTogether( + int count, + Func> send) + { + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var tasks = Enumerable.Range(0, count).Select(async i => + { + await gate.Task; + return await send(i); + }).ToArray(); + gate.SetResult(); + return await Task.WhenAll(tasks); + } + + private static void NumberCalls(ConcurrencyRaceCall[] calls, int start) + { + for (var i = 0; i < calls.Length; i++) + { + calls[i] = calls[i] with { RequestNumber = start + i }; + } + } + + private async Task SendAsync( + InProcessActivityCapture capture, + string behavior, + string idempotencyKey, + int quantity) + { + var result = await HttpOrderCreate.PostAsync( + _http, + capture, + idempotencyKey, + quantity, + jsonOptions: JsonOptions); + + return new ConcurrencyRaceCall( + 0, + behavior, + idempotencyKey, + result.HttpStatus, + result.OrderId, + result.Quantity, + result.CachedResponseHeader, + result.ClientDurationMs, + result.TraceIdHex, + result.Spans.Count(IsMediator), + result.Spans.Count(IsNpgsql), + result.Body); + } + + private static bool IsProductionWinner(ConcurrencyRaceCall call) + => call.HttpStatus == 200 && !call.CachedResponseHeader; + + private static bool IsCachedReplay(ConcurrencyRaceCall call) + => call.HttpStatus == 200 && call.CachedResponseHeader; + + private static string DescribeRole(ConcurrencyRaceCall call) + => call.HttpStatus switch + { + 200 when call.CachedResponseHeader => "cached-replay", + 200 => "production-winner", + 409 => "in-flight-conflict", + 500 => "lock-failure", + _ => $"http-{call.HttpStatus}" + }; + + private static string DescribeCalls(IEnumerable calls) + => string.Join("; ", calls.Select(c => + $"{DescribeRole(c)} status={c.HttpStatus} mediator={c.MediatorSpanCount} npgsql={c.NpgsqlSpanCount} order={c.OrderId}")); + + private static string DescribeSources(IReadOnlyList spans) + => string.Join(",", spans.Select(s => s.Source).Distinct().OrderBy(s => s, StringComparer.Ordinal)); + + private sealed record ConcurrencyRaceCall( + int RequestNumber, + string Behavior, + string IdempotencyKey, + int HttpStatus, + Guid OrderId, + int? Quantity, + bool CachedResponseHeader, + double ClientDurationMs, + string TraceId, + int MediatorSpanCount, + int NpgsqlSpanCount, + string Body); + + private sealed record ConcurrencyRaceConfiguration( + string Path, + int SameKeyConcurrency, + int IndependentConcurrency, + string Mechanism); + + private sealed record ConcurrencyRaceObservations( + int[] CoordinatedStatuses, + int CoordinatedProductionWinners, + int CoordinatedCachedReplays, + int CoordinatedConflicts, + int CoordinatedLockFailures, + int CoordinatedUniqueOrderIds, + int CoordinatedMediatorCallers, + int CoordinatedNpgsqlCallers, + int[] IndependentStatuses, + int IndependentUniqueOrderIds, + int IndependentMediatorCallers, + IReadOnlyList Notes); + + private sealed record ConcurrencyRaceExperimentResult( + string Name, + DateTimeOffset StartedUtc, + string GitSha, + string Environment, + ConcurrencyRaceConfiguration Configuration, + IReadOnlyList Calls, + ConcurrencyRaceObservations Observations); +} diff --git a/tests/Lab/IntegrationTests/Experiments/DuplicateDelivery/DuplicateDeliveryExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/DuplicateDelivery/DuplicateDeliveryExperimentTests.cs new file mode 100644 index 0000000..a3389d3 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/DuplicateDelivery/DuplicateDeliveryExperimentTests.cs @@ -0,0 +1,350 @@ +using System.Diagnostics; +using System.Text.Json; +using EventBusRabbitMQ; +using EventBusRabbitMQ.Domain; +using EventBusRabbitMQ.Infrastructure.EventBus; +using FeatureFusion.Features.Order.IntegrationEvents.Events; +using FeatureFusion.Infrastructure.Context; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Async; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Xunit.Abstractions; + +namespace IntegrationTests.Experiments.DuplicateDelivery; + +/// +/// Experiment 7: duplicate integration-event delivery / consumer deduplication. +/// Hypothesis: when the same identity +/// (IntegrationEvent.Id / RabbitMQ MessageId) is published twice through +/// the real consumer path, the inbox layer suppresses a second handler execution even though +/// EventBusOptions.EnableDeduplication is false in the lab fixture. +/// A second event with a new identity but the same OrderId payload is processed independently. +/// is test observation infrastructure (decorator on the +/// real handler); inbox_messages rows are production persistence written by +/// MessageProcessor. +/// +[Collection(AspireCollection.Name)] +public sealed class DuplicateDeliveryExperimentTests : IAsyncLifetime +{ + private static readonly TimeSpan DuplicateObservationWindow = TimeSpan.FromSeconds(3); + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly AspireFixture _fixture; + private readonly IServiceProvider _services; + private readonly ITestOutputHelper _output; + + public DuplicateDeliveryExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + _ = fixture.CreateClient(); + _services = fixture.Services; + } + + public async Task InitializeAsync() => await _fixture.ResetRabbitMQ(); + + public Task DisposeAsync() => Task.CompletedTask; + + [Fact] + public async Task Duplicate_integration_event_delivery_is_suppressed_by_inbox_not_handler_replay() + { + _fixture.ProcessedEvents.Clear(); + + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var calls = new List(); + var inboxObservations = new List(); + + var eventBusOptions = GetRequiredService>().Value; + var sharedOrderId = Guid.NewGuid(); + const decimal total = 88.25m; + + var baselineEvent = new OrderCreatedIntegrationEvent(sharedOrderId, total); + + var baseline = await PublishAndObserveAsync( + capture, + calls, + behavior: "BaselineDelivery", + @event: baselineEvent); + + baseline.ProcessedEventCountForMessageId.Should().Be(1, + "baseline should execute the real handler once for IntegrationEvent.Id={0}", baselineEvent.Id); + + await WaitForInboxProcessedAsync(baselineEvent.Id); + + var inboxAfterBaseline = await QueryInboxAsync(baselineEvent.Id); + inboxObservations.Add(inboxAfterBaseline with { Phase = "AfterBaseline" }); + + inboxAfterBaseline.InboxRowCount.Should().Be(1); + inboxAfterBaseline.IsProcessed.Should().BeTrue(); + inboxAfterBaseline.ProcessedMessageRowCount.Should().Be(0, + "MessageDeduplicationService / processed_messages is inactive when EnableDeduplication=false"); + + var duplicate = await PublishAndObserveAsync( + capture, + calls, + behavior: "DuplicateDelivery", + @event: baselineEvent, + priorProcessedCount: baseline.ProcessedEventCountForMessageId, + waitForObservationWindow: true); + + var inboxAfterDuplicate = await QueryInboxAsync(baselineEvent.Id); + inboxObservations.Add(inboxAfterDuplicate with { Phase = "AfterDuplicate" }); + + var controlEvent = new OrderCreatedIntegrationEvent(sharedOrderId, total); + + var control = await PublishAndObserveAsync( + capture, + calls, + behavior: "ControlDifferentIdentity", + @event: controlEvent); + + control.ProcessedEventCountForMessageId.Should().Be(1, + "a new IntegrationEvent.Id should run the handler again even when OrderId matches baseline"); + + await WaitForInboxProcessedAsync(controlEvent.Id); + + var inboxAfterControl = await QueryInboxAsync(controlEvent.Id); + inboxObservations.Add(inboxAfterControl with { Phase = "AfterControl" }); + + var processedForBaselineId = _fixture.ProcessedEvents.Count(e => e.Id == baselineEvent.Id); + var processedForControlId = _fixture.ProcessedEvents.Count(e => e.Id == controlEvent.Id); + var processedForSharedOrderId = _fixture.ProcessedEvents.Count(e => e.OrderId == sharedOrderId); + + var eventBusAttemptsBaselineId = CountEventBusProcessAttempts(capture, baselineEvent.Id); + var eventBusAttemptsControlId = CountEventBusProcessAttempts(capture, controlEvent.Id); + + var result = new DuplicateDeliveryExperimentResult( + Name: "duplicate-integration-event-delivery-v1", + StartedUtc: startedUtc, + GitSha: LabRunInfo.ReadGitSha(), + Environment: "Development", + Configuration: new DuplicateDeliveryConfiguration( + ConsumerPath: "IEventBus.PublishDirect → RabbitMQ (feature_fusion queue) → EventBus.MessagetHandler → MessageProcessor.ProcessMessageAsync → inbox_messages → OrderCreatedIntegrationEventHandler (via TestEventHandlerDecorator)", + EnableDeduplication: eventBusOptions.EnableDeduplication, + SubscriptionClientName: eventBusOptions.SubscriptionClientName, + DeduplicationNote: "inbox.IsDuplicateAsync(messageId) runs regardless of EnableDeduplication; MessageDeduplicationService / processed_messages runs only when EnableDeduplication=true", + ProcessedEventsNote: "AspireFixture.ProcessedEvents records handler invocations via TestEventHandlerDecorator; OrderCreatedIntegrationEventHandler business effect is an in-memory ReceivedEvents list only"), + Calls: calls, + InboxObservations: inboxObservations, + Observations: new DuplicateDeliveryObservations( + BaselineIntegrationEventId: baselineEvent.Id, + ControlIntegrationEventId: controlEvent.Id, + SharedOrderId: sharedOrderId, + SharedTotal: total, + ProcessedCountForBaselineId: processedForBaselineId, + ProcessedCountForControlId: processedForControlId, + ProcessedCountForSharedOrderId: processedForSharedOrderId, + EventBusProcessAttemptsForBaselineId: eventBusAttemptsBaselineId, + EventBusProcessAttemptsForControlId: eventBusAttemptsControlId, + DuplicateDeliveryIncreasedHandlerCount: duplicate.ProcessedEventCountForMessageId > baseline.ProcessedEventCountForMessageId, + InboxRowCountAfterDuplicateForBaselineId: inboxAfterDuplicate.InboxRowCount, + Notes: + [ + $"EnableDeduplication={eventBusOptions.EnableDeduplication} in AspireFixture (matches lab appsettings).", + $"Baseline IntegrationEvent.Id={baselineEvent.Id}; control IntegrationEvent.Id={controlEvent.Id}; both OrderId={sharedOrderId}.", + $"After duplicate delivery: EventBus ProcessMessage attempts for baseline Id={eventBusAttemptsBaselineId}; ProcessedEvents for that Id={processedForBaselineId}.", + "Second delivery with the same message identity is ACKed as Success by MessageProcessor before DispatchToHandlers when inbox_messages is already processed (IsProcessed=true).", + "Control proves deduplication is identity-based (IntegrationEvent.Id / MessageId), not payload-based (OrderId/Total)." + ])); + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + duplicate.ProcessedEventCountForMessageId.Should().Be(1, + "duplicate delivery with the same IntegrationEvent.Id must not invoke the handler a second time"); + eventBusAttemptsBaselineId.Should().BeGreaterThanOrEqualTo(2, + "duplicate delivery should still reach the real consumer (EventBus ProcessMessage) even when suppressed"); + inboxAfterDuplicate.InboxRowCount.Should().Be(1, + "inbox_messages should retain a single row for the baseline message identity"); + processedForSharedOrderId.Should().Be(2, + "baseline and control identities should each produce one handler observation for the same OrderId"); + processedForBaselineId.Should().Be(1); + processedForControlId.Should().Be(1); + inboxAfterControl.InboxRowCount.Should().Be(1); + inboxAfterControl.IsProcessed.Should().BeTrue(); + } + + private async Task PublishAndObserveAsync( + InProcessActivityCapture capture, + List calls, + string behavior, + OrderCreatedIntegrationEvent @event, + int priorProcessedCount = 0, + bool waitForObservationWindow = false) + { + var eventBus = GetRequiredService(); + var publishedUtc = DateTimeOffset.UtcNow; + var processAttemptsBefore = CountEventBusProcessAttempts(capture, @event.Id); + + await eventBus.PublishDirect(@event); + + if (waitForObservationWindow) + { + await Wait.UntilAsync( + () => DateTimeOffset.UtcNow - publishedUtc >= DuplicateObservationWindow, + TimeSpan.FromSeconds(10)); + } + else + { + await Wait.UntilAsync( + () => _fixture.ProcessedEvents.Count(e => e.Id == @event.Id) > priorProcessedCount, + TimeSpan.FromSeconds(20)); + } + + var completedUtc = DateTimeOffset.UtcNow; + var processedForMessageId = _fixture.ProcessedEvents.Count(e => e.Id == @event.Id); + var processAttemptsAfter = CountEventBusProcessAttempts(capture, @event.Id); + var matchingProcessed = _fixture.ProcessedEvents + .Where(e => e.Id == @event.Id) + .ToList(); + + var call = new DuplicateDeliveryCall( + RequestNumber: calls.Count + 1, + Behavior: behavior, + IntegrationEventId: @event.Id, + OrderId: @event.OrderId, + Total: @event.Total, + CreationDate: @event.CreationDate, + PublishedUtc: publishedUtc, + CompletedUtc: completedUtc, + DurationMs: (completedUtc - publishedUtc).TotalMilliseconds, + ProcessedEventCountForMessageId: processedForMessageId, + ProcessedEventCountDeltaFromPrior: processedForMessageId - priorProcessedCount, + EventBusProcessAttemptsBefore: processAttemptsBefore, + EventBusProcessAttemptsAfter: processAttemptsAfter, + EventBusProcessAttemptsDelta: processAttemptsAfter - processAttemptsBefore, + LatestProcessedOrderId: matchingProcessed.LastOrDefault()?.OrderId, + LatestProcessedTotal: matchingProcessed.LastOrDefault()?.Total); + + calls.Add(call); + return call; + } + + private async Task WaitForInboxProcessedAsync(Guid messageId) + { + var stopwatch = Stopwatch.StartNew(); + while (stopwatch.Elapsed < TimeSpan.FromSeconds(20)) + { + var observation = await QueryInboxAsync(messageId); + if (observation.InboxRowCount == 1 + && observation.IsProcessed + && observation.InboxStatus == MessageStatus.Processed.ToString()) + { + return; + } + + await Task.Delay(100); + } + + throw new TimeoutException($"Inbox message {messageId} was not marked processed within timeout"); + } + + private async Task QueryInboxAsync(Guid messageId) + { + await using var scope = _services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var inboxRows = await db.InboxMessages + .AsNoTracking() + .Where(m => m.Id == messageId) + .ToListAsync(); + + var processedRows = await db.ProcessedMessages + .AsNoTracking() + .Where(m => m.Id == messageId) + .ToListAsync(); + + var row = inboxRows.SingleOrDefault(); + + return new InboxObservation( + Phase: string.Empty, + ObservedUtc: DateTimeOffset.UtcNow, + MessageId: messageId, + InboxRowCount: inboxRows.Count, + InboxStatus: row?.Status.ToString(), + IsProcessed: row?.IsProcessed ?? false, + InboxProcessedAtUtc: row?.ProcessedAt, + ProcessedMessageRowCount: processedRows.Count); + } + + private static int CountEventBusProcessAttempts(InProcessActivityCapture capture, Guid messageId) + { + var id = messageId.ToString(); + return capture.All.Count(span => + span.Source == "EventBus" + && span.DisplayName == "ProcessMessage" + && span.Tags.TryGetValue("message.id", out var mid) + && mid == id); + } + + private T GetRequiredService() where T : notnull => + _services.GetRequiredService(); + + private sealed record DuplicateDeliveryCall( + int RequestNumber, + string Behavior, + Guid IntegrationEventId, + Guid OrderId, + decimal Total, + DateTime CreationDate, + DateTimeOffset PublishedUtc, + DateTimeOffset CompletedUtc, + double DurationMs, + int ProcessedEventCountForMessageId, + int ProcessedEventCountDeltaFromPrior, + int EventBusProcessAttemptsBefore, + int EventBusProcessAttemptsAfter, + int EventBusProcessAttemptsDelta, + Guid? LatestProcessedOrderId, + decimal? LatestProcessedTotal); + + private sealed record InboxObservation( + string Phase, + DateTimeOffset ObservedUtc, + Guid MessageId, + int InboxRowCount, + string? InboxStatus, + bool IsProcessed, + DateTime? InboxProcessedAtUtc, + int ProcessedMessageRowCount); + + private sealed record DuplicateDeliveryConfiguration( + string ConsumerPath, + bool EnableDeduplication, + string SubscriptionClientName, + string DeduplicationNote, + string ProcessedEventsNote); + + private sealed record DuplicateDeliveryObservations( + Guid BaselineIntegrationEventId, + Guid ControlIntegrationEventId, + Guid SharedOrderId, + decimal SharedTotal, + int ProcessedCountForBaselineId, + int ProcessedCountForControlId, + int ProcessedCountForSharedOrderId, + int EventBusProcessAttemptsForBaselineId, + int EventBusProcessAttemptsForControlId, + bool DuplicateDeliveryIncreasedHandlerCount, + int InboxRowCountAfterDuplicateForBaselineId, + IReadOnlyList Notes); + + private sealed record DuplicateDeliveryExperimentResult( + string Name, + DateTimeOffset StartedUtc, + string GitSha, + string Environment, + DuplicateDeliveryConfiguration Configuration, + IReadOnlyList Calls, + IReadOnlyList InboxObservations, + DuplicateDeliveryObservations Observations); +} diff --git a/tests/Lab/IntegrationTests/Experiments/EventBusFailure/EventBusFailureExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/EventBusFailure/EventBusFailureExperimentTests.cs new file mode 100644 index 0000000..7d37b0c --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/EventBusFailure/EventBusFailureExperimentTests.cs @@ -0,0 +1,569 @@ +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using EventBusRabbitMQ; +using EventBusRabbitMQ.Domain; +using EventBusRabbitMQ.Infrastructure; +using EventBusRabbitMQ.Infrastructure.EventBus; +using FeatureFusion.Infrastructure.Context; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.EventBus; +using IntegrationTests.Infrastructure.Async; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using RabbitMQ.Client; +using Xunit.Abstractions; + +namespace IntegrationTests.Experiments.EventBusFailure; + +/// +/// Experiment 9: EventBus failure outcome / retry-DLQ behavior (discovery-oriented). +/// Hypothesis: when a failing integration event is published through the real consumer, +/// what consumer, inbox, retry-header, ACK/NACK, and DLQ behavior actually occurs? +/// Does not assume RetryCount=3 produces three handler attempts. +/// FailingIntegrationEventHandler.InvocationCount is test-only observation. +/// RabbitMQ queue/DLQ state, inbox persistence, and EventBus Activities are production evidence. +/// +[Collection(AspireCollection.Name)] +public sealed class EventBusFailureExperimentTests : IAsyncLifetime +{ + private static readonly TimeSpan StableOutcomeTimeout = TimeSpan.FromSeconds(60); + private static readonly TimeSpan StabilityWindow = TimeSpan.FromSeconds(3); + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly AspireFixture _fixture; + private readonly IServiceProvider _services; + private readonly ITestOutputHelper _output; + + public EventBusFailureExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + _ = fixture.CreateClient(); + _services = fixture.Services; + } + + public async Task InitializeAsync() => await _fixture.ResetRabbitMQ(); + + public Task DisposeAsync() => Task.CompletedTask; + + [Fact] + public async Task Failing_integration_event_outcome_is_observed_against_permanent_failure_control() + { + FailingIntegrationEventHandler.ResetInvocationCount(); + + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var calls = new List(); + var observations = new List(); + + var eventBusOptions = GetRequiredService>().Value; + await using var channel = await CreateChannelAsync(); + var mainQueue = eventBusOptions.SubscriptionClientName; + var dlqName = $"{mainQueue}{RabbitMQConstants.DeadLetterQueueSuffix}"; + + await channel.QueuePurgeAsync(dlqName); + + var failingEvent = new FailingIntegrationEvent(Guid.NewGuid(), 110.0m); + var publishUtc = DateTimeOffset.UtcNow; + + calls.Add(new FailurePhaseCall( + 1, + "FailingHandlerPublish", + failingEvent.Id, + "FailingIntegrationEvent", + PublishedUtc: publishUtc, + InitialRetryHeader: 0)); + + await GetRequiredService().PublishDirect(failingEvent); + + var failingOutcome = await WaitForStableFailingOutcomeAsync( + capture, + channel, + mainQueue, + dlqName, + failingEvent.Id); + + var failingInbox = await QueryInboxAsync(failingEvent.Id); + var failingActivities = GetProcessMessageActivities(capture, failingEvent.Id); + + observations.Add(new FailureObservation( + Phase: "AfterFailingHandler", + ObservedUtc: DateTimeOffset.UtcNow, + MessageId: failingEvent.Id, + Outcome: failingOutcome.Outcome.ToString(), + HandlerInvocationCount: FailingIntegrationEventHandler.InvocationCount, + ConsumerProcessMessageCount: failingActivities.Count, + ProcessMessageStatuses: failingActivities.Select(a => GetTag(a, "message.status")).Distinct().ToList(), + ProcessMessageRetryCounts: failingActivities.Select(a => GetTag(a, "message.retry_count")).Distinct().ToList(), + InitialPublishRetryHeader: 0, + FinalDlqRetryHeader: failingOutcome.DlqRetryHeader, + MainQueueMessageCount: failingOutcome.MainQueueMessageCount, + DlqMessageCount: failingOutcome.DlqMessageCount, + DlqContainsMessage: failingOutcome.DlqContainsMessage, + InboxRowCount: failingInbox.InboxRowCount, + InboxMessageStatus: failingInbox.MessageStatus, + InboxIsProcessed: failingInbox.IsProcessed, + InboxSubscriberName: failingInbox.SubscriberName, + InboxSubscriberStatus: failingInbox.SubscriberStatus, + StableObservationDurationMs: failingOutcome.StableObservationDurationMs, + Notes: failingOutcome.Notes)); + + var invalidMessageId = Guid.NewGuid(); + var invalidPublishUtc = DateTimeOffset.UtcNow; + + calls.Add(new FailurePhaseCall( + 2, + "PermanentFailureControl", + invalidMessageId, + "OrderCreatedIntegrationEvent", + PublishedUtc: invalidPublishUtc, + InitialRetryHeader: 0)); + + await channel.QueuePurgeAsync(dlqName); + + var invalidProps = new BasicProperties + { + MessageId = invalidMessageId.ToString(), + Headers = new Dictionary + { + [RabbitMQConstants.EventTypeHeader] = "NonExistentEventType", + [RabbitMQConstants.SourceServiceHeader] = "TestService", + [RabbitMQConstants.RetryCountHeaderKey] = 0 + } + }; + + await channel.BasicPublishAsync( + exchange: RabbitMQConstants.MainExchangeName, + routingKey: "OrderCreatedIntegrationEvent", + mandatory: true, + basicProperties: invalidProps, + body: Encoding.UTF8.GetBytes("{ invalid json }")); + + await Wait.UntilAsync( + () => GetQueueMessageCount(channel, dlqName) >= 1, + TimeSpan.FromSeconds(30)); + + var controlDlqMessage = await GetDlqMessageByIdAsync(channel, dlqName, invalidMessageId); + var controlInbox = await QueryInboxAsync(invalidMessageId); + var controlActivities = GetProcessMessageActivities(capture, invalidMessageId); + + observations.Add(new FailureObservation( + Phase: "AfterPermanentFailureControl", + ObservedUtc: DateTimeOffset.UtcNow, + MessageId: invalidMessageId, + Outcome: "DeadLettered", + HandlerInvocationCount: FailingIntegrationEventHandler.InvocationCount, + ConsumerProcessMessageCount: controlActivities.Count, + ProcessMessageStatuses: controlActivities.Select(a => GetTag(a, "message.status")).Distinct().ToList(), + ProcessMessageRetryCounts: controlActivities.Select(a => GetTag(a, "message.retry_count")).Distinct().ToList(), + InitialPublishRetryHeader: 0, + FinalDlqRetryHeader: controlDlqMessage?.RetryHeader, + MainQueueMessageCount: GetQueueMessageCount(channel, mainQueue), + DlqMessageCount: GetQueueMessageCount(channel, dlqName), + DlqContainsMessage: controlDlqMessage is not null, + InboxRowCount: controlInbox.InboxRowCount, + InboxMessageStatus: controlInbox.MessageStatus, + InboxIsProcessed: controlInbox.IsProcessed, + InboxSubscriberName: controlInbox.SubscriberName, + InboxSubscriberStatus: controlInbox.SubscriberStatus, + StableObservationDurationMs: null, + Notes: + [ + "Invalid JSON + unregistered Event-Type header; fails before handler dispatch.", + controlDlqMessage is null + ? "DLQ message not retrieved by MessageId (unexpected)." + : $"DLQ MessageId={controlDlqMessage.MessageId}; x-retry-count={controlDlqMessage.RetryHeader}." + ])); + + var conclusion = BuildConclusion( + failingOutcome, + failingInbox, + FailingIntegrationEventHandler.InvocationCount, + failingActivities, + eventBusOptions); + + var result = new EventBusFailureExperimentResult( + Name: "event-bus-failure-outcome-v1", + StartedUtc: startedUtc, + GitSha: LabRunInfo.ReadGitSha(), + Environment: "Development", + Configuration: new EventBusFailureConfiguration( + MainQueue: mainQueue, + DlqQueue: dlqName, + EnableDeduplication: eventBusOptions.EnableDeduplication, + ConfiguredRetryCount: eventBusOptions.RetryCount, + IntendedRetryMechanism: "MessagetHandler: RetryLater + attemptNumber < RetryCount+1 → BasicNack(requeue:true); else BasicNack(requeue:false) → DLQ via x-dead-letter-exchange", + ImplementationCaveats: + [ + "x-retry-count header is published as 0 and is not incremented on requeue in consumer code", + "inbox dedup short-circuits only when the message is successfully processed (IsProcessed / Status=Processed); pending/failed rows may redispatch (Exp 7 successful duplicate vs Exp 11 retry)", + "CalculateRetryDelay is computed but not applied before requeue" + ], + HandlerInvocationNote: "FailingIntegrationEventHandler.InvocationCount is test-only", + RetryLaterPhaseOmittedReason: "FailingIntegrationEvent throws plain Exception (PermanentFailure); TransientException retry path is covered in Experiment 11"), + Calls: calls, + Observations: observations, + Conclusion: conclusion); + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + failingOutcome.Outcome.Should().Be( + FailureOutcomeKind.DeadLettered, + "plain handler exception should propagate PermanentFailure to DLQ"); + + FailingIntegrationEventHandler.InvocationCount.Should().Be(1, + "handler should execute once before permanent failure dead-letters"); + + failingOutcome.DlqContainsMessage.Should().BeTrue(); + failingInbox.InboxRowCount.Should().BeGreaterThanOrEqualTo(1, + "handler reached path should store inbox before DispatchToHandlers"); + failingInbox.IsProcessed.Should().BeFalse( + "permanent handler failure must not mark inbox message processed on first attempt"); + failingInbox.SubscriberStatus.Should().Be(MessageStatus.Failed.ToString()); + + controlDlqMessage.Should().NotBeNull("permanent pre-handler failure should dead-letter"); + controlInbox.InboxRowCount.Should().Be(0, "invalid/unregistered message should not store inbox before handler"); + } + + private async Task WaitForStableFailingOutcomeAsync( + InProcessActivityCapture capture, + IChannel channel, + string mainQueue, + string dlqName, + Guid messageId) + { + var stopwatch = Stopwatch.StartNew(); + var lastProcessCount = -1; + var stableSinceUtc = DateTimeOffset.UtcNow; + var notes = new List(); + + while (stopwatch.Elapsed < StableOutcomeTimeout) + { + var dlqMessage = await GetDlqMessageByIdAsync(channel, dlqName, messageId, peekOnly: true); + if (dlqMessage is not null) + { + return new StableFailureOutcome( + FailureOutcomeKind.DeadLettered, + GetQueueMessageCount(channel, mainQueue), + GetQueueMessageCount(channel, dlqName), + DlqContainsMessage: true, + DlqRetryHeader: dlqMessage.RetryHeader, + StableObservationDurationMs: stopwatch.Elapsed.TotalMilliseconds, + Notes: + [ + ..notes, + "Message observed in DLQ with matching MessageId.", + $"DLQ x-retry-count={dlqMessage.RetryHeader}." + ]); + } + + var processCount = GetProcessMessageActivities(capture, messageId).Count; + var mainCount = GetQueueMessageCount(channel, mainQueue); + + if (processCount != lastProcessCount) + { + lastProcessCount = processCount; + stableSinceUtc = DateTimeOffset.UtcNow; + notes.Add($"ProcessMessage count changed to {processCount} at {stopwatch.Elapsed.TotalMilliseconds:F0}ms."); + } + + if (processCount > 0 && mainCount == 0) + { + var stableFor = DateTimeOffset.UtcNow - stableSinceUtc; + if (stableFor >= StabilityWindow) + { + var statuses = GetProcessMessageActivities(capture, messageId) + .Select(a => GetTag(a, "message.status")) + .ToList(); + + return new StableFailureOutcome( + FailureOutcomeKind.Acked, + mainCount, + GetQueueMessageCount(channel, dlqName), + DlqContainsMessage: false, + DlqRetryHeader: null, + StableObservationDurationMs: stopwatch.Elapsed.TotalMilliseconds, + Notes: + [ + ..notes, + $"Main queue empty; {processCount} ProcessMessage span(s); statuses=[{string.Join(",", statuses)}].", + "No matching DLQ message observed within timeout window." + ]); + } + } + + if (mainCount > 0 && processCount == 0 && stopwatch.Elapsed >= StabilityWindow) + { + return new StableFailureOutcome( + FailureOutcomeKind.MainQueuePending, + mainCount, + GetQueueMessageCount(channel, dlqName), + DlqContainsMessage: false, + DlqRetryHeader: null, + StableObservationDurationMs: stopwatch.Elapsed.TotalMilliseconds, + Notes: + [ + ..notes, + "Message remains on main queue without observed ProcessMessage activity." + ]); + } + + await Task.Delay(100); + } + + return new StableFailureOutcome( + FailureOutcomeKind.Inconclusive, + GetQueueMessageCount(channel, mainQueue), + GetQueueMessageCount(channel, dlqName), + DlqContainsMessage: false, + DlqRetryHeader: null, + StableObservationDurationMs: stopwatch.Elapsed.TotalMilliseconds, + Notes: + [ + ..notes, + "Timed out before stable ACK or DLQ outcome." + ]); + } + + private static FailureConclusion BuildConclusion( + StableFailureOutcome failingOutcome, + InboxObservation failingInbox, + int handlerInvocations, + IReadOnlyList failingActivities, + EventBusOptions options) + { + var retryObserved = failingActivities.Count > 1 + || (failingOutcome.DlqRetryHeader ?? 0) > 0 + || failingActivities.Any(a => GetTag(a, "message.status") == "retrying"); + + var lines = new List + { + $"Configured RetryCount={options.RetryCount}; EnableDeduplication={options.EnableDeduplication}.", + $"Failing-event outcome={failingOutcome.Outcome}; handler invocations={handlerInvocations}; consumer ProcessMessage spans={failingActivities.Count}.", + handlerInvocations == 0 && failingOutcome.Outcome == FailureOutcomeKind.DeadLettered + ? "FailingIntegrationEvent reached DLQ with zero handler invocations — failure occurred before DispatchToHandlers (not a post-handler retry/DLQ path)." + : handlerInvocations > 0 && failingOutcome.Outcome == FailureOutcomeKind.DeadLettered + ? "Handler threw plain Exception; DispatchToHandlers returned PermanentFailure; consumer dead-lettered after one handler attempt." + : "See ProcessMessage statuses and inbox rows in observations.", + failingInbox.InboxRowCount > 0 + ? $"Inbox row present: message Status={failingInbox.MessageStatus}, IsProcessed={failingInbox.IsProcessed}, subscriber {failingInbox.SubscriberName} Status={failingInbox.SubscriberStatus}." + : "No inbox row for failing message.", + "Permanent-failure control (invalid JSON) dead-letters without inbox row.", + retryObserved + ? "Multiple consumer attempts or retrying status observed." + : "No retry-header progression and no multi-attempt consumer processing observed (x-retry-count remained 0)." + }; + + string? designVsActual = null; + if (handlerInvocations == 0 && failingOutcome.Outcome == FailureOutcomeKind.DeadLettered) + { + designVsActual = + "FailingIntegrationEvent does not currently reach the handler; DLQ outcome reflects PermanentFailure before handler dispatch, not the intended handler-failure retry loop."; + } + else if (handlerInvocations > 0 && failingOutcome.Outcome == FailureOutcomeKind.DeadLettered) + { + designVsActual = + "Handler threw plain Exception; PermanentFailure propagated; consumer dead-lettered as intended."; + } + + return new FailureConclusion( + FailingHandlerOutcome: failingOutcome.Outcome.ToString(), + HandlerInvocationCount: handlerInvocations, + RetryObservedForFailingEvent: retryObserved, + InboxStoredBeforeHandler: failingInbox.InboxRowCount > 0, + DlqReachedForFailingEvent: failingOutcome.DlqContainsMessage, + ApparentDesignVsActual: designVsActual, + SummaryLines: lines); + } + + private async Task QueryInboxAsync(Guid messageId) + { + await using var scope = _services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var message = await db.InboxMessages + .AsNoTracking() + .FirstOrDefaultAsync(m => m.Id == messageId); + + var subscriber = await db.InboxSubscriber + .AsNoTracking() + .FirstOrDefaultAsync(s => s.MessageId == messageId); + + return new InboxObservation( + InboxRowCount: message is null ? 0 : 1, + MessageStatus: message?.Status.ToString(), + IsProcessed: message?.IsProcessed ?? false, + SubscriberName: subscriber?.SubscriberName, + SubscriberStatus: subscriber?.Status.ToString()); + } + + private static IReadOnlyList GetProcessMessageActivities( + InProcessActivityCapture capture, + Guid messageId) + { + var id = messageId.ToString(); + return capture.All + .Where(span => + span.Source == "EventBus" + && span.DisplayName == "ProcessMessage" + && span.Tags.TryGetValue("message.id", out var mid) + && mid == id) + .ToList(); + } + + private static string? GetTag(CapturedActivity activity, string key) => + activity.Tags.TryGetValue(key, out var value) ? value : null; + + private static uint GetQueueMessageCount(IChannel channel, string queueName) + { + var queue = channel.QueueDeclarePassiveAsync(queueName).GetAwaiter().GetResult(); + return queue.MessageCount; + } + + private async Task GetDlqMessageByIdAsync( + IChannel channel, + string dlqName, + Guid messageId, + bool peekOnly = false) + { + var maxScans = 50; + for (var i = 0; i < maxScans; i++) + { + var message = await channel.BasicGetAsync(dlqName, autoAck: false); + if (message is null) + return null; + + if (Guid.Parse(message.BasicProperties.MessageId!) == messageId) + { + if (!peekOnly) + await channel.BasicAckAsync(message.DeliveryTag, multiple: false); + else + await channel.BasicNackAsync(message.DeliveryTag, multiple: false, requeue: true); + + return new DlqMessageObservation( + messageId, + ReadRetryHeader(message.BasicProperties)); + } + + await channel.BasicNackAsync(message.DeliveryTag, multiple: false, requeue: true); + } + + return null; + } + + private static int ReadRetryHeader(IReadOnlyBasicProperties properties) + { + if (properties.Headers?.TryGetValue(RabbitMQConstants.RetryCountHeaderKey, out var value) == true) + { + return value switch + { + int count => count, + long l => (int)l, + byte[] bytes when bytes.Length > 0 => bytes[0], + _ => 0 + }; + } + + return 0; + } + + private async Task CreateChannelAsync() => + await GetRequiredService().CreateChannelAsync(); + + private T GetRequiredService() where T : notnull => + _services.GetRequiredService(); + + private enum FailureOutcomeKind + { + Acked, + DeadLettered, + MainQueuePending, + Inconclusive + } + + private sealed record StableFailureOutcome( + FailureOutcomeKind Outcome, + uint MainQueueMessageCount, + uint DlqMessageCount, + bool DlqContainsMessage, + int? DlqRetryHeader, + double StableObservationDurationMs, + IReadOnlyList Notes); + + private sealed record DlqMessageObservation(Guid MessageId, int RetryHeader); + + private sealed record FailurePhaseCall( + int RequestNumber, + string Behavior, + Guid MessageId, + string RoutingKey, + DateTimeOffset PublishedUtc, + int InitialRetryHeader); + + private sealed record InboxObservation( + int InboxRowCount, + string? MessageStatus, + bool IsProcessed, + string? SubscriberName, + string? SubscriberStatus); + + private sealed record FailureObservation( + string Phase, + DateTimeOffset ObservedUtc, + Guid MessageId, + string Outcome, + int HandlerInvocationCount, + int ConsumerProcessMessageCount, + IReadOnlyList ProcessMessageStatuses, + IReadOnlyList ProcessMessageRetryCounts, + int InitialPublishRetryHeader, + int? FinalDlqRetryHeader, + uint MainQueueMessageCount, + uint DlqMessageCount, + bool DlqContainsMessage, + int InboxRowCount, + string? InboxMessageStatus, + bool InboxIsProcessed, + string? InboxSubscriberName, + string? InboxSubscriberStatus, + double? StableObservationDurationMs, + IReadOnlyList Notes); + + private sealed record EventBusFailureConfiguration( + string MainQueue, + string DlqQueue, + bool EnableDeduplication, + int ConfiguredRetryCount, + string IntendedRetryMechanism, + IReadOnlyList ImplementationCaveats, + string HandlerInvocationNote, + string RetryLaterPhaseOmittedReason); + + private sealed record FailureConclusion( + string FailingHandlerOutcome, + int HandlerInvocationCount, + bool RetryObservedForFailingEvent, + bool InboxStoredBeforeHandler, + bool DlqReachedForFailingEvent, + string? ApparentDesignVsActual, + IReadOnlyList SummaryLines); + + private sealed record EventBusFailureExperimentResult( + string Name, + DateTimeOffset StartedUtc, + string GitSha, + string Environment, + EventBusFailureConfiguration Configuration, + IReadOnlyList Calls, + IReadOnlyList Observations, + FailureConclusion Conclusion); +} diff --git a/tests/Lab/IntegrationTests/Experiments/EventBusObservationBaseline/EventBusObservationBaselineExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/EventBusObservationBaseline/EventBusObservationBaselineExperimentTests.cs new file mode 100644 index 0000000..dc00591 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/EventBusObservationBaseline/EventBusObservationBaselineExperimentTests.cs @@ -0,0 +1,220 @@ +using System.Text.Json; +using FeatureFusion.Infrastructure.Context; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Async; +using IntegrationTests.Infrastructure.EventBusLab; +using IntegrationTests.Infrastructure.Orders; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Xunit.Abstractions; + +namespace IntegrationTests.Experiments.EventBusObservationBaseline; + +/// +/// Experiment 19: EventBus observation-seam baseline (happy path). +/// Hypothesis: for a cache-miss HTTP order create, the Lab journal + DB + handler +/// observation can establish the ordered lifecycle +/// outbox → publish → processor → handler, distinguishing Observed vs Inferred facts. +/// Does not inject faults. Does not add W3C propagation. Does not change production semantics +/// when the Lab hook is registered but faults are disarmed. +/// +[Collection(AspireCollection.Name)] +public sealed class EventBusObservationBaselineExperimentTests +{ + private const int Quantity = 2; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly AspireFixture _fixture; + private readonly IServiceProvider _services; + private readonly HttpClient _http; + private readonly ITestOutputHelper _output; + + public EventBusObservationBaselineExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + _services = fixture.Services; + _http = fixture.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + } + + [Fact] + public async Task Http_order_eventbus_lifecycle_stages_are_journaled_as_observed_vs_inferred() + { + _fixture.ProcessedEvents.Clear(); + _fixture.EventBusJournal.Clear(); + _fixture.EventBusFaults.Clear(); + + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var key = System.Ulid.NewUlid().ToString(); + + var http = await HttpOrderCreate.PostAsync( + _http, + capture, + key, + Quantity, + jsonOptions: JsonOptions); + + http.HttpStatus.Should().Be(200, http.Body); + http.OrderId.Should().NotBeEmpty(); + + var outboxAfterInsert = await OrderOutboxObserver.WaitUntilExistsAsync(_services, http.OrderId); + var outboxInsertedPending = outboxAfterInsert.WorkerPending || !outboxAfterInsert.WorkerProcessed; + + await Wait.UntilAsync( + () => _fixture.EventBusJournal.Snapshot().Any(r => + r.Stage == EventBusLabStages.AfterPublishBeforeOutboxMark + && r.OrderId == http.OrderId), + TimeSpan.FromSeconds(20)); + + var processedOutbox = await OrderOutboxObserver.WaitUntilProcessedAsync(_services, http.OrderId); + + await Wait.UntilAsync( + () => _fixture.ProcessedEvents.Any(e => e.OrderId == http.OrderId), + TimeSpan.FromSeconds(20)); + + await Wait.UntilAsync( + () => _fixture.EventBusJournal.ForMessage(processedOutbox.IntegrationEventId) + .Any(r => r.Stage == EventBusLabStages.ProcessorCompleted), + TimeSpan.FromSeconds(10)); + + var delivered = _fixture.ProcessedEvents.Single(e => e.OrderId == http.OrderId); + delivered.Id.Should().Be(processedOutbox.IntegrationEventId); + + var inbox = await QueryInboxAsync(delivered.Id); + var journalForMessage = _fixture.EventBusJournal.ForMessage(delivered.Id); + var processSpans = capture.All + .Where(s => s.Source == "EventBus" + && s.DisplayName == "ProcessMessage" + && s.Tags.TryGetValue("message.id", out var mid) + && mid == delivered.Id.ToString()) + .ToList(); + + var stages = new + { + outboxInserted = new + { + evidenceKind = "Observed", + observed = outboxInsertedPending || outboxAfterInsert.OutboxMessageId != Guid.Empty, + outboxMessageId = outboxAfterInsert.OutboxMessageId, + status = outboxAfterInsert.Status, + note = "OrderOutboxObserver row for OrderId (pending may race if worker is fast)" + }, + publishCompleted = new + { + evidenceKind = "Observed", + observed = journalForMessage.Any(r => r.Stage == EventBusLabStages.AfterPublishBeforeOutboxMark), + note = "IEventBusLabHook after PublishDirect, before MarkProcessed" + }, + outboxMarkedProcessed = new + { + evidenceKind = "Observed", + observed = processedOutbox.WorkerProcessed, + status = processedOutbox.Status, + note = "OutboxWorker MarkProcessed after hook returned" + }, + processorEntered = new + { + evidenceKind = "Observed", + observed = journalForMessage.Any(r => r.Stage == EventBusLabStages.ProcessorEntered), + note = "LabMessageProcessorDecorator enter" + }, + processorCompleted = new + { + evidenceKind = "Observed", + observed = journalForMessage.Any(r => r.Stage == EventBusLabStages.ProcessorCompleted), + detail = journalForMessage.LastOrDefault(r => r.Stage == EventBusLabStages.ProcessorCompleted)?.Detail + }, + handlerInvoked = new + { + evidenceKind = "Observed", + observed = _fixture.ProcessedEvents.Count(e => e.Id == delivered.Id) == 1, + note = "TestEventHandlerDecorator / ProcessedEvents" + }, + inboxProcessed = new + { + evidenceKind = "Observed", + observed = inbox.IsProcessed, + inboxRowCount = inbox.InboxRowCount, + status = inbox.Status + }, + consumerAck = new + { + evidenceKind = "Inferred", + inferred = processSpans.Any(s => + s.Tags.TryGetValue("message.status", out var st) + && string.Equals(st, "processed", StringComparison.Ordinal)), + processMessageSpanCount = processSpans.Count, + note = "EventBus sets message.status=processed then SafeAckAsync — ack itself is not separately instrumented" + }, + dedupGate = new + { + evidenceKind = "Inferred", + exercised = false, + note = "Happy-path first delivery; inbox/processed_messages dedup characterized by Exp 7/17 — not asserted here" + } + }; + + var result = new + { + name = "eventbus-observation-baseline-v1", + startedUtc, + gitSha = LabRunInfo.ReadGitSha(), + environment = "Development", + configuration = new + { + path = HttpOrderCreate.Path, + labSeam = "IEventBusLabHook + LabMessageProcessorDecorator (faults disarmed)", + w3cPropagation = false, + asyncPath = "HTTP order → outbox → OutBoxWorker PublishDirect → Lab hook → MarkProcessed → RabbitMQ → MessageProcessor → handler → Ack" + }, + orderId = http.OrderId, + integrationEventId = delivered.Id, + stages, + journal = journalForMessage, + notes = new[] + { + "Observed = direct Lab journal or DB/handler evidence.", + "Inferred = derived from ProcessMessage tags / lifecycle adjacency (ack).", + "Does not prove exactly-once broker delivery or crash consistency." + } + }; + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + http.CachedResponseHeader.Should().BeFalse(); + stages.outboxInserted.observed.Should().BeTrue(); + stages.publishCompleted.observed.Should().BeTrue( + "Lab hook must observe AfterPublishBeforeOutboxMark for OrderId={0}", http.OrderId); + stages.outboxMarkedProcessed.observed.Should().BeTrue(); + stages.processorEntered.observed.Should().BeTrue(); + stages.processorCompleted.observed.Should().BeTrue(); + stages.handlerInvoked.observed.Should().BeTrue(); + inbox.InboxRowCount.Should().BeGreaterThanOrEqualTo(1); + inbox.IsProcessed.Should().BeTrue(); + stages.consumerAck.inferred.Should().BeTrue( + "ProcessMessage message.status=processed should be present when handler succeeded"); + _fixture.EventBusJournal.Snapshot() + .Should().NotContain(r => r.Stage == EventBusLabStages.SimulatedCrashAfterPublish); + } + + private async Task<(int InboxRowCount, bool IsProcessed, string? Status)> QueryInboxAsync(Guid messageId) + { + await using var scope = _services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var rows = await db.InboxMessages.AsNoTracking().Where(m => m.Id == messageId).ToListAsync(); + var row = rows.SingleOrDefault(); + return (rows.Count, row?.IsProcessed ?? false, row?.Status.ToString()); + } +} diff --git a/tests/Lab/IntegrationTests/Experiments/EventBusPublishCrash/EventBusPublishCrashExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/EventBusPublishCrash/EventBusPublishCrashExperimentTests.cs new file mode 100644 index 0000000..968c8ae --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/EventBusPublishCrash/EventBusPublishCrashExperimentTests.cs @@ -0,0 +1,226 @@ +using System.Text.Json; +using FeatureFusion.Features.Order.IntegrationEvents.Events; +using FeatureFusion.Infrastructure.Context; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Async; +using IntegrationTests.Infrastructure.EventBusLab; +using IntegrationTests.Infrastructure.Orders; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Xunit.Abstractions; + +namespace IntegrationTests.Experiments.EventBusPublishCrash; + +/// +/// Experiment 20: Lab fault B — after successful PublishDirect / before outbox MarkProcessed. +/// Hypothesis: when the Lab hook simulates a crash after broker publish, the outbox row remains +/// pending and a later worker poll may publish again; with fixture +/// EnableDeduplication=false, inbox completion dedup may still suppress a second handler +/// for the same IntegrationEvent.Id. Characterizes; does not fix duplicate-publish risk. +/// No W3C propagation. +/// +[Collection(AspireCollection.Name)] +public sealed class EventBusPublishCrashExperimentTests +{ + private const int Quantity = 2; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly AspireFixture _fixture; + private readonly IServiceProvider _services; + private readonly HttpClient _http; + private readonly ITestOutputHelper _output; + + public EventBusPublishCrashExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + _services = fixture.Services; + _http = fixture.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + } + + [Fact] + public async Task Publish_then_crash_before_outbox_mark_is_characterized() + { + _fixture.ProcessedEvents.Clear(); + _fixture.EventBusJournal.Clear(); + _fixture.EventBusFaults.Clear(); + + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var key = System.Ulid.NewUlid().ToString(); + + var http = await HttpOrderCreate.PostAsync( + _http, + capture, + key, + Quantity, + jsonOptions: JsonOptions); + + http.HttpStatus.Should().Be(200, http.Body); + http.OrderId.Should().NotBeEmpty(); + + // Arm immediately after OrderId is known — before OutBoxWorker (~2s) typically publishes. + _fixture.EventBusFaults.ArmCrashAfterPublishOnceForOrderId(http.OrderId); + + await Wait.UntilAsync( + () => _fixture.EventBusJournal.Snapshot().Any(r => + r.Stage == EventBusLabStages.SimulatedCrashAfterPublish + && r.OrderId == http.OrderId), + TimeSpan.FromSeconds(20)); + + var crashRecord = _fixture.EventBusJournal.Snapshot() + .Single(r => r.Stage == EventBusLabStages.SimulatedCrashAfterPublish + && r.OrderId == http.OrderId); + var messageId = crashRecord.MessageId + ?? throw new InvalidOperationException("Crash journal missing MessageId"); + + var outboxAfterCrash = await OrderOutboxObserver.FindByOrderIdAsync(_services, http.OrderId); + outboxAfterCrash.Should().ContainSingle(); + var pendingAfterCrash = outboxAfterCrash[0]; + var stillPendingAfterCrash = pendingAfterCrash.WorkerPending && !pendingAfterCrash.WorkerProcessed; + + await Wait.UntilAsync( + () => _fixture.ProcessedEvents.Any(e => e.OrderId == http.OrderId) + || _fixture.EventBusJournal.ForMessage(messageId) + .Any(r => r.Stage == EventBusLabStages.ProcessorEntered), + TimeSpan.FromSeconds(20)); + + var processedOutbox = await OrderOutboxObserver.WaitUntilProcessedAsync(_services, http.OrderId); + + await Wait.UntilAsync( + () => _fixture.ProcessedEvents.Any(e => e.OrderId == http.OrderId), + TimeSpan.FromSeconds(20)); + + var settleStart = DateTimeOffset.UtcNow; + await Wait.UntilAsync( + () => DateTimeOffset.UtcNow - settleStart >= TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(10)); + + var journal = _fixture.EventBusJournal.ForMessage(messageId); + var publishAttempts = journal.Count(r => r.Stage == EventBusLabStages.AfterPublishBeforeOutboxMark); + var crashCount = journal.Count(r => r.Stage == EventBusLabStages.SimulatedCrashAfterPublish); + var processorEnters = journal.Count(r => r.Stage == EventBusLabStages.ProcessorEntered); + var processorCompletes = journal.Count(r => r.Stage == EventBusLabStages.ProcessorCompleted); + var handlerCount = _fixture.ProcessedEvents.Count(e => e.Id == messageId); + var handlerCountByOrder = _fixture.ProcessedEvents.Count(e => e.OrderId == http.OrderId); + + var inbox = await QueryInboxAsync(messageId); + var processSpans = capture.All.Count(s => + s.Source == "EventBus" + && s.DisplayName == "ProcessMessage" + && s.Tags.TryGetValue("message.id", out var mid) + && mid == messageId.ToString()); + + var characterization = new + { + brokerReceivedAtLeastOnce = processorEnters >= 1 || handlerCount >= 1, + outboxRemainedPendingAfterCrash = stillPendingAfterCrash, + outboxEventuallyMarkedProcessed = processedOutbox.WorkerProcessed, + publishAttemptsObserved = publishAttempts, + republishObserved = publishAttempts >= 2, + consumerProcessAttempts = processorEnters, + handlerInvocationsForMessageId = handlerCount, + handlerInvocationsForOrderId = handlerCountByOrder, + inboxDedupLikelyPreventedDuplicateHandler = + processorEnters >= 2 && handlerCount == 1, + inboxProcessed = inbox.IsProcessed + }; + + var result = new + { + name = "eventbus-publish-then-crash-v1", + startedUtc, + gitSha = LabRunInfo.ReadGitSha(), + environment = "Development", + configuration = new + { + fault = "B: AfterPublishBeforeOutboxMark → EventBusLabSimulatedCrashException (one-shot by OrderId)", + enableDeduplication = false, + w3cPropagation = false + }, + orderId = http.OrderId, + integrationEventId = messageId, + observations = new + { + stillPendingAfterCrash, + pendingStatus = pendingAfterCrash.Status, + processedOutboxStatus = processedOutbox.Status, + publishAttempts, + crashCount, + processorEnters, + processorCompletes, + handlerCount, + handlerCountByOrder, + processMessageSpans = processSpans, + inbox + }, + characterization, + journal, + observedVsInferred = new + { + observed = new[] + { + "SimulatedCrashAfterPublish journal", + "AfterPublishBeforeOutboxMark count", + "Outbox pending then processed", + "ProcessorEntered/Completed", + "ProcessedEvents handler count", + "Inbox row state" + }, + inferred = new[] + { + "inboxDedupLikelyPreventedDuplicateHandler (processor>1 && handler==1)", + "Broker received message because PublishDirect returned before crash" + }, + unobservable = new[] + { + "Exact RabbitMQ confirm bit without broker admin API", + "In-process memory state at simulated crash (no real process kill)", + "W3C correlation across republish (not in scope)" + } + }, + notes = new[] + { + "Does not fix duplicate-publish risk when MarkProcessed is skipped after broker send.", + "One-shot fault: second OutBoxWorker poll publishes without crashing.", + "Exp 19 remains the no-fault baseline." + } + }; + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + crashCount.Should().Be(1); + stillPendingAfterCrash.Should().BeTrue( + "after Lab crash the outbox row must remain pending (ProcessedAt null)"); + publishAttempts.Should().BeGreaterThanOrEqualTo(1); + processedOutbox.WorkerProcessed.Should().BeTrue( + "worker should eventually MarkProcessed on a later poll after the one-shot fault"); + handlerCount.Should().Be(1, + "same IntegrationEvent.Id should invoke the order handler once (inbox dedup). handlers={0}; processorEnters={1}", + handlerCount, + processorEnters); + handlerCountByOrder.Should().Be(1); + inbox.IsProcessed.Should().BeTrue(); + characterization.brokerReceivedAtLeastOnce.Should().BeTrue(); + } + + private async Task<(int InboxRowCount, bool IsProcessed, string? Status)> QueryInboxAsync(Guid messageId) + { + await using var scope = _services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var rows = await db.InboxMessages.AsNoTracking().Where(m => m.Id == messageId).ToListAsync(); + var row = rows.SingleOrDefault(); + return (rows.Count, row?.IsProcessed ?? false, row?.Status.ToString()); + } +} diff --git a/tests/Lab/IntegrationTests/Experiments/HandlerThrowFailure/HandlerThrowFailureExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/HandlerThrowFailure/HandlerThrowFailureExperimentTests.cs new file mode 100644 index 0000000..edc5347 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/HandlerThrowFailure/HandlerThrowFailureExperimentTests.cs @@ -0,0 +1,620 @@ +using System.Diagnostics; +using System.Text.Json; +using EventBusRabbitMQ; +using EventBusRabbitMQ.Domain; +using EventBusRabbitMQ.Events; +using EventBusRabbitMQ.Infrastructure; +using EventBusRabbitMQ.Infrastructure.EventBus; +using FeatureFusion.Infrastructure.Context; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.EventBus; +using IntegrationTests.Infrastructure.Async; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using RabbitMQ.Client; +using Xunit.Abstractions; + +namespace IntegrationTests.Experiments.HandlerThrowFailure; + +/// +/// Experiment 11: handler-throw failure outcome (discovery-oriented). +/// Hypothesis: when a deserializable integration event reaches a registered handler that throws, +/// what does MessageProcessor return, does +/// EventBus.MessagetHandler ACK/NACK/requeue/DLQ, and what inbox/telemetry evidence remains? +/// Does not increment broker x-retry-count or apply retry delay — those remain documented caveats. +/// +[Collection(AspireCollection.Name)] +public sealed class HandlerThrowFailureExperimentTests : IAsyncLifetime +{ + private static readonly TimeSpan StableOutcomeTimeout = TimeSpan.FromSeconds(60); + private static readonly TimeSpan StabilityWindow = TimeSpan.FromSeconds(3); + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly AspireFixture _fixture; + private readonly IServiceProvider _services; + private readonly ITestOutputHelper _output; + + public HandlerThrowFailureExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + _ = fixture.CreateClient(); + _services = fixture.Services; + } + + public async Task InitializeAsync() => await _fixture.ResetRabbitMQ(); + + public Task DisposeAsync() => Task.CompletedTask; + + [Fact] + public async Task Handler_throw_outcome_is_observed_for_transient_business_and_plain_failures() + { + FailingIntegrationEventHandler.ResetInvocationCount(); + TransientThrowingIntegrationEventHandler.ResetInvocationCount(); + BusinessFailureIntegrationEventHandler.ResetInvocationCount(); + OnceTransientThenSucceedIntegrationEventHandler.Reset(); + + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var eventBusOptions = GetRequiredService>().Value; + await using var channel = await CreateChannelAsync(); + var mainQueue = eventBusOptions.SubscriptionClientName; + var dlqName = $"{mainQueue}{RabbitMQConstants.DeadLetterQueueSuffix}"; + + var phases = new List(); + var observations = new List(); + + await ObservePhaseAsync( + capture, + channel, + mainQueue, + dlqName, + phases, + observations, + behavior: "PlainExceptionHandlerThrow", + eventFactory: id => new FailingIntegrationEvent(id, 110.0m), + routingKey: nameof(FailingIntegrationEvent), + getHandlerInvocations: () => FailingIntegrationEventHandler.InvocationCount, + expectedHandlerClassification: "PermanentFailure (plain Exception in DispatchToHandlers)", + minHandlerInvocations: 1); + + await channel.QueuePurgeAsync(mainQueue); + await channel.QueuePurgeAsync(dlqName); + + await ObservePhaseAsync( + capture, + channel, + mainQueue, + dlqName, + phases, + observations, + behavior: "TransientExceptionHandlerThrow", + eventFactory: id => new TransientThrowingIntegrationEvent(id, 120.0m), + routingKey: nameof(TransientThrowingIntegrationEvent), + getHandlerInvocations: () => TransientThrowingIntegrationEventHandler.InvocationCount, + expectedHandlerClassification: "RetryLater until Attempts>=RetryCount then PermanentFailure / DLQ", + minHandlerInvocations: eventBusOptions.RetryCount); + + await channel.QueuePurgeAsync(mainQueue); + await channel.QueuePurgeAsync(dlqName); + + await ObservePhaseAsync( + capture, + channel, + mainQueue, + dlqName, + phases, + observations, + behavior: "TransientThenSucceedHandlerRetry", + eventFactory: id => new OnceTransientThenSucceedIntegrationEvent(id, 140.0m), + routingKey: nameof(OnceTransientThenSucceedIntegrationEvent), + getHandlerInvocations: () => + { + var call = phases.Last(p => p.Behavior == "TransientThenSucceedHandlerRetry"); + return OnceTransientThenSucceedIntegrationEventHandler.InvocationCountFor(call.MessageId); + }, + expectedHandlerClassification: "RetryLater then Success", + minHandlerInvocations: 2); + + await channel.QueuePurgeAsync(mainQueue); + await channel.QueuePurgeAsync(dlqName); + + await ObservePhaseAsync( + capture, + channel, + mainQueue, + dlqName, + phases, + observations, + behavior: "BusinessExceptionHandlerThrow", + eventFactory: id => new BusinessFailureIntegrationEvent(id, 130.0m), + routingKey: nameof(BusinessFailureIntegrationEvent), + getHandlerInvocations: () => BusinessFailureIntegrationEventHandler.InvocationCount, + expectedHandlerClassification: "PermanentFailure (BusinessException in DispatchToHandlers)", + minHandlerInvocations: 1); + + var result = new HandlerThrowFailureExperimentResult( + Name: "handler-throw-failure-outcome-v1", + StartedUtc: startedUtc, + GitSha: LabRunInfo.ReadGitSha(), + Environment: "Development", + Configuration: new HandlerThrowFailureConfiguration( + MainQueue: mainQueue, + DlqQueue: dlqName, + ConfiguredRetryCount: eventBusOptions.RetryCount, + EnableDeduplication: eventBusOptions.EnableDeduplication, + ConsumerPath: "PublishDirect → RabbitMQ → EventBus.MessagetHandler → MessageProcessor.ProcessMessageAsync → inbox_messages → DispatchToHandlers", + ImplementationCaveats: + [ + "x-retry-count is the RabbitMQ broker header and is not incremented on requeue; it is not the inbox_subscribers.Attempts budget", + "message.retry_count ProcessMessage tag is the broker header (typically 0), not DB Attempts", + "CalculateRetryDelay is computed on RetryLater branch but not applied before BasicNack(requeue:true)", + "RetryCount bounds handler executions via inbox_subscribers.Attempts; exhausted RetryLater is converted to PermanentFailure before ACK/NACK" + ]), + Phases: phases, + Observations: observations, + Conclusion: BuildConclusion(observations, eventBusOptions)); + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + var plain = observations.Single(o => o.Phase == "PlainExceptionHandlerThrow"); + var transient = observations.Single(o => o.Phase == "TransientExceptionHandlerThrow"); + var business = observations.Single(o => o.Phase == "BusinessExceptionHandlerThrow"); + var recovered = observations.Single(o => o.Phase == "TransientThenSucceedHandlerRetry"); + + AssertPermanentFailurePhase(plain, expectedAttempts: 1); + AssertBoundedTransientFailurePhase(transient, eventBusOptions.RetryCount); + AssertPermanentFailurePhase(business, expectedAttempts: 1); + AssertSuccessfulTransientRetryPhase(recovered); + } + + private static void AssertPermanentFailurePhase(HandlerThrowObservation obs, int expectedAttempts) + { + obs.HandlerInvocationCount.Should().Be(1, + "{0} should invoke the handler once before permanent failure", obs.Phase); + obs.ConsumerOutcome.Should().Be( + ConsumerOutcomeKind.DeadLettered, + "{0}: PermanentFailure should dead-letter", obs.Phase); + obs.ProcessMessageStatuses.Should().Contain("failed", + "{0}: ProcessMessage should record failed status on DLQ path", obs.Phase); + obs.DlqContainsMessage.Should().BeTrue( + "{0}: permanent handler failure should reach DLQ", obs.Phase); + obs.InboxIsProcessed.Should().BeFalse( + "{0}: permanent handler failure must not mark inbox processed on first attempt", obs.Phase); + obs.SubscriberStatus.Should().Be(MessageStatus.Failed.ToString(), + "{0}: subscriber status should reflect handler failure", obs.Phase); + obs.SubscriberAttempts.Should().Be(expectedAttempts, + "{0}: permanent failure should record one handler attempt", obs.Phase); + } + + private static void AssertBoundedTransientFailurePhase(HandlerThrowObservation obs, int configuredRetryCount) + { + obs.HandlerInvocationCount.Should().Be(configuredRetryCount, + "{0}: RetryCount={1} must allow exactly {1} handler executions", obs.Phase, configuredRetryCount); + obs.ConsumerProcessMessageCount.Should().Be(configuredRetryCount, + "{0}: each handler execution should correspond to one ProcessMessage delivery", obs.Phase); + obs.ConsumerOutcome.Should().Be( + ConsumerOutcomeKind.DeadLettered, + "{0}: exhausted RetryLater should dead-letter", obs.Phase); + obs.ProcessMessageStatuses.Should().Contain("retrying", + "{0}: early attempts should record retrying before requeue", obs.Phase); + obs.ProcessMessageStatuses.Should().Contain("failed", + "{0}: final exhausted attempt should record failed on DLQ path", obs.Phase); + obs.DlqContainsMessage.Should().BeTrue( + "{0}: transient failure must DLQ after RetryCount handler executions", obs.Phase); + obs.InboxIsProcessed.Should().BeFalse( + "{0}: exhausted transient failure must not mark inbox processed", obs.Phase); + obs.SubscriberStatus.Should().Be(MessageStatus.Failed.ToString(), + "{0}: subscriber status should reflect handler failure", obs.Phase); + obs.SubscriberAttempts.Should().Be(configuredRetryCount, + "{0}: inbox_subscribers.Attempts must equal RetryCount", obs.Phase); + obs.ProcessMessageRetryCounts + .Where(v => v is not null) + .Should() + .AllBe("0", + "{0}: message.retry_count tag is the broker x-retry-count header (still 0), not DB Attempts", obs.Phase); + } + + private static void AssertSuccessfulTransientRetryPhase(HandlerThrowObservation obs) + { + obs.HandlerInvocationCount.Should().Be(2, + "{0}: first TransientException then success should invoke the handler twice", obs.Phase); + obs.ConsumerOutcome.Should().Be( + ConsumerOutcomeKind.Acked, + "{0}: successful retry should ACK", obs.Phase); + obs.ProcessMessageStatuses.Should().Contain("retrying"); + obs.ProcessMessageStatuses.Should().Contain("processed"); + obs.DlqContainsMessage.Should().BeFalse( + "{0}: successful retry must not dead-letter", obs.Phase); + obs.InboxIsProcessed.Should().BeTrue( + "{0}: successful retry should mark inbox processed", obs.Phase); + obs.SubscriberStatus.Should().Be(MessageStatus.Processed.ToString(), + "{0}: subscriber should be Processed after successful retry", obs.Phase); + obs.SubscriberAttempts.Should().Be(2, + "{0}: Attempts should count both the failed and successful handler executions", obs.Phase); + } + + private async Task ObservePhaseAsync( + InProcessActivityCapture capture, + IChannel channel, + string mainQueue, + string dlqName, + List phases, + List observations, + string behavior, + Func eventFactory, + string routingKey, + Func getHandlerInvocations, + string expectedHandlerClassification, + int minHandlerInvocations = 1, + bool waitForRetryTelemetry = false) + { + var @event = eventFactory(Guid.NewGuid()); + var publishUtc = DateTimeOffset.UtcNow; + + phases.Add(new HandlerThrowPhaseCall( + RequestNumber: phases.Count + 1, + Behavior: behavior, + MessageId: @event.Id, + RoutingKey: routingKey, + PublishedUtc: publishUtc, + ExpectedHandlerClassification: expectedHandlerClassification)); + + await GetRequiredService().PublishDirect(@event); + + await Wait.UntilAsync( + () => getHandlerInvocations() >= minHandlerInvocations, + TimeSpan.FromSeconds(30)); + + if (waitForRetryTelemetry) + { + await Wait.UntilAsync( + () => + { + var activities = GetProcessMessageActivities(capture, @event.Id); + return activities.Any(a => GetTag(a, "message.status") == "retrying") + || activities.Count >= 2; + }, + TimeSpan.FromSeconds(30)); + } + + var outcome = await WaitForStableConsumerOutcomeAsync(capture, channel, mainQueue, dlqName, @event.Id); + var inbox = await QueryInboxAsync(@event.Id); + var activities = GetProcessMessageActivities(capture, @event.Id); + + observations.Add(new HandlerThrowObservation( + Phase: behavior, + ObservedUtc: DateTimeOffset.UtcNow, + MessageId: @event.Id, + ExpectedHandlerClassification: expectedHandlerClassification, + HandlerInvocationCount: getHandlerInvocations(), + ConsumerOutcome: outcome.Outcome, + ConsumerProcessMessageCount: activities.Count, + ProcessMessageStatuses: activities.Select(a => GetTag(a, "message.status")).Distinct().ToList(), + ProcessMessageRetryCounts: activities.Select(a => GetTag(a, "message.retry_count")).Distinct().ToList(), + ProcessMessageRetryDelayMs: activities + .Select(a => GetTag(a, "message.retry_delay_ms")) + .Where(v => v is not null) + .Distinct() + .ToList(), + MainQueueMessageCountAfterStable: outcome.MainQueueMessageCount, + DlqMessageCountAfterStable: outcome.DlqMessageCount, + DlqContainsMessage: outcome.DlqContainsMessage, + InboxRowCount: inbox.InboxRowCount, + InboxMessageStatus: inbox.MessageStatus, + InboxIsProcessed: inbox.IsProcessed, + SubscriberName: inbox.SubscriberName, + SubscriberStatus: inbox.SubscriberStatus, + SubscriberAttempts: inbox.SubscriberAttempts, + StableObservationDurationMs: outcome.StableObservationDurationMs, + Notes: outcome.Notes)); + } + + private async Task WaitForStableConsumerOutcomeAsync( + InProcessActivityCapture capture, + IChannel channel, + string mainQueue, + string dlqName, + Guid messageId) + { + var stopwatch = Stopwatch.StartNew(); + var lastProcessCount = -1; + var stableSinceUtc = DateTimeOffset.UtcNow; + var notes = new List(); + + while (stopwatch.Elapsed < StableOutcomeTimeout) + { + var dlqMessage = await GetDlqMessageByIdAsync(channel, dlqName, messageId, peekOnly: true); + if (dlqMessage is not null) + { + return new StableConsumerOutcome( + ConsumerOutcomeKind.DeadLettered, + GetQueueMessageCount(channel, mainQueue), + GetQueueMessageCount(channel, dlqName), + DlqContainsMessage: true, + StableObservationDurationMs: stopwatch.Elapsed.TotalMilliseconds, + Notes: + [ + ..notes, + "Message observed in DLQ.", + $"DLQ x-retry-count={dlqMessage.RetryHeader}." + ]); + } + + var processCount = GetProcessMessageActivities(capture, messageId).Count; + var mainCount = GetQueueMessageCount(channel, mainQueue); + + if (processCount != lastProcessCount) + { + lastProcessCount = processCount; + stableSinceUtc = DateTimeOffset.UtcNow; + notes.Add($"ProcessMessage count={processCount} at {stopwatch.Elapsed.TotalMilliseconds:F0}ms."); + } + + if (processCount > 0 && mainCount == 0) + { + if (DateTimeOffset.UtcNow - stableSinceUtc >= StabilityWindow) + { + var statuses = GetProcessMessageActivities(capture, messageId) + .Select(a => GetTag(a, "message.status")) + .ToList(); + var processed = statuses.Contains("processed"); + var failed = statuses.Contains("failed"); + + return new StableConsumerOutcome( + processed && !failed ? ConsumerOutcomeKind.Acked : ConsumerOutcomeKind.Inconclusive, + mainCount, + GetQueueMessageCount(channel, dlqName), + DlqContainsMessage: false, + StableObservationDurationMs: stopwatch.Elapsed.TotalMilliseconds, + Notes: + [ + ..notes, + $"Stable outcome after {stopwatch.Elapsed.TotalMilliseconds:F0}ms; ProcessMessage statuses=[{string.Join(",", statuses)}]." + ]); + } + } + + await Task.Delay(100); + } + + return new StableConsumerOutcome( + ConsumerOutcomeKind.Inconclusive, + GetQueueMessageCount(channel, mainQueue), + GetQueueMessageCount(channel, dlqName), + DlqContainsMessage: false, + StableObservationDurationMs: stopwatch.Elapsed.TotalMilliseconds, + Notes: [..notes, "Timed out before stable consumer outcome."]); + } + + private static HandlerThrowFailureConclusion BuildConclusion( + IReadOnlyList observations, + EventBusOptions options) + { + var permanentPhases = observations.Where(o => + o.Phase is "PlainExceptionHandlerThrow" or "BusinessExceptionHandlerThrow").ToList(); + var transientPhase = observations.Single(o => o.Phase == "TransientExceptionHandlerThrow"); + var recoveredPhase = observations.SingleOrDefault(o => o.Phase == "TransientThenSucceedHandlerRetry"); + + var permanentDeadLettered = permanentPhases.All(o => + o.ConsumerOutcome == ConsumerOutcomeKind.DeadLettered && o.DlqContainsMessage); + var transientBounded = transientPhase.HandlerInvocationCount == options.RetryCount + && transientPhase.DlqContainsMessage + && transientPhase.SubscriberAttempts == options.RetryCount; + var recovered = recoveredPhase is { HandlerInvocationCount: 2, InboxIsProcessed: true, DlqContainsMessage: false }; + var anyDlq = observations.Any(o => o.DlqContainsMessage); + var retryHeaderProgressed = observations.Any(o => + o.ProcessMessageRetryCounts.Any(v => v is not null && v != "0")); + + return new HandlerThrowFailureConclusion( + AllPhasesAckedDespiteHandlerThrow: observations.All(o => o.ConsumerOutcome == ConsumerOutcomeKind.Acked), + AnyRetryingStatusObserved: observations.Any(o => + o.ProcessMessageStatuses.Contains("retrying") || o.ConsumerOutcome == ConsumerOutcomeKind.Requeued), + AnyDlqObserved: anyDlq, + RetryHeaderProgressed: retryHeaderProgressed, + RetryDelayTagObserved: observations.Any(o => o.ProcessMessageRetryDelayMs.Count > 0), + SummaryLines: + [ + $"Configured RetryCount={options.RetryCount}; EnableDeduplication={options.EnableDeduplication}.", + permanentDeadLettered + ? "Plain and BusinessException handler failures dead-lettered after one handler attempt; inbox not marked processed." + : "Permanent-failure phases did not all dead-letter — see per-phase ConsumerOutcome.", + transientBounded + ? $"TransientException executed the handler RetryCount={options.RetryCount} times (Attempts={transientPhase.SubscriberAttempts}) then dead-lettered; ProcessMessage spans={transientPhase.ConsumerProcessMessageCount}." + : "TransientException did not observe bounded RetryCount → DLQ.", + recovered + ? "Once-transient-then-succeed recovered on attempt 2; inbox processed; no DLQ." + : "Successful-retry phase was not observed or did not complete as processed.", + retryHeaderProgressed + ? "x-retry-count / message.retry_count broker header progressed on at least one attempt." + : "message.retry_count remained 0 (broker x-retry-count); inbox_subscribers.Attempts is the retry budget.", + "DispatchToHandlers still aggregates RetryLater > PermanentFailure > Success; exhausted RetryLater is converted to PermanentFailure using Attempts >= RetryCount." + ]); + } + + private async Task QueryInboxAsync(Guid messageId) + { + await using var scope = _services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var message = await db.InboxMessages + .AsNoTracking() + .FirstOrDefaultAsync(m => m.Id == messageId); + + var subscriber = await db.InboxSubscriber + .AsNoTracking() + .FirstOrDefaultAsync(s => s.MessageId == messageId); + + return new InboxObservation( + InboxRowCount: message is null ? 0 : 1, + MessageStatus: message?.Status.ToString(), + IsProcessed: message?.IsProcessed ?? false, + SubscriberName: subscriber?.SubscriberName, + SubscriberStatus: subscriber?.Status.ToString(), + SubscriberAttempts: subscriber?.Attempts ?? 0); + } + + private static IReadOnlyList GetProcessMessageActivities( + InProcessActivityCapture capture, + Guid messageId) + { + var id = messageId.ToString(); + return capture.All + .Where(span => + span.Source == "EventBus" + && span.DisplayName == "ProcessMessage" + && span.Tags.TryGetValue("message.id", out var mid) + && mid == id) + .ToList(); + } + + private static string? GetTag(CapturedActivity activity, string key) => + activity.Tags.TryGetValue(key, out var value) ? value : null; + + private static uint GetQueueMessageCount(IChannel channel, string queueName) + { + var queue = channel.QueueDeclarePassiveAsync(queueName).GetAwaiter().GetResult(); + return queue.MessageCount; + } + + private async Task GetDlqMessageByIdAsync( + IChannel channel, + string dlqName, + Guid messageId, + bool peekOnly = false) + { + for (var i = 0; i < 50; i++) + { + var message = await channel.BasicGetAsync(dlqName, autoAck: false); + if (message is null) + return null; + + if (Guid.Parse(message.BasicProperties.MessageId!) == messageId) + { + if (!peekOnly) + await channel.BasicAckAsync(message.DeliveryTag, multiple: false); + else + await channel.BasicNackAsync(message.DeliveryTag, multiple: false, requeue: true); + + return new DlqMessageObservation(messageId, ReadRetryHeader(message.BasicProperties)); + } + + await channel.BasicNackAsync(message.DeliveryTag, multiple: false, requeue: true); + } + + return null; + } + + private static int ReadRetryHeader(IReadOnlyBasicProperties properties) + { + if (properties.Headers?.TryGetValue(RabbitMQConstants.RetryCountHeaderKey, out var value) == true) + { + return value switch + { + int count => count, + long l => (int)l, + byte[] bytes when bytes.Length > 0 => bytes[0], + _ => 0 + }; + } + + return 0; + } + + private async Task CreateChannelAsync() => + await GetRequiredService().CreateChannelAsync(); + + private T GetRequiredService() where T : notnull => + _services.GetRequiredService(); + + private enum ConsumerOutcomeKind + { + Acked, + Requeued, + DeadLettered, + Inconclusive + } + + private sealed record StableConsumerOutcome( + ConsumerOutcomeKind Outcome, + uint MainQueueMessageCount, + uint DlqMessageCount, + bool DlqContainsMessage, + double StableObservationDurationMs, + IReadOnlyList Notes); + + private sealed record DlqMessageObservation(Guid MessageId, int RetryHeader); + + private sealed record InboxObservation( + int InboxRowCount, + string? MessageStatus, + bool IsProcessed, + string? SubscriberName, + string? SubscriberStatus, + int SubscriberAttempts); + + private sealed record HandlerThrowPhaseCall( + int RequestNumber, + string Behavior, + Guid MessageId, + string RoutingKey, + DateTimeOffset PublishedUtc, + string ExpectedHandlerClassification); + + private sealed record HandlerThrowObservation( + string Phase, + DateTimeOffset ObservedUtc, + Guid MessageId, + string ExpectedHandlerClassification, + int HandlerInvocationCount, + ConsumerOutcomeKind ConsumerOutcome, + int ConsumerProcessMessageCount, + IReadOnlyList ProcessMessageStatuses, + IReadOnlyList ProcessMessageRetryCounts, + IReadOnlyList ProcessMessageRetryDelayMs, + uint MainQueueMessageCountAfterStable, + uint DlqMessageCountAfterStable, + bool DlqContainsMessage, + int InboxRowCount, + string? InboxMessageStatus, + bool InboxIsProcessed, + string? SubscriberName, + string? SubscriberStatus, + int SubscriberAttempts, + double StableObservationDurationMs, + IReadOnlyList Notes); + + private sealed record HandlerThrowFailureConfiguration( + string MainQueue, + string DlqQueue, + int ConfiguredRetryCount, + bool EnableDeduplication, + string ConsumerPath, + IReadOnlyList ImplementationCaveats); + + private sealed record HandlerThrowFailureConclusion( + bool AllPhasesAckedDespiteHandlerThrow, + bool AnyRetryingStatusObserved, + bool AnyDlqObserved, + bool RetryHeaderProgressed, + bool RetryDelayTagObserved, + IReadOnlyList SummaryLines); + + private sealed record HandlerThrowFailureExperimentResult( + string Name, + DateTimeOffset StartedUtc, + string GitSha, + string Environment, + HandlerThrowFailureConfiguration Configuration, + IReadOnlyList Phases, + IReadOnlyList Observations, + HandlerThrowFailureConclusion Conclusion); +} diff --git a/tests/Lab/IntegrationTests/Experiments/IdempotencyFingerprint/IdempotencyFingerprintExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/IdempotencyFingerprint/IdempotencyFingerprintExperimentTests.cs new file mode 100644 index 0000000..c16c826 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/IdempotencyFingerprint/IdempotencyFingerprintExperimentTests.cs @@ -0,0 +1,146 @@ +using System.Diagnostics; +using System.Text.Json; +using BuildingBlocks.Idempotency; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Orders; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Xunit.Abstractions; +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Experiments.IdempotencyFingerprint; + +/// +/// Experiment 12 — Request fingerprint (opt-in, richer BuildingBlocks.Idempotency v1). +/// Proves : same key + same +/// body replays; same key + different body → 422 without production. Enabled only via +/// test-host PostConfigure; Lab default remains fingerprint off (Exp 3). +/// Not part of the original extraction proof set — added for the richer v1 capability. +/// Does not re-prove Exp 3/4 concurrency or default fingerprint-off body tolerance. +/// +[Collection(AspireCollection.Name)] +public sealed class IdempotencyFingerprintExperimentTests +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly HttpClient _http; + private readonly ITestOutputHelper _output; + + public IdempotencyFingerprintExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _output = output; + _http = fixture.WithWebHostBuilder(builder => + { + builder.ConfigureTestServices(services => + { + services.PostConfigure(o => + { + o.EnableRequestFingerprint = true; + }); + }); + }).CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + } + + [Fact] + public async Task Order_create_fingerprint_conflict_and_same_body_replay_are_observed() + { + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var key = System.Ulid.NewUlid().ToString(); + var calls = new List(); + + var miss = await SendAsync(capture, calls, "MissPopulate", key, quantity: 2); + var sameBody = await SendAsync(capture, calls, "SameKeySameBody", key, quantity: 2); + var differentBody = await SendAsync(capture, calls, "SameKeyDifferentBody", key, quantity: 5); + + var result = new + { + name = "order-create-idempotency-fingerprint-v1", + startedUtc, + gitSha = LabRunInfo.ReadGitSha(), + calls, + observations = new + { + miss.HttpStatus, + miss.OrderId, + sameBodyReplay = sameBody.CachedResponseHeader && sameBody.HttpStatus == 200, + sameBodyOrderId = sameBody.OrderId, + fingerprintConflictStatus = differentBody.HttpStatus + } + }; + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + miss.HttpStatus.Should().Be(200); + miss.OrderId.Should().NotBeEmpty(); + miss.MediatorSpanCount.Should().BeGreaterThan(0); + + sameBody.HttpStatus.Should().Be(200); + sameBody.OrderId.Should().Be(miss.OrderId); + sameBody.CachedResponseHeader.Should().BeTrue(); + sameBody.MediatorSpanCount.Should().Be(0); + + differentBody.HttpStatus.Should().Be(422, + "EnableRequestFingerprint must reject same Idempotency-Key with a different body"); + differentBody.CachedResponseHeader.Should().BeFalse(); + differentBody.MediatorSpanCount.Should().Be(0, + "fingerprint conflict must not run production. Spans: {0}", + Describe(capture.ForTraceHex(differentBody.TraceId))); + } + + private async Task SendAsync( + InProcessActivityCapture capture, + List calls, + string behavior, + string idempotencyKey, + int quantity) + { + var result = await HttpOrderCreate.PostAsync( + _http, + capture, + idempotencyKey, + quantity, + jsonOptions: JsonOptions); + + var call = new FingerprintCall( + behavior, + idempotencyKey, + quantity, + result.HttpStatus, + result.OrderId, + result.CachedResponseHeader, + result.Spans.Count(IsMediator), + result.Spans.Count(IsNpgsql), + result.TraceIdHex, + result.ClientDurationMs, + result.Body); + calls.Add(call); + return call; + } + + private static string Describe(IReadOnlyList spans) + => string.Join("; ", spans.Select(s => $"{s.Source}:{s.DisplayName}")); + + private sealed record FingerprintCall( + string Behavior, + string IdempotencyKey, + int RequestedQuantity, + int HttpStatus, + Guid OrderId, + bool CachedResponseHeader, + int MediatorSpanCount, + int NpgsqlSpanCount, + string TraceId, + double ClientDurationMs, + string Body); +} diff --git a/tests/Lab/IntegrationTests/Experiments/IdempotencyProcessingLease/IdempotencyProcessingLeaseExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/IdempotencyProcessingLease/IdempotencyProcessingLeaseExperimentTests.cs new file mode 100644 index 0000000..8db8541 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/IdempotencyProcessingLease/IdempotencyProcessingLeaseExperimentTests.cs @@ -0,0 +1,382 @@ +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using BuildingBlocks.Idempotency; +using BuildingBlocks.Mediator; +using FeatureFusion.Features.Orders.Commands; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Orders; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Caching.Distributed; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Newtonsoft.Json; +using Xunit.Abstractions; +using static FeatureFusion.Features.Orders.Commands.CreateOrderCommandHandler; +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Experiments.IdempotencyProcessingLease; + +/// +/// Experiment 15 — HTTP idempotency ProcessingTtl lease overlap. +/// Hypothesis: if the first same-key request remains in Processing longer than +/// ProcessingTtl, a second request may acquire the expired lease and execute +/// production concurrently (duplicate order / outbox). Does not change Lab defaults — +/// short TTL and a test-only handler gate are applied only on this experiment host. +/// Does not add lease renewal. Completes the HTTP idempotency matrix after Exp 3/4/12. +/// +[Collection(AspireCollection.Name)] +public sealed class IdempotencyProcessingLeaseExperimentTests +{ + private const string UserIdFallback = "123"; + private const int Quantity = 2; + + /// Test-host only. Lab / package default remains 2 minutes. + private static readonly TimeSpan TestProcessingTtl = TimeSpan.FromSeconds(1); + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly AspireFixture _fixture; + private readonly WebApplicationFactory _factory; + private readonly HttpClient _http; + private readonly CreateOrderLeaseHoldGate _gate; + private readonly ITestOutputHelper _output; + + public IdempotencyProcessingLeaseExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + + _factory = fixture.WithWebHostBuilder(builder => + { + builder.ConfigureTestServices(services => + { + services.PostConfigure(o => + { + o.ProcessingTtl = TestProcessingTtl; + }); + + services.RemoveAll>>(); + services.AddSingleton(); + services.AddScoped(); + services.AddScoped>>(sp => + new GatedCreateOrderCommandHandler( + sp.GetRequiredService(), + sp.GetRequiredService())); + }); + }); + + _http = _factory.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + _gate = _factory.Services.GetRequiredService(); + } + + [Fact] + public async Task Same_key_after_ProcessingTtl_may_run_production_while_first_still_in_flight() + { + _fixture.ProcessedEvents.Clear(); + _gate.Reset(); + + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var key = System.Ulid.NewUlid().ToString(); + var cacheKey = $"Idempotency_{UserIdFallback}_{key}"; + var calls = new List(); + var timeline = new List(); + + void Mark(string phase, string detail) + => timeline.Add(new TimelineEvent(DateTimeOffset.UtcNow, phase, detail)); + + var firstTask = SendAsync(capture, calls, "FirstLeaseHolder", key, Quantity); + + await _gate.FirstEntered.WaitAsync(TimeSpan.FromSeconds(30)); + var leaseAnchorUtc = DateTimeOffset.UtcNow; + Mark("first_handler_entered", + $"HandlerEntries={_gate.HandlerEntries}; Processing cache should be live until ~{leaseAnchorUtc.Add(TestProcessingTtl):O}."); + + var probe = await SendAsync(capture, calls, "ProcessingProbeWhileHeld", key, Quantity); + Mark("processing_probe", + $"HttpStatus={probe.HttpStatus}; expected Conflict while Processing unexpired."); + + var lease = await WaitForProcessingLeaseExpiredAsync(cacheKey, leaseAnchorUtc); + Mark("lease_expired", + $"CachePresent={lease.CachePresent}; Status={lease.Status}; ExpiresAt={lease.ProcessingExpiresAtUtc}; WaitedMs={lease.WaitedMs:F0}."); + + var second = await SendAsync(capture, calls, "ExpiredLeaseAcquirer", key, Quantity); + Mark("second_completed", + $"HttpStatus={second.HttpStatus}; OrderId={second.OrderId}; HandlerEntries={_gate.HandlerEntries}; Mediator={second.MediatorSpanCount}."); + + _gate.ReleaseFirst.TrySetResult(); + var first = await firstTask.WaitAsync(TimeSpan.FromSeconds(30)); + Mark("first_completed", + $"HttpStatus={first.HttpStatus}; OrderId={first.OrderId}; HandlerEntries={_gate.HandlerEntries}."); + + var orderIds = new[] { first.OrderId, second.OrderId } + .Where(id => id != Guid.Empty) + .Distinct() + .ToList(); + + var outboxByOrder = new Dictionary(); + foreach (var orderId in orderIds) + outboxByOrder[orderId] = (await OrderOutboxObserver.FindByOrderIdAsync(_factory.Services, orderId)).Count; + + var finalCache = await ReadCacheEntryAsync(cacheKey); + var productionCallers = calls.Count(c => + c.Behavior is "FirstLeaseHolder" or "ExpiredLeaseAcquirer" + && c.HttpStatus == 200 + && c.MediatorSpanCount > 0 + && !c.CachedResponseHeader); + + var hypothesisConfirmed = + probe.HttpStatus == 409 + && first.HttpStatus == 200 + && second.HttpStatus == 200 + && orderIds.Count == 2 + && _gate.HandlerEntries >= 2 + && productionCallers == 2; + + var result = new + { + name = "order-create-idempotency-processing-lease-v1", + startedUtc, + gitSha = LabRunInfo.ReadGitSha(), + environment = "Development", + configuration = new + { + path = HttpOrderCreate.Path, + testHostProcessingTtl = TestProcessingTtl.ToString(), + labDefaultProcessingTtlUnchanged = "00:02:00", + delayMechanism = "Test-only GatedCreateOrderCommandHandler awaits ReleaseFirst after FirstEntered; production CreateOrderCommandHandler unchanged", + cacheKey + }, + timeline, + calls, + observations = new + { + probeHttpStatus = probe.HttpStatus, + firstHttpStatus = first.HttpStatus, + secondHttpStatus = second.HttpStatus, + firstOrderId = first.OrderId, + secondOrderId = second.OrderId, + distinctOrderIds = orderIds.Count, + handlerEntries = _gate.HandlerEntries, + productionCallers, + outboxRowsByOrderId = outboxByOrder, + finalCacheStatus = finalCache?.Status, + finalCacheHasResponse = !string.IsNullOrEmpty(finalCache?.Response), + hypothesisConfirmed, + hypothesis = "Expired ProcessingTtl allows a second same-key production execution while the first is still in flight." + }, + notes = new[] + { + "Probe while first held must be HTTP 409 Processing conflict (unexpired lease).", + "After ProcessingTtl, ExpiredLeaseAcquirer may run production concurrently with FirstLeaseHolder.", + "Package lock covers GetOrCreate only — not the full handler duration (documented lease tradeoff).", + hypothesisConfirmed + ? "Hypothesis CONFIRMED: two production executions / distinct orderIds under expired Processing lease." + : "Hypothesis FALSIFIED or inconclusive: overlap did not produce two production outcomes as packaged semantics allow." + } + }; + + _output.WriteLine(System.Text.Json.JsonSerializer.Serialize(result, JsonOptions)); + + probe.HttpStatus.Should().Be(409, + "while the first request is held in Processing and the lease is unexpired, the same key must Conflict"); + probe.MediatorSpanCount.Should().Be(0); + probe.CachedResponseHeader.Should().BeFalse(); + + first.HttpStatus.Should().Be(200); + first.OrderId.Should().NotBeEmpty(); + first.CachedResponseHeader.Should().BeFalse(); + first.MediatorSpanCount.Should().BeGreaterThan(0); + + second.HttpStatus.Should().Be(200, + "after ProcessingTtl expiry the filter must admit a new production execution (ExpiredProcessing_AllowsNewExecution)"); + second.OrderId.Should().NotBeEmpty(); + second.CachedResponseHeader.Should().BeFalse( + "second request should not be a Completed replay of the first (first still in flight)"); + second.MediatorSpanCount.Should().BeGreaterThan(0, + "expired lease acquirer must execute production. Spans: {0}", + Describe(capture.ForTraceHex(second.TraceId))); + + orderIds.Should().HaveCount(2, + "concurrent production under expired Processing must create two distinct orders. first={0} second={1}", + first.OrderId, second.OrderId); + _gate.HandlerEntries.Should().BeGreaterThanOrEqualTo(2); + productionCallers.Should().Be(2); + + foreach (var orderId in orderIds) + { + outboxByOrder[orderId].Should().BeGreaterThanOrEqualTo(1, + "each production order should leave at least one outbox row. orderId={0}", orderId); + } + } + + private async Task WaitForProcessingLeaseExpiredAsync( + string cacheKey, + DateTimeOffset leaseAnchorUtc) + { + var cache = _factory.Services.GetRequiredService(); + var deadline = leaseAnchorUtc.Add(TestProcessingTtl).Add(TimeSpan.FromSeconds(5)); + var stopwatch = Stopwatch.StartNew(); + + while (DateTimeOffset.UtcNow < deadline) + { + var entry = await ReadCacheEntryAsync(cache, cacheKey); + var expiredByClock = DateTimeOffset.UtcNow >= leaseAnchorUtc.Add(TestProcessingTtl); + var expiredByEntry = entry is null + || (entry.Status == "Processing" + && entry.ProcessingExpiresAtUtc is { } expires + && expires <= DateTimeOffset.UtcNow); + + if (expiredByClock && expiredByEntry) + { + return new LeaseObservation( + entry is not null, + entry?.Status, + entry?.ProcessingExpiresAtUtc, + stopwatch.Elapsed.TotalMilliseconds); + } + + await Task.Delay(50); + } + + var final = await ReadCacheEntryAsync(cache, cacheKey); + return new LeaseObservation( + final is not null, + final?.Status, + final?.ProcessingExpiresAtUtc, + stopwatch.Elapsed.TotalMilliseconds); + } + + private Task ReadCacheEntryAsync(string cacheKey) + => ReadCacheEntryAsync(_factory.Services.GetRequiredService(), cacheKey); + + private static async Task ReadCacheEntryAsync( + IDistributedCache cache, + string cacheKey) + { + var bytes = await cache.GetAsync(cacheKey); + if (bytes is null || bytes.Length == 0) + return null; + + return JsonConvert.DeserializeObject(Encoding.UTF8.GetString(bytes)); + } + + private async Task SendAsync( + InProcessActivityCapture capture, + List calls, + string behavior, + string idempotencyKey, + int quantity) + { + var result = await HttpOrderCreate.PostAsync( + _http, + capture, + idempotencyKey, + quantity, + jsonOptions: JsonOptions); + + var call = new LeaseCall( + Behavior: behavior, + IdempotencyKey: idempotencyKey, + HttpStatus: result.HttpStatus, + OrderId: result.OrderId, + CachedResponseHeader: result.CachedResponseHeader, + MediatorSpanCount: result.Spans.Count(IsMediator), + NpgsqlSpanCount: result.Spans.Count(IsNpgsql), + TraceId: result.TraceIdHex, + ClientDurationMs: result.ClientDurationMs, + Body: result.Body); + + calls.Add(call); + return call; + } + + private static string Describe(IReadOnlyList spans) + => string.Join("; ", spans.Select(s => $"{s.Source}:{s.DisplayName}")); + + /// Test-only synchronization. Not production code. + public sealed class CreateOrderLeaseHoldGate + { + private int _handlerEntries; + private TaskCompletionSource _firstEntered = NewTcs(); + private TaskCompletionSource _releaseFirst = NewTcs(); + + public Task FirstEntered => _firstEntered.Task; + public TaskCompletionSource ReleaseFirst => _releaseFirst; + public int HandlerEntries => Volatile.Read(ref _handlerEntries); + + public void Reset() + { + Volatile.Write(ref _handlerEntries, 0); + _firstEntered = NewTcs(); + _releaseFirst = NewTcs(); + } + + public async Task WaitIfFirstAsync() + { + var n = Interlocked.Increment(ref _handlerEntries); + if (n == 1) + { + _firstEntered.TrySetResult(); + await _releaseFirst.Task.ConfigureAwait(false); + } + } + + private static TaskCompletionSource NewTcs() + => new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + private sealed class GatedCreateOrderCommandHandler + : ICommandHandler> + { + private readonly CreateOrderCommandHandler _inner; + private readonly CreateOrderLeaseHoldGate _gate; + + public GatedCreateOrderCommandHandler( + CreateOrderCommandHandler inner, + CreateOrderLeaseHoldGate gate) + { + _inner = inner; + _gate = gate; + } + + public async Task> Handle( + CreateOrderCommand request, + CancellationToken cancellationToken) + { + await _gate.WaitIfFirstAsync().ConfigureAwait(false); + return await _inner.Handle(request, cancellationToken).ConfigureAwait(false); + } + } + + private sealed record LeaseCall( + string Behavior, + string IdempotencyKey, + int HttpStatus, + Guid OrderId, + bool CachedResponseHeader, + int MediatorSpanCount, + int NpgsqlSpanCount, + string TraceId, + double ClientDurationMs, + string Body); + + private sealed record TimelineEvent(DateTimeOffset AtUtc, string Phase, string Detail); + + private sealed record LeaseObservation( + bool CachePresent, + string? Status, + DateTimeOffset? ProcessingExpiresAtUtc, + double WaitedMs); +} diff --git a/tests/Lab/IntegrationTests/Experiments/MafMcpPrototype/MafMcpPrototypeTests.cs b/tests/Lab/IntegrationTests/Experiments/MafMcpPrototype/MafMcpPrototypeTests.cs new file mode 100644 index 0000000..c044d3a --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/MafMcpPrototype/MafMcpPrototypeTests.cs @@ -0,0 +1,407 @@ +using System.Diagnostics; +using System.Text.Json; +using BuildingBlocks.Mcp; +using EventBusRabbitMQ.Infrastructure; +using FeatureFusion.Features.Order.IntegrationEvents.Events; +using FeatureFusion.Infrastructure.Context; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Agent; +using IntegrationTests.Infrastructure.Async; +using IntegrationTests.Infrastructure.Mcp; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.Agents.AI; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using ModelContextProtocol.Client; +using OpenAI; +using OpenAI.Chat; +using System.ClientModel; +using Xunit.Abstractions; +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Experiments.MafMcpPrototype; + +/// +/// MAF prototype spike (not Exp 15): a real Microsoft Agent Framework agent in front of the existing +/// FeatureFusion /mcp endpoint. Characterizes whether MAF can drive observable MCP tool sequences +/// against the production Lab application without changing BuildingBlocks.Mcp. +/// +[Collection(AspireCollection.Name)] +public sealed class MafMcpPrototypeTests +{ + private const string Goal = + "Create an order for product 1, quantity 2, for customer 1."; + + private const string AgentInstructions = + """ + You are a lab agent that completes tasks using only the MCP tools provided. + When a write tool requires idempotencyKey, generate a new UUID/ULID for each distinct operation. + When a write tool requires confirmed=true, set it before invoking the tool. + Prefer the smallest number of tool calls needed to complete the goal. + """; + + private const string OrdersCreateTool = "orders.create"; + private const string OrderCreatedEventType = nameof(OrderCreatedIntegrationEvent); + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly AspireFixture _fixture; + private readonly IServiceProvider _services; + private readonly HttpClient _http; + private readonly ITestOutputHelper _output; + + public MafMcpPrototypeTests(AspireFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + _services = fixture.Services; + _http = fixture.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + } + + [Fact] + public async Task Maf_mcp_transport_connects_and_lists_featurefusion_tools() + { + await using var mcp = await LabMcpClient.CreateAsync(_http); + var tools = await mcp.ListToolsAsync(); + var names = tools.Select(t => t.Name).ToArray(); + + names.Should().Contain(["demo.echo", "products.list", OrdersCreateTool, "lab.ping"]); + } + + [Fact] + public async Task Maf_agent_runs_goal_and_records_observed_tool_sequence() + { + if (!TryCreateChatClient(out var chatClient, out var provider, out var model, out var skipReason)) + { + _output.WriteLine(skipReason); + return; + } + + const int runCount = 3; + _fixture.ProcessedEvents.Clear(); + + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var (transportTraceId, transportSpanId) = NewTraceParent(); + _http.DefaultRequestHeaders.Remove("traceparent"); + _http.DefaultRequestHeaders.TryAddWithoutValidation( + "traceparent", + FormatTraceParent(transportTraceId, transportSpanId)); + + await using var mcp = await LabMcpClient.CreateAsync(_http); + var mcpTools = await mcp.ListToolsAsync(); + + var agent = chatClient.AsAIAgent( + name: "FeatureFusionLabAgent", + instructions: AgentInstructions, + tools: [.. mcpTools.Cast()]); + + var runs = new List(); + for (var i = 0; i < runCount; i++) + { + capture.Clear(); + var runStarted = DateTimeOffset.UtcNow; + var runStopwatch = Stopwatch.StartNew(); + var session = await agent.CreateSessionAsync(); + using var runCts = new CancellationTokenSource(TimeSpan.FromMinutes(4)); + var response = await agent.RunAsync(Goal, session, cancellationToken: runCts.Token); + runStopwatch.Stop(); + + var toolCalls = MafAgentRunJournal.ReadToolCalls(response); + var sequence = toolCalls.Select(c => c.ToolName).ToArray(); + var orderId = await TryResolveOrderIdAsync(toolCalls, response.Text); + var outboxRows = orderId == Guid.Empty + ? [] + : await WaitForOutboxRowsAsync(orderId); + + if (orderId != Guid.Empty) + { + await Wait.UntilAsync( + () => _fixture.ProcessedEvents.Any(e => e.OrderId == orderId), + TimeSpan.FromSeconds(20)); + } + + var mcpToolSpans = capture.All + .Where(s => s.DisplayName == "mcp.tool") + .ToList(); + + var ordersCreateSpans = mcpToolSpans + .Where(s => HasToolTag(s, OrdersCreateTool)) + .ToList(); + + runs.Add(new MafPrototypeRun( + RunNumber: i + 1, + StartedUtc: runStarted, + DurationMs: runStopwatch.ElapsedMilliseconds, + FinalText: response.Text, + ToolCallCount: toolCalls.Count, + ToolSequence: sequence, + ToolCalls: toolCalls, + OrderId: orderId == Guid.Empty ? null : orderId, + BusinessOperationSucceeded: orderId != Guid.Empty, + OutboxRowCount: outboxRows.Count, + ProcessedEventCount: orderId == Guid.Empty + ? 0 + : _fixture.ProcessedEvents.Count(e => e.OrderId == orderId), + McpToolSpanCount: mcpToolSpans.Count, + OrdersCreateSpanCount: ordersCreateSpans.Count, + MediatorSpanCount: capture.All.Count(IsMediator), + NpgsqlSpanCount: capture.All.Count(IsNpgsql))); + } + + var artifact = new + { + name = "maf-mcp-prototype-v1", + startedUtc, + gitSha = LabRunInfo.ReadGitSha(), + environment = "Development", + configuration = new + { + provider, + model, + goal = Goal, + systemInstructions = AgentInstructions, + runCount, + mcpEndpoint = "/mcp", + transportTraceId = transportTraceId.ToHexString() + }, + runs, + observations = new + { + Sequences = runs.Select(r => r.ToolSequence).ToArray(), + AllRunsSucceeded = runs.All(r => r.BusinessOperationSucceeded), + SequenceVariance = DescribeSequenceVariance(runs) + }, + notes = new[] + { + "Prototype spike only — not a numbered Lab experiment.", + "BuildingBlocks.Mcp and production application code are unchanged.", + "Agent chooses tool order; sequences are recorded as observed." + } + }; + + _output.WriteLine(JsonSerializer.Serialize(artifact, JsonOptions)); + + runs.Should().NotBeEmpty("at least one MAF run should be recorded when a chat provider is configured"); + runs.Should().OnlyContain( + r => r.ToolCallCount > 0, + "each MAF run should invoke at least one MCP tool"); + + // Characterize outcome; do not assert a specific tool order. + var succeeded = runs.Count(r => r.BusinessOperationSucceeded); + _output.WriteLine($"MAF prototype: {succeeded}/{runs.Count} runs produced an order."); + } + + private static bool TryCreateChatClient( + out ChatClient chatClient, + out string provider, + out string model, + out string skipReason) + { + chatClient = null!; + provider = ""; + model = ""; + skipReason = ""; + + var openAiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY"); + if (!string.IsNullOrWhiteSpace(openAiKey)) + { + model = Environment.GetEnvironmentVariable("OPENAI_MODEL") ?? "gpt-4o-mini"; + provider = "OpenAI"; + chatClient = new OpenAIClient(openAiKey).GetChatClient(model); + return true; + } + + var azureEndpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT"); + var azureKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_API_KEY"); + if (!string.IsNullOrWhiteSpace(azureEndpoint) && !string.IsNullOrWhiteSpace(azureKey)) + { + model = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + provider = "AzureOpenAI"; + chatClient = new OpenAIClient( + new ApiKeyCredential(azureKey), + new OpenAIClientOptions { Endpoint = new Uri(azureEndpoint) }) + .GetChatClient(model); + return true; + } + + var ollamaBase = Environment.GetEnvironmentVariable("OLLAMA_BASE_URL"); + var ollamaModel = Environment.GetEnvironmentVariable("OLLAMA_MODEL"); + if (!string.IsNullOrWhiteSpace(ollamaBase) && !string.IsNullOrWhiteSpace(ollamaModel)) + { + model = ollamaModel; + provider = "Ollama"; + chatClient = new OpenAIClient( + new ApiKeyCredential("ollama"), + new OpenAIClientOptions { Endpoint = ToOpenAiCompatibleEndpoint(ollamaBase) }) + .GetChatClient(model); + return true; + } + + skipReason = + "SKIP: Set OPENAI_API_KEY, AZURE_OPENAI_ENDPOINT + AZURE_OPENAI_API_KEY, or OLLAMA_BASE_URL + OLLAMA_MODEL for live MAF agent runs."; + return false; + } + + private static Uri ToOpenAiCompatibleEndpoint(string baseUrl) + { + var trimmed = baseUrl.TrimEnd('/'); + if (!trimmed.EndsWith("/v1", StringComparison.OrdinalIgnoreCase)) + trimmed += "/v1"; + + return new Uri(trimmed); + } + + private static string DescribeSequenceVariance(IReadOnlyList runs) + { + if (runs.Count <= 1) + return "single-run"; + + var first = string.Join("→", runs[0].ToolSequence); + if (runs.All(r => string.Join("→", r.ToolSequence) == first)) + return "identical-sequences"; + + var sameLength = runs.Select(r => r.ToolSequence.Count).Distinct().Count() == 1; + return sameLength ? "partially-different-sequences" : "substantially-different-sequences"; + } + + private async Task TryResolveOrderIdAsync( + IReadOnlyList toolCalls, + string? finalText) + { + foreach (var call in toolCalls.Where(c => c.ToolName == OrdersCreateTool).Reverse()) + { + var fromResult = TryParseOrderId(call.ResultJson); + if (fromResult != Guid.Empty) + return fromResult; + } + + return TryParseOrderId(finalText); + } + + private static Guid TryParseOrderId(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + return Guid.Empty; + + try + { + using var doc = JsonDocument.Parse(text); + if (TryReadOrderId(doc.RootElement, out var orderId)) + return orderId; + } + catch (JsonException) + { + // fall through to regex + } + + const string guidPattern = + @"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"; + var match = System.Text.RegularExpressions.Regex.Match(text, guidPattern); + return match.Success && Guid.TryParse(match.Value, out var parsed) ? parsed : Guid.Empty; + } + + private static bool TryReadOrderId(JsonElement element, out Guid orderId) + { + orderId = Guid.Empty; + if (element.ValueKind == JsonValueKind.Object) + { + if (element.TryGetProperty("orderId", out var direct) + && direct.TryGetGuid(out orderId)) + return true; + + if (element.TryGetProperty("value", out var value) + && TryReadOrderId(value, out orderId)) + return true; + } + + return false; + } + + private async Task> WaitForOutboxRowsAsync(Guid orderId) + { + var stopwatch = Stopwatch.StartNew(); + while (stopwatch.Elapsed < TimeSpan.FromSeconds(20)) + { + var rows = await FindOutboxRowsForOrderIdAsync(orderId); + if (rows.Count > 0) + return rows; + + await Task.Delay(100); + } + + return []; + } + + private async Task> FindOutboxRowsForOrderIdAsync(Guid orderId) + { + await using var scope = _services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var serializerOptions = scope.ServiceProvider + .GetRequiredService>().Value.JsonSerializerOptions; + + var rows = await db.OutboxMessages + .AsNoTracking() + .Where(m => m.EventType == OrderCreatedEventType) + .ToListAsync(); + + var matches = new List(); + foreach (var row in rows) + { + if (row.Payload is null) + continue; + + var evt = JsonSerializer.Deserialize( + row.Payload, + serializerOptions); + + if (evt is null || evt.OrderId != orderId) + continue; + + matches.Add(new OutboxRowObservation( + row.Id, + evt.Id, + evt.OrderId, + row.Status.ToString())); + } + + return matches; + } + + private static bool HasToolTag(CapturedActivity span, string toolName) => + span.Tags.TryGetValue("mcp.tool.name", out var name) && name == toolName; + + private sealed record OutboxRowObservation( + Guid OutboxMessageId, + Guid IntegrationEventId, + Guid OrderId, + string Status); + + private sealed record MafPrototypeRun( + int RunNumber, + DateTimeOffset StartedUtc, + long DurationMs, + string? FinalText, + int ToolCallCount, + IReadOnlyList ToolSequence, + IReadOnlyList ToolCalls, + Guid? OrderId, + bool BusinessOperationSucceeded, + int OutboxRowCount, + int ProcessedEventCount, + int McpToolSpanCount, + int OrdersCreateSpanCount, + int MediatorSpanCount, + int NpgsqlSpanCount); +} diff --git a/tests/Lab/IntegrationTests/Experiments/McpAgentKeyRegeneration/McpAgentKeyRegenerationExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/McpAgentKeyRegeneration/McpAgentKeyRegenerationExperimentTests.cs new file mode 100644 index 0000000..0667de4 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/McpAgentKeyRegeneration/McpAgentKeyRegenerationExperimentTests.cs @@ -0,0 +1,338 @@ +using System.Diagnostics; +using System.Text.Json; +using BuildingBlocks.Mcp; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Async; +using IntegrationTests.Infrastructure.Mcp; +using IntegrationTests.Infrastructure.Orders; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using ModelContextProtocol.Client; +using Xunit.Abstractions; +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Experiments.McpAgentKeyRegeneration; + +/// +/// Experiment 13: agent write retry with regenerated MCP idempotency keys. +/// Hypothesis: when an unreliable agent loses a successful orders.create result and +/// retries the same logical intent with new idempotency keys, how much real business-side +/// amplification occurs? Same-key replay should not amplify; regenerated keys are expected +/// to create additional valid orders under the current MCP idempotency contract. +/// Does not prove HTTP Redis idempotency (Exp 3 / BuildingBlocks.Idempotency). Builds on +/// Exp 6 MCP semantics and Exp 10 outbox/handler observation patterns. +/// +[Collection(AspireCollection.Name)] +public sealed class McpAgentKeyRegenerationExperimentTests +{ + private const string ToolName = "orders.create"; + private const int ProductId = 1; + private const int CustomerId = 1; + private const int Quantity = 2; + + private static readonly TimeSpan ReplayObservationWindow = TimeSpan.FromSeconds(5); + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly AspireFixture _fixture; + private readonly IServiceProvider _services; + private readonly HttpClient _http; + private readonly ITestOutputHelper _output; + + public McpAgentKeyRegenerationExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + _services = fixture.Services; + _http = fixture.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + } + + [Fact] + public async Task Agent_regenerated_idempotency_keys_amplify_mcp_writes_and_downstream_work() + { + _fixture.ProcessedEvents.Clear(); + + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var (transportTraceId, transportSpanId) = NewTraceParent(); + _http.DefaultRequestHeaders.Remove("traceparent"); + _http.DefaultRequestHeaders.TryAddWithoutValidation( + "traceparent", + FormatTraceParent(transportTraceId, transportSpanId)); + + await using var mcp = await LabMcpClient.CreateAsync(_http); + var seenToolTraces = new HashSet(StringComparer.Ordinal); + var calls = new List(); + + var k1 = System.Ulid.NewUlid().ToString(); + var k2 = System.Ulid.NewUlid().ToString(); + var k3 = System.Ulid.NewUlid().ToString(); + + var missK1 = await CallAsync( + mcp, capture, seenToolTraces, calls, + behavior: "K1_ConfirmedMiss", + idempotencyKey: k1); + + var replayK1 = await CallAsync( + mcp, capture, seenToolTraces, calls, + behavior: "K1_SameKeyReplay", + idempotencyKey: k1); + + var agentRetryK2 = await CallAsync( + mcp, capture, seenToolTraces, calls, + behavior: "K2_AgentRegeneratedKey", + idempotencyKey: k2); + + var agentRetryK3 = await CallAsync( + mcp, capture, seenToolTraces, calls, + behavior: "K3_AgentRegeneratedKey", + idempotencyKey: k3); + + var distinctOrderIds = calls + .Where(c => !c.IsError && c.OrderId != Guid.Empty) + .Select(c => c.OrderId) + .Distinct() + .ToList(); + + foreach (var orderId in distinctOrderIds) + { + await OrderOutboxObserver.WaitUntilExistsAsync(_services, orderId); + await Wait.UntilAsync( + () => _fixture.ProcessedEvents.Any(e => e.OrderId == orderId), + TimeSpan.FromSeconds(20)); + } + + var replayCompletedUtc = DateTimeOffset.UtcNow; + await Wait.UntilAsync( + () => DateTimeOffset.UtcNow - replayCompletedUtc >= ReplayObservationWindow, + TimeSpan.FromSeconds(10)); + + var outboxByOrder = new Dictionary(); + foreach (var orderId in distinctOrderIds) + outboxByOrder[orderId] = (await OrderOutboxObserver.FindByOrderIdAsync(_services, orderId)).Count; + + var processedByOrder = distinctOrderIds.ToDictionary( + id => id, + id => _fixture.ProcessedEvents.Count(e => e.OrderId == id)); + + // AspireFixture.ProcessedEvents is collection-scoped. Clear() drops prior list entries but + // OutBoxWorker / RabbitMQ may still deliver OrderCreated events from earlier experiments + // into the same list. Amplification evidence is owned OrderIds only — not global Count. + var ownedProcessedEvents = _fixture.ProcessedEvents + .Where(e => distinctOrderIds.Contains(e.OrderId)) + .ToList(); + var foreignProcessedEventCount = _fixture.ProcessedEvents.Count - ownedProcessedEvents.Count; + + var productionCalls = calls.Where(c => c.SawNewToolSpan).ToList(); + var mediatorExecutions = calls.Sum(c => c.MediatorSpanCount); + var npgsqlExecutions = calls.Sum(c => c.NpgsqlSpanCount); + + var result = new AgentKeyRegenerationExperimentResult( + Name: "mcp-orders-create-agent-key-regeneration-v1", + StartedUtc: startedUtc, + GitSha: LabRunInfo.ReadGitSha(), + Environment: "Development", + Configuration: new AgentKeyRegenerationConfiguration( + ToolName, + McpDefaults.ConfirmedArgument, + McpDefaults.IdempotencyKeyArgument, + Store: "MemoryIdempotencyStore (in-process; distinct from HTTP Redis idempotency)", + AsyncPath: "orders.create → McpInvoker → CreateOrderCommandHandler → outbox_messages → OutBoxWorker → handler", + ComparedTo: "Experiment 6 (MCP idempotency envelope); Experiment 10 (outbox/handler observation)"), + Calls: calls, + Observations: new AgentKeyRegenerationObservations( + McpCallCount: calls.Count, + ProductionToolSpanCount: productionCalls.Count, + DistinctOrderIds: distinctOrderIds, + MediatorExecutionCount: mediatorExecutions, + NpgsqlExecutionCount: npgsqlExecutions, + OutboxRowCountByOrderId: outboxByOrder, + ProcessedEventCountByOrderId: processedByOrder, + TotalProcessedEvents: ownedProcessedEvents.Count, + ForeignProcessedEventCount: foreignProcessedEventCount, + K1OrderId: missK1.OrderId, + ReplaySameOrderAsK1: replayK1.OrderId == missK1.OrderId, + K2DifferentOrderFromK1: agentRetryK2.OrderId != missK1.OrderId, + K3DifferentOrderFromK1AndK2: agentRetryK3.OrderId != missK1.OrderId + && agentRetryK3.OrderId != agentRetryK2.OrderId, + AmplificationFactor: distinctOrderIds.Count, + Notes: + [ + "One logical agent intent ('create this order') simulated with K1, then agent retries with K2/K3 after losing the result.", + "Regenerated-key retries are client behavior against the idempotency-key contract — not an idempotency bug.", + $"Transport TraceId={transportTraceId.ToHexString()}; tool traces tracked separately.", + "ProcessedEvents is test-only handler observation via TestEventHandlerDecorator.", + "TotalProcessedEvents counts handlers for this experiment's OrderIds only; ForeignProcessedEventCount is late suite contamination on the shared fixture list." + ])); + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + missK1.IsError.Should().BeFalse(missK1.Error); + missK1.OrderId.Should().NotBeEmpty(); + missK1.SawNewToolSpan.Should().BeTrue(); + missK1.MediatorSpanCount.Should().BeGreaterThan(0); + missK1.NpgsqlSpanCount.Should().BeGreaterThan(0); + + replayK1.IsError.Should().BeFalse(replayK1.Error); + replayK1.OrderId.Should().Be(missK1.OrderId); + replayK1.SawNewToolSpan.Should().BeFalse( + "same-key replay should return cached MCP result without a new mcp.tool span"); + replayK1.MediatorSpanCount.Should().Be(0); + replayK1.NpgsqlSpanCount.Should().Be(0); + + agentRetryK2.IsError.Should().BeFalse(agentRetryK2.Error); + agentRetryK2.OrderId.Should().NotBe(missK1.OrderId); + agentRetryK2.SawNewToolSpan.Should().BeTrue(); + agentRetryK2.MediatorSpanCount.Should().BeGreaterThan(0); + agentRetryK2.NpgsqlSpanCount.Should().BeGreaterThan(0); + + agentRetryK3.IsError.Should().BeFalse(agentRetryK3.Error); + agentRetryK3.OrderId.Should().NotBe(missK1.OrderId); + agentRetryK3.OrderId.Should().NotBe(agentRetryK2.OrderId); + agentRetryK3.SawNewToolSpan.Should().BeTrue(); + agentRetryK3.MediatorSpanCount.Should().BeGreaterThan(0); + agentRetryK3.NpgsqlSpanCount.Should().BeGreaterThan(0); + + distinctOrderIds.Should().HaveCount(3, + "three production writes (K1, K2, K3) should yield three distinct orders. Ids: {0}", + string.Join(",", distinctOrderIds)); + + productionCalls.Should().HaveCount(3, + "only K1, K2, and K3 should start new mcp.tool spans; K1 replay should not"); + + productionCalls.Sum(c => c.MediatorSpanCount).Should().Be(3); + productionCalls.Should().OnlyContain(c => c.MediatorSpanCount >= 1 && c.NpgsqlSpanCount >= 1, + "each production tool call should reach Mediator and Npgsql at least once"); + productionCalls.Sum(c => c.NpgsqlSpanCount).Should().BeGreaterThanOrEqualTo(3, + "K1 may emit more than one Npgsql span per production path (catalog + outbox); total observed was {0}", + productionCalls.Sum(c => c.NpgsqlSpanCount)); + + foreach (var orderId in distinctOrderIds) + { + outboxByOrder[orderId].Should().Be(1, + "each production order should persist exactly one outbox row. OrderId={0}", orderId); + processedByOrder[orderId].Should().Be(1, + "each production order should be observed once by ProcessedEvents. OrderId={0}", orderId); + } + + ownedProcessedEvents.Should().HaveCount(3, + "exactly three handler observations for this experiment's OrderIds (global ProcessedEvents.Count can include late deliveries from earlier suite tests). Owned={0}; foreign={1}; global={2}", + ownedProcessedEvents.Count, + foreignProcessedEventCount, + _fixture.ProcessedEvents.Count); + processedByOrder[missK1.OrderId].Should().Be(1, + "K1 replay and later regenerated-key writes must not duplicate handler work for the first order"); + } + + private async Task CallAsync( + McpClient mcp, + InProcessActivityCapture capture, + HashSet seenToolTraces, + List calls, + string behavior, + string idempotencyKey) + { + var args = new Dictionary + { + ["productId"] = ProductId, + ["quantity"] = Quantity, + ["customerId"] = CustomerId, + [McpDefaults.IdempotencyKeyArgument] = idempotencyKey, + [McpDefaults.ConfirmedArgument] = true + }; + + var clock = Stopwatch.StartNew(); + var result = await mcp.CallToolAsync(ToolName, args); + clock.Stop(); + + var isError = result.IsError ?? false; + var errorText = isError ? McpToolResults.Truncate(McpToolResults.GetText(result)) : null; + var order = !isError ? McpToolResults.TryParseOrder(result, JsonOptions) : null; + + var toolSpan = capture.All.FirstOrDefault(s => + s.DisplayName == "mcp.tool" + && HasToolTag(s, ToolName) + && seenToolTraces.Add(s.TraceId)); + + var toolTrace = toolSpan?.TraceId; + var related = toolTrace is null + ? [] + : capture.All.Where(s => s.TraceId == toolTrace).ToList(); + + var call = new AgentKeyRegenerationCall( + RequestNumber: calls.Count + 1, + Behavior: behavior, + IdempotencyKey: idempotencyKey, + IsError: isError, + Error: errorText, + OrderId: order?.OrderId ?? Guid.Empty, + QuantityReturned: order?.Quantity, + ClientDurationMs: clock.ElapsedMilliseconds, + ToolTraceId: toolTrace, + SawNewToolSpan: toolSpan is not null, + MediatorSpanCount: related.Count(IsMediator), + NpgsqlSpanCount: related.Count(IsNpgsql)); + + calls.Add(call); + return call; + } + + private static bool HasToolTag(CapturedActivity span, string toolName) => + span.Tags.TryGetValue("mcp.tool.name", out var name) && name == toolName; + + private sealed record AgentKeyRegenerationCall( + int RequestNumber, + string Behavior, + string IdempotencyKey, + bool IsError, + string? Error, + Guid OrderId, + int? QuantityReturned, + double ClientDurationMs, + string? ToolTraceId, + bool SawNewToolSpan, + int MediatorSpanCount, + int NpgsqlSpanCount); + + private sealed record AgentKeyRegenerationConfiguration( + string Tool, + string ConfirmedArgument, + string IdempotencyKeyArgument, + string Store, + string AsyncPath, + string ComparedTo); + + private sealed record AgentKeyRegenerationObservations( + int McpCallCount, + int ProductionToolSpanCount, + IReadOnlyList DistinctOrderIds, + int MediatorExecutionCount, + int NpgsqlExecutionCount, + IReadOnlyDictionary OutboxRowCountByOrderId, + IReadOnlyDictionary ProcessedEventCountByOrderId, + int TotalProcessedEvents, + int ForeignProcessedEventCount, + Guid K1OrderId, + bool ReplaySameOrderAsK1, + bool K2DifferentOrderFromK1, + bool K3DifferentOrderFromK1AndK2, + int AmplificationFactor, + IReadOnlyList Notes); + + private sealed record AgentKeyRegenerationExperimentResult( + string Name, + DateTimeOffset StartedUtc, + string GitSha, + string Environment, + AgentKeyRegenerationConfiguration Configuration, + IReadOnlyList Calls, + AgentKeyRegenerationObservations Observations); +} diff --git a/tests/Lab/IntegrationTests/Experiments/McpConcurrentSameKey/McpConcurrentSameKeyExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/McpConcurrentSameKey/McpConcurrentSameKeyExperimentTests.cs new file mode 100644 index 0000000..93f5ba2 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/McpConcurrentSameKey/McpConcurrentSameKeyExperimentTests.cs @@ -0,0 +1,299 @@ +using System.Diagnostics; +using System.Collections.Concurrent; +using System.Text.Json; +using BuildingBlocks.Mcp; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Async; +using IntegrationTests.Infrastructure.Mcp; +using IntegrationTests.Infrastructure.Orders; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using ModelContextProtocol.Client; +using ModelContextProtocol.Protocol; +using Xunit.Abstractions; + +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Experiments.McpConcurrentSameKey; + +/// +/// Experiment 14: concurrent agent behavior — the same MCP orders.create write sent concurrently +/// with identical arguments and a shared idempotency key (K1). +/// Hypothesis: MCP idempotency + in-flight gating results in exactly one production business operation +/// (one order, one outbox row, one downstream handler observation), with other callers replaying. +/// This experiment characterizes behavior; it does not change MCP or any BuildingBlock. +/// +[Collection(AspireCollection.Name)] +public sealed class McpConcurrentSameKeyExperimentTests +{ + private const string ToolName = "orders.create"; + private const int ProductId = 1; + private const int CustomerId = 1; + private const int Quantity = 2; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly AspireFixture _fixture; + private readonly IServiceProvider _services; + private readonly HttpClient _http; + private readonly ITestOutputHelper _output; + + public McpConcurrentSameKeyExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + _services = fixture.Services; + _http = fixture.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + } + + [Fact] + public async Task Concurrent_same_key_mcp_write_produces_exactly_one_business_operation() + { + _fixture.ProcessedEvents.Clear(); + + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var (transportTraceId, transportSpanId) = NewTraceParent(); + _http.DefaultRequestHeaders.Remove("traceparent"); + _http.DefaultRequestHeaders.TryAddWithoutValidation( + "traceparent", + FormatTraceParent(transportTraceId, transportSpanId)); + + var k1 = System.Ulid.NewUlid().ToString(); + var seenToolTraces = new ConcurrentDictionary(StringComparer.Ordinal); + var transportTraceIdHex = transportTraceId.ToHexString(); + var calls = await StartTogether( + count: 3, + send: i => CallOnceAsync( + requestNumber: i + 1, + behavior: $"ConcurrentCall_{i + 1}", + idempotencyKey: k1, + capture, + transportTraceIdHex, + seenToolTraces)); + + var distinctOrderIds = calls + .Where(c => !c.IsError && c.OrderId != Guid.Empty) + .Select(c => c.OrderId) + .Distinct() + .ToList(); + + // Tool-level production execution: only the first concurrent in-flight winner should execute mcp.tool. + var toolTraces = capture.All + .Where(s => s.DisplayName == "mcp.tool" && HasToolTag(s, ToolName)) + .Select(s => s.TraceId) + .Distinct() + .ToList(); + + var productionToolExecutionCount = toolTraces.Count; + + // If production is exactly-once for this write key, all successful callers should get the same orderId. + var expectedSingleOrder = distinctOrderIds.SingleOrDefault(); + calls.Where(c => !c.IsError).Select(c => c.OrderId).Distinct().Count().Should().Be(distinctOrderIds.Count, + "orderId uniqueness should match distinctOrderIds derived from successful callers"); + + // Outbox + downstream handler evidence. + var outboxByOrderId = new Dictionary>(); + var processedCountByOrderId = new Dictionary(); + + foreach (var orderId in distinctOrderIds) + { + var outboxRows = MapObservations(await OrderOutboxObserver.WaitUntilAnyAsync(_services, orderId)); + outboxByOrderId[orderId] = outboxRows; + + await Wait.UntilAsync( + () => _fixture.ProcessedEvents.Any(e => e.OrderId == orderId), + TimeSpan.FromSeconds(20)); + + processedCountByOrderId[orderId] = _fixture.ProcessedEvents.Count(e => e.OrderId == orderId); + } + + var totalProcessedEvents = _fixture.ProcessedEvents.Count; + var totalOutboxRows = outboxByOrderId.Values.Sum(list => list.Count); + + var result = new + { + name = "mcp-orders-create-concurrent-same-key-v1", + startedUtc, + gitSha = LabRunInfo.ReadGitSha(), + environment = "Development", + configuration = new + { + Tool = ToolName, + ConfirmedArgument = McpDefaults.ConfirmedArgument, + IdempotencyKeyArgument = McpDefaults.IdempotencyKeyArgument, + Concurrency = "3 parallel calls", + SharedIdempotencyKey = k1, + Store = "MemoryIdempotencyStore (in-process)", + AsyncPath = "orders.create → McpInvoker → CreateOrderCommandHandler → outbox → OutBoxWorker → handler", + }, + transport = new + { + traceparent = transportTraceId.ToHexString(), + toolExecutionCount = productionToolExecutionCount + }, + calls, + observations = new + { + DistinctOrderIds = distinctOrderIds, + OutboxRowCountByOrderId = outboxByOrderId.ToDictionary( + kvp => kvp.Key.ToString(), + kvp => kvp.Value.Count), + ProcessedEventCountByOrderId = processedCountByOrderId.ToDictionary( + kvp => kvp.Key.ToString(), + kvp => kvp.Value), + TotalOutboxRows = totalOutboxRows, + TotalProcessedEvents = totalProcessedEvents + } + }; + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + // Critical assertions. + calls.Should().OnlyContain(c => !c.IsError, "concurrent idempotency waiters should replay, not fail"); + distinctOrderIds.Should().HaveCount(1, "concurrent same-key MCP calls should converge to a single business order. Orders: {0}", + string.Join(",", distinctOrderIds)); + + productionToolExecutionCount.Should().Be(1, + "only one concurrent invoker should execute mcp.tool for a shared idempotency key"); + + var winnerOrderId = expectedSingleOrder; + outboxByOrderId.Should().ContainKey(winnerOrderId); + outboxByOrderId[winnerOrderId].Should().HaveCount(1, "one production order should persist exactly one outbox row. OrderId={0}", winnerOrderId); + + processedCountByOrderId.Should().ContainKey(winnerOrderId); + processedCountByOrderId[winnerOrderId].Should().Be(1, + "downstream handler observation should happen exactly once for the winner order. OrderId={0}", winnerOrderId); + + // Compare guidance (no code change): Exp 4 is HTTP/Redis, Exp 6 is same-key replay, Exp 13 is regenerated-key amplification. + // If this fails, MCP concurrency behavior is inconsistent with already-proven expectations. + } + + private async Task CallOnceAsync( + int requestNumber, + string behavior, + string idempotencyKey, + InProcessActivityCapture capture, + string transportTraceId, + ConcurrentDictionary seenToolTraces) + { + // Create a new MCP client per task to avoid client concurrency assumptions. + await using var mcp = await LabMcpClient.CreateAsync(_http); + + var args = new Dictionary + { + ["productId"] = ProductId, + ["quantity"] = Quantity, + ["customerId"] = CustomerId, + [McpDefaults.IdempotencyKeyArgument] = idempotencyKey, + [McpDefaults.ConfirmedArgument] = true + }; + + var clock = Stopwatch.StartNew(); + var result = await mcp.CallToolAsync(ToolName, args); + clock.Stop(); + + var isError = result.IsError ?? false; + var errorCode = isError ? TryReadErrorCode(result) : null; + var errorText = isError ? McpToolResults.Truncate(McpToolResults.GetText(result)) : null; + var order = !isError ? McpToolResults.TryParseOrder(result, JsonOptions) : null; + + // Claim the (at most) one production mcp.tool trace in a thread-safe way. + // In a successful same-key concurrent run, cached waiters should not execute mcp.tool, + // so they should remain SawNewToolSpan=false and have mediator/npdgsql counts of 0. + var toolSpan = capture.All.FirstOrDefault(s => + s.DisplayName == "mcp.tool" + && HasToolTag(s, ToolName) + && seenToolTraces.TryAdd(s.TraceId, 0)); + + var toolTraceId = toolSpan?.TraceId; + var related = toolTraceId is null + ? [] + : capture.All.Where(s => s.TraceId == toolTraceId).ToList(); + + return new McpConcurrentSameKeyCall( + RequestNumber: requestNumber, + Behavior: behavior, + IdempotencyKey: idempotencyKey, + IsError: isError, + ErrorCode: errorCode, + Error: errorText, + OrderId: order?.OrderId ?? Guid.Empty, + QuantityReturned: order?.Quantity, + ClientDurationMs: clock.ElapsedMilliseconds, + TransportTraceId: transportTraceId, + SawNewToolSpan: toolSpan is not null, + MediatorSpanCount: related.Count(IsMediator), + NpgsqlSpanCount: related.Count(IsNpgsql)); + } + + private static IReadOnlyList MapObservations(IReadOnlyList rows) => + rows.Select(row => new OutboxRowObservation( + OutboxMessageId: row.OutboxMessageId, + IntegrationEventId: row.IntegrationEventId, + OrderId: row.OrderId, + Status: row.Status, + ProcessedAtUtc: row.ProcessedAtUtc, + CompletedAtUtc: row.CompletedAtUtc)).ToList(); + + private static string? TryReadErrorCode(CallToolResult result) + { + var code = McpToolResults.TryReadJsonErrorCode(result); + if (code is not null) + return code; + + var text = McpToolResults.GetText(result); + return text.Contains("ConfirmationRequired", StringComparison.OrdinalIgnoreCase) + ? nameof(McpErrorCode.ConfirmationRequired) + : null; + } + + private static bool HasToolTag(CapturedActivity span, string toolName) => + span.Tags.TryGetValue("mcp.tool.name", out var name) && name == toolName; + + private static async Task StartTogether( + int count, + Func> send) + { + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var tasks = Enumerable.Range(0, count).Select(async i => + { + await gate.Task; + return await send(i); + }).ToArray(); + gate.SetResult(); + return await Task.WhenAll(tasks); + } + + private sealed record OutboxRowObservation( + Guid OutboxMessageId, + Guid IntegrationEventId, + Guid OrderId, + string Status, + DateTime? ProcessedAtUtc, + DateTime? CompletedAtUtc); + + private sealed record McpConcurrentSameKeyCall( + int RequestNumber, + string Behavior, + string IdempotencyKey, + bool IsError, + string? ErrorCode, + string? Error, + Guid OrderId, + int? QuantityReturned, + long ClientDurationMs, + string TransportTraceId, + bool SawNewToolSpan, + int MediatorSpanCount, + int NpgsqlSpanCount); +} + diff --git a/tests/Lab/IntegrationTests/Experiments/McpOrderIdempotency/McpOrderIdempotencyExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/McpOrderIdempotency/McpOrderIdempotencyExperimentTests.cs new file mode 100644 index 0000000..b33efbb --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/McpOrderIdempotency/McpOrderIdempotencyExperimentTests.cs @@ -0,0 +1,368 @@ +using System.Diagnostics; +using System.Text.Json; +using BuildingBlocks.Mcp; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Mcp; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using ModelContextProtocol.Client; +using ModelContextProtocol.Protocol; +using Xunit.Abstractions; +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Experiments.McpOrderIdempotency; + +/// +/// Experiment 6: MCP orders.create confirmation gate and in-memory idempotency envelope. +/// Hypothesis: the MCP surface uses and +/// RequireConfirmation semantics that differ from the HTTP Redis +/// IdempotentAttributeFilter path in Experiment 3. +/// Investigates confirmation gating, idempotency-key replay, replay bypass of +/// CreateOrderCommand, same-key/different-args behavior, and fresh execution on a new key. +/// Does not re-prove HTTP Redis idempotency (Exp 3), outbox/async handler delivery (Exp 5), +/// or gateway behavior. +/// +[Collection(AspireCollection.Name)] +public sealed class McpOrderIdempotencyExperimentTests +{ + private const string ToolName = "orders.create"; + private const int ProductId = 1; + private const int CustomerId = 1; + private const int BaselineQuantity = 2; + private const int MutatedQuantity = 5; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly HttpClient _http; + private readonly ITestOutputHelper _output; + + public McpOrderIdempotencyExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _output = output; + _http = fixture.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + } + + [Fact] + public async Task Mcp_orders_create_confirmation_and_memory_idempotency_are_observed() + { + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var (transportTraceId, transportSpanId) = NewTraceParent(); + _http.DefaultRequestHeaders.Remove("traceparent"); + _http.DefaultRequestHeaders.TryAddWithoutValidation( + "traceparent", + FormatTraceParent(transportTraceId, transportSpanId)); + + await using var mcp = await LabMcpClient.CreateAsync(_http); + var seenToolTraces = new HashSet(StringComparer.Ordinal); + var calls = new List(); + var cachedKey = System.Ulid.NewUlid().ToString(); + var freshKey = System.Ulid.NewUlid().ToString(); + + var requestNumber = 0; + + calls.Add(await CallAsync( + mcp, + capture, + seenToolTraces, + ++requestNumber, + behavior: "Unconfirmed", + idempotencyKey: System.Ulid.NewUlid().ToString(), + confirmed: false, + quantity: BaselineQuantity)); + + calls.Add(await CallAsync( + mcp, + capture, + seenToolTraces, + ++requestNumber, + behavior: "ConfirmedMiss", + idempotencyKey: cachedKey, + confirmed: true, + quantity: BaselineQuantity)); + + calls.Add(await CallAsync( + mcp, + capture, + seenToolTraces, + ++requestNumber, + behavior: "SameKeyReplay", + idempotencyKey: cachedKey, + confirmed: true, + quantity: BaselineQuantity)); + + calls.Add(await CallAsync( + mcp, + capture, + seenToolTraces, + ++requestNumber, + behavior: "SameKeyDifferentQuantity", + idempotencyKey: cachedKey, + confirmed: true, + quantity: MutatedQuantity)); + + calls.Add(await CallAsync( + mcp, + capture, + seenToolTraces, + ++requestNumber, + behavior: "NewKeyFresh", + idempotencyKey: freshKey, + confirmed: true, + quantity: MutatedQuantity)); + + var unconfirmed = calls[0]; + var miss = calls[1]; + var replay = calls[2]; + var mutated = calls[3]; + var fresh = calls[4]; + + var result = new McpOrderIdempotencyExperimentResult( + Name: "mcp-orders-create-idempotency-v1", + StartedUtc: startedUtc, + GitSha: LabRunInfo.ReadGitSha(), + Environment: "Development", + Configuration: new McpOrderIdempotencyConfiguration( + ToolName, + McpDefaults.ConfirmedArgument, + McpDefaults.IdempotencyKeyArgument, + Store: "MemoryIdempotencyStore (UseMemoryIdempotency, TTL 1h)", + CacheKey: "toolName + U+001F + idempotencyKey (body not included)", + Command: "CreateOrderCommand → CreateOrderCommandHandler"), + Calls: calls, + Observations: new McpOrderIdempotencyObservations( + UnconfirmedIsError: unconfirmed.IsError, + UnconfirmedErrorCode: unconfirmed.ErrorCode, + UnconfirmedSawNewToolSpan: unconfirmed.SawNewToolSpan, + UnconfirmedMediatorSpans: unconfirmed.MediatorSpanCount, + UnconfirmedNpgsqlSpans: unconfirmed.NpgsqlSpanCount, + MissIsError: miss.IsError, + MissOrderId: miss.OrderId, + MissQuantity: miss.Quantity, + MissSawNewToolSpan: miss.SawNewToolSpan, + MissMediatorSpans: miss.MediatorSpanCount, + MissNpgsqlSpans: miss.NpgsqlSpanCount, + ReplayIsError: replay.IsError, + ReplayOrderId: replay.OrderId, + ReplayQuantity: replay.Quantity, + ReplaySawNewToolSpan: replay.SawNewToolSpan, + ReplayMediatorSpans: replay.MediatorSpanCount, + ReplayNpgsqlSpans: replay.NpgsqlSpanCount, + ReplaySameOrderIdAsMiss: replay.OrderId == miss.OrderId && miss.OrderId != Guid.Empty, + MutatedIsError: mutated.IsError, + MutatedOrderId: mutated.OrderId, + MutatedQuantity: mutated.Quantity, + MutatedSawNewToolSpan: mutated.SawNewToolSpan, + MutatedMediatorSpans: mutated.MediatorSpanCount, + MutatedNpgsqlSpans: mutated.NpgsqlSpanCount, + MutatedKeptOriginalQuantity: mutated.Quantity == miss.Quantity, + FreshIsError: fresh.IsError, + FreshOrderId: fresh.OrderId, + FreshQuantity: fresh.Quantity, + FreshSawNewToolSpan: fresh.SawNewToolSpan, + FreshMediatorSpans: fresh.MediatorSpanCount, + FreshNpgsqlSpans: fresh.NpgsqlSpanCount, + FreshDifferentOrderId: fresh.OrderId != miss.OrderId && fresh.OrderId != Guid.Empty, + Notes: + [ + $"Transport TraceId={transportTraceId.ToHexString()}.", + $"Unconfirmed: isError={unconfirmed.IsError}, errorCode={unconfirmed.ErrorCode ?? "(none)"}, newToolSpan={unconfirmed.SawNewToolSpan}.", + $"ConfirmedMiss toolTrace={miss.ToolTraceId}; mediator={miss.MediatorSpanCount}; npgsql={miss.NpgsqlSpanCount}.", + $"SameKeyReplay toolTrace={replay.ToolTraceId ?? "(none)"}; newToolSpan={replay.SawNewToolSpan}; cached quantity={replay.Quantity}.", + $"SameKeyDifferentQuantity requested={MutatedQuantity}, returned quantity={mutated.Quantity}, orderId={mutated.OrderId}.", + $"NewKeyFresh orderId={fresh.OrderId}, quantity={fresh.Quantity}.", + "MCP idempotency cache key is namespaced by tool + idempotencyKey only; request body changes do not bust the cache." + ])); + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + unconfirmed.IsError.Should().BeTrue( + "orders.create requires confirmed=true before execution"); + unconfirmed.ErrorCode.Should().Be("ConfirmationRequired", + "McpInvoker rejects unconfirmed writes with {0}", unconfirmed.Error); + unconfirmed.SawNewToolSpan.Should().BeFalse( + "confirmation failure returns before InvokeCoreAsync / mcp.tool"); + unconfirmed.MediatorSpanCount.Should().Be(0); + unconfirmed.NpgsqlSpanCount.Should().Be(0); + + miss.IsError.Should().BeFalse(miss.Error); + miss.OrderId.Should().NotBeEmpty(); + miss.Quantity.Should().Be(BaselineQuantity); + miss.SawNewToolSpan.Should().BeTrue(); + miss.MediatorSpanCount.Should().BeGreaterThan(0, + "confirmed miss should dispatch CreateOrderCommand. Tool trace: {0}", + miss.ToolTraceId); + miss.NpgsqlSpanCount.Should().BeGreaterThan(0, + "confirmed miss should persist catalog/outbox. Tool trace: {0}", + miss.ToolTraceId); + + replay.IsError.Should().BeFalse(replay.Error); + replay.OrderId.Should().Be(miss.OrderId); + replay.Quantity.Should().Be(BaselineQuantity); + replay.SawNewToolSpan.Should().BeFalse( + "memory idempotency hit returns cached payload without starting mcp.tool"); + replay.MediatorSpanCount.Should().Be(0); + replay.NpgsqlSpanCount.Should().Be(0); + + mutated.IsError.Should().BeFalse(mutated.Error); + mutated.OrderId.Should().Be(miss.OrderId); + mutated.Quantity.Should().Be(BaselineQuantity, + "MemoryIdempotencyStore replays by tool+key, not by serialized command body"); + mutated.SawNewToolSpan.Should().BeFalse(); + mutated.MediatorSpanCount.Should().Be(0); + mutated.NpgsqlSpanCount.Should().Be(0); + + fresh.IsError.Should().BeFalse(fresh.Error); + fresh.OrderId.Should().NotBe(miss.OrderId); + fresh.Quantity.Should().Be(MutatedQuantity); + fresh.SawNewToolSpan.Should().BeTrue(); + fresh.MediatorSpanCount.Should().BeGreaterThan(0); + fresh.NpgsqlSpanCount.Should().BeGreaterThan(0); + } + + private async Task CallAsync( + McpClient mcp, + InProcessActivityCapture capture, + HashSet seenToolTraces, + int requestNumber, + string behavior, + string idempotencyKey, + bool confirmed, + int quantity) + { + var args = new Dictionary + { + ["productId"] = ProductId, + ["quantity"] = quantity, + ["customerId"] = CustomerId, + [McpDefaults.IdempotencyKeyArgument] = idempotencyKey + }; + if (confirmed) + args[McpDefaults.ConfirmedArgument] = true; + + var clock = Stopwatch.StartNew(); + var result = await mcp.CallToolAsync(ToolName, args); + clock.Stop(); + + var isError = result.IsError ?? false; + var errorText = isError ? McpToolResults.Truncate(McpToolResults.GetText(result)) : null; + var errorCode = isError ? TryReadErrorCode(result) : null; + var order = !isError ? McpToolResults.TryParseOrder(result, JsonOptions) : null; + + var toolSpan = capture.All.FirstOrDefault(s => + s.DisplayName == "mcp.tool" + && HasTag(s, "mcp.tool.name", ToolName) + && seenToolTraces.Add(s.TraceId)); + + var toolTrace = toolSpan?.TraceId; + var related = toolTrace is null + ? [] + : capture.All.Where(s => s.TraceId == toolTrace).ToList(); + + return new McpOrderIdempotencyCall( + requestNumber, + behavior, + idempotencyKey, + confirmed, + quantity, + isError, + errorCode, + errorText, + order?.OrderId ?? Guid.Empty, + order?.Quantity, + clock.ElapsedMilliseconds, + toolTrace, + toolSpan is not null, + related.Count(IsMediator), + related.Count(IsNpgsql)); + } + + private static string? TryReadErrorCode(CallToolResult result) + { + var code = McpToolResults.TryReadJsonErrorCode(result); + if (code is not null) + return code; + + var text = McpToolResults.GetText(result); + return text.Contains("ConfirmationRequired", StringComparison.OrdinalIgnoreCase) + ? nameof(McpErrorCode.ConfirmationRequired) + : null; + } + + private sealed record McpOrderIdempotencyCall( + int RequestNumber, + string Behavior, + string IdempotencyKey, + bool Confirmed, + int RequestedQuantity, + bool IsError, + string? ErrorCode, + string? Error, + Guid OrderId, + int? Quantity, + long ClientDurationMs, + string? ToolTraceId, + bool SawNewToolSpan, + int MediatorSpanCount, + int NpgsqlSpanCount); + + private sealed record McpOrderIdempotencyConfiguration( + string Tool, + string ConfirmedArgument, + string IdempotencyKeyArgument, + string Store, + string CacheKey, + string Command); + + private sealed record McpOrderIdempotencyObservations( + bool UnconfirmedIsError, + string? UnconfirmedErrorCode, + bool UnconfirmedSawNewToolSpan, + int UnconfirmedMediatorSpans, + int UnconfirmedNpgsqlSpans, + bool MissIsError, + Guid MissOrderId, + int? MissQuantity, + bool MissSawNewToolSpan, + int MissMediatorSpans, + int MissNpgsqlSpans, + bool ReplayIsError, + Guid ReplayOrderId, + int? ReplayQuantity, + bool ReplaySawNewToolSpan, + int ReplayMediatorSpans, + int ReplayNpgsqlSpans, + bool ReplaySameOrderIdAsMiss, + bool MutatedIsError, + Guid MutatedOrderId, + int? MutatedQuantity, + bool MutatedSawNewToolSpan, + int MutatedMediatorSpans, + int MutatedNpgsqlSpans, + bool MutatedKeptOriginalQuantity, + bool FreshIsError, + Guid FreshOrderId, + int? FreshQuantity, + bool FreshSawNewToolSpan, + int FreshMediatorSpans, + int FreshNpgsqlSpans, + bool FreshDifferentOrderId, + IReadOnlyList Notes); + + private sealed record McpOrderIdempotencyExperimentResult( + string Name, + DateTimeOffset StartedUtc, + string GitSha, + string Environment, + McpOrderIdempotencyConfiguration Configuration, + IReadOnlyList Calls, + McpOrderIdempotencyObservations Observations); +} diff --git a/tests/Lab/IntegrationTests/Experiments/McpOrderOutbox/McpOrderOutboxExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/McpOrderOutbox/McpOrderOutboxExperimentTests.cs new file mode 100644 index 0000000..0a9ce28 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/McpOrderOutbox/McpOrderOutboxExperimentTests.cs @@ -0,0 +1,552 @@ +using System.Diagnostics; +using System.Text.Json; +using BuildingBlocks.Mcp; +using FeatureFusion.Infrastructure.Context; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Async; +using IntegrationTests.Infrastructure.Mcp; +using IntegrationTests.Infrastructure.Orders; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using ModelContextProtocol.Client; +using ModelContextProtocol.Protocol; +using Xunit.Abstractions; +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Experiments.McpOrderOutbox; + +/// +/// Experiment 10: MCP confirmed orders.create → outbox → worker → RabbitMQ → inbox → handler parity. +/// Hypothesis: after a confirmed MCP orders.create cache miss, does the real production workflow +/// persist an outbox_messages row and eventually deliver the same +/// OrderCreatedIntegrationEvent through OutBoxWorker, RabbitMQ, inbox, and handler +/// as the HTTP workflow characterized in Experiment 8? +/// MCP idempotency replay (Exp 6) should not add outbox rows or handler observations. +/// AspireFixture.ProcessedEvents is test observation infrastructure, not production telemetry. +/// Does not claim parity merely because CreateOrderCommandHandler is shared — proves the observed pipeline. +/// +[Collection(AspireCollection.Name)] +public sealed class McpOrderOutboxExperimentTests +{ + private const string ToolName = "orders.create"; + private const int ProductId = 1; + private const int CustomerId = 1; + private const int Quantity = 2; + + private static readonly TimeSpan ReplayObservationWindow = TimeSpan.FromSeconds(5); + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly AspireFixture _fixture; + private readonly IServiceProvider _services; + private readonly HttpClient _http; + private readonly ITestOutputHelper _output; + + public McpOrderOutboxExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + _services = fixture.Services; + _http = fixture.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + } + + [Fact] + public async Task Mcp_confirmed_orders_create_follows_outbox_to_handler_pipeline_and_replay_skips_async_work() + { + _fixture.ProcessedEvents.Clear(); + + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var (transportTraceId, transportSpanId) = NewTraceParent(); + _http.DefaultRequestHeaders.Remove("traceparent"); + _http.DefaultRequestHeaders.TryAddWithoutValidation( + "traceparent", + FormatTraceParent(transportTraceId, transportSpanId)); + + await using var mcp = await LabMcpClient.CreateAsync(_http); + var seenToolTraces = new HashSet(StringComparer.Ordinal); + var baselineKey = System.Ulid.NewUlid().ToString(); + var controlKey = System.Ulid.NewUlid().ToString(); + var calls = new List(); + var outboxObservations = new List(); + var pipelineObservations = new List(); + + var miss = await CallMcpAsync( + mcp, + capture, + seenToolTraces, + calls, + behavior: "ConfirmedCacheMiss", + idempotencyKey: baselineKey, + quantity: Quantity); + + miss.IsError.Should().BeFalse(miss.Error); + miss.OrderId.Should().NotBeEmpty(); + miss.SawNewToolSpan.Should().BeTrue(); + miss.MediatorSpanCount.Should().BeGreaterThan(0); + miss.NpgsqlSpanCount.Should().BeGreaterThan(0); + + var outboxAfterCreate = await FindOutboxRowsForOrderIdAsync(miss.OrderId); + outboxAfterCreate.Should().ContainSingle( + "confirmed MCP cache miss should persist exactly one outbox row for the order"); + + var createdRow = outboxAfterCreate[0]; + outboxObservations.Add(ToSnapshot(createdRow, "AfterOutboxInsert")); + + createdRow.EventType.Should().Be(OrderOutboxObserver.OrderCreatedEventType); + createdRow.IntegrationEventId.Should().Be(createdRow.OutboxMessageId); + createdRow.OrderId.Should().Be(miss.OrderId); + + var processedOutbox = await OrderOutboxObserver.WaitUntilProcessedAsync(_services, miss.OrderId); + outboxObservations.Add(ToSnapshot(processedOutbox, "AfterWorkerProcessed")); + + await Wait.UntilAsync( + () => _fixture.ProcessedEvents.Any(e => e.OrderId == miss.OrderId), + TimeSpan.FromSeconds(20)); + + var handlerEvents = _fixture.ProcessedEvents + .Where(e => e.OrderId == miss.OrderId) + .ToList(); + + handlerEvents.Should().ContainSingle(); + var delivered = handlerEvents[0]; + delivered.Id.Should().Be(processedOutbox.IntegrationEventId); + + await Wait.UntilAsync( + () => GetProcessMessageActivities(capture, processedOutbox.IntegrationEventId).Count > 0 + || CountAllProcessMessageActivities(capture) > 0, + TimeSpan.FromSeconds(5)); + + var inboxAfterDelivery = await WaitForInboxRowAsync(processedOutbox.IntegrationEventId); + var processMessageActivities = GetProcessMessageActivities(capture, processedOutbox.IntegrationEventId); + var totalProcessMessageActivities = CountAllProcessMessageActivities(capture); + + pipelineObservations.Add(new PipelineObservation( + Phase: "AfterAsyncPipeline", + ObservedUtc: DateTimeOffset.UtcNow, + OrderId: miss.OrderId, + IntegrationEventId: processedOutbox.IntegrationEventId, + OutboxWorkerProcessed: processedOutbox.WorkerProcessed, + ProcessMessageCount: processMessageActivities.Count, + ProcessMessageStatuses: processMessageActivities + .Select(a => GetTag(a, "message.status")) + .Distinct() + .ToList(), + TotalEventBusProcessMessageCount: totalProcessMessageActivities, + InboxRowCount: inboxAfterDelivery.InboxRowCount, + InboxProcessed: inboxAfterDelivery.IsProcessed, + ProcessedEventCountForOrder: handlerEvents.Count, + Notes: + [ + $"Outbox Id={processedOutbox.OutboxMessageId} equals IntegrationEvent.Id; OrderId={miss.OrderId}.", + $"EventBus ProcessMessage spans for message.id={processedOutbox.IntegrationEventId}: {processMessageActivities.Count} (total EventBus ProcessMessage={totalProcessMessageActivities}).", + processMessageActivities.Count == 0 + ? "ProcessMessage Activity was not observed in-process for this outbox-triggered delivery; consumer path inferred from inbox row + ProcessedEvents + outbox processed state." + : "ProcessMessage Activity observed for the baseline IntegrationEvent.Id.", + "ProcessedEvents is test-only handler observation via TestEventHandlerDecorator." + ])); + + var outboxCountBeforeReplay = (await FindOutboxRowsForOrderIdAsync(miss.OrderId)).Count; + var processedCountBeforeReplay = _fixture.ProcessedEvents.Count(e => e.OrderId == miss.OrderId); + + var replay = await CallMcpAsync( + mcp, + capture, + seenToolTraces, + calls, + behavior: "McpIdempotencyReplay", + idempotencyKey: baselineKey, + quantity: Quantity); + + var replayCompletedUtc = DateTimeOffset.UtcNow; + + await Wait.UntilAsync( + () => + { + var count = _fixture.ProcessedEvents.Count(e => e.OrderId == miss.OrderId); + return count == processedCountBeforeReplay + && DateTimeOffset.UtcNow - replayCompletedUtc >= ReplayObservationWindow; + }, + TimeSpan.FromSeconds(20)); + + var outboxAfterReplay = await FindOutboxRowsForOrderIdAsync(miss.OrderId); + outboxObservations.Add(ToSnapshot(outboxAfterReplay[0], "AfterMcpIdempotencyReplay")); + + var control = await CallMcpAsync( + mcp, + capture, + seenToolTraces, + calls, + behavior: "ControlNewIdempotencyKey", + idempotencyKey: controlKey, + quantity: Quantity); + + control.IsError.Should().BeFalse(); + control.OrderId.Should().NotBe(miss.OrderId); + control.SawNewToolSpan.Should().BeTrue(); + control.MediatorSpanCount.Should().BeGreaterThan(0); + control.NpgsqlSpanCount.Should().BeGreaterThan(0); + + var controlOutbox = await FindOutboxRowsForOrderIdAsync(control.OrderId); + controlOutbox.Should().ContainSingle(); + + await Wait.UntilAsync( + () => _fixture.ProcessedEvents.Any(e => e.OrderId == control.OrderId), + TimeSpan.FromSeconds(20)); + + var result = new McpOrderOutboxExperimentResult( + Name: "mcp-order-outbox-parity-v1", + StartedUtc: startedUtc, + GitSha: LabRunInfo.ReadGitSha(), + Environment: "Development", + Configuration: new McpOrderOutboxConfiguration( + ToolName, + McpDefaults.ConfirmedArgument, + McpDefaults.IdempotencyKeyArgument, + McpIdempotencyStore: "MemoryIdempotencyStore (in-process; distinct from HTTP Redis idempotency)", + AsyncPath: "orders.create → McpInvoker → CreateOrderCommandHandler → IntegrationEventService → outbox_messages → OutBoxWorker → PublishDirect → RabbitMQ → MessageProcessor → inbox_messages → OrderCreatedIntegrationEventHandler", + ComparedTo: "Experiment 8 HTTP outbox lifecycle fingerprint", + ProcessedEventsNote: "AspireFixture.ProcessedEvents is test-only handler observation"), + Calls: calls, + OutboxObservations: outboxObservations, + PipelineObservations: pipelineObservations, + Observations: new McpOrderOutboxObservations( + BaselineOrderId: miss.OrderId, + BaselineIntegrationEventId: processedOutbox.IntegrationEventId, + OutboxCountAfterMcpMiss: outboxAfterCreate.Count, + OutboxInitiallyPending: createdRow.WorkerPending, + OutboxAlreadyProcessedAtCreateObservation: createdRow.WorkerProcessed, + OutboxWorkerProcessed: processedOutbox.WorkerProcessed, + ProcessMessageCountForBaseline: processMessageActivities.Count, + TotalEventBusProcessMessageCount: totalProcessMessageActivities, + ConsumerEvidenceFromProcessMessageSpan: processMessageActivities.Count > 0, + ConsumerEvidenceFromInboxAndHandler: inboxAfterDelivery.InboxRowCount == 1 && handlerEvents.Count == 1, + InboxRowCountForBaseline: inboxAfterDelivery.InboxRowCount, + InboxProcessedForBaseline: inboxAfterDelivery.IsProcessed, + ProcessedEventCountAfterBaseline: handlerEvents.Count, + ReplaySameOrderId: replay.OrderId == miss.OrderId, + ReplaySawNewToolSpan: replay.SawNewToolSpan, + ReplayMediatorSpans: replay.MediatorSpanCount, + ReplayNpgsqlSpans: replay.NpgsqlSpanCount, + OutboxCountBeforeReplay: outboxCountBeforeReplay, + OutboxCountAfterReplay: outboxAfterReplay.Count, + ProcessedEventCountAfterReplay: _fixture.ProcessedEvents.Count(e => e.OrderId == miss.OrderId), + ControlOrderId: control.OrderId, + ControlIntegrationEventId: controlOutbox[0].IntegrationEventId, + ControlOutboxCount: controlOutbox.Count, + ControlProcessedEventCount: _fixture.ProcessedEvents.Count(e => e.OrderId == control.OrderId), + Notes: + [ + $"Transport TraceId={transportTraceId.ToHexString()}; tool traces tracked separately from transport.", + $"MCP miss toolTrace={miss.ToolTraceId}; replay newToolSpan={replay.SawNewToolSpan}.", + "Does not claim exactly-once RabbitMQ delivery or worker crash consistency.", + "Intermediate Pending outbox state may be skipped if OutBoxWorker poll wins immediately after HTTP-equivalent timing." + ])); + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + replay.IsError.Should().BeFalse(); + replay.OrderId.Should().Be(miss.OrderId); + replay.SawNewToolSpan.Should().BeFalse( + "MCP memory idempotency replay should not start a new mcp.tool span"); + replay.MediatorSpanCount.Should().Be(0); + replay.NpgsqlSpanCount.Should().Be(0); + + outboxAfterReplay.Should().ContainSingle(); + outboxAfterReplay[0].OutboxMessageId.Should().Be(processedOutbox.OutboxMessageId); + _fixture.ProcessedEvents.Count(e => e.OrderId == miss.OrderId).Should().Be(1); + + controlOutbox[0].IntegrationEventId.Should().NotBe(processedOutbox.IntegrationEventId); + _fixture.ProcessedEvents.Count(e => e.OrderId == control.OrderId).Should().Be(1); + + var consumerEvidenceObserved = processMessageActivities.Count > 0 + || (inboxAfterDelivery.InboxRowCount == 1 + && handlerEvents.Count == 1 + && processedOutbox.WorkerProcessed); + consumerEvidenceObserved.Should().BeTrue( + "OutBoxWorker → RabbitMQ → consumer should leave inbox/handler evidence; ProcessMessage span is optional in-process telemetry"); + inboxAfterDelivery.InboxRowCount.Should().Be(1); + delivered.Id.Should().Be(processedOutbox.IntegrationEventId); + } + + private async Task CallMcpAsync( + McpClient mcp, + InProcessActivityCapture capture, + HashSet seenToolTraces, + List calls, + string behavior, + string idempotencyKey, + int quantity) + { + var args = new Dictionary + { + ["productId"] = ProductId, + ["quantity"] = quantity, + ["customerId"] = CustomerId, + [McpDefaults.IdempotencyKeyArgument] = idempotencyKey, + [McpDefaults.ConfirmedArgument] = true + }; + + var clock = Stopwatch.StartNew(); + var result = await mcp.CallToolAsync(ToolName, args); + clock.Stop(); + var completedUtc = DateTimeOffset.UtcNow; + + var isError = result.IsError ?? false; + var errorText = isError ? McpToolResults.Truncate(McpToolResults.GetText(result)) : null; + var order = !isError ? McpToolResults.TryParseOrder(result, JsonOptions) : null; + + var toolSpan = capture.All.FirstOrDefault(s => + s.DisplayName == "mcp.tool" + && HasToolTag(s, ToolName) + && seenToolTraces.Add(s.TraceId)); + + var toolTrace = toolSpan?.TraceId; + var related = toolTrace is null + ? [] + : capture.All.Where(s => s.TraceId == toolTrace).ToList(); + + var call = new McpOrderOutboxCall( + RequestNumber: calls.Count + 1, + Behavior: behavior, + IdempotencyKey: idempotencyKey, + Quantity: quantity, + IsError: isError, + Error: errorText, + OrderId: order?.OrderId ?? Guid.Empty, + QuantityReturned: order?.Quantity, + CompletedUtc: completedUtc, + ClientDurationMs: clock.ElapsedMilliseconds, + ToolTraceId: toolTrace, + SawNewToolSpan: toolSpan is not null, + MediatorSpanCount: related.Count(IsMediator), + NpgsqlSpanCount: related.Count(IsNpgsql)); + + calls.Add(call); + return call; + } + + private async Task> FindOutboxRowsForOrderIdAsync(Guid orderId) + { + var rows = await OrderOutboxObserver.FindByOrderIdAsync(_services, orderId); + return rows.Select(MapObservation).ToList(); + } + + private async Task QueryInboxAsync(Guid integrationEventId) + { + await using var scope = _services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var inboxRows = await db.InboxMessages + .AsNoTracking() + .Where(m => m.Id == integrationEventId) + .ToListAsync(); + + var row = inboxRows.SingleOrDefault(); + + return new InboxObservation( + InboxRowCount: inboxRows.Count, + InboxStatus: row?.Status.ToString(), + IsProcessed: row?.IsProcessed ?? false, + ProcessedAtUtc: row?.ProcessedAt); + } + + private async Task WaitForInboxRowAsync(Guid integrationEventId) + { + var stopwatch = Stopwatch.StartNew(); + while (stopwatch.Elapsed < TimeSpan.FromSeconds(20)) + { + var observation = await QueryInboxAsync(integrationEventId); + if (observation.InboxRowCount == 1) + return observation; + + await Task.Delay(100); + } + + throw new TimeoutException( + $"Inbox message {integrationEventId} was not stored within timeout"); + } + + private static int CountAllProcessMessageActivities(InProcessActivityCapture capture) => + capture.All.Count(span => + span.Source == "EventBus" + && span.DisplayName == "ProcessMessage"); + + private static IReadOnlyList GetProcessMessageActivities( + InProcessActivityCapture capture, + Guid integrationEventId) + { + var id = integrationEventId.ToString(); + return capture.All + .Where(span => + span.Source == "EventBus" + && span.DisplayName == "ProcessMessage" + && span.Tags.TryGetValue("message.id", out var mid) + && mid == id) + .ToList(); + } + + private static OutboxSnapshot ToSnapshot(OrderOutboxRow row, string phase) => + ToSnapshot(MapObservation(row), phase); + + private static OutboxSnapshot ToSnapshot(OutboxRowObservation row, string phase) => + new( + Phase: phase, + ObservedUtc: DateTimeOffset.UtcNow, + OutboxMessageId: row.OutboxMessageId, + IntegrationEventId: row.IntegrationEventId, + OrderId: row.OrderId, + Total: row.Total, + EventType: row.EventType, + Status: row.Status, + ProcessedAtUtc: row.ProcessedAtUtc, + CompletedAtUtc: row.CompletedAtUtc, + RetryCount: row.RetryCount, + WorkerPending: row.WorkerPending, + WorkerProcessed: row.WorkerProcessed, + CreatedAtUtc: row.CreatedAtUtc); + + private static OutboxRowObservation MapObservation(OrderOutboxRow row) => + new( + OutboxMessageId: row.OutboxMessageId, + IntegrationEventId: row.IntegrationEventId, + OrderId: row.OrderId, + Total: row.Total, + EventType: row.EventType, + Status: row.Status, + ProcessedAtUtc: row.ProcessedAtUtc, + CompletedAtUtc: row.CompletedAtUtc, + RetryCount: row.RetryCount, + WorkerPending: row.WorkerPending, + WorkerProcessed: row.WorkerProcessed, + CreatedAtUtc: row.CreatedAtUtc); + + private static bool HasToolTag(CapturedActivity span, string toolName) => + span.Tags.TryGetValue("mcp.tool.name", out var name) && name == toolName; + + private static string? GetTag(CapturedActivity activity, string key) => + activity.Tags.TryGetValue(key, out var value) ? value : null; + + private sealed record McpOrderOutboxCall( + int RequestNumber, + string Behavior, + string IdempotencyKey, + int Quantity, + bool IsError, + string? Error, + Guid OrderId, + int? QuantityReturned, + DateTimeOffset CompletedUtc, + double ClientDurationMs, + string? ToolTraceId, + bool SawNewToolSpan, + int MediatorSpanCount, + int NpgsqlSpanCount); + + private sealed record OutboxRowObservation( + Guid OutboxMessageId, + Guid IntegrationEventId, + Guid OrderId, + decimal Total, + string EventType, + string Status, + DateTime? ProcessedAtUtc, + DateTime? CompletedAtUtc, + int RetryCount, + bool WorkerPending, + bool WorkerProcessed, + DateTime CreatedAtUtc); + + private sealed record OutboxSnapshot( + string Phase, + DateTimeOffset ObservedUtc, + Guid OutboxMessageId, + Guid IntegrationEventId, + Guid OrderId, + decimal Total, + string EventType, + string Status, + DateTime? ProcessedAtUtc, + DateTime? CompletedAtUtc, + int RetryCount, + bool WorkerPending, + bool WorkerProcessed, + DateTime CreatedAtUtc); + + private sealed record InboxObservation( + int InboxRowCount, + string? InboxStatus, + bool IsProcessed, + DateTime? ProcessedAtUtc); + + private sealed record PipelineObservation( + string Phase, + DateTimeOffset ObservedUtc, + Guid OrderId, + Guid IntegrationEventId, + bool OutboxWorkerProcessed, + int ProcessMessageCount, + IReadOnlyList ProcessMessageStatuses, + int TotalEventBusProcessMessageCount, + int InboxRowCount, + bool InboxProcessed, + int ProcessedEventCountForOrder, + IReadOnlyList Notes); + + private sealed record McpOrderOutboxConfiguration( + string Tool, + string ConfirmedArgument, + string IdempotencyKeyArgument, + string McpIdempotencyStore, + string AsyncPath, + string ComparedTo, + string ProcessedEventsNote); + + private sealed record McpOrderOutboxObservations( + Guid BaselineOrderId, + Guid BaselineIntegrationEventId, + int OutboxCountAfterMcpMiss, + bool OutboxInitiallyPending, + bool OutboxAlreadyProcessedAtCreateObservation, + bool OutboxWorkerProcessed, + int ProcessMessageCountForBaseline, + int TotalEventBusProcessMessageCount, + bool ConsumerEvidenceFromProcessMessageSpan, + bool ConsumerEvidenceFromInboxAndHandler, + int InboxRowCountForBaseline, + bool InboxProcessedForBaseline, + int ProcessedEventCountAfterBaseline, + bool ReplaySameOrderId, + bool ReplaySawNewToolSpan, + int ReplayMediatorSpans, + int ReplayNpgsqlSpans, + int OutboxCountBeforeReplay, + int OutboxCountAfterReplay, + int ProcessedEventCountAfterReplay, + Guid ControlOrderId, + Guid ControlIntegrationEventId, + int ControlOutboxCount, + int ControlProcessedEventCount, + IReadOnlyList Notes); + + private sealed record McpOrderOutboxExperimentResult( + string Name, + DateTimeOffset StartedUtc, + string GitSha, + string Environment, + McpOrderOutboxConfiguration Configuration, + IReadOnlyList Calls, + IReadOnlyList OutboxObservations, + IReadOnlyList PipelineObservations, + McpOrderOutboxObservations Observations); +} diff --git a/tests/Lab/IntegrationTests/Experiments/McpToolStormRateLimit/McpToolStormRateLimitExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/McpToolStormRateLimit/McpToolStormRateLimitExperimentTests.cs new file mode 100644 index 0000000..5ecc59d --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/McpToolStormRateLimit/McpToolStormRateLimitExperimentTests.cs @@ -0,0 +1,380 @@ +using System.Diagnostics; +using System.Text.Json; +using BuildingBlocks.Mcp; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Mcp; +using IntegrationTests.Infrastructure.Orders; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using ModelContextProtocol.Client; +using ModelContextProtocol.Protocol; +using Xunit.Abstractions; +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Experiments.McpToolStormRateLimit; + +/// +/// Experiment 16 — MCP tool-storm bounded by . +/// Hypothesis: a burst of orders.create calls with distinct idempotency keys +/// cannot be collapsed by the idempotency store; a configured limiter rejects surplus calls +/// with before Mediator/Npgsql/business execution. +/// This is not an idempotency experiment. Test-host limiter only (Lab default remains +/// ). Does not change package rate-limiter code or add a NuGet. +/// +[Collection(AspireCollection.Name)] +public sealed class McpToolStormRateLimitExperimentTests +{ + private const string ToolName = "orders.create"; + private const int ProductId = 1; + private const int CustomerId = 1; + private const int Quantity = 2; + private const int MaxPermits = 2; + private const int StormSize = 5; + private const int RetryAfterSeconds = 1; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly AspireFixture _fixture; + private readonly WebApplicationFactory _factory; + private readonly HttpClient _http; + private readonly FixedPermitMcpRateLimiter _limiter; + private readonly ITestOutputHelper _output; + + public McpToolStormRateLimitExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + + _limiter = new FixedPermitMcpRateLimiter( + toolName: ToolName, + maxPermits: MaxPermits, + retryAfterSeconds: RetryAfterSeconds); + + _factory = fixture.WithWebHostBuilder(builder => + { + builder.ConfigureTestServices(services => + { + services.RemoveAll(); + services.AddSingleton(_limiter); + }); + }); + + _http = _factory.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + } + + [Fact] + public async Task Distinct_key_mcp_write_storm_is_bounded_by_rate_limiter_before_production() + { + _fixture.ProcessedEvents.Clear(); + _limiter.Reset(); + + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var (transportTraceId, transportSpanId) = NewTraceParent(); + _http.DefaultRequestHeaders.Remove("traceparent"); + _http.DefaultRequestHeaders.TryAddWithoutValidation( + "traceparent", + FormatTraceParent(transportTraceId, transportSpanId)); + + await using var mcp = await LabMcpClient.CreateAsync(_http); + var seenToolTraces = new HashSet(StringComparer.Ordinal); + var calls = new List(); + + for (var i = 0; i < StormSize; i++) + { + var key = System.Ulid.NewUlid().ToString(); + await CallAsync( + mcp, + capture, + seenToolTraces, + calls, + behavior: $"Storm_{i + 1}_DistinctKey", + idempotencyKey: key); + } + + var accepted = calls.Where(c => !c.IsError).ToList(); + var rateLimited = calls.Where(c => + c.IsError && string.Equals(c.ErrorCode, nameof(McpErrorCode.RateLimited), StringComparison.Ordinal)).ToList(); + var otherErrors = calls.Where(c => c.IsError && c.ErrorCode != nameof(McpErrorCode.RateLimited)).ToList(); + + var acceptedOrderIds = accepted + .Where(c => c.OrderId != Guid.Empty) + .Select(c => c.OrderId) + .Distinct() + .ToList(); + + var outboxByOrder = new Dictionary(); + foreach (var orderId in acceptedOrderIds) + outboxByOrder[orderId] = (await OrderOutboxObserver.FindByOrderIdAsync(_factory.Services, orderId)).Count; + + var mcpToolSpansForOrdersCreate = capture.All.Count(s => + s.DisplayName == "mcp.tool" && HasToolTag(s, ToolName)); + + var hypothesisConfirmed = + accepted.Count == MaxPermits + && rateLimited.Count == StormSize - MaxPermits + && otherErrors.Count == 0 + && accepted.All(c => c.SawNewToolSpan && c.MediatorSpanCount > 0 && c.NpgsqlSpanCount > 0) + && rateLimited.All(c => + !c.SawNewToolSpan + && c.MediatorSpanCount == 0 + && c.NpgsqlSpanCount == 0 + && c.OrderId == Guid.Empty) + && acceptedOrderIds.Count == MaxPermits + && mcpToolSpansForOrdersCreate == MaxPermits; + + var result = new + { + name = "mcp-orders-create-tool-storm-rate-limit-v1", + startedUtc, + gitSha = LabRunInfo.ReadGitSha(), + environment = "Development", + configuration = new + { + tool = ToolName, + stormSize = StormSize, + maxPermits = MaxPermits, + retryAfterSeconds = RetryAfterSeconds, + limiter = "Test-only FixedPermitMcpRateLimiter (per toolName, fixed permit count); Lab default remains NoOpRateLimiter", + idempotencyNote = "Distinct Ulid keys per call so MemoryIdempotencyStore cannot collapse the storm", + pipelineNote = "McpInvoker checks IMcpRateLimiter after visibility and before confirmation/idempotency/InvokeCore (mcp.tool Activity + dispatcher)" + }, + calls, + observations = new + { + attempted = calls.Count, + accepted = accepted.Count, + rateLimited = rateLimited.Count, + otherErrors = otherErrors.Count, + limiterAcquireAttempts = _limiter.AcquireAttempts, + distinctAcceptedOrderIds = acceptedOrderIds.Count, + outboxRowsByOrderId = outboxByOrder, + mcpToolSpansForOrdersCreate, + acceptedMediatorTotal = accepted.Sum(c => c.MediatorSpanCount), + acceptedNpgsqlTotal = accepted.Sum(c => c.NpgsqlSpanCount), + rateLimitedMediatorTotal = rateLimited.Sum(c => c.MediatorSpanCount), + rateLimitedNpgsqlTotal = rateLimited.Sum(c => c.NpgsqlSpanCount), + hypothesisConfirmed, + hypothesis = "Distinct-key MCP write storm is bounded by IMcpRateLimiter before Mediator/Npgsql/business execution." + }, + notes = new[] + { + "Distinct idempotency keys ensure the storm is not suppressed by MCP idempotency (contrast Exp 13 amplification without a limiter).", + "Package semantics: IMcpRateLimiter receives toolName + McpInvokeContext; scope is implementation-defined. This experiment uses a fixed per-tool permit budget.", + "Rejected calls must not emit mcp.tool (Activity starts only in InvokeCore) and must not reach Mediator/Npgsql.", + hypothesisConfirmed + ? "Hypothesis CONFIRMED: limiter accepted MaxPermits production writes and RateLimited the surplus before production." + : "Hypothesis FALSIFIED or inconclusive relative to packaged rate-limit boundary." + } + }; + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + calls.Should().HaveCount(StormSize); + otherErrors.Should().BeEmpty( + "storm outcomes should be success or RateLimited only. Unexpected: {0}", + string.Join("; ", otherErrors.Select(c => $"{c.Behavior}:{c.ErrorCode}:{c.Error}"))); + + accepted.Should().HaveCount(MaxPermits, + "fixed permit limiter should allow exactly {0} orders.create calls", MaxPermits); + rateLimited.Should().HaveCount(StormSize - MaxPermits); + + foreach (var call in accepted) + { + call.ErrorCode.Should().BeNull(); + call.OrderId.Should().NotBeEmpty(); + call.SawNewToolSpan.Should().BeTrue( + "accepted call must enter InvokeCore and emit mcp.tool. {0}", call.Behavior); + call.MediatorSpanCount.Should().BeGreaterThan(0, call.Behavior); + call.NpgsqlSpanCount.Should().BeGreaterThan(0, call.Behavior); + } + + foreach (var call in rateLimited) + { + call.ErrorCode.Should().Be(nameof(McpErrorCode.RateLimited), call.Behavior); + call.OrderId.Should().BeEmpty(call.Behavior); + call.SawNewToolSpan.Should().BeFalse( + "RateLimited must return before InvokeCore (no mcp.tool). {0} error={1}", + call.Behavior, call.Error); + call.MediatorSpanCount.Should().Be(0, call.Behavior); + call.NpgsqlSpanCount.Should().Be(0, call.Behavior); + call.RetryAfterSeconds.Should().Be(RetryAfterSeconds); + } + + acceptedOrderIds.Should().HaveCount(MaxPermits); + mcpToolSpansForOrdersCreate.Should().Be(MaxPermits); + _limiter.AcquireAttempts.Should().Be(StormSize); + + foreach (var orderId in acceptedOrderIds) + { + outboxByOrder[orderId].Should().BeGreaterThanOrEqualTo(1, + "each accepted production order should leave an outbox row. orderId={0}", orderId); + } + + rateLimited.Sum(c => c.MediatorSpanCount).Should().Be(0); + rateLimited.Sum(c => c.NpgsqlSpanCount).Should().Be(0); + } + + private async Task CallAsync( + McpClient mcp, + InProcessActivityCapture capture, + HashSet seenToolTraces, + List calls, + string behavior, + string idempotencyKey) + { + var args = new Dictionary + { + ["productId"] = ProductId, + ["quantity"] = Quantity, + ["customerId"] = CustomerId, + [McpDefaults.IdempotencyKeyArgument] = idempotencyKey, + [McpDefaults.ConfirmedArgument] = true + }; + + var clock = Stopwatch.StartNew(); + var result = await mcp.CallToolAsync(ToolName, args); + clock.Stop(); + + var isError = result.IsError ?? false; + var errorText = isError ? McpToolResults.Truncate(McpToolResults.GetText(result), 240) : null; + var errorCode = isError ? TryReadErrorCode(result) : null; + var retryAfter = isError ? TryReadRetryAfterSeconds(result) : null; + var order = !isError ? McpToolResults.TryParseOrder(result, JsonOptions) : null; + + var toolSpan = capture.All.FirstOrDefault(s => + s.DisplayName == "mcp.tool" + && HasToolTag(s, ToolName) + && seenToolTraces.Add(s.TraceId)); + + var toolTrace = toolSpan?.TraceId; + var related = toolTrace is null + ? (IReadOnlyList)Array.Empty() + : capture.All.Where(s => s.TraceId == toolTrace).ToList(); + + var call = new StormCall( + RequestNumber: calls.Count + 1, + Behavior: behavior, + IdempotencyKey: idempotencyKey, + IsError: isError, + ErrorCode: errorCode, + Error: errorText, + RetryAfterSeconds: retryAfter, + OrderId: order?.OrderId ?? Guid.Empty, + ClientDurationMs: clock.ElapsedMilliseconds, + ToolTraceId: toolTrace, + SawNewToolSpan: toolSpan is not null, + MediatorSpanCount: related.Count(IsMediator), + NpgsqlSpanCount: related.Count(IsNpgsql)); + + calls.Add(call); + return call; + } + + private static string? TryReadErrorCode(CallToolResult result) + { + var code = McpToolResults.TryReadJsonErrorCode(result); + if (code is not null) + return code; + + var text = McpToolResults.GetText(result); + return text.Contains("RateLimited", StringComparison.OrdinalIgnoreCase) + ? nameof(McpErrorCode.RateLimited) + : null; + } + + private static int? TryReadRetryAfterSeconds(CallToolResult result) + { + var text = McpToolResults.GetText(result); + if (string.IsNullOrWhiteSpace(text)) + return null; + + try + { + using var doc = JsonDocument.Parse(text); + if (doc.RootElement.TryGetProperty("retryAfterSeconds", out var retry) + && retry.ValueKind == JsonValueKind.Number + && retry.TryGetInt32(out var seconds)) + { + return seconds; + } + } + catch (JsonException) + { + // ignore + } + + return null; + } + + private static bool HasToolTag(CapturedActivity span, string toolName) + => span.Tags.Any(t => + t.Key == "mcp.tool.name" + && string.Equals(t.Value, toolName, StringComparison.Ordinal)); + + /// + /// Test-only fixed-budget limiter. Allows the first acquires for + /// one tool name; other tools always allow. Not production code. + /// + public sealed class FixedPermitMcpRateLimiter : IMcpRateLimiter + { + private readonly string _toolName; + private readonly int _maxPermits; + private readonly int? _retryAfterSeconds; + private int _acquireAttempts; + + public FixedPermitMcpRateLimiter(string toolName, int maxPermits, int? retryAfterSeconds) + { + _toolName = toolName; + _maxPermits = maxPermits; + _retryAfterSeconds = retryAfterSeconds; + } + + public int MaxPermits => _maxPermits; + public int AcquireAttempts => Volatile.Read(ref _acquireAttempts); + + public void Reset() => Volatile.Write(ref _acquireAttempts, 0); + + public ValueTask TryAcquireAsync( + string toolName, + McpInvokeContext context, + CancellationToken cancellationToken) + { + if (!string.Equals(toolName, _toolName, StringComparison.OrdinalIgnoreCase)) + return ValueTask.FromResult(McpRateLimitDecision.Allow); + + var attempt = Interlocked.Increment(ref _acquireAttempts); + if (attempt <= _maxPermits) + return ValueTask.FromResult(McpRateLimitDecision.Allow); + + return ValueTask.FromResult(McpRateLimitDecision.Deny(_retryAfterSeconds)); + } + } + + private sealed record StormCall( + int RequestNumber, + string Behavior, + string IdempotencyKey, + bool IsError, + string? ErrorCode, + string? Error, + int? RetryAfterSeconds, + Guid OrderId, + long ClientDurationMs, + string? ToolTraceId, + bool SawNewToolSpan, + int MediatorSpanCount, + int NpgsqlSpanCount); + +} diff --git a/tests/Lab/IntegrationTests/Experiments/OutboxDelivery/OutboxDeliveryExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/OutboxDelivery/OutboxDeliveryExperimentTests.cs new file mode 100644 index 0000000..1cd4409 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/OutboxDelivery/OutboxDeliveryExperimentTests.cs @@ -0,0 +1,304 @@ +using System.Diagnostics; +using System.Text.Json; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Async; +using IntegrationTests.Infrastructure.Orders; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using Xunit.Abstractions; +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Experiments.OutboxDelivery; + +/// +/// Experiment 5: HTTP order create → transactional outbox → OutBoxWorker → RabbitMQ → handler. +/// Hypothesis: a successful cache-miss POST /api/v2/Order/order persists catalog/outbox +/// in one transaction; OutBoxWorker eventually publishes OrderCreatedIntegrationEvent; +/// the real consumer runs once. Replaying the same Idempotency-Key returns the cached HTTP +/// body and does not produce a second integration event. +/// AspireFixture.ProcessedEvents is test observation infrastructure (decorator on the real +/// handler), not production behavior. +/// +[Collection(AspireCollection.Name)] +public sealed class OutboxDeliveryExperimentTests +{ + private const int Quantity = 2; + + /// + /// Covers several OutBoxWorker poll intervals (2s) without asserting exact worker timing. + /// + private static readonly TimeSpan ReplayObservationWindow = TimeSpan.FromSeconds(5); + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly AspireFixture _fixture; + private readonly HttpClient _http; + private readonly ITestOutputHelper _output; + + public OutboxDeliveryExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + _http = fixture.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + } + + [Fact] + public async Task Http_order_create_delivers_outbox_event_once_and_idempotent_replay_does_not_redeliver() + { + _fixture.ProcessedEvents.Clear(); + + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var idempotencyKey = System.Ulid.NewUlid().ToString(); + var calls = new List(); + var processedObservations = new List(); + + var baseline = await SendAsync( + capture, + calls, + behavior: "BaselineProduction", + idempotencyKey: idempotencyKey, + quantity: Quantity); + + baseline.HttpStatus.Should().Be(200); + baseline.OrderId.Should().NotBeEmpty(); + baseline.CachedResponseHeader.Should().BeFalse( + "the first request with a new Idempotency-Key should execute production"); + baseline.MediatorSpanCount.Should().BeGreaterThan(0, + "baseline should reach ISender.Send(CreateOrderCommand). Spans: {0}", + Describe(capture.ForTraceHex(baseline.TraceId))); + baseline.NpgsqlSpanCount.Should().BeGreaterThan(0, + "baseline should persist catalog/outbox via SaveChanges. Spans: {0}", + Describe(capture.ForTraceHex(baseline.TraceId))); + + var httpCompletedUtc = baseline.CompletedUtc; + var deliveryWaitStartedUtc = DateTimeOffset.UtcNow; + + await Wait.UntilAsync( + () => _fixture.ProcessedEvents.Any(e => e.OrderId == baseline.OrderId), + TimeSpan.FromSeconds(20)); + + var deliveryCompletedUtc = DateTimeOffset.UtcNow; + var matchingEvents = _fixture.ProcessedEvents + .Where(e => e.OrderId == baseline.OrderId) + .ToList(); + + matchingEvents.Should().ContainSingle( + "OutBoxWorker should eventually deliver exactly one OrderCreatedIntegrationEvent for the order"); + + var delivered = matchingEvents[0]; + processedObservations.Add(new ProcessedEventObservation( + Phase: "AfterBaseline", + ObservedUtc: deliveryCompletedUtc, + IntegrationEventId: delivered.Id, + OrderId: delivered.OrderId, + Total: delivered.Total, + CreationDate: delivered.CreationDate)); + + var asyncDeliveryDurationMs = (deliveryCompletedUtc - httpCompletedUtc).TotalMilliseconds; + + var replay = await SendAsync( + capture, + calls, + behavior: "IdempotentReplay", + idempotencyKey: idempotencyKey, + quantity: Quantity); + + var replayCompletedUtc = replay.CompletedUtc; + + await Wait.UntilAsync( + () => + { + var count = _fixture.ProcessedEvents.Count(e => e.OrderId == baseline.OrderId); + return count == 1 && DateTimeOffset.UtcNow - replayCompletedUtc >= ReplayObservationWindow; + }, + TimeSpan.FromSeconds(20)); + + var eventsAfterReplay = _fixture.ProcessedEvents + .Where(e => e.OrderId == baseline.OrderId) + .ToList(); + + processedObservations.Add(new ProcessedEventObservation( + Phase: "AfterReplayObservationWindow", + ObservedUtc: DateTimeOffset.UtcNow, + IntegrationEventId: eventsAfterReplay[0].Id, + OrderId: eventsAfterReplay[0].OrderId, + Total: eventsAfterReplay[0].Total, + CreationDate: eventsAfterReplay[0].CreationDate)); + + var result = new OutboxDeliveryExperimentResult( + Name: "http-order-outbox-delivery-v1", + StartedUtc: startedUtc, + GitSha: LabRunInfo.ReadGitSha(), + Environment: "Development", + Configuration: new OutboxDeliveryConfiguration( + HttpOrderCreate.Path, + HttpOrderCreate.IdempotencyHeader, + HttpOrderCreate.CachedResponseHeader, + AsyncPath: "OrderController.CreateOrder → CreateOrderCommandHandler → IntegrationEventService.PublishThroughEventBusAsync → outbox_messages → OutBoxWorker → EventBus.PublishDirect → RabbitMQ → OrderCreatedIntegrationEventHandler", + ProcessedEventsNote: "AspireFixture.ProcessedEvents is populated by TestEventHandlerDecorator wrapping the real OrderCreatedIntegrationEventHandler"), + Calls: calls, + ProcessedEventObservations: processedObservations, + Observations: new OutboxDeliveryObservations( + BaselineHttpStatus: baseline.HttpStatus, + BaselineOrderId: baseline.OrderId, + BaselineQuantity: baseline.Quantity, + BaselineCachedHeader: baseline.CachedResponseHeader, + BaselineMediatorSpans: baseline.MediatorSpanCount, + BaselineNpgsqlSpans: baseline.NpgsqlSpanCount, + BaselineHttpDurationMs: baseline.ClientDurationMs, + AsyncDeliveryDurationMs: asyncDeliveryDurationMs, + DeliveryWaitStartedUtc: deliveryWaitStartedUtc, + DeliveryCompletedUtc: deliveryCompletedUtc, + ProcessedEventCountAfterBaseline: matchingEvents.Count, + ReplayHttpStatus: replay.HttpStatus, + ReplayOrderId: replay.OrderId, + ReplayQuantity: replay.Quantity, + ReplayCachedHeader: replay.CachedResponseHeader, + ReplayMediatorSpans: replay.MediatorSpanCount, + ReplayNpgsqlSpans: replay.NpgsqlSpanCount, + ReplayHttpDurationMs: replay.ClientDurationMs, + ProcessedEventCountAfterReplay: eventsAfterReplay.Count, + ReplaySameOrderIdAsBaseline: replay.OrderId == baseline.OrderId, + Notes: + [ + $"Baseline TraceId={baseline.TraceId}; sources={DescribeSources(capture.ForTraceHex(baseline.TraceId))}.", + $"Replay TraceId={replay.TraceId}; sources={DescribeSources(capture.ForTraceHex(replay.TraceId))}.", + $"ProcessedEvents after baseline: {matchingEvents.Count} event(s) for OrderId={baseline.OrderId}; IntegrationEvent.Id={delivered.Id} (distinct from OrderId).", + $"Async delivery observed {asyncDeliveryDurationMs:F0} ms after HTTP completion; OutBoxWorker poll interval is ~2s (not asserted).", + $"Replay observation window: {ReplayObservationWindow.TotalSeconds}s after HTTP completion before asserting no second event.", + "ProcessedEvents is test-only observation infrastructure; it proves the real handler executed." + ])); + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + replay.HttpStatus.Should().Be(200); + replay.OrderId.Should().Be(baseline.OrderId); + replay.Quantity.Should().Be(Quantity); + replay.CachedResponseHeader.Should().BeTrue( + "replay of a completed Idempotency-Key should set {0}", HttpOrderCreate.CachedResponseHeader); + replay.MediatorSpanCount.Should().Be(0, + "idempotent replay short-circuits before the controller action. Spans: {0}", + Describe(capture.ForTraceHex(replay.TraceId))); + replay.NpgsqlSpanCount.Should().Be(0, + "idempotent replay should not re-run catalog SaveChanges/outbox insert. Spans: {0}", + Describe(capture.ForTraceHex(replay.TraceId))); + + eventsAfterReplay.Should().ContainSingle( + "idempotent HTTP replay must not produce a second OrderCreatedIntegrationEvent delivery"); + } + + private async Task SendAsync( + InProcessActivityCapture capture, + List calls, + string behavior, + string idempotencyKey, + int quantity) + { + var result = await HttpOrderCreate.PostAsync( + _http, + capture, + idempotencyKey, + quantity, + jsonOptions: JsonOptions); + + var call = new OutboxDeliveryCall( + calls.Count + 1, + behavior, + idempotencyKey, + quantity, + result.HttpStatus, + result.OrderId, + result.Quantity, + result.TotalAmount, + result.CachedResponseHeader, + result.ClientDurationMs, + result.CompletedUtc, + result.TraceIdHex, + result.Spans.Count(IsMediator), + result.Spans.Count(IsNpgsql), + result.Body); + + calls.Add(call); + return call; + } + + private static string Describe(IReadOnlyList spans) + => string.Join("; ", spans.Select(s => $"{s.Source}:{s.DisplayName}")); + + private static string DescribeSources(IReadOnlyList spans) + => string.Join(",", spans.Select(s => s.Source).Distinct().OrderBy(s => s, StringComparer.Ordinal)); + + private sealed record OutboxDeliveryCall( + int RequestNumber, + string Behavior, + string IdempotencyKey, + int RequestedQuantity, + int HttpStatus, + Guid OrderId, + int? Quantity, + decimal? TotalAmount, + bool CachedResponseHeader, + double ClientDurationMs, + DateTimeOffset CompletedUtc, + string TraceId, + int MediatorSpanCount, + int NpgsqlSpanCount, + string Body); + + private sealed record ProcessedEventObservation( + string Phase, + DateTimeOffset ObservedUtc, + Guid IntegrationEventId, + Guid OrderId, + decimal Total, + DateTime CreationDate); + + private sealed record OutboxDeliveryConfiguration( + string Path, + string IdempotencyHeader, + string CachedResponseHeader, + string AsyncPath, + string ProcessedEventsNote); + + private sealed record OutboxDeliveryObservations( + int BaselineHttpStatus, + Guid BaselineOrderId, + int? BaselineQuantity, + bool BaselineCachedHeader, + int BaselineMediatorSpans, + int BaselineNpgsqlSpans, + double BaselineHttpDurationMs, + double AsyncDeliveryDurationMs, + DateTimeOffset DeliveryWaitStartedUtc, + DateTimeOffset DeliveryCompletedUtc, + int ProcessedEventCountAfterBaseline, + int ReplayHttpStatus, + Guid ReplayOrderId, + int? ReplayQuantity, + bool ReplayCachedHeader, + int ReplayMediatorSpans, + int ReplayNpgsqlSpans, + double ReplayHttpDurationMs, + int ProcessedEventCountAfterReplay, + bool ReplaySameOrderIdAsBaseline, + IReadOnlyList Notes); + + private sealed record OutboxDeliveryExperimentResult( + string Name, + DateTimeOffset StartedUtc, + string GitSha, + string Environment, + OutboxDeliveryConfiguration Configuration, + IReadOnlyList Calls, + IReadOnlyList ProcessedEventObservations, + OutboxDeliveryObservations Observations); +} diff --git a/tests/Lab/IntegrationTests/Experiments/OutboxLifecycle/OutboxLifecycleExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/OutboxLifecycle/OutboxLifecycleExperimentTests.cs new file mode 100644 index 0000000..e3e5bc3 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/OutboxLifecycle/OutboxLifecycleExperimentTests.cs @@ -0,0 +1,385 @@ +using System.Diagnostics; +using System.Text.Json; +using EventBusRabbitMQ.Domain; +using FeatureFusion.Infrastructure.Context; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Async; +using IntegrationTests.Infrastructure.Orders; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Xunit.Abstractions; +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Experiments.OutboxLifecycle; + +/// +/// Experiment 8: HTTP order create → transactional outbox row lifecycle fingerprint. +/// Hypothesis: a cache-miss POST /api/v2/Order/order inserts one +/// outbox_messages row (Status=Pending, ProcessedAt=null); +/// the real OutBoxWorker publishes via PublishDirect then marks the row +/// processed (Status=Processed, ProcessedAt/CompletedAt set); +/// the consumer/handler run once. Idempotent HTTP replay does not insert another outbox row +/// or produce a second handler observation for the same order. +/// Outbox row Id equals IntegrationEvent.Id and is distinct from +/// OrderCreatedIntegrationEvent.OrderId. +/// AspireFixture.ProcessedEvents is test observation infrastructure, not production telemetry. +/// Does not claim exactly-once RabbitMQ delivery or crash consistency between publish and mark-processed. +/// +[Collection(AspireCollection.Name)] +public sealed class OutboxLifecycleExperimentTests +{ + private const int Quantity = 2; + + /// + /// Covers several OutBoxWorker poll intervals (2s) after replay without asserting exact timing. + /// + private static readonly TimeSpan ReplayObservationWindow = TimeSpan.FromSeconds(5); + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly AspireFixture _fixture; + private readonly IServiceProvider _services; + private readonly HttpClient _http; + private readonly ITestOutputHelper _output; + + public OutboxLifecycleExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + _services = fixture.Services; + _http = fixture.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + } + + [Fact] + public async Task Http_order_create_fingerprints_outbox_lifecycle_and_replay_does_not_add_rows() + { + _fixture.ProcessedEvents.Clear(); + + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var baselineKey = System.Ulid.NewUlid().ToString(); + var controlKey = System.Ulid.NewUlid().ToString(); + var calls = new List(); + var outboxObservations = new List(); + var processedObservations = new List(); + + var baseline = await SendAsync( + capture, + calls, + behavior: "CacheMissOutboxCreation", + idempotencyKey: baselineKey, + quantity: Quantity); + + baseline.HttpStatus.Should().Be(200); + baseline.OrderId.Should().NotBeEmpty(); + baseline.CachedResponseHeader.Should().BeFalse(); + baseline.MediatorSpanCount.Should().BeGreaterThan(0); + baseline.NpgsqlSpanCount.Should().BeGreaterThan(0); + + var outboxAfterCreate = await OrderOutboxObserver.FindByOrderIdAsync(_services, baseline.OrderId); + outboxAfterCreate.Should().ContainSingle( + "cache miss should persist exactly one OrderCreatedIntegrationEvent outbox row for the order"); + + var createdRow = outboxAfterCreate[0]; + outboxObservations.Add(ToSnapshot(createdRow, "AfterCacheMiss")); + + createdRow.EventType.Should().Be(OrderOutboxObserver.OrderCreatedEventType); + createdRow.IntegrationEventId.Should().Be(createdRow.OutboxMessageId, + "outbox row primary key equals IntegrationEvent.Id in payload"); + createdRow.OrderId.Should().Be(baseline.OrderId, + "OrderId in payload is the HTTP order identity, distinct from IntegrationEvent.Id"); + + var processedRow = await OrderOutboxObserver.WaitUntilProcessedAsync(_services, baseline.OrderId); + var processedOutbox = ToSnapshot(processedRow, "AfterWorkerProcessed"); + outboxObservations.Add(processedOutbox); + + processedOutbox.WorkerProcessed.Should().BeTrue( + "OutBoxWorker sets Status=Processed and ProcessedAt/CompletedAt after publish"); + processedOutbox.Status.Should().Be(MessageStatus.Processed.ToString()); + + await Wait.UntilAsync( + () => _fixture.ProcessedEvents.Any(e => e.OrderId == baseline.OrderId), + TimeSpan.FromSeconds(20)); + + var handlerEvents = _fixture.ProcessedEvents + .Where(e => e.OrderId == baseline.OrderId) + .ToList(); + + handlerEvents.Should().ContainSingle(); + var delivered = handlerEvents[0]; + + delivered.Id.Should().Be(processedOutbox.IntegrationEventId, + "handler receives the same IntegrationEvent.Id stored in outbox_messages"); + + var inboxAfterDelivery = await QueryInboxAsync(processedOutbox.IntegrationEventId); + + processedObservations.Add(new ProcessedEventObservation( + Phase: "AfterWorkerAndHandler", + ObservedUtc: DateTimeOffset.UtcNow, + IntegrationEventId: delivered.Id, + OrderId: delivered.OrderId, + Total: delivered.Total, + InboxRowCount: inboxAfterDelivery.InboxRowCount, + InboxProcessed: inboxAfterDelivery.IsProcessed)); + + var outboxCountBeforeReplay = (await OrderOutboxObserver.FindByOrderIdAsync(_services, baseline.OrderId)).Count; + + var replay = await SendAsync( + capture, + calls, + behavior: "IdempotentReplay", + idempotencyKey: baselineKey, + quantity: Quantity); + + var replayCompletedUtc = replay.CompletedUtc; + + await Wait.UntilAsync( + () => + { + var count = _fixture.ProcessedEvents.Count(e => e.OrderId == baseline.OrderId); + return count == 1 && DateTimeOffset.UtcNow - replayCompletedUtc >= ReplayObservationWindow; + }, + TimeSpan.FromSeconds(20)); + + var outboxAfterReplay = await OrderOutboxObserver.FindByOrderIdAsync(_services, baseline.OrderId); + outboxObservations.Add(ToSnapshot(outboxAfterReplay[0], "AfterIdempotentReplay")); + + var control = await SendAsync( + capture, + calls, + behavior: "ControlNewIdempotencyKey", + idempotencyKey: controlKey, + quantity: Quantity); + + control.HttpStatus.Should().Be(200); + control.OrderId.Should().NotBe(baseline.OrderId); + control.CachedResponseHeader.Should().BeFalse(); + + var controlOutbox = await OrderOutboxObserver.FindByOrderIdAsync(_services, control.OrderId); + controlOutbox.Should().ContainSingle( + "a new Idempotency-Key should create a new order and its own outbox row"); + + var result = new OutboxLifecycleExperimentResult( + Name: "http-order-outbox-lifecycle-v1", + StartedUtc: startedUtc, + GitSha: LabRunInfo.ReadGitSha(), + Environment: "Development", + Configuration: new OutboxLifecycleConfiguration( + HttpOrderCreate.Path, + HttpOrderCreate.IdempotencyHeader, + HttpOrderCreate.CachedResponseHeader, + AsyncPath: "POST order → IdempotentAttributeFilter (miss) → CreateOrderCommandHandler → IntegrationEventService.PublishThroughEventBusAsync → outbox_messages (transactional) → OutBoxWorker (poll ProcessedAt==null) → PublishDirect → MarkMessageAsProcessedAsync → RabbitMQ → MessageProcessor → inbox_messages → OrderCreatedIntegrationEventHandler", + WorkerOrderingNote: "OutBoxWorker calls PublishDirect before MarkMessageAsProcessedAsync (Status=Processed, ProcessedAt, CompletedAt). Worker selects pending rows by ProcessedAt==null.", + IdentityNote: "outbox_messages.Id == IntegrationEvent.Id != OrderCreatedIntegrationEvent.OrderId", + ProcessedEventsNote: "AspireFixture.ProcessedEvents is test-only handler observation"), + Calls: calls, + OutboxObservations: outboxObservations, + ProcessedEventObservations: processedObservations, + Observations: new OutboxLifecycleObservations( + BaselineOrderId: baseline.OrderId, + BaselineIntegrationEventId: processedOutbox.IntegrationEventId, + OutboxCountAfterCreate: outboxAfterCreate.Count, + OutboxInitiallyPending: createdRow.WorkerPending, + OutboxAlreadyProcessedAtCreateObservation: createdRow.WorkerProcessed, + OutboxCountBeforeReplay: outboxCountBeforeReplay, + OutboxCountAfterReplay: outboxAfterReplay.Count, + ProcessedEventCountForBaselineOrder: _fixture.ProcessedEvents.Count(e => e.OrderId == baseline.OrderId), + ReplayHttpStatus: replay.HttpStatus, + ReplaySameOrderId: replay.OrderId == baseline.OrderId, + ReplayCachedHeader: replay.CachedResponseHeader, + ReplayMediatorSpans: replay.MediatorSpanCount, + ReplayNpgsqlSpans: replay.NpgsqlSpanCount, + ControlOrderId: control.OrderId, + ControlOutboxCount: controlOutbox.Count, + ControlIntegrationEventId: controlOutbox[0].IntegrationEventId, + Notes: + [ + $"After cache miss, outbox row Status={createdRow.Status}; ProcessedAt null={createdRow.ProcessedAtUtc is null} (worker may process within poll interval).", + $"After worker: Status={processedOutbox.Status}; ProcessedAt={processedOutbox.ProcessedAtUtc:O}; CompletedAt={processedOutbox.CompletedAtUtc:O}.", + $"Handler IntegrationEvent.Id={delivered.Id} matches outbox_messages.Id; OrderId={delivered.OrderId}.", + $"Inbox row for IntegrationEvent.Id: count={inboxAfterDelivery.InboxRowCount}, processed={inboxAfterDelivery.IsProcessed}.", + "Does not assert exactly-once RabbitMQ delivery or behavior if worker crashes between publish and mark-processed." + ])); + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + replay.HttpStatus.Should().Be(200); + replay.OrderId.Should().Be(baseline.OrderId); + replay.CachedResponseHeader.Should().BeTrue(); + replay.MediatorSpanCount.Should().Be(0); + replay.NpgsqlSpanCount.Should().Be(0); + + outboxAfterReplay.Should().ContainSingle(); + outboxAfterReplay[0].OutboxMessageId.Should().Be(processedOutbox.OutboxMessageId); + outboxCountBeforeReplay.Should().Be(1); + _fixture.ProcessedEvents.Count(e => e.OrderId == baseline.OrderId).Should().Be(1); + } + + private async Task SendAsync( + InProcessActivityCapture capture, + List calls, + string behavior, + string idempotencyKey, + int quantity) + { + var result = await HttpOrderCreate.PostAsync( + _http, + capture, + idempotencyKey, + quantity, + jsonOptions: JsonOptions); + + var call = new OutboxLifecycleCall( + RequestNumber: calls.Count + 1, + Behavior: behavior, + IdempotencyKey: idempotencyKey, + Quantity: quantity, + HttpStatus: result.HttpStatus, + OrderId: result.OrderId, + QuantityReturned: result.Quantity, + TotalAmount: result.TotalAmount, + CachedResponseHeader: result.CachedResponseHeader, + ClientDurationMs: result.ClientDurationMs, + CompletedUtc: result.CompletedUtc, + TraceId: result.TraceIdHex, + MediatorSpanCount: result.Spans.Count(IsMediator), + NpgsqlSpanCount: result.Spans.Count(IsNpgsql), + Body: result.Body); + + calls.Add(call); + return call; + } + + private static OutboxSnapshot ToSnapshot(OrderOutboxRow row, string phase) + => new( + Phase: phase, + ObservedUtc: DateTimeOffset.UtcNow, + OutboxMessageId: row.OutboxMessageId, + IntegrationEventId: row.IntegrationEventId, + OrderId: row.OrderId, + Total: row.Total, + EventType: row.EventType, + Status: row.Status, + ProcessedAtUtc: row.ProcessedAtUtc, + CompletedAtUtc: row.CompletedAtUtc, + RetryCount: row.RetryCount, + WorkerPending: row.WorkerPending, + WorkerProcessed: row.WorkerProcessed, + CreatedAtUtc: row.CreatedAtUtc); + + private async Task QueryInboxAsync(Guid integrationEventId) + { + await using var scope = _services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var inboxRows = await db.InboxMessages + .AsNoTracking() + .Where(m => m.Id == integrationEventId) + .ToListAsync(); + + var row = inboxRows.SingleOrDefault(); + + return new InboxObservation( + InboxRowCount: inboxRows.Count, + InboxStatus: row?.Status.ToString(), + IsProcessed: row?.IsProcessed ?? false, + ProcessedAtUtc: row?.ProcessedAt); + } + + private sealed record OutboxLifecycleCall( + int RequestNumber, + string Behavior, + string IdempotencyKey, + int Quantity, + int HttpStatus, + Guid OrderId, + int? QuantityReturned, + decimal? TotalAmount, + bool CachedResponseHeader, + double ClientDurationMs, + DateTimeOffset CompletedUtc, + string TraceId, + int MediatorSpanCount, + int NpgsqlSpanCount, + string Body); + + private sealed record OutboxSnapshot( + string Phase, + DateTimeOffset ObservedUtc, + Guid OutboxMessageId, + Guid IntegrationEventId, + Guid OrderId, + decimal Total, + string EventType, + string Status, + DateTime? ProcessedAtUtc, + DateTime? CompletedAtUtc, + int RetryCount, + bool WorkerPending, + bool WorkerProcessed, + DateTime CreatedAtUtc); + + private sealed record InboxObservation( + int InboxRowCount, + string? InboxStatus, + bool IsProcessed, + DateTime? ProcessedAtUtc); + + private sealed record ProcessedEventObservation( + string Phase, + DateTimeOffset ObservedUtc, + Guid IntegrationEventId, + Guid OrderId, + decimal Total, + int InboxRowCount, + bool InboxProcessed); + + private sealed record OutboxLifecycleConfiguration( + string Path, + string IdempotencyHeader, + string CachedResponseHeader, + string AsyncPath, + string WorkerOrderingNote, + string IdentityNote, + string ProcessedEventsNote); + + private sealed record OutboxLifecycleObservations( + Guid BaselineOrderId, + Guid BaselineIntegrationEventId, + int OutboxCountAfterCreate, + bool OutboxInitiallyPending, + bool OutboxAlreadyProcessedAtCreateObservation, + int OutboxCountBeforeReplay, + int OutboxCountAfterReplay, + int ProcessedEventCountForBaselineOrder, + int ReplayHttpStatus, + bool ReplaySameOrderId, + bool ReplayCachedHeader, + int ReplayMediatorSpans, + int ReplayNpgsqlSpans, + Guid ControlOrderId, + int ControlOutboxCount, + Guid ControlIntegrationEventId, + IReadOnlyList Notes); + + private sealed record OutboxLifecycleExperimentResult( + string Name, + DateTimeOffset StartedUtc, + string GitSha, + string Environment, + OutboxLifecycleConfiguration Configuration, + IReadOnlyList Calls, + IReadOnlyList OutboxObservations, + IReadOnlyList ProcessedEventObservations, + OutboxLifecycleObservations Observations); +} diff --git a/tests/Lab/IntegrationTests/Experiments/PaginationAbuse/CarelessPaginationClient.cs b/tests/Lab/IntegrationTests/Experiments/PaginationAbuse/CarelessPaginationClient.cs new file mode 100644 index 0000000..c6c1532 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/PaginationAbuse/CarelessPaginationClient.cs @@ -0,0 +1,138 @@ +using System.Diagnostics; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; + +namespace IntegrationTests.Experiments.PaginationAbuse; + +/// +/// Deterministic HTTP client that pages like a careless tool caller. +/// Forgery uses public Base64url + JSON only — not CursorCodec.Encode. +/// +internal sealed class CarelessPaginationClient +{ + internal const string Path = "/api/v2/products-page"; + internal const string SortBy = "Id"; + internal const string SortDirection = "Ascending"; + internal const string PageDirection = "Forward"; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly HttpClient _http; + + public CarelessPaginationClient(HttpClient http) => _http = http; + + public async Task GetAsync( + int limit, + string cursor, + string pageDirection, + string traceparent) + { + var url = $"{Path}?limit={limit}&sortBy={SortBy}&sortDirection={SortDirection}&pageDirection={pageDirection}"; + if (!string.IsNullOrEmpty(cursor)) + url += $"&cursor={Uri.EscapeDataString(cursor)}"; + + using var request = new HttpRequestMessage(HttpMethod.Get, url); + request.Headers.TryAddWithoutValidation("traceparent", traceparent); + + var clock = Stopwatch.StartNew(); + var response = await _http.SendAsync(request); + clock.Stop(); + + var body = await response.Content.ReadAsStringAsync(); + ProductsPage? page = null; + string? error = null; + if (response.IsSuccessStatusCode) + { + page = JsonSerializer.Deserialize(body, JsonOptions); + } + else + { + error = body.Length > 500 ? body[..500] : body; + } + + return new PaginationAbuseHttpResult( + (int)response.StatusCode, + page, + clock.ElapsedMilliseconds, + error); + } + + /// + /// Decode v1.{"{payload}"}, bump the Id-sort seek integer, re-encode unsigned. + /// For sortBy=Id the payload has one vals slot (t=System.Int32). + /// + public static string TamperSeekId(string cursor, int delta) + { + ArgumentException.ThrowIfNullOrWhiteSpace(cursor); + if (!cursor.StartsWith("v1.", StringComparison.Ordinal)) + throw new InvalidOperationException($"Cursor is not v1: {cursor[..Math.Min(cursor.Length, 24)]}"); + + var rest = cursor[3..]; + var dot = rest.IndexOf('.'); + var payload = dot < 0 ? rest : rest[..dot]; + var json = Encoding.UTF8.GetString(Base64UrlDecode(payload)); + using var doc = JsonDocument.Parse(json); + if (!doc.RootElement.TryGetProperty("vals", out var vals) + || vals.ValueKind != JsonValueKind.Array + || vals.GetArrayLength() < 1) + { + throw new InvalidOperationException($"Cursor payload has no vals: {json}"); + } + + var slot = vals[0]; + if (!slot.TryGetProperty("v", out var seek) || seek.ValueKind != JsonValueKind.Number) + throw new InvalidOperationException($"Id-sort seek value is not a number: {json}"); + + var original = seek.GetInt32(); + var forged = original + delta; + + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream)) + { + writer.WriteStartObject(); + CopyProperty(writer, doc.RootElement, "v"); + CopyProperty(writer, doc.RootElement, "fp"); + CopyProperty(writer, doc.RootElement, "walk"); + writer.WritePropertyName("vals"); + writer.WriteStartArray(); + writer.WriteStartObject(); + CopyProperty(writer, slot, "t"); + writer.WriteNumber("v", forged); + writer.WriteEndObject(); + for (var i = 1; i < vals.GetArrayLength(); i++) + vals[i].WriteTo(writer); + writer.WriteEndArray(); + writer.WriteEndObject(); + } + + return "v1." + Base64UrlEncode(stream.ToArray()); + } + + private static void CopyProperty(Utf8JsonWriter writer, JsonElement obj, string name) + { + if (!obj.TryGetProperty(name, out var value)) + return; + writer.WritePropertyName(name); + value.WriteTo(writer); + } + + private static string Base64UrlEncode(byte[] data) + => Convert.ToBase64String(data).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + + private static byte[] Base64UrlDecode(string text) + { + var padded = text.Replace('-', '+').Replace('_', '/'); + padded += (4 - padded.Length % 4) % 4 == 0 ? "" : new string('=', (4 - padded.Length % 4) % 4); + return Convert.FromBase64String(padded); + } +} + +internal sealed record PaginationAbuseHttpResult( + int HttpStatus, + ProductsPage? Page, + long ClientDurationMs, + string? Error); diff --git a/tests/Lab/IntegrationTests/Experiments/PaginationAbuse/McpPaginationAbuseExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/PaginationAbuse/McpPaginationAbuseExperimentTests.cs new file mode 100644 index 0000000..9a2ab15 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/PaginationAbuse/McpPaginationAbuseExperimentTests.cs @@ -0,0 +1,243 @@ +using System.Diagnostics; +using System.Text.Json; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Mcp; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using ModelContextProtocol.Client; +using ModelContextProtocol.Protocol; +using Xunit.Abstractions; +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Experiments.PaginationAbuse; + +/// +/// Experiment 2: same careless pagination behaviors through MCP products.list. +/// Compares the MCP envelope with HTTP Experiment 1. Does not re-prove keyset SQL. +/// +[Collection(AspireCollection.Name)] +public sealed class McpPaginationAbuseExperimentTests +{ + private const string ToolName = "products.list"; + private const int Limit = 7; + private const int WalkPages = 8; + private const int TamperDelta = 50; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly HttpClient _http; + private readonly ITestOutputHelper _output; + + public McpPaginationAbuseExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _output = output; + _http = fixture.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + } + + [Fact] + public async Task Cursor_abuse_against_mcp_products_list_is_observed() + { + using var capture = new InProcessActivityCapture(); + var (transportTraceId, transportSpanId) = NewTraceParent(); + _http.DefaultRequestHeaders.Remove("traceparent"); + _http.DefaultRequestHeaders.TryAddWithoutValidation( + "traceparent", + FormatTraceParent(transportTraceId, transportSpanId)); + + await using var mcp = await LabMcpClient.CreateAsync(_http); + var seenToolTraces = new HashSet(StringComparer.Ordinal); + var calls = new List(); + var requestNumber = 0; + + async Task CallAsync(string behavior, string cursor) + { + requestNumber++; + var args = new Dictionary + { + ["limit"] = Limit, + ["sortBy"] = "Id", + ["sortDirection"] = "Ascending", + ["pageDirection"] = "Forward" + }; + if (!string.IsNullOrEmpty(cursor)) + args["cursor"] = cursor; + + var clock = Stopwatch.StartNew(); + var result = await mcp.CallToolAsync(ToolName, args); + clock.Stop(); + + var isError = result.IsError ?? false; + var errorText = isError ? McpToolResults.Truncate(McpToolResults.GetText(result)) : null; + var page = !isError && result.StructuredContent is { } structured + ? ParseProductsPage(structured) + : null; + + var toolSpan = capture.All.FirstOrDefault(s => + s.DisplayName == "mcp.tool" + && HasTag(s, "mcp.tool.name", ToolName) + && seenToolTraces.Add(s.TraceId)); + + var toolTrace = toolSpan?.TraceId; + var related = toolTrace is null + ? [] + : capture.All.Where(s => s.TraceId == toolTrace).ToList(); + var mediator = related.Where(IsMediator).ToList(); + + return new McpPaginationAbuseCall( + requestNumber, + behavior, + cursor, + isError, + errorText, + page?.Items.Select(i => i.Id).ToArray() ?? [], + page?.NextCursor ?? "", + page?.PreviousCursor ?? "", + page?.HasMore ?? false, + page?.HasPrevious ?? false, + page?.TotalCount ?? 0, + clock.ElapsedMilliseconds, + transportTraceId.ToHexString(), + toolTrace, + toolSpan is not null, + mediator.Count, + mediator.Count == 0 ? null : mediator.Max(s => s.Duration.TotalMilliseconds), + related.Count(IsNpgsql)); + } + + for (var i = 0; i < WalkPages; i++) + { + var cursorIn = i == 0 ? "" : calls[^1].NextCursor; + calls.Add(await CallAsync("Walk", cursorIn)); + } + + var walks = calls.Where(c => c.Behavior == "Walk").ToList(); + var walkIds = walks.SelectMany(c => c.ItemIds).ToList(); + var walkUnique = walkIds.Distinct().ToList(); + + var walk2 = walks[1]; + var replay = await CallAsync("Replay", walk2.CursorIn); + calls.Add(replay); + + var forged = CarelessPaginationClient.TamperSeekId(walks[0].NextCursor, TamperDelta); + var tamper = await CallAsync("Tamper", forged); + calls.Add(tamper); + + var malformed = await CallAsync("MalformedProbe", "not-a-valid-cursor"); + calls.Add(malformed); + + var transportHttp = capture.ForTrace(transportTraceId).Where(IsAspNetCore).ToList(); + var firstToolTrace = walks[0].ToolTraceId; + var toolVsTransport = firstToolTrace is not null + && !string.Equals(firstToolTrace, transportTraceId.ToHexString(), StringComparison.Ordinal); + + var payload = new + { + name = "mcp-products-list-cursor-abuse-v1", + startedUtc = DateTimeOffset.UtcNow, + gitSha = LabRunInfo.ReadGitSha(), + tool = ToolName, + transportTraceId = transportTraceId.ToHexString(), + calls, + observations = new + { + walkCalls = walks.Count, + walkUniqueIds = walkUnique.Count, + walkDuplicateIds = walkIds.Count - walkUnique.Count, + replaySameIdsAsOriginal = replay.ItemIds.SequenceEqual(walk2.ItemIds), + tamperIsError = tamper.IsError, + tamperReturnedIds = tamper.ItemIds, + malformedIsError = malformed.IsError, + malformedError = malformed.Error, + malformedMediatorSpanCount = malformed.MediatorSpanCount, + malformedNpgsqlSpanCount = malformed.NpgsqlSpanCount, + transportAspNetCoreSpanCount = transportHttp.Count, + firstToolTraceId = firstToolTrace, + toolTraceDistinctFromTransport = toolVsTransport, + notes = new[] + { + $"Walk unique={walkUnique.Count}, duplicates={walkIds.Count - walkUnique.Count}.", + $"Replay {(replay.ItemIds.SequenceEqual(walk2.ItemIds) ? "matched" : "did not match")} walk page 2.", + $"Tamper isError={tamper.IsError}; ids=[{string.Join(",", tamper.ItemIds)}].", + $"Malformed isError={malformed.IsError}; mediator={malformed.MediatorSpanCount}; npgsql={malformed.NpgsqlSpanCount}.", + $"POST /mcp transport TraceId={transportTraceId.ToHexString()}; first mcp.tool TraceId={firstToolTrace ?? "(none)"}; distinct={toolVsTransport}." + } + } + }; + _output.WriteLine(JsonSerializer.Serialize(payload, new JsonSerializerOptions { WriteIndented = true })); + + foreach (var walk in walks) + { + walk.IsError.Should().BeFalse(walk.Error); + walk.ItemIds.Should().HaveCount(Limit); + } + + walkUnique.Should().HaveCount(56); + (walkIds.Count - walkUnique.Count).Should().Be(0, "clean MCP walk should not overlap pages"); + walks[0].HasPrevious.Should().BeFalse(); + + replay.IsError.Should().BeFalse(replay.Error); + replay.ItemIds.Should().Equal(walk2.ItemIds); + + tamper.IsError.Should().BeFalse(tamper.Error); + tamper.ItemIds.Should().Equal(58, 59, 60, 61, 62, 63, 64); + + malformed.IsError.Should().BeTrue("MCP should report a tool-level error for a malformed cursor"); + malformed.NpgsqlSpanCount.Should().Be(0, "malformed cursor should not query PostgreSQL"); + + transportHttp.Should().NotBeEmpty("Streamable HTTP POST /mcp should continue the injected transport traceparent"); + firstToolTrace.Should().NotBeNullOrWhiteSpace(); + toolVsTransport.Should().BeTrue( + "mcp.tool TraceId should not be the POST /mcp transport TraceId (Phase 1)"); + walks[0].SawMcpToolSpan.Should().BeTrue(); + walks[0].MediatorSpanCount.Should().BeGreaterThan(0); + walks[0].NpgsqlSpanCount.Should().BeGreaterThan(0); + walks[0].ToolTraceId.Should().NotBeNull(); + capture.All.Where(s => s.TraceId == walks[0].ToolTraceId && IsMediator(s)) + .Should().Contain(s => s.DisplayName.Contains("GetProductsQuery", StringComparison.Ordinal)); + } + + private static ProductsPage ParseProductsPage(JsonElement structured) + { + var json = structured.GetRawText(); + var page = JsonSerializer.Deserialize(json, JsonOptions); + if (page?.Items is { Count: > 0 }) + return page; + + if (structured.ValueKind == JsonValueKind.Object && structured.TryGetProperty("value", out var value)) + { + page = JsonSerializer.Deserialize(value.GetRawText(), JsonOptions); + if (page?.Items is { Count: > 0 }) + return page; + } + + throw new InvalidOperationException($"Could not parse products page from MCP structured content: {json}"); + } + + private sealed record McpPaginationAbuseCall( + int RequestNumber, + string Behavior, + string CursorIn, + bool IsError, + string? Error, + IReadOnlyList ItemIds, + string NextCursor, + string PreviousCursor, + bool HasMore, + bool HasPrevious, + int TotalCount, + long ClientDurationMs, + string TransportTraceId, + string? ToolTraceId, + bool SawMcpToolSpan, + int MediatorSpanCount, + double? MediatorDurationMs, + int NpgsqlSpanCount); +} diff --git a/tests/Lab/IntegrationTests/Experiments/PaginationAbuse/PaginationAbuseExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/PaginationAbuse/PaginationAbuseExperimentTests.cs new file mode 100644 index 0000000..58c2320 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/PaginationAbuse/PaginationAbuseExperimentTests.cs @@ -0,0 +1,164 @@ +using System.Diagnostics; +using System.Text.Json; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.AspNetCore.Mvc.Testing; +using Xunit.Abstractions; +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Experiments.PaginationAbuse; + +/// +/// Observation experiment: a deterministic careless client against +/// GET /api/v2/products-page. Not a pagination correctness suite. +/// +[Collection(AspireCollection.Name)] +public sealed class PaginationAbuseExperimentTests +{ + private const int Limit = 7; + private const int WalkPages = 8; + private const int TamperDelta = 50; + + private readonly CarelessPaginationClient _client; + private readonly ITestOutputHelper _output; + + public PaginationAbuseExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _output = output; + _client = new CarelessPaginationClient(fixture.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + })); + } + + [Fact] + public async Task Cursor_abuse_against_products_page_is_observed() + { + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var calls = new List(); + var requestNumber = 0; + + async Task SendAsync(string behavior, string cursor) + { + requestNumber++; + var (traceId, spanId) = NewTraceParent(); + var http = await _client.GetAsync( + Limit, + cursor, + CarelessPaginationClient.PageDirection, + FormatTraceParent(traceId, spanId)); + var page = http.Page; + var spans = capture.ForTrace(traceId); + var mediator = spans.Where(IsMediator).ToList(); + return new PaginationAbuseCall( + requestNumber, + behavior, + Limit, + CarelessPaginationClient.SortBy, + CarelessPaginationClient.SortDirection, + CarelessPaginationClient.PageDirection, + cursor, + http.HttpStatus, + page?.Items.Select(i => i.Id).ToArray() ?? [], + page?.NextCursor ?? "", + page?.PreviousCursor ?? "", + page?.HasMore ?? false, + page?.HasPrevious ?? false, + page?.TotalCount ?? 0, + http.ClientDurationMs, + http.Error, + traceId.ToHexString(), + mediator.Count, + mediator.Count == 0 ? null : mediator.Max(s => s.Duration.TotalMilliseconds), + spans.Count(IsNpgsql)); + } + + for (var i = 0; i < WalkPages; i++) + { + var cursorIn = i == 0 ? "" : calls[^1].NextCursor; + calls.Add(await SendAsync("Walk", cursorIn)); + } + + var walks = calls.Where(c => c.Behavior == "Walk").ToList(); + walks.Should().HaveCount(WalkPages); + + var walk2 = walks[1]; + var replay = await SendAsync("Replay", walk2.CursorIn); + calls.Add(replay); + + var stale = await SendAsync("StaleReuse", walk2.CursorIn); + calls.Add(stale); + + var tamperSource = walks[0].NextCursor; + tamperSource.Should().NotBeNullOrWhiteSpace(); + var forged = CarelessPaginationClient.TamperSeekId(tamperSource, TamperDelta); + var tamper = await SendAsync("Tamper", forged); + calls.Add(tamper); + + var malformed = await SendAsync("MalformedProbe", "not-a-valid-cursor"); + calls.Add(malformed); + + var walkIds = walks.SelectMany(c => c.ItemIds).ToList(); + var walkUnique = walkIds.Distinct().ToList(); + var replaySame = replay.ItemIds.SequenceEqual(walk2.ItemIds); + var staleSame = stale.ItemIds.SequenceEqual(walk2.ItemIds); + var cursorIns = walks.Select(c => c.CursorIn).ToList(); + var cursorLoop = cursorIns + .Select((c, i) => (c, i)) + .Any(x => x.i > 0 && cursorIns.Take(x.i).Contains(x.c, StringComparer.Ordinal)); + + var observations = new PaginationAbuseObservations( + WalkCalls: walks.Count, + WalkUniqueIds: walkUnique.Count, + WalkDuplicateIds: walkIds.Count - walkUnique.Count, + WalkEmptyPages: walks.Count(c => c.ItemIds.Count == 0), + ReplaySameIdsAsOriginal: replaySame, + ReplayDuplicateIdCount: replay.ItemIds.Intersect(walk2.ItemIds).Count(), + StaleReuseSameIdsAsOriginalPage: staleSame, + TamperHttpStatus: tamper.HttpStatus, + TamperReturnedIds: tamper.ItemIds, + MalformedHttpStatus: malformed.HttpStatus, + CursorLoopDetected: cursorLoop, + TotalCalls: calls.Count, + FailureCount: calls.Count(c => c.HttpStatus >= 400), + TotalClientDurationMs: calls.Sum(c => c.ClientDurationMs), + Notes: + [ + $"Replay ids {(replaySame ? "matched" : "did not match")} walk call #2.", + $"Stale reuse ids {(staleSame ? "matched" : "did not match")} walk call #2 (seek position vs session ticket).", + $"Tamper HTTP {tamper.HttpStatus}; seek Id bumped by +{TamperDelta}; returned [{string.Join(",", tamper.ItemIds)}].", + $"Walk npgsql spans: {string.Join(",", walks.Select(c => c.NpgsqlSpanCount))}.", + $"First-page totalCount={walks[0].TotalCount}." + ]); + + var result = new PaginationAbuseExperimentResult( + Name: "pagination-cursor-abuse-v1", + StartedUtc: startedUtc, + GitSha: LabRunInfo.ReadGitSha(), + Environment: "Development", + CatalogProductCount: walks[0].TotalCount, + Configuration: new PaginationAbuseConfiguration( + CarelessPaginationClient.Path, + CarelessPaginationClient.SortBy, + CarelessPaginationClient.SortDirection, + Limit, + WalkPages), + Calls: calls, + Observations: observations); + + _output.WriteLine(JsonSerializer.Serialize(result, PaginationAbuseJsonContext.Default.PaginationAbuseExperimentResult)); + + foreach (var walk in walks) + { + walk.HttpStatus.Should().Be(200); + walk.ItemIds.Should().HaveCount(Limit, "non-final walk pages of a 1000-row catalog should be full"); + } + + walks[0].HasPrevious.Should().BeFalse(); + walks[0].TotalCount.Should().BeGreaterThan(Limit); + malformed.HttpStatus.Should().Be(400); + } +} diff --git a/tests/Lab/IntegrationTests/Experiments/PaginationAbuse/PaginationAbuseModels.cs b/tests/Lab/IntegrationTests/Experiments/PaginationAbuse/PaginationAbuseModels.cs new file mode 100644 index 0000000..165ab13 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/PaginationAbuse/PaginationAbuseModels.cs @@ -0,0 +1,73 @@ +using System.Text.Json.Serialization; + +namespace IntegrationTests.Experiments.PaginationAbuse; + +internal sealed record PaginationAbuseExperimentResult( + string Name, + DateTimeOffset StartedUtc, + string GitSha, + string Environment, + int CatalogProductCount, + PaginationAbuseConfiguration Configuration, + IReadOnlyList Calls, + PaginationAbuseObservations Observations); + +internal sealed record PaginationAbuseConfiguration( + string Path, + string SortBy, + string SortDirection, + int Limit, + int WalkPages); + +internal sealed record PaginationAbuseCall( + int RequestNumber, + string Behavior, + int Limit, + string SortBy, + string SortDirection, + string PageDirection, + string CursorIn, + int HttpStatus, + IReadOnlyList ItemIds, + string NextCursor, + string PreviousCursor, + bool HasMore, + bool HasPrevious, + int TotalCount, + long ClientDurationMs, + string? Error, + string TraceId, + int MediatorSpanCount, + double? MediatorDurationMs, + int NpgsqlSpanCount); + +internal sealed record PaginationAbuseObservations( + int WalkCalls, + int WalkUniqueIds, + int WalkDuplicateIds, + int WalkEmptyPages, + bool ReplaySameIdsAsOriginal, + int ReplayDuplicateIdCount, + bool StaleReuseSameIdsAsOriginalPage, + int TamperHttpStatus, + IReadOnlyList TamperReturnedIds, + int MalformedHttpStatus, + bool CursorLoopDetected, + int TotalCalls, + int FailureCount, + long TotalClientDurationMs, + IReadOnlyList Notes); + +internal sealed record ProductsPage( + IReadOnlyList Items, + string NextCursor, + string PreviousCursor, + bool HasMore, + bool HasPrevious, + int TotalCount); + +internal sealed record ProductItem(int Id, string Name, decimal Price, DateTime CreatedAt); + +[JsonSourceGenerationOptions(WriteIndented = true, PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)] +[JsonSerializable(typeof(PaginationAbuseExperimentResult))] +internal partial class PaginationAbuseJsonContext : JsonSerializerContext; diff --git a/tests/Lab/IntegrationTests/Experiments/ProcessedMessageDeduplication/ProcessedMessageDeduplicationExperimentTests.cs b/tests/Lab/IntegrationTests/Experiments/ProcessedMessageDeduplication/ProcessedMessageDeduplicationExperimentTests.cs new file mode 100644 index 0000000..1965ab9 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/ProcessedMessageDeduplication/ProcessedMessageDeduplicationExperimentTests.cs @@ -0,0 +1,383 @@ +using System.Diagnostics; +using System.Text.Json; +using EventBusRabbitMQ; +using EventBusRabbitMQ.Domain; +using EventBusRabbitMQ.Infrastructure.EventBus; +using FeatureFusion.Features.Order.IntegrationEvents.Events; +using FeatureFusion.Infrastructure.Context; +using FluentAssertions; +using IntegrationTests.Aspire; +using IntegrationTests.Infrastructure.Async; +using IntegrationTests.Infrastructure.Reporting; +using IntegrationTests.Infrastructure.Telemetry; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Xunit.Abstractions; + +namespace IntegrationTests.Experiments.ProcessedMessageDeduplication; + +/// +/// Experiment 17 — EnableDeduplication=true + duplicate message delivery. +/// Hypothesis: with the lab EventBus option enabled, the first delivery of an +/// runs the handler and records +/// processed_messages; a second PublishDirect with the same +/// IntegrationEvent.Id reaches the consumer again but is suppressed before +/// handler dispatch (MessageProcessor checks processed_messages before inbox +/// and before DispatchToHandlers), yielding exactly one handler observation. +/// Contrasts Exp 7 (EnableDeduplication=false, inbox completion dedup only, +/// processed_messages unused). Test-host flips the option on the shared +/// instance (same reference EventBus reads per message); +/// Lab/fixture default remains false. No production EventBus changes. +/// +[Collection(AspireCollection.Name)] +public sealed class ProcessedMessageDeduplicationExperimentTests : IAsyncLifetime +{ + private static readonly TimeSpan DuplicateObservationWindow = TimeSpan.FromSeconds(3); + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly AspireFixture _fixture; + private readonly IServiceProvider _services; + private readonly ITestOutputHelper _output; + private bool _priorEnableDeduplication; + + public ProcessedMessageDeduplicationExperimentTests(AspireFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + _ = fixture.CreateClient(); + _services = fixture.Services; + } + + public async Task InitializeAsync() + { + await _fixture.ResetRabbitMQ(); + + var options = GetRequiredService>().Value; + _priorEnableDeduplication = options.EnableDeduplication; + options.EnableDeduplication = true; + } + + public Task DisposeAsync() + { + GetRequiredService>().Value.EnableDeduplication = _priorEnableDeduplication; + return Task.CompletedTask; + } + + [Fact] + public async Task EnableDeduplication_records_processed_messages_and_suppresses_duplicate_handler_dispatch() + { + _fixture.ProcessedEvents.Clear(); + + var startedUtc = DateTimeOffset.UtcNow; + using var capture = new InProcessActivityCapture(); + var calls = new List(); + var stateObservations = new List(); + + var eventBusOptions = GetRequiredService>().Value; + eventBusOptions.EnableDeduplication.Should().BeTrue( + "Exp 17 must run with EnableDeduplication=true on the live EventBusOptions instance"); + + var sharedOrderId = Guid.NewGuid(); + const decimal total = 91.50m; + var baselineEvent = new OrderCreatedIntegrationEvent(sharedOrderId, total); + + var baseline = await PublishAndObserveAsync( + capture, + calls, + behavior: "BaselineDelivery", + @event: baselineEvent); + + baseline.ProcessedEventCountForMessageId.Should().Be(1, + "baseline must execute the handler once for IntegrationEvent.Id={0}", baselineEvent.Id); + + await WaitForProcessedLayersAsync(baselineEvent.Id); + + var afterBaseline = await QueryStateAsync(baselineEvent.Id); + stateObservations.Add(afterBaseline with { Phase = "AfterBaseline" }); + + afterBaseline.InboxRowCount.Should().Be(1); + afterBaseline.IsProcessed.Should().BeTrue(); + afterBaseline.ProcessedMessageRowCount.Should().Be(1, + "EnableDeduplication=true must record processed_messages after successful handler completion"); + + var processAttemptsAfterBaseline = CountEventBusProcessAttempts(capture, baselineEvent.Id); + + var duplicate = await PublishAndObserveAsync( + capture, + calls, + behavior: "DuplicateSameIdentity", + @event: baselineEvent, + priorProcessedCount: baseline.ProcessedEventCountForMessageId, + waitForObservationWindow: true); + + var afterDuplicate = await QueryStateAsync(baselineEvent.Id); + stateObservations.Add(afterDuplicate with { Phase = "AfterDuplicate" }); + + var processAttemptsAfterDuplicate = CountEventBusProcessAttempts(capture, baselineEvent.Id); + var processStatuses = GetProcessMessageStatuses(capture, baselineEvent.Id); + + var controlEvent = new OrderCreatedIntegrationEvent(sharedOrderId, total); + var control = await PublishAndObserveAsync( + capture, + calls, + behavior: "ControlDifferentIdentity", + @event: controlEvent); + + control.ProcessedEventCountForMessageId.Should().Be(1, + "a new IntegrationEvent.Id must run the handler again even with EnableDeduplication=true"); + + await WaitForProcessedLayersAsync(controlEvent.Id); + var afterControl = await QueryStateAsync(controlEvent.Id); + stateObservations.Add(afterControl with { Phase = "AfterControl" }); + + var handlerCountBaselineId = _fixture.ProcessedEvents.Count(e => e.Id == baselineEvent.Id); + var handlerCountControlId = _fixture.ProcessedEvents.Count(e => e.Id == controlEvent.Id); + var handlerCountSharedOrder = _fixture.ProcessedEvents.Count(e => e.OrderId == sharedOrderId); + + var hypothesisConfirmed = + handlerCountBaselineId == 1 + && duplicate.ProcessedEventCountDeltaFromPrior == 0 + && processAttemptsAfterDuplicate >= processAttemptsAfterBaseline + 1 + && afterBaseline.ProcessedMessageRowCount == 1 + && afterDuplicate.ProcessedMessageRowCount == 1 + && afterDuplicate.InboxRowCount == 1 + && afterControl.ProcessedMessageRowCount == 1 + && handlerCountControlId == 1; + + var result = new + { + name = "event-bus-enable-deduplication-processed-messages-v1", + startedUtc, + gitSha = LabRunInfo.ReadGitSha(), + environment = "Development", + configuration = new + { + consumerPath = "IEventBus.PublishDirect → RabbitMQ → EventBus.MessagetHandler → MessageProcessor.ProcessMessageAsync(deduplication:true) → OrderCreatedIntegrationEventHandler", + enableDeduplication = eventBusOptions.EnableDeduplication, + subscriptionClientName = eventBusOptions.SubscriptionClientName, + optionMechanism = "Mutate shared EventBusOptions.EnableDeduplication for this test only (same instance EventBus reads per delivery); restore in DisposeAsync. Avoids a second competing WithWebHostBuilder consumer.", + guaranteedOrdering = new[] + { + "1. if EnableDeduplication && processed_messages hit → ProcessingResult.Success (no DispatchToHandlers)", + "2. else if inbox IsDuplicate (already processed) → Success (no dispatch)", + "3. else store inbox → DispatchToHandlers → on Success Mark processed_messages then Mark inbox processed → return Success → ACK" + }, + contrastWithExp7 = "Exp 7: EnableDeduplication=false → processed_messages unused; duplicate suppressed by inbox IsProcessed only." + }, + calls, + stateObservations, + observations = new + { + baselineIntegrationEventId = baselineEvent.Id, + controlIntegrationEventId = controlEvent.Id, + sharedOrderId, + handlerCountBaselineId, + handlerCountControlId, + handlerCountSharedOrder, + processAttemptsAfterBaseline, + processAttemptsAfterDuplicate, + processMessageStatusesForBaselineId = processStatuses, + duplicateIncreasedHandlerCount = duplicate.ProcessedEventCountDeltaFromPrior > 0, + processedMessagesAfterBaseline = afterBaseline.ProcessedMessageRowCount, + processedMessagesAfterDuplicate = afterDuplicate.ProcessedMessageRowCount, + inboxRowsAfterDuplicate = afterDuplicate.InboxRowCount, + hypothesisConfirmed, + hypothesis = "EnableDeduplication=true records processed_messages on first success and suppresses a same-Id second delivery before handler dispatch." + }, + notes = new[] + { + "First delivery: processed_messages miss → inbox store → handler → Mark processed_messages → Mark inbox → ACK.", + "Second delivery (same IntegrationEvent.Id): consumer runs again (ProcessMessage), but MessageProcessor returns Success from the processed_messages check before DispatchToHandlers.", + "After a successful first delivery, inbox IsProcessed is also true; the code checks processed_messages first, so that layer is the guaranteed early exit when EnableDeduplication=true.", + "Control with a new IntegrationEvent.Id proves suppression is identity-based, not OrderId-based.", + "Handler business effect remains the test decorator + in-memory ReceivedEvents (same as Exp 7); no HTTP/outbox order create in this experiment.", + hypothesisConfirmed + ? "Hypothesis CONFIRMED." + : "Hypothesis FALSIFIED or inconclusive relative to packaged EnableDeduplication semantics." + } + }; + + _output.WriteLine(JsonSerializer.Serialize(result, JsonOptions)); + + handlerCountBaselineId.Should().Be(1, + "duplicate same-Id delivery must not dispatch the handler a second time"); + duplicate.ProcessedEventCountDeltaFromPrior.Should().Be(0); + processAttemptsAfterDuplicate.Should().BeGreaterThanOrEqualTo( + processAttemptsAfterBaseline + 1, + "duplicate PublishDirect must still reach EventBus ProcessMessage even when suppressed"); + afterDuplicate.ProcessedMessageRowCount.Should().Be(1); + afterDuplicate.InboxRowCount.Should().Be(1); + afterDuplicate.IsProcessed.Should().BeTrue(); + + handlerCountControlId.Should().Be(1); + handlerCountSharedOrder.Should().Be(2, + "baseline + control identities → two handler observations for the same OrderId"); + afterControl.ProcessedMessageRowCount.Should().Be(1, + "control identity must also record its own processed_messages row"); + + foreach (var status in processStatuses) + { + (status == "processed" || status == null).Should().BeTrue( + "successful and early-Success duplicate paths ACK with message.status=processed (or unset). Statuses: [{0}]", + string.Join(",", processStatuses)); + } + } + + private async Task PublishAndObserveAsync( + InProcessActivityCapture capture, + List calls, + string behavior, + OrderCreatedIntegrationEvent @event, + int priorProcessedCount = 0, + bool waitForObservationWindow = false) + { + var eventBus = GetRequiredService(); + var publishedUtc = DateTimeOffset.UtcNow; + var processAttemptsBefore = CountEventBusProcessAttempts(capture, @event.Id); + + await eventBus.PublishDirect(@event); + + if (waitForObservationWindow) + { + await Wait.UntilAsync( + () => DateTimeOffset.UtcNow - publishedUtc >= DuplicateObservationWindow, + TimeSpan.FromSeconds(10)); + } + else + { + await Wait.UntilAsync( + () => _fixture.ProcessedEvents.Count(e => e.Id == @event.Id) > priorProcessedCount, + TimeSpan.FromSeconds(20)); + } + + var completedUtc = DateTimeOffset.UtcNow; + var processedForMessageId = _fixture.ProcessedEvents.Count(e => e.Id == @event.Id); + var processAttemptsAfter = CountEventBusProcessAttempts(capture, @event.Id); + + var call = new DedupCall( + RequestNumber: calls.Count + 1, + Behavior: behavior, + IntegrationEventId: @event.Id, + OrderId: @event.OrderId, + Total: @event.Total, + PublishedUtc: publishedUtc, + CompletedUtc: completedUtc, + DurationMs: (completedUtc - publishedUtc).TotalMilliseconds, + ProcessedEventCountForMessageId: processedForMessageId, + ProcessedEventCountDeltaFromPrior: processedForMessageId - priorProcessedCount, + EventBusProcessAttemptsBefore: processAttemptsBefore, + EventBusProcessAttemptsAfter: processAttemptsAfter, + EventBusProcessAttemptsDelta: processAttemptsAfter - processAttemptsBefore); + + calls.Add(call); + return call; + } + + private async Task WaitForProcessedLayersAsync(Guid messageId) + { + var stopwatch = Stopwatch.StartNew(); + while (stopwatch.Elapsed < TimeSpan.FromSeconds(20)) + { + var state = await QueryStateAsync(messageId); + if (state.InboxRowCount == 1 + && state.IsProcessed + && state.InboxStatus == MessageStatus.Processed.ToString() + && state.ProcessedMessageRowCount == 1) + { + return; + } + + await Task.Delay(100); + } + + throw new TimeoutException( + $"Message {messageId} was not fully recorded in inbox (processed) + processed_messages within timeout"); + } + + private async Task QueryStateAsync(Guid messageId) + { + await using var scope = _services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var inboxRows = await db.InboxMessages + .AsNoTracking() + .Where(m => m.Id == messageId) + .ToListAsync(); + + var processedRows = await db.ProcessedMessages + .AsNoTracking() + .Where(m => m.Id == messageId) + .ToListAsync(); + + var row = inboxRows.SingleOrDefault(); + var processed = processedRows.SingleOrDefault(); + + return new DedupStateObservation( + Phase: string.Empty, + ObservedUtc: DateTimeOffset.UtcNow, + MessageId: messageId, + InboxRowCount: inboxRows.Count, + InboxStatus: row?.Status.ToString(), + IsProcessed: row?.IsProcessed ?? false, + InboxProcessedAtUtc: row?.ProcessedAt, + ProcessedMessageRowCount: processedRows.Count, + ProcessedMessageProcessedAtUtc: processed?.ProcessedAt); + } + + private static int CountEventBusProcessAttempts(InProcessActivityCapture capture, Guid messageId) + { + var id = messageId.ToString(); + return capture.All.Count(span => + span.Source == "EventBus" + && span.DisplayName == "ProcessMessage" + && span.Tags.TryGetValue("message.id", out var mid) + && mid == id); + } + + private static IReadOnlyList GetProcessMessageStatuses(InProcessActivityCapture capture, Guid messageId) + { + var id = messageId.ToString(); + return capture.All + .Where(span => + span.Source == "EventBus" + && span.DisplayName == "ProcessMessage" + && span.Tags.TryGetValue("message.id", out var mid) + && mid == id) + .Select(span => span.Tags.TryGetValue("message.status", out var status) ? status : null) + .ToList(); + } + + private T GetRequiredService() where T : notnull => + _services.GetRequiredService(); + + private sealed record DedupCall( + int RequestNumber, + string Behavior, + Guid IntegrationEventId, + Guid OrderId, + decimal Total, + DateTimeOffset PublishedUtc, + DateTimeOffset CompletedUtc, + double DurationMs, + int ProcessedEventCountForMessageId, + int ProcessedEventCountDeltaFromPrior, + int EventBusProcessAttemptsBefore, + int EventBusProcessAttemptsAfter, + int EventBusProcessAttemptsDelta); + + private sealed record DedupStateObservation( + string Phase, + DateTimeOffset ObservedUtc, + Guid MessageId, + int InboxRowCount, + string? InboxStatus, + bool IsProcessed, + DateTime? InboxProcessedAtUtc, + int ProcessedMessageRowCount, + DateTime? ProcessedMessageProcessedAtUtc); +} diff --git a/tests/Lab/IntegrationTests/Experiments/README.md b/tests/Lab/IntegrationTests/Experiments/README.md new file mode 100644 index 0000000..269c600 --- /dev/null +++ b/tests/Lab/IntegrationTests/Experiments/README.md @@ -0,0 +1,445 @@ +# Lab integration experiments + +Behavioral experiments against **real** FeatureFusion workflows hosted by [`AspireFixture`](../Aspire/AspireFixture.cs). Each experiment is an integration test with a stated hypothesis, labeled invocations, captured evidence, a JSON artifact, and explicit invariants. + +**Lab charter (what the laboratory is for, evidence families, when to extract a BuildingBlock):** [`docs/lab/README.md`](../../../../docs/lab/README.md). + +This document describes what the Lab **does today**. It is not a framework specification and does not prescribe a Scenario/Actor engine. + +## Shared observation helpers (Phase 2) + +Lab-only, not NuGet. Used when ≥3 experiments share identical observation semantics: + +| Helper | Location | Consumers (Exp #) | +|--------|----------|-------------------| +| `HttpOrderCreate` | [`Infrastructure/Orders/HttpOrderCreate.cs`](../Infrastructure/Orders/HttpOrderCreate.cs) | 3, 4, 5, 8, 12, 15, 18 | +| `OrderOutboxObserver` | [`Infrastructure/Orders/OrderOutboxObserver.cs`](../Infrastructure/Orders/OrderOutboxObserver.cs) | 8, 10, 13, 14, 15, 16, 18 | +| `McpToolResults` | [`Infrastructure/Mcp/McpToolResults.cs`](../Infrastructure/Mcp/McpToolResults.cs) | 2, 6, 10, 13, 14, 16 | + +Also: prefer `LabTrace` / `LabRunInfo` over local copies of `IsMediator` / `IsNpgsql` / `ReadGitSha`. + +## Evidence families (summary) + +| Family | Experiments | Notes | +|--------|-------------|-------| +| Pagination abuse | 1–2 | HTTP + MCP keyset clients | +| HTTP Redis idempotency | 3, 4, 12, 15 | → BuildingBlocks.Idempotency provenance | +| MCP writes / agent clients | 6, 13, 14, 16 | Idempotency + rate-limit bound | +| Outbox happy path | 5, 8, 10 | HTTP + MCP parity | +| Consumer dedup / DLQ | 7, 9, 11, 17 | Inbox + `processed_messages` | +| Telemetry / correlation | TraceEvidence, 18 | W3C does not cross RabbitMQ today | +| MAF + Ollama | Prototype only | Not numbered | + +## Purpose + +- **Behavioral experiments**, not unit tests of libraries in isolation. The application under test is the in-process lab API (`WebApplicationFactory`) with Aspire containers for Postgres, Redis, RabbitMQ, and Memcached. +- **Production code unchanged.** Experiments observe production paths; only test infrastructure (fixture decoration, observation helpers) may differ. +- **Integration evidence over mocks.** Prefer HTTP/MCP responses, in-process `Activity` capture, and fixture observation hooks over synthetic stand-ins for collaborators. +- **One falsifiable hypothesis per experiment**, expressed in the test class XML documentation and asserted at the end of the `[Fact]`. + +Related but separate: [`FeatureFusionTraceEvidenceTests`](../Api/FeatureFusionTraceEvidenceTests.cs) (Phase 1) proves the in-process trace evidence path before the numbered experiment series. It lives under `Api/`, not `Experiments/`. + +Smoke and direct EventBus tests live under [`EventBus/`](../EventBus/) and [`Api/`](../Api/). They validate wiring; experiments validate **observable production behavior** under deliberate client/envelope conditions. + +--- + +## Experiment catalog + +| # | Artifact name | Folder / test class | +|---|---------------|---------------------| +| 1 | `pagination-cursor-abuse-v1` | [`PaginationAbuse/PaginationAbuseExperimentTests.cs`](PaginationAbuse/PaginationAbuseExperimentTests.cs) | +| 2 | `mcp-products-list-cursor-abuse-v1` | [`PaginationAbuse/McpPaginationAbuseExperimentTests.cs`](PaginationAbuse/McpPaginationAbuseExperimentTests.cs) | +| 3 | `order-create-redis-idempotency-v1` | [`CacheVsProduction/CacheVsProductionExperimentTests.cs`](CacheVsProduction/CacheVsProductionExperimentTests.cs) | +| 4 | `order-create-idempotency-concurrency-v1` | [`Concurrency/ConcurrencyRaceExperimentTests.cs`](Concurrency/ConcurrencyRaceExperimentTests.cs) | +| 5 | `http-order-outbox-delivery-v1` | [`OutboxDelivery/OutboxDeliveryExperimentTests.cs`](OutboxDelivery/OutboxDeliveryExperimentTests.cs) | +| 6 | `mcp-orders-create-idempotency-v1` | [`McpOrderIdempotency/McpOrderIdempotencyExperimentTests.cs`](McpOrderIdempotency/McpOrderIdempotencyExperimentTests.cs) | +| 7 | `duplicate-integration-event-delivery-v1` | [`DuplicateDelivery/DuplicateDeliveryExperimentTests.cs`](DuplicateDelivery/DuplicateDeliveryExperimentTests.cs) | +| 8 | `http-order-outbox-lifecycle-v1` | [`OutboxLifecycle/OutboxLifecycleExperimentTests.cs`](OutboxLifecycle/OutboxLifecycleExperimentTests.cs) | +| 9 | `event-bus-failure-outcome-v1` | [`EventBusFailure/EventBusFailureExperimentTests.cs`](EventBusFailure/EventBusFailureExperimentTests.cs) | +| 10 | `mcp-order-outbox-parity-v1` | [`McpOrderOutbox/McpOrderOutboxExperimentTests.cs`](McpOrderOutbox/McpOrderOutboxExperimentTests.cs) | +| 11 | `handler-throw-failure-outcome-v1` | [`HandlerThrowFailure/HandlerThrowFailureExperimentTests.cs`](HandlerThrowFailure/HandlerThrowFailureExperimentTests.cs) | +| 12 | `order-create-idempotency-fingerprint-v1` | [`IdempotencyFingerprint/IdempotencyFingerprintExperimentTests.cs`](IdempotencyFingerprint/IdempotencyFingerprintExperimentTests.cs) | +| 13 | `mcp-orders-create-agent-key-regeneration-v1` | [`McpAgentKeyRegeneration/McpAgentKeyRegenerationExperimentTests.cs`](McpAgentKeyRegeneration/McpAgentKeyRegenerationExperimentTests.cs) | +| 14 | `mcp-orders-create-concurrent-same-key-v1` | [`McpConcurrentSameKey/McpConcurrentSameKeyExperimentTests.cs`](McpConcurrentSameKey/McpConcurrentSameKeyExperimentTests.cs) | +| 15 | `order-create-idempotency-processing-lease-v1` | [`IdempotencyProcessingLease/IdempotencyProcessingLeaseExperimentTests.cs`](IdempotencyProcessingLease/IdempotencyProcessingLeaseExperimentTests.cs) | +| 16 | `mcp-orders-create-tool-storm-rate-limit-v1` | [`McpToolStormRateLimit/McpToolStormRateLimitExperimentTests.cs`](McpToolStormRateLimit/McpToolStormRateLimitExperimentTests.cs) | +| 17 | `event-bus-enable-deduplication-processed-messages-v1` | [`ProcessedMessageDeduplication/ProcessedMessageDeduplicationExperimentTests.cs`](ProcessedMessageDeduplication/ProcessedMessageDeduplicationExperimentTests.cs) | +| 18 | `http-order-async-trace-correlation-v1` | [`AsyncTraceCorrelation/AsyncTraceCorrelationExperimentTests.cs`](AsyncTraceCorrelation/AsyncTraceCorrelationExperimentTests.cs) | +| 19 | `eventbus-observation-baseline-v1` | [`EventBusObservationBaseline/EventBusObservationBaselineExperimentTests.cs`](EventBusObservationBaseline/EventBusObservationBaselineExperimentTests.cs) | +| 20 | `eventbus-publish-then-crash-v1` | [`EventBusPublishCrash/EventBusPublishCrashExperimentTests.cs`](EventBusPublishCrash/EventBusPublishCrashExperimentTests.cs) | + +**Hard end:** Exp 20 closes the MCP/EventBus research branch. No Exp 21 in this line. + +### MAF prototype spike (not a numbered experiment) + +**Status:** wiring verified; live agent runs require a chat provider (OpenAI, Azure OpenAI, or Ollama). + +[`MafMcpPrototype/MafMcpPrototypeTests.cs`](MafMcpPrototype/MafMcpPrototypeTests.cs) places a real [Microsoft Agent Framework](https://github.com/microsoft/agent-framework) agent in front of the existing FeatureFusion `/mcp` endpoint (Aspire + `WebApplicationFactory`, unchanged `BuildingBlocks.Mcp`). It records ordered MCP tool-call sequences from `AgentResponse.Messages` and correlates them with in-process `mcp.tool` / Mediator / Npgsql spans plus outbox/handler evidence. + +Run: + +```bash +# Transport-only (no LLM key required) +dotnet test tests/Lab/IntegrationTests/IntegrationTests.csproj \ + --filter "FullyQualifiedName~Maf_mcp_transport_connects" + +# Live agent — OpenAI / Azure +export OPENAI_API_KEY=... +export OPENAI_MODEL=gpt-4o-mini # optional +dotnet test tests/Lab/IntegrationTests/IntegrationTests.csproj \ + --filter "FullyQualifiedName~Maf_agent_runs_goal" + +# Live agent — local/self-hosted Ollama (OpenAI-compatible /v1) +export OLLAMA_BASE_URL=http://127.0.0.1:11434 +export OLLAMA_MODEL=qwen3.5:4b-q4_K_M +dotnet test tests/Lab/IntegrationTests/IntegrationTests.csproj \ + --filter "FullyQualifiedName~Maf_agent_runs_goal" +``` + +Ollama is a Lab research convenience, not a FeatureFusion dependency. Choose a model that advertises `tools` capability. Streaming is unused. + +This is research infrastructure only — not Exp 15 and not a BuildingBlock. + +### Workstream status — HTTP Redis idempotency (Exp 3, 4, 12) + +**COMPLETE — BuildingBlocks.Idempotency 1.0.0** + +Reusable implementation: [`src/BuildingBlocks/Idempotency`](../../../../src/BuildingBlocks/Idempotency). Lab hosts the package on `POST /api/v2/Order/order`; these three experiments are the **evidence / provenance trail**, not unfinished extraction work. Package unit tests live under `tests/BuildingBlocks/Idempotency.Tests`. MCP `orders.create` idempotency (Exp 6) is a separate in-memory store and is **not** part of this BuildingBlock. + +### Experiment 1 — HTTP pagination cursor abuse + +- **Hypothesis / problem:** A deterministic “careless” client walking `GET /api/v2/products-page` produces observable cursor semantics (replay, stale reuse, tamper, malformed cursor) without claiming full pagination correctness. +- **Surface:** HTTP `GET /api/v2/products-page` → Mediator `GetProductsQuery` → PostgreSQL. +- **Primary behavior:** Clean walk yields 56 unique IDs across 8 pages; replay is stable; unsigned tamper shifts seek window; malformed cursor returns HTTP 400 before Mediator/Npgsql. +- **Limitation / non-goal:** Not a pagination correctness or performance suite. Host pagination signing key is not configured in the lab. + +### Experiment 2 — MCP pagination cursor abuse + +- **Hypothesis / problem:** Same careless pagination behaviors through MCP `products.list`, comparing the MCP envelope to Experiment 1. +- **Surface:** MCP `products.list` → same `GetProductsQuery` path as Exp 1. +- **Primary behavior:** Walk/replay/tamper parallels Exp 1; malformed cursor is MCP `isError` with Mediator but **no** Npgsql; POST `/mcp` transport TraceId ≠ `mcp.tool` TraceId. +- **Limitation / non-goal:** Does not re-prove keyset SQL correctness (Exp 1 already exercises the query). JSON artifact uses an anonymous payload (cosmetic difference from other experiments). + +### Experiment 3 — HTTP Redis idempotency vs production (CacheVsProduction) + +- **Role:** Originally characterized Lab HTTP idempotency; now **regression / provenance** that extracted `BuildingBlocks.Idempotency` preserves the intended miss/hit workflow (fingerprint **off**, Lab default). +- **Workstream:** COMPLETE — BuildingBlocks.Idempotency 1.0.0 (extraction proof set). +- **Hypothesis / problem:** Package filter (Redis `IDistributedCache`) separates cache replay from production execution on `POST /api/v2/Order/order`. +- **Surface:** HTTP order create → `[Idempotent(useLock: true)]` → `CreateOrderCommandHandler` (Mediator + catalog SaveChanges + outbox insert on miss). +- **Primary behavior:** Miss runs production (Mediator + Npgsql); hit replays with `X-Idempotent-Response`; same key + different body keeps original order (body not part of key when fingerprint off); new key runs production again. Replay body casing reflects package Newtonsoft cache serialization. +- **Limitation / non-goal:** Does not cover MCP `orders.create` (Exp 6). Does not assert async handler delivery (Exp 5). Does not prove concurrency lock races (Exp 4) or opt-in fingerprinting (Exp 12). + +### Experiment 4 — HTTP idempotency concurrency (ConcurrencyRace) + +- **Role:** Originally proved the Redis same-key concurrency problem and lock behavior; now **regression / provenance** for `RedisIdempotencyLock` + Processing/Completed coordination under concurrent clients. +- **Workstream:** COMPLETE — BuildingBlocks.Idempotency 1.0.0 (extraction proof set). +- **Hypothesis / problem:** Concurrent clients with the **same** `Idempotency-Key` coordinate through Redis lock/status; different keys run independently. +- **Surface:** Same HTTP order-create path as Exp 3 (`BuildingBlocks.Idempotency`). +- **Primary behavior:** Three concurrent same-key requests → exactly one production winner, one unique `orderId`; losers are cached replay, HTTP 409, or lock-failure HTTP 500 (as observed). Two different keys both run production. +- **Limitation / non-goal:** `StartTogether` gate is local to this experiment. Does not prove MCP or outbox semantics. Does not prove fingerprinting (Exp 12). ProcessingTtl lease overlap while a handler is still in flight is **Exp 15**. + +### Experiment 5 — HTTP order create → outbox → handler + +- **Hypothesis / problem:** Cache-miss order create eventually delivers exactly one `OrderCreatedIntegrationEvent` to the consumer; idempotent HTTP replay does not deliver a second event. +- **Surface:** HTTP order create → outbox insert → `OutBoxWorker` → RabbitMQ → `OrderCreatedIntegrationEventHandler`. +- **Primary behavior:** Baseline HTTP success + Mediator/Npgsql; bounded wait until one handler observation; replay is cached HTTP with no Mediator/Npgsql and no second event. +- **Limitation / non-goal:** **`ProcessedEvents` is test observation infrastructure** (decorator on the real handler), not production telemetry. Proves **eventual handler observation**, not direct proof that a specific `outbox_messages` row transitioned pending→processed or that RabbitMQ was the only publish path. + +### Experiment 6 — MCP orders.create confirmation + idempotency + +- **Hypothesis / problem:** MCP `orders.create` has its own confirmation gate and `MemoryIdempotencyStore` replay semantics, distinct from HTTP Redis idempotency (Exp 3). +- **Surface:** MCP `orders.create` → `McpInvoker` → `CreateOrderCommand` (same handler as HTTP on cache miss). +- **Primary behavior:** Unconfirmed call fails with `ConfirmationRequired` before `mcp.tool`; confirmed miss runs Mediator + Npgsql; same-key replay returns cached result with **no new** `mcp.tool` span; same key + different quantity replays original; new key runs fresh production. +- **Limitation / non-goal:** Does not re-prove HTTP Redis idempotency (Exp 3) or outbox/async delivery (Exp 5). Does not use `ProcessedEvents`. + +### Experiment 7 — duplicate integration-event delivery / consumer deduplication + +- **Hypothesis / problem:** When the same `OrderCreatedIntegrationEvent` identity (`IntegrationEvent.Id` / RabbitMQ `MessageId`) is delivered twice through the real consumer, does the handler run once or twice? +- **Surface:** `IEventBus.PublishDirect` → RabbitMQ → `MessageProcessor` → `inbox_messages` → `OrderCreatedIntegrationEventHandler` (decorated by `TestEventHandlerDecorator` in the fixture). +- **Observed behavior:** Baseline runs the handler once and stores one `inbox_messages` row. Duplicate publish with the **same** event identity after successful processing reaches the consumer again (`EventBus` `ProcessMessage` spans) but does **not** invoke the handler a second time — suppressed by `inbox.IsDuplicateAsync` when `IsProcessed=true`. Control publish with a **new** `IntegrationEvent.Id` but the same `OrderId`/`Total` runs the handler again. `EnableDeduplication=false` in the lab; `processed_messages` is unused; inbox completion deduplication still applies. +- **Limitation / non-goal:** Does not prove outbox publish path (Exp 5). Does not enable or exercise `MessageDeduplicationService` / `EnableDeduplication=true` — that is **Exp 17**. Handler has no durable business side effect beyond logging and an in-memory list — `ProcessedEvents` and `inbox_messages` are the primary evidence. Does not simulate RabbitMQ redelivery flags (`redelivered=true`); uses a second explicit publish with preserved message identity. + +### Experiment 8 — HTTP order outbox lifecycle fingerprint + +- **Hypothesis / problem:** Exp 5 proves eventual handler delivery but not `outbox_messages` persistence. Does a cache-miss HTTP order insert one outbox row, does `OutBoxWorker` mark it processed after publish, and does idempotent replay avoid a second row? +- **Surface:** `POST /api/v2/Order/order` (miss) → `CreateOrderCommandHandler` → `IntegrationEventService` → transactional `outbox_messages` → `OutBoxWorker` → `PublishDirect` → RabbitMQ → inbox → handler. +- **Observed behavior:** One outbox row per order (`Id == IntegrationEvent.Id` ≠ `OrderId`); worker sets `Status=Processed` with `ProcessedAt`/`CompletedAt` after `PublishDirect`; one handler observation correlated by `IntegrationEvent.Id`; idempotent replay returns cached HTTP with no Mediator/Npgsql and no additional outbox row; new idempotency key creates a separate order and outbox row (control). +- **Evidence:** `CatalogDbContext.OutboxMessages` queries (production persistence); `ProcessedEvents` (test decorator); optional `inbox_messages` correlation; Mediator/Npgsql spans on miss vs replay. +- **Limitation / non-goal:** Does not claim exactly-once RabbitMQ delivery or crash consistency if the worker fails between publish and mark-processed. May observe the row already processed immediately after HTTP if the worker poll wins the race (`ProcessedAt==null` is the worker’s pending selector). Does not enable `EnableDeduplication=true`. Does not re-prove consumer duplicate suppression (Exp 7). + +### Experiment 9 — EventBus failure outcome / retry-DLQ behavior + +- **Hypothesis / problem (discovery-oriented):** When a failing integration event is published, what consumer, inbox, retry-header, ACK/NACK, and DLQ behavior actually occurs? Does **not** assume `RetryCount=3` produces three handler attempts. +- **Surface:** `IEventBus.PublishDirect` → RabbitMQ (`feature_fusion` / `feature_fusion_dlq`) → `EventBus.MessagetHandler` → `MessageProcessor` → inbox → handler; control uses invalid JSON (same pattern as `RabbitMQEventBusTests.Should_Not_Requeue_Invalid_Messages`). +- **Observed behavior (lab, after fixture fix):** `FailingIntegrationEvent` **reaches the handler once**, then **dead-letters** (`ProcessMessage` status `failed`) with inbox **not** marked processed and subscriber **Failed**. Permanent-failure control (invalid JSON) still dead-letters immediately without inbox. Prior runs with mis-registered handlers showed pre-handler DLQ (zero handler invocations); that was a **test fixture issue**, not production DLQ semantics for handler throws. +- **Limitation / non-goal:** Characterizes mixed pre-handler control + legacy failing-event phase. **Experiment 11** is the dedicated handler-throw characterization (plain, `TransientException`, `BusinessException`). Does not change `RetryCount`, enable deduplication, or claim exactly-once/at-least-once semantics. + +### Experiment 10 — MCP confirmed order → outbox → handler parity + +- **Hypothesis / problem:** After a confirmed MCP `orders.create` cache miss, does the real workflow persist `outbox_messages` and deliver `OrderCreatedIntegrationEvent` through worker → RabbitMQ → inbox → handler as HTTP (Exp 8)? +- **Surface:** MCP `orders.create` (confirmed) → `McpInvoker` → `CreateOrderCommandHandler` → `IntegrationEventService` → outbox → `OutBoxWorker` → consumer → handler. +- **Observed behavior:** Confirmed miss runs `mcp.tool` + Mediator + Npgsql; one outbox row (`Id == IntegrationEvent.Id` ≠ `OrderId`); worker marks processed; inbox row + `ProcessedEvents` handler observation (ProcessMessage Activity may be absent in-process for outbox-triggered delivery); MCP idempotency replay returns same order with no new tool/Mediator/Npgsql/outbox/handler work; new idempotency key creates independent order/outbox/handler path (control). +- **Evidence:** MCP tool spans, `outbox_messages`, `inbox_messages`, `EventBus` `ProcessMessage` Activities when captured, `ProcessedEvents` (test decorator). +- **Limitation / non-goal:** Proves observed pipeline parity with Exp 8, not merely shared handler type. Does not claim exactly-once RabbitMQ delivery or worker crash consistency. Intermediate pending outbox state may be skipped if worker poll is fast. In-process `ProcessMessage` Activity capture is best-effort for async outbox worker delivery — consumer path is also inferred from inbox + handler evidence. Does not re-prove MCP confirmation/idempotency mechanics (Exp 6) or HTTP Redis idempotency (Exp 3). + +### Experiment 11 — handler-throw failure outcome + +- **Hypothesis / problem:** When a deserializable event reaches a registered handler that throws, what does `MessageProcessor` return and does `MessagetHandler` ACK/NACK/requeue/DLQ? After Phase 6, does `RetryCount` bound transient handler executions via `inbox_subscribers.Attempts`? +- **Surface:** `PublishDirect` → consumer → inbox → `DispatchToHandlers` for plain `Exception`, `TransientException`, `BusinessException`, and once-transient-then-succeed test handlers. +- **Observed behavior (lab):** Plain `Exception` and `BusinessException` → handler once → **DLQ**, `Attempts=1`. Always-transient → **exactly RetryCount=3** handler executions (`retrying` then `failed`) → **DLQ**, `Attempts=3`, inbox not processed. Once-transient-then-succeed → 2 executions → **ACK**, inbox processed. Broker `x-retry-count` / `message.retry_count` stays **0** (not the DB attempt budget). +- **Evidence:** Test-only handler counters, `inbox_subscribers.Attempts`, queue/DLQ depths, `EventBus` `ProcessMessage` Activities, `inbox_messages`. +- **Limitation / non-goal:** Does not increment `x-retry-count` or apply `CalculateRetryDelay`. `message.retry_count` remains the broker header. + +### Experiment 12 — HTTP idempotency request fingerprint (opt-in) + +- **Role:** Proves the **opt-in request fingerprint** capability added in richer BuildingBlocks.Idempotency v1 (not part of the original extraction proof set). Remains a **regression / proof gate** for that feature. +- **Workstream:** COMPLETE — BuildingBlocks.Idempotency 1.0.0 (richer v1 evidence). +- **Hypothesis / problem:** With `EnableRequestFingerprint=true`, same key + same body replays; same key + different body conflicts (422) without production. +- **Surface:** Same order-create path as Exp 3; fingerprint enabled via test-host `PostConfigure` only (Lab app default stays off). +- **Primary behavior:** Miss runs production; same-body replay sets `X-Idempotent-Response`; different quantity → HTTP 422, no Mediator. +- **Limitation / non-goal:** Does not change Lab default (fingerprint off — Exp 3 remains the fingerprint-off compatibility gate). Does not re-prove Exp 4 concurrency. + +### Experiment 13 — MCP agent write retry with regenerated idempotency keys + +- **Role:** Characterizes **unreliable agent client behavior** — one logical write intent retried with new idempotency keys after a successful result (simulated context loss). +- **Hypothesis / problem:** If an agent retries the same logical `orders.create` with regenerated keys (K2, K3) after K1 succeeded, how much business-side amplification occurs (orders, Mediator/Npgsql, outbox, handler observations)? +- **Surface:** MCP `orders.create` (`confirmed=true`) → `MemoryIdempotencyStore` → `CreateOrderCommandHandler` → outbox → async handler (`ProcessedEvents` test decorator). +- **Primary behavior:** K1 miss → production; K1 replay → cached replay (no new tool/Mediator/Npgsql); K2/K3 with new keys → additional production paths. Measures distinct `orderId`s, outbox rows, and downstream handler counts per order. +- **Relationship:** Extends Exp 6 (same-key replay / fresh-key control) with **retry-after-success** agent semantics. Uses Exp 10 outbox/handler observation patterns. **Not** HTTP `BuildingBlocks.Idempotency` (Exp 3/4/12). +- **Limitation / non-goal:** Does not claim idempotency is broken when keys are regenerated — deduplication is keyed by idempotency key by design. Does not prove concurrent same-key MCP calls (deferred). Does not change MCP or package code. + +### Experiment 14 — Concurrent Same-Key MCP write (McpConcurrentSameKey) + +- **Role:** Characterizes unreliable/agentic client behavior under **in-flight** concurrency: the same MCP write is submitted concurrently with identical arguments and a shared idempotency key. +- **Hypothesis / problem:** If multiple concurrent agent/tool calls represent the same write and share an idempotency key, does MCP preserve exactly one production outcome (one order, one outbox row, one downstream handler observation), with other callers replaying? +- **Surface:** MCP `orders.create` (`confirmed=true`) → `McpInvoker` → in-process `MemoryIdempotencyStore` in-flight gating → CreateOrderCommandHandler → outbox → `OutBoxWorker` → RabbitMQ → inbox/handler (`ProcessedEvents` test decorator). +- **Observed behavior (asserted by this experiment’s invariants):** All 3 parallel calls succeeded; they all returned a single distinct `orderId`; the capture shows exactly **one** `mcp.tool` production execution (tool trace count); and production evidence is `outbox_messages=1` with downstream `ProcessedEvents` for the winner order `=1`. +- **Relationship / compare:** Exp 4 proves HTTP/Redis concurrency; Exp 6 proves same-key replay; Exp 13 proves regenerated-key amplification. Exp 14 completes the matrix for concurrent same-key MCP calls. +- **Limitation / non-goal:** Does not test regenerated-key behavior (Exp 13), HTTP/Redis semantics (Exp 4), or pagination misuse/cursor abuse (Exp 1/2). + +### Experiment 15 — HTTP idempotency ProcessingTtl lease overlap + +- **Role:** Closes the lease-expiry gap deferred by Exp 4 / package docs: what happens when the first same-key request stays in `Processing` longer than `ProcessingTtl`. +- **Hypothesis / problem:** After the Processing lease expires, a second client with the same `Idempotency-Key` may run production while the first handler is still in flight → duplicate orders/outbox. +- **Surface:** HTTP `POST /api/v2/Order/order` → `BuildingBlocks.Idempotency` (`useLock: true`) → Mediator `CreateOrderCommand` → outbox. Test host only: short `ProcessingTtl` via `PostConfigure` + test-only gated handler wrapper (production handler unchanged). +- **Primary behavior (characterized by this experiment):** Probe while first held → HTTP 409; after lease expiry → second request executes production; both complete with distinct `orderId`s and outbox rows when the packaged lease tradeoff admits overlap. +- **Limitation / non-goal:** Does not add lease renewal or change Lab/package `ProcessingTtl` defaults. Does not re-prove Exp 3/4/12. Gate is test observation infrastructure only. + +### Experiment 16 — MCP tool-storm → IMcpRateLimiter + +- **Role:** Complements Exp 13 (distinct-key amplification): when idempotency cannot collapse a write storm, does `IMcpRateLimiter` bound execution at the MCP boundary? +- **Hypothesis / problem:** A burst of confirmed `orders.create` calls with **distinct** idempotency keys is rate-limited after a fixed permit budget; surplus returns `McpErrorCode.RateLimited` before Mediator/Npgsql/outbox. +- **Surface:** MCP `orders.create` → `McpInvoker` (`IMcpRateLimiter` before confirmation/idempotency/`InvokeCore`) → dispatcher only for allowed calls. Test host replaces `NoOpRateLimiter` with a fixed-permit limiter; Lab default stays no-op. +- **Primary behavior:** With budget N and storm size M (M>N), exactly N production paths (mcp.tool + Mediator + Npgsql + order/outbox); M−N errors with `RateLimited` and no tool/Mediator/Npgsql spans. +- **Limitation / non-goal:** Not an idempotency experiment. Does not ship a rate-limiter NuGet or change package limiter code. Scope of the test limiter is per-tool fixed permits (package interface is implementation-defined). No MAF/LLM. + +### Experiment 17 — EnableDeduplication=true / processed_messages + +- **Role:** Completes the consumer dedup family Exp 7 left open: when `EnableDeduplication=true`, does `processed_messages` record successful deliveries and suppress a same-`IntegrationEvent.Id` redelivery before handler dispatch? +- **Hypothesis / problem:** First `PublishDirect` of an identity runs the handler and inserts `processed_messages`; second publish with the **same** id reaches `ProcessMessage` again but returns Success from the `processed_messages` check before `DispatchToHandlers` — exactly one handler observation. +- **Surface:** `IEventBus.PublishDirect` → RabbitMQ → `MessageProcessor.ProcessMessageAsync(deduplication:true)`. Test flips `EventBusOptions.EnableDeduplication` on the shared options instance for this test only (restored in dispose); fixture default stays false. +- **Primary behavior:** After baseline: inbox processed + `processed_messages=1` + handler=1. After duplicate: ProcessMessage attempts increase, handler count stays 1, still one inbox row and one `processed_messages` row. Control new id → second handler + its own `processed_messages` row. +- **Limitation / non-goal:** Does not simulate outbox publish-then-crash. Does not change production EventBus. After a successful first delivery both inbox and `processed_messages` would suppress; the code checks `processed_messages` first when the flag is on. Exp 7 remains the inbox-only (flag off) gate. + +### Experiment 18 — Async trace correlation (HTTP → outbox → consumer) + +- **Role:** Characterizes whether W3C trace context crosses the async messaging boundary, or whether only business ids remain correlatable. +- **Hypothesis / problem:** After HTTP order create → outbox → RabbitMQ → consumer, does `EventBus` `ProcessMessage` share the originating TraceId / parent / Activity links, or start a separate trace? +- **Surface:** `POST /api/v2/Order/order` (Lab `traceparent` like Exp 8) → Mediator → outbox → `OutBoxWorker` → `PublishDirect` → consumer `ProcessMessage`. +- **Observed production instrumentation:** No `traceparent` on RabbitMQ headers; `OutBoxWorker`/`PublishDirect` emit no spans; `ProcessMessage` starts a root `EventBus` Activity tagged with `message.id`. Correlation across the boundary is via `IntegrationEvent.Id` / `OrderId`. +- **Limitation / non-goal:** Does not add propagation or change Telemetry/EventBus. Documents the gap if separate consumer traces are observed. + +### Experiment 19 — EventBus observation baseline + +- **Role:** Proves the Lab-only EventBus observation seam on the happy path (no faults). +- **Hypothesis / problem:** Can we journal Observed stages for outbox → publish → processor → handler, and label Ack as Inferred? +- **Surface:** HTTP order create + `IEventBusLabHook` (disarmed) + `LabMessageProcessorDecorator` + outbox/inbox/`ProcessedEvents`. +- **Primary behavior:** AfterPublishBeforeOutboxMark, ProcessorEntered/Completed, handler once, inbox processed; `message.status=processed` inferred near Ack. +- **Limitation / non-goal:** Dedup not exercised (see Exp 7/17). No W3C. No Scenario engine. + +### Experiment 20 — Publish-then-crash (fault B) — **branch end** + +- **Role:** Final EventBus research experiment. Characterizes crash after broker publish before outbox MarkProcessed. +- **Hypothesis / problem:** Outbox stays pending; later poll may republish; inbox may still run the handler once for the same `IntegrationEvent.Id`. +- **Surface:** One-shot Lab fault B via `EventBusLabFaultController.ArmCrashAfterPublishOnceForOrderId`. +- **Limitation / non-goal:** Does not fix duplicate-publish risk. Does not add W3C. No Exp 21 in this branch. + +--- + +## Established experiment convention + +Recurring pattern (not a base class): + +``` +hypothesis (XML docs) + → labeled invocations (behavior string per call) + → real production path (HTTP or MCP through AspireFixture) + → telemetry / evidence capture + → JSON artifact (_output.WriteLine) + → FluentAssertions on invariants +``` + +### Why one `[Fact]` + +Each experiment is a single end-to-end journal: one hypothesis, one ordered sequence of invocations, one artifact, one assertion block. Splitting phases into multiple facts would fragment evidence and complicate fixture state (e.g. idempotency keys, `ProcessedEvents`). + +### Why labeled behaviors + +Every invocation carries a `behavior` label (`Walk`, `MissPopulate`, `SameKeyReplay`, `BaselineProduction`, …). Labels make artifacts readable and assertions traceable without a DSL or runner type. + +### Why experiment-specific call records + +Call shape differs by domain (pagination cursors, MCP `isError`, idempotency headers, concurrency roles). Shared call types would force false uniformity. + +### Why artifact before assertions + +The JSON artifact is the experiment’s audit trail. It is emitted to `ITestOutputHelper` **before** assertions so failed runs still leave useful output in test logs. + +Typical artifact envelope (exact property naming varies by experiment): + +- `name`, `startedUtc`, `gitSha`, optional `environment` / `configuration` +- `calls[]` — per-invocation evidence +- `observations` — roll-up metrics +- `notes[]` — human-readable interpretation + +--- + +## Evidence model + +### In-process telemetry (production Activities, test-process capture) + +| Piece | Location | Role | +|-------|----------|------| +| `InProcessActivityCapture` | [`Infrastructure/Telemetry/`](../Infrastructure/Telemetry/InProcessActivityCapture.cs) | `ActivityListener` in the test process; collects stopped Activities while FeatureFusion runs in-process via WAF. `Clear()` drops spans so a later independent run in the same test does not inherit cumulative counts. | +| `CapturedActivity` | [`Infrastructure/Telemetry/CapturedActivity.cs`](../Infrastructure/Telemetry/CapturedActivity.cs) | Snapshot of one span (source, display name, trace/span/parent ids, tags, duration, Activity links) | +| `LabTrace` | [`Infrastructure/Telemetry/LabTrace.cs`](../Infrastructure/Telemetry/LabTrace.cs) | W3C `traceparent` helpers and generic tag predicates | + +**`traceparent` propagation:** HTTP experiments inject `LabTrace.NewTraceParent()` + `FormatTraceParent()` on each request. MCP Experiment 2 sets transport `traceparent` on the shared `HttpClient`; tool-level traces are discovered separately (Exp 2, Exp 6). + +**Narrow local predicates:** Experiments 2–6 often use source-only `IsMediator` / `IsNpgsql` (`BuildingBlocks.Mediator`, `Npgsql`) for stable span counts. `LabTrace` also offers broader tag-based predicates; experiments keep local narrow helpers when counting semantics must not drift. + +### Test observation infrastructure (not production telemetry) + +| Piece | Location | Role | +|-------|----------|------| +| `AspireFixture.ProcessedEvents` | [`AspireFixture.cs`](../Aspire/AspireFixture.cs) | List populated by `TestEventHandlerDecorator` wrapping `OrderCreatedIntegrationEventHandler` | +| `Wait.UntilAsync` | [`Infrastructure/Async/Wait.cs`](../Infrastructure/Async/Wait.cs) | Bounded condition polling (100 ms interval); used for eventual handler delivery (Exp 5) and EventBus tests | + +**`ProcessedEvents`:** Proves the **decorated handler ran** in the integration test host after HTTP/outbox/RabbitMQ processing. It is **not** how operators observe production. Exp 5 documents this explicitly in XML docs, configuration notes, and artifact `notes`. + +**Production telemetry** (OTLP, SigNoz, etc.) is configured in the lab app but is **not** required for experiments. In-process capture substitutes for collector-based proof in the test process. + +--- + +## Shared vs local infrastructure + +### Intentionally shared + +| Helper | Path | Used for | +|--------|------|----------| +| Telemetry capture | `Infrastructure/Telemetry/*` | All experiments + TraceEvidence | +| Async wait | `Infrastructure/Async/Wait.cs` | Exp 5, EventBus tests | +| MCP client factory | `Infrastructure/Mcp/LabMcpClient.cs` | Exp 2, Exp 6, MCP smoke, TraceEvidence | +| Run metadata | `Infrastructure/Reporting/LabRunInfo.cs` | Exp 6 today (`ReadGitSha()`); older experiments still use local copies | + +### Intentionally local + +- Call record types and observation roll-ups +- Invocation orchestration (walk loops, cache phases, concurrency gates) +- Domain clients (`CarelessPaginationClient` in Exp 1) +- Experiment-specific assertion sets +- Concurrency gate (`StartTogether` in Exp 4 only) +- Artifact serialization choices (source-generated JSON in Exp 1, records elsewhere, anonymous object in Exp 2) + +Extract a helper only when semantics are **stable across experiments**. Duplication that reflects different hypotheses should stay local. + +--- + +## Anti-patterns / non-goals + +The Lab **does not** currently use: + +- Scenario / Actor models +- Experiment DSL or invocation language +- `ExperimentEngine` or generic invocation runner +- Shared experiment base class +- Universal artifact schema or shared `IExperimentResult` type + +These are intentionally avoided. Fourteen numbered experiments share a **convention**, not a framework. Premature abstraction would force uniform call shapes and hide domain-specific evidence. Revisit only if multiple future experiments demonstrate stable semantics that genuinely require shared types. + +--- + +## Current limitations + +- **Exp 5 — handler vs outbox proof:** Superseded for outbox persistence by Exp 8; Exp 5 still documents handler-only observation without `outbox_messages` fingerprinting. +- **HTTP vs MCP idempotency:** Exp 3/4/12 (Redis / `BuildingBlocks.Idempotency`) and Exp 6 (`MemoryIdempotencyStore` / `McpInvoker`) are different stores, gates, and replay signals (`X-Idempotent-Response` vs absence of new `mcp.tool` span). HTTP Redis workstream is COMPLETE; Exp 6 is MCP-only. +- **Gateway / rate limiting:** YARP + Memcached tests live in `FeatureFusion.ApiGateway.Tests`, not the Aspire `IntegrationTests` experiment host. +- **Inbox deduplication:** Experiment 7 exercises duplicate delivery with `EnableDeduplication=false` (inbox completion). **Exp 17** exercises `EnableDeduplication=true` / `processed_messages`. +- **Recommendation cache middleware, feature-flag paths, SigNoz/OTLP in tests:** Present in the lab app but not covered by Experiments 1–18. +- **Test isolation:** `[Collection(AspireCollection.Name)]` with `DisableTestParallelization = true`. Experiments that use `ProcessedEvents` should clear or scope observations (Exp 5 clears at start). +- **Minor artifact inconsistency:** Exp 2 captures `startedUtc` at artifact build time; others capture at test start. JSON property casing differs (anonymous camelCase vs record PascalCase). + +--- + +## How to add a new experiment + +1. **Inspect the real production workflow** — routes, MCP tools, handlers, filters, stores. Do not assume behavior from prior experiments. +2. **Formulate one falsifiable hypothesis** — document in the test class XML summary with explicit non-goals. +3. **Add a folder under `Experiments/`** — one test class, one `[Fact]`, `[Collection(AspireCollection.Name)]`. +4. **Reuse shared observation infrastructure** — `InProcessActivityCapture`, `LabTrace`, `Wait.UntilAsync`, `LabMcpClient` when applicable. +5. **Keep domain semantics local** — call records, invocation sequence, helpers, assertions. +6. **Label every invocation** with a `behavior` string. +7. **Emit JSON artifact before assertions** — include `name`, `startedUtc`, `gitSha`, `calls`, `observations`, `notes`. +8. **Assert application-visible behavior first** — HTTP status, MCP `isError`, response bodies, headers. +9. **Assert collaborator evidence second** — Mediator/Npgsql span counts, `ProcessedEvents`, tool trace presence. +10. **Avoid fixed sleeps** for eventual behavior — use `Wait.UntilAsync` with a bounded timeout. +11. **Do not modify production code** unless explicitly requested for a separate task. +12. **Validate:** + +```bash +dotnet test tests/Lab/IntegrationTests/IntegrationTests.csproj \ + --filter "FullyQualifiedName~YourExperimentTests" \ + --verbosity normal + +dotnet test tests/Lab/IntegrationTests/IntegrationTests.csproj \ + --filter "FullyQualifiedName~FeatureFusionTraceEvidenceTests" \ + --verbosity normal + +dotnet test tests/Lab/IntegrationTests/IntegrationTests.csproj \ + --filter "FullyQualifiedName~PaginationAbuseExperimentTests|FullyQualifiedName~McpPaginationAbuseExperimentTests|FullyQualifiedName~CacheVsProductionExperimentTests|FullyQualifiedName~ConcurrencyRaceExperimentTests|FullyQualifiedName~OutboxDeliveryExperimentTests|FullyQualifiedName~McpOrderIdempotencyExperimentTests|FullyQualifiedName~DuplicateDeliveryExperimentTests|FullyQualifiedName~OutboxLifecycleExperimentTests|FullyQualifiedName~EventBusFailureExperimentTests|FullyQualifiedName~McpOrderOutboxExperimentTests|FullyQualifiedName~HandlerThrowFailureExperimentTests|FullyQualifiedName~IdempotencyFingerprintExperimentTests|FullyQualifiedName~McpAgentKeyRegenerationExperimentTests|FullyQualifiedName~McpConcurrentSameKeyExperimentTests|FullyQualifiedName~IdempotencyProcessingLeaseExperimentTests|FullyQualifiedName~McpToolStormRateLimitExperimentTests|FullyQualifiedName~ProcessedMessageDeduplicationExperimentTests|FullyQualifiedName~AsyncTraceCorrelationExperimentTests|FullyQualifiedName~YourExperimentTests" \ + --verbosity normal +``` + +Note: `FullyQualifiedName~PaginationAbuseExperimentTests` matches **both** Exp 1 and Exp 2 class names by substring; use exact class names when filtering a single experiment. + +--- + +## Folder layout + +``` +Experiments/ + README.md ← this file + PaginationAbuse/ ← Exp 1 (HTTP), Exp 2 (MCP) + CacheVsProduction/ ← Exp 3 (HTTP Redis idempotency provenance) + Concurrency/ ← Exp 4 (Redis lock concurrency provenance) + OutboxDelivery/ ← Exp 5 + McpOrderIdempotency/ ← Exp 6 + DuplicateDelivery/ ← Exp 7 + OutboxLifecycle/ ← Exp 8 + EventBusFailure/ ← Exp 9 + McpOrderOutbox/ ← Exp 10 + HandlerThrowFailure/ ← Exp 11 + IdempotencyFingerprint/ ← Exp 12 (opt-in fingerprint, richer v1) + McpAgentKeyRegeneration/ ← Exp 13 (agent regenerated idempotency keys) + McpConcurrentSameKey/ ← Exp 14 (concurrent same-key MCP idempotency) + IdempotencyProcessingLease/ ← Exp 15 (HTTP ProcessingTtl lease overlap) + McpToolStormRateLimit/ ← Exp 16 (MCP distinct-key storm + IMcpRateLimiter) + ProcessedMessageDeduplication/ ← Exp 17 (EnableDeduplication + processed_messages) + AsyncTraceCorrelation/ ← Exp 18 (HTTP→outbox→consumer TraceId correlation) + +Infrastructure/ ← shared observation helpers (not experiments) + Telemetry/ + Async/ + Mcp/ + Reporting/ +``` diff --git a/tests/Lab/IntegrationTests/Infrastructure/Agent/MafAgentRunJournal.cs b/tests/Lab/IntegrationTests/Infrastructure/Agent/MafAgentRunJournal.cs new file mode 100644 index 0000000..3aea873 --- /dev/null +++ b/tests/Lab/IntegrationTests/Infrastructure/Agent/MafAgentRunJournal.cs @@ -0,0 +1,111 @@ +using System.Text.Json; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace IntegrationTests.Infrastructure.Agent; + +/// +/// Extracts ordered tool-call evidence from a MAF . +/// +public static class MafAgentRunJournal +{ + public static IReadOnlyList ReadToolCalls(AgentResponse response) + { + var calls = new List(); + var ordinal = 0; + + foreach (var message in response.Messages) + { + foreach (var content in message.Contents) + { + switch (content) + { + case FunctionCallContent call: + calls.Add(new MafObservedToolCall( + Ordinal: ++ordinal, + Kind: "call", + ToolName: call.Name, + ArgumentsJson: SerializeArguments(call.Arguments), + ResultJson: null, + Error: null)); + break; + case FunctionResultContent result: + if (calls.Count == 0 || calls[^1].Kind != "call") + { + calls.Add(new MafObservedToolCall( + Ordinal: ++ordinal, + Kind: "result", + ToolName: result.CallId, + ArgumentsJson: null, + ResultJson: SerializeResult(result.Result), + Error: null)); + break; + } + + var pending = calls[^1]; + calls[^1] = pending with + { + Kind = "call+result", + ResultJson = SerializeResult(result.Result), + Error = TryReadError(result.Result) + }; + break; + } + } + } + + return calls; + } + + public static string DescribeSequence(IEnumerable calls) + => string.Join(" → ", calls.Select(c => c.ToolName)); + + private static string? SerializeArguments(IDictionary? arguments) + { + if (arguments is null || arguments.Count == 0) + return "{}"; + + return JsonSerializer.Serialize(arguments); + } + + private static string? SerializeResult(object? result) + { + if (result is null) + return null; + + return result switch + { + string text => text, + _ => JsonSerializer.Serialize(result) + }; + } + + private static string? TryReadError(object? result) + { + if (result is null) + return null; + + var text = result switch + { + string s => s, + _ => JsonSerializer.Serialize(result) + }; + + if (text.Contains("error", StringComparison.OrdinalIgnoreCase) + || text.Contains("ConfirmationRequired", StringComparison.OrdinalIgnoreCase) + || text.Contains("IdempotencyKeyRequired", StringComparison.OrdinalIgnoreCase)) + { + return text.Length <= 500 ? text : text[..500]; + } + + return null; + } +} + +public sealed record MafObservedToolCall( + int Ordinal, + string Kind, + string ToolName, + string? ArgumentsJson, + string? ResultJson, + string? Error); diff --git a/tests/Lab/IntegrationTests/Infrastructure/Async/Wait.cs b/tests/Lab/IntegrationTests/Infrastructure/Async/Wait.cs new file mode 100644 index 0000000..68115ff --- /dev/null +++ b/tests/Lab/IntegrationTests/Infrastructure/Async/Wait.cs @@ -0,0 +1,17 @@ +using System.Diagnostics; + +namespace IntegrationTests.Infrastructure.Async; + +public static class Wait +{ + public static async Task UntilAsync(Func condition, TimeSpan timeout) + { + var stopwatch = Stopwatch.StartNew(); + while (stopwatch.Elapsed < timeout) + { + if (condition()) return; + await Task.Delay(100); + } + throw new TimeoutException("Condition not met within timeout"); + } +} diff --git a/tests/Lab/IntegrationTests/Infrastructure/EventBusLab/EventBusLabHook.cs b/tests/Lab/IntegrationTests/Infrastructure/EventBusLab/EventBusLabHook.cs new file mode 100644 index 0000000..895b8a3 --- /dev/null +++ b/tests/Lab/IntegrationTests/Infrastructure/EventBusLab/EventBusLabHook.cs @@ -0,0 +1,115 @@ +using EventBusRabbitMQ.Events; +using EventBusRabbitMQ.Infrastructure.Messaging; +using FeatureFusion.Features.Order.IntegrationEvents.Events; + +namespace IntegrationTests.Infrastructure.EventBusLab; + +/// +/// Lab : journals AfterPublish; optional one-shot crash at point B. +/// +public sealed class EventBusLabHook : IEventBusLabHook +{ + private readonly EventBusLabJournal _journal; + private readonly EventBusLabFaultController _faults; + + public EventBusLabHook(EventBusLabJournal journal, EventBusLabFaultController faults) + { + _journal = journal; + _faults = faults; + } + + public Task OnAfterPublishBeforeOutboxMarkAsync( + Guid messageId, + string eventType, + IntegrationEvent @event, + CancellationToken cancellationToken = default) + { + Guid? orderId = @event is OrderCreatedIntegrationEvent order ? order.OrderId : null; + + _journal.Record( + EventBusLabStages.AfterPublishBeforeOutboxMark, + messageId, + eventType, + evidenceKind: "Observed", + detail: "PublishDirect returned; outbox MarkProcessed not yet called", + orderId: orderId); + + if (_faults.ShouldSimulateCrashAfterPublish(messageId, eventType, orderId)) + { + _journal.Record( + EventBusLabStages.SimulatedCrashAfterPublish, + messageId, + eventType, + evidenceKind: "Observed", + detail: "Lab fault B: skip MarkProcessed (pending outbox)", + orderId: orderId); + throw new EventBusLabSimulatedCrashException(messageId); + } + + return Task.CompletedTask; + } +} + +/// Deterministic Lab fault switches. Default: all off. +public sealed class EventBusLabFaultController +{ + private Guid? _crashOnceForMessageId; + private Guid? _crashOnceForOrderId; + private string? _crashOnceForEventType; + private int _crashArmedCount; + + public void Clear() + { + _crashOnceForMessageId = null; + _crashOnceForOrderId = null; + _crashOnceForEventType = null; + _crashArmedCount = 0; + } + + /// Arm point-B crash for the next matching outbox publish (by message id). + public void ArmCrashAfterPublishOnce(Guid messageId) + { + Clear(); + _crashOnceForMessageId = messageId; + _crashArmedCount = 1; + } + + /// Arm point-B crash for the first publish whose payload OrderId matches. + public void ArmCrashAfterPublishOnceForOrderId(Guid orderId) + { + Clear(); + _crashOnceForOrderId = orderId; + _crashArmedCount = 1; + } + + /// + /// Arm point-B crash for the next publish of + /// (e.g. OrderCreatedIntegrationEvent). Safe to call before OrderId is known. + /// + public void ArmCrashAfterPublishOnceForEventType(string eventType) + { + Clear(); + _crashOnceForEventType = eventType; + _crashArmedCount = 1; + } + + internal bool ShouldSimulateCrashAfterPublish(Guid messageId, string eventType, Guid? orderId) + { + if (_crashArmedCount <= 0) + return false; + + var match = (_crashOnceForMessageId is { } mid && mid == messageId) + || (_crashOnceForOrderId is { } oid && orderId == oid) + || (_crashOnceForEventType is { } et + && string.Equals(et, eventType, StringComparison.Ordinal)); + + if (!match) + return false; + + _crashArmedCount = 0; + _crashOnceForMessageId = null; + _crashOnceForOrderId = null; + _crashOnceForEventType = null; + return true; + } +} diff --git a/tests/Lab/IntegrationTests/Infrastructure/EventBusLab/EventBusLabJournal.cs b/tests/Lab/IntegrationTests/Infrastructure/EventBusLab/EventBusLabJournal.cs new file mode 100644 index 0000000..4f0a9a6 --- /dev/null +++ b/tests/Lab/IntegrationTests/Infrastructure/EventBusLab/EventBusLabJournal.cs @@ -0,0 +1,55 @@ +using System.Collections.Concurrent; + +namespace IntegrationTests.Infrastructure.EventBusLab; + +/// +/// Lab-only append-only journal of EventBus pipeline stages. Observation only — no assertions. +/// +public sealed class EventBusLabJournal +{ + private readonly ConcurrentQueue _records = new(); + + public IReadOnlyList Snapshot() => _records.ToArray(); + + public void Clear() => _records.Clear(); + + public void Record( + string stage, + Guid? messageId, + string? eventType, + string evidenceKind, + string? detail = null, + Guid? orderId = null) + { + _records.Enqueue(new EventBusLabStageRecord( + Utc: DateTimeOffset.UtcNow, + Stage: stage, + MessageId: messageId, + OrderId: orderId, + EventType: eventType, + EvidenceKind: evidenceKind, + Detail: detail)); + } + + public IReadOnlyList ForMessage(Guid messageId) + => _records.Where(r => r.MessageId == messageId).ToList(); +} + +/// Observed vs Inferred. +public sealed record EventBusLabStageRecord( + DateTimeOffset Utc, + string Stage, + Guid? MessageId, + Guid? OrderId, + string? EventType, + string EvidenceKind, + string? Detail); + +/// Well-known stage names for Exp 19/20 artifacts. +public static class EventBusLabStages +{ + public const string AfterPublishBeforeOutboxMark = "AfterPublishBeforeOutboxMark"; + public const string ProcessorEntered = "ProcessorEntered"; + public const string ProcessorCompleted = "ProcessorCompleted"; + public const string SimulatedCrashAfterPublish = "SimulatedCrashAfterPublish"; +} diff --git a/tests/Lab/IntegrationTests/Infrastructure/EventBusLab/LabMessageProcessorDecorator.cs b/tests/Lab/IntegrationTests/Infrastructure/EventBusLab/LabMessageProcessorDecorator.cs new file mode 100644 index 0000000..b6f3ee7 --- /dev/null +++ b/tests/Lab/IntegrationTests/Infrastructure/EventBusLab/LabMessageProcessorDecorator.cs @@ -0,0 +1,53 @@ +using EventBusRabbitMQ.Domain; +using EventBusRabbitMQ.Infrastructure.Messaging; +using RabbitMQ.Client.Events; + +namespace IntegrationTests.Infrastructure.EventBusLab; + +/// +/// Lab-only decorator around . Journals enter/exit only — +/// does not change processing results. +/// +public sealed class LabMessageProcessorDecorator : IMessageProcessor +{ + private readonly IMessageProcessor _inner; + private readonly EventBusLabJournal _journal; + + public LabMessageProcessorDecorator(IMessageProcessor inner, EventBusLabJournal journal) + { + _inner = inner; + _journal = journal; + } + + public async Task ProcessMessageAsync( + BasicDeliverEventArgs args, + bool deduplication) + { + var messageId = TryParseMessageId(args); + var eventType = args.RoutingKey; + + _journal.Record( + EventBusLabStages.ProcessorEntered, + messageId, + eventType, + evidenceKind: "Observed", + detail: $"deduplicationFlag={deduplication}; redelivered={args.Redelivered}"); + + var result = await _inner.ProcessMessageAsync(args, deduplication).ConfigureAwait(false); + + _journal.Record( + EventBusLabStages.ProcessorCompleted, + messageId, + eventType, + evidenceKind: "Observed", + detail: $"result={result}; deduplicationFlag={deduplication}"); + + return result; + } + + private static Guid? TryParseMessageId(BasicDeliverEventArgs args) + { + var raw = args.BasicProperties?.MessageId; + return Guid.TryParse(raw, out var id) ? id : null; + } +} diff --git a/tests/Lab/IntegrationTests/Infrastructure/Mcp/LabMcpClient.cs b/tests/Lab/IntegrationTests/Infrastructure/Mcp/LabMcpClient.cs new file mode 100644 index 0000000..a119912 --- /dev/null +++ b/tests/Lab/IntegrationTests/Infrastructure/Mcp/LabMcpClient.cs @@ -0,0 +1,16 @@ +using ModelContextProtocol.Client; + +namespace IntegrationTests.Infrastructure.Mcp; + +public static class LabMcpClient +{ + public static async Task CreateAsync(HttpClient http) + { + var endpoint = new Uri(http.BaseAddress ?? new Uri("http://localhost"), "mcp"); + var transport = new HttpClientTransport( + new HttpClientTransportOptions { Endpoint = endpoint }, + http, + ownsHttpClient: false); + return await McpClient.CreateAsync(transport); + } +} diff --git a/tests/Lab/IntegrationTests/Infrastructure/Mcp/McpToolResults.cs b/tests/Lab/IntegrationTests/Infrastructure/Mcp/McpToolResults.cs new file mode 100644 index 0000000..e276b95 --- /dev/null +++ b/tests/Lab/IntegrationTests/Infrastructure/Mcp/McpToolResults.cs @@ -0,0 +1,81 @@ +using System.Text.Json; +using BuildingBlocks.Mcp; +using ModelContextProtocol.Protocol; + +namespace IntegrationTests.Infrastructure.Mcp; + +/// +/// Lab-only parsing of MCP payloads. +/// Observation only — no assertions; experiment-specific error fallbacks stay in tests. +/// +public static class McpToolResults +{ + private static readonly JsonSerializerOptions DefaultJsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + public static string GetText(CallToolResult result) + => string.Join("\n", result.Content.OfType().Select(b => b.Text)); + + public static string Truncate(string text, int maxLength = 500) + => text.Length <= maxLength ? text : text[..maxLength]; + + /// + /// Parses orders.create structured content (root or wrapped value). + /// + public static McpOrderBody? TryParseOrder( + CallToolResult result, + JsonSerializerOptions? jsonOptions = null) + { + if (result.StructuredContent is not { } structured) + return null; + + var options = jsonOptions ?? DefaultJsonOptions; + var order = JsonSerializer.Deserialize(structured.GetRawText(), options); + if (order?.OrderId != Guid.Empty) + return order; + + if (structured.ValueKind == JsonValueKind.Object + && structured.TryGetProperty("value", out var value)) + { + order = JsonSerializer.Deserialize(value.GetRawText(), options); + if (order?.OrderId != Guid.Empty) + return order; + } + + return null; + } + + /// + /// Reads a JSON code property from tool error text when present. + /// Does not apply experiment-specific string heuristics (ConfirmationRequired / RateLimited). + /// + public static string? TryReadJsonErrorCode(CallToolResult result) + { + var text = GetText(result); + if (string.IsNullOrWhiteSpace(text)) + return null; + + try + { + using var doc = JsonDocument.Parse(text); + if (!doc.RootElement.TryGetProperty("code", out var code)) + return null; + + return code.ValueKind switch + { + JsonValueKind.String => code.GetString(), + JsonValueKind.Number => Enum.GetName(typeof(McpErrorCode), code.GetInt32()), + _ => code.GetRawText() + }; + } + catch (JsonException) + { + return null; + } + } +} + +/// Shared shape for MCP orders.create structured content. +public sealed record McpOrderBody(Guid OrderId, int Quantity, decimal? TotalAmount); diff --git a/tests/Lab/IntegrationTests/Infrastructure/Orders/HttpOrderCreate.cs b/tests/Lab/IntegrationTests/Infrastructure/Orders/HttpOrderCreate.cs new file mode 100644 index 0000000..b2f1ce5 --- /dev/null +++ b/tests/Lab/IntegrationTests/Infrastructure/Orders/HttpOrderCreate.cs @@ -0,0 +1,103 @@ +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using IntegrationTests.Infrastructure.Telemetry; +using static IntegrationTests.Infrastructure.Telemetry.LabTrace; + +namespace IntegrationTests.Infrastructure.Orders; + +/// +/// Lab-only HTTP POST to FeatureFusion order create. Observation only — no assertions. +/// +public static class HttpOrderCreate +{ + public const string Path = "/api/v2/Order/order"; + public const string IdempotencyHeader = "Idempotency-Key"; + public const string CachedResponseHeader = "X-Idempotent-Response"; + + private static readonly JsonSerializerOptions DefaultJsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + /// + /// Posts {"productId","quantity","customerId"} with Idempotency-Key and W3C traceparent. + /// When / are omitted, a new trace parent is created + /// (same as prior experiment-local NewTraceParent() usage). + /// + public static async Task PostAsync( + HttpClient http, + InProcessActivityCapture capture, + string idempotencyKey, + int quantity, + int productId = 1, + int customerId = 1, + ActivityTraceId? traceId = null, + ActivitySpanId? parentSpanId = null, + JsonSerializerOptions? jsonOptions = null, + CancellationToken cancellationToken = default) + { + ActivityTraceId tid; + ActivitySpanId sid; + if (traceId is { } t && parentSpanId is { } s) + { + tid = t; + sid = s; + } + else + { + (tid, sid) = NewTraceParent(); + } + + using var request = new HttpRequestMessage(HttpMethod.Post, Path); + request.Headers.TryAddWithoutValidation(IdempotencyHeader, idempotencyKey); + request.Headers.TryAddWithoutValidation("traceparent", FormatTraceParent(tid, sid)); + request.Content = new StringContent( + $$"""{"productId":{{productId}},"quantity":{{quantity}},"customerId":{{customerId}}}""", + Encoding.UTF8, + "application/json"); + + var started = Stopwatch.StartNew(); + using var response = await http.SendAsync(request, cancellationToken); + started.Stop(); + var completedUtc = DateTimeOffset.UtcNow; + + var body = await response.Content.ReadAsStringAsync(cancellationToken); + HttpOrderBody? order = null; + if (response.IsSuccessStatusCode) + { + order = JsonSerializer.Deserialize( + body, + jsonOptions ?? DefaultJsonOptions); + } + + var spans = capture.ForTrace(tid); + return new HttpOrderCreateResult( + HttpStatus: (int)response.StatusCode, + Body: body, + OrderId: order?.OrderId ?? Guid.Empty, + Quantity: order?.Quantity, + TotalAmount: order?.TotalAmount, + CachedResponseHeader: response.Headers.Contains(CachedResponseHeader), + ClientDurationMs: started.Elapsed.TotalMilliseconds, + CompletedUtc: completedUtc, + TraceId: tid, + TraceIdHex: tid.ToHexString(), + Spans: spans); + } +} + +public sealed record HttpOrderBody(Guid OrderId, int Quantity, decimal TotalAmount); + +public sealed record HttpOrderCreateResult( + int HttpStatus, + string Body, + Guid OrderId, + int? Quantity, + decimal? TotalAmount, + bool CachedResponseHeader, + double ClientDurationMs, + DateTimeOffset CompletedUtc, + ActivityTraceId TraceId, + string TraceIdHex, + IReadOnlyList Spans); diff --git a/tests/Lab/IntegrationTests/Infrastructure/Orders/OrderOutboxObserver.cs b/tests/Lab/IntegrationTests/Infrastructure/Orders/OrderOutboxObserver.cs new file mode 100644 index 0000000..3086f2a --- /dev/null +++ b/tests/Lab/IntegrationTests/Infrastructure/Orders/OrderOutboxObserver.cs @@ -0,0 +1,168 @@ +using System.Diagnostics; +using System.Text.Json; +using EventBusRabbitMQ.Domain; +using EventBusRabbitMQ.Infrastructure; +using FeatureFusion.Features.Order.IntegrationEvents.Events; +using FeatureFusion.Infrastructure.Context; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace IntegrationTests.Infrastructure.Orders; + +/// +/// Lab-only observation of outbox_messages rows for +/// . No assertions; callers interpret evidence. +/// +public static class OrderOutboxObserver +{ + public static string OrderCreatedEventType { get; } = nameof(OrderCreatedIntegrationEvent); + + public static async Task> FindByOrderIdAsync( + IServiceProvider services, + Guid orderId, + CancellationToken cancellationToken = default) + { + await using var scope = services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var serializerOptions = scope.ServiceProvider + .GetRequiredService>().Value.JsonSerializerOptions; + + var rows = await db.OutboxMessages + .AsNoTracking() + .Where(m => m.EventType == OrderCreatedEventType) + .ToListAsync(cancellationToken); + + var matches = new List(); + foreach (var row in rows) + { + if (row.Payload is null) + continue; + + var evt = JsonSerializer.Deserialize( + row.Payload, + serializerOptions); + if (evt is null || evt.OrderId != orderId) + continue; + + matches.Add(Map(row, evt)); + } + + return matches; + } + + /// + /// Polls until exactly one outbox row exists for (any status). + /// Default timeout 20s / 100ms delay — matches prior experiment-local waits. + /// + public static async Task WaitUntilExistsAsync( + IServiceProvider services, + Guid orderId, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var limit = timeout ?? TimeSpan.FromSeconds(20); + var stopwatch = Stopwatch.StartNew(); + while (stopwatch.Elapsed < limit) + { + cancellationToken.ThrowIfCancellationRequested(); + var rows = await FindByOrderIdAsync(services, orderId, cancellationToken); + if (rows.Count == 1) + return rows[0]; + + await Task.Delay(100, cancellationToken); + } + + throw new TimeoutException( + $"Outbox message for OrderId={orderId} was not observed within timeout"); + } + + /// + /// Polls until exactly one outbox row exists and is true. + /// Default timeout 20s / 100ms delay — matches prior experiment-local waits. + /// + public static async Task WaitUntilProcessedAsync( + IServiceProvider services, + Guid orderId, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var limit = timeout ?? TimeSpan.FromSeconds(20); + var stopwatch = Stopwatch.StartNew(); + while (stopwatch.Elapsed < limit) + { + cancellationToken.ThrowIfCancellationRequested(); + var rows = await FindByOrderIdAsync(services, orderId, cancellationToken); + if (rows.Count == 1 && rows[0].WorkerProcessed) + return rows[0]; + + await Task.Delay(100, cancellationToken); + } + + throw new TimeoutException( + $"Outbox message for OrderId={orderId} did not reach processed state within timeout"); + } + + /// + /// Polls until at least one outbox row exists for . + /// Default timeout 20s / 100ms delay — matches prior experiment-local waits. + /// + public static async Task> WaitUntilAnyAsync( + IServiceProvider services, + Guid orderId, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var limit = timeout ?? TimeSpan.FromSeconds(20); + var stopwatch = Stopwatch.StartNew(); + while (stopwatch.Elapsed < limit) + { + cancellationToken.ThrowIfCancellationRequested(); + var rows = await FindByOrderIdAsync(services, orderId, cancellationToken); + if (rows.Count >= 1) + return rows; + + await Task.Delay(100, cancellationToken); + } + + throw new TimeoutException( + $"Outbox message for OrderId={orderId} was not observed within timeout"); + } + + private static OrderOutboxRow Map(OutboxMessage row, OrderCreatedIntegrationEvent evt) + { + var workerPending = row.ProcessedAt is null; + var workerProcessed = row.Status == MessageStatus.Processed + && row.ProcessedAt is not null + && row.CompletedAt is not null; + + return new OrderOutboxRow( + OutboxMessageId: row.Id, + IntegrationEventId: evt.Id, + OrderId: evt.OrderId, + Total: evt.Total, + EventType: row.EventType, + Status: row.Status.ToString(), + ProcessedAtUtc: row.ProcessedAt, + CompletedAtUtc: row.CompletedAt, + RetryCount: row.RetryCount, + WorkerPending: workerPending, + WorkerProcessed: workerProcessed, + CreatedAtUtc: row.CreatedAt); + } +} + +/// Observed outbox row for an order-created integration event. +public sealed record OrderOutboxRow( + Guid OutboxMessageId, + Guid IntegrationEventId, + Guid OrderId, + decimal Total, + string EventType, + string Status, + DateTime? ProcessedAtUtc, + DateTime? CompletedAtUtc, + int RetryCount, + bool WorkerPending, + bool WorkerProcessed, + DateTime CreatedAtUtc); diff --git a/tests/Lab/IntegrationTests/Infrastructure/Reporting/LabRunInfo.cs b/tests/Lab/IntegrationTests/Infrastructure/Reporting/LabRunInfo.cs new file mode 100644 index 0000000..6fd39de --- /dev/null +++ b/tests/Lab/IntegrationTests/Infrastructure/Reporting/LabRunInfo.cs @@ -0,0 +1,14 @@ +using System.Reflection; + +namespace IntegrationTests.Infrastructure.Reporting; + +public static class LabRunInfo +{ + public static string ReadGitSha() + { + var info = typeof(LabRunInfo).Assembly + .GetCustomAttribute() + ?.InformationalVersion; + return string.IsNullOrWhiteSpace(info) ? "unknown" : info; + } +} diff --git a/tests/Lab/IntegrationTests/Infrastructure/Telemetry/CapturedActivity.cs b/tests/Lab/IntegrationTests/Infrastructure/Telemetry/CapturedActivity.cs new file mode 100644 index 0000000..d861773 --- /dev/null +++ b/tests/Lab/IntegrationTests/Infrastructure/Telemetry/CapturedActivity.cs @@ -0,0 +1,46 @@ +using System.Diagnostics; + +namespace IntegrationTests.Infrastructure.Telemetry; + +/// +/// Immutable snapshot of a stopped for in-process test observation. +/// +public sealed record CapturedActivity( + string TraceId, + string SpanId, + string? ParentSpanId, + string Source, + string DisplayName, + TimeSpan Duration, + IReadOnlyDictionary Tags, + IReadOnlyList Links) +{ + public static CapturedActivity From(Activity activity) + { + var tags = new Dictionary(StringComparer.Ordinal); + foreach (var tag in activity.Tags) + { + if (tag.Value is not null) + tags[tag.Key] = tag.Value; + } + + var links = activity.Links + .Select(link => new ActivityLinkSnapshot( + link.Context.TraceId.ToHexString(), + link.Context.SpanId.ToHexString())) + .ToList(); + + return new CapturedActivity( + activity.TraceId.ToHexString(), + activity.SpanId.ToHexString(), + activity.ParentSpanId == default ? null : activity.ParentSpanId.ToHexString(), + activity.Source.Name, + activity.DisplayName, + activity.Duration, + tags, + links); + } +} + +/// Snapshot of an context for correlation experiments. +public sealed record ActivityLinkSnapshot(string TraceId, string SpanId); diff --git a/tests/Lab/IntegrationTests/Infrastructure/Telemetry/InProcessActivityCapture.cs b/tests/Lab/IntegrationTests/Infrastructure/Telemetry/InProcessActivityCapture.cs new file mode 100644 index 0000000..f748675 --- /dev/null +++ b/tests/Lab/IntegrationTests/Infrastructure/Telemetry/InProcessActivityCapture.cs @@ -0,0 +1,42 @@ +using System.Collections.Concurrent; +using System.Diagnostics; + +namespace IntegrationTests.Infrastructure.Telemetry; + +/// +/// Captures stopped Activities from the test process via . +/// Used when FeatureFusion runs in-process through WebApplicationFactory. +/// +public sealed class InProcessActivityCapture : IDisposable +{ + private readonly ActivityListener _listener; + private readonly ConcurrentBag _stopped = []; + + public InProcessActivityCapture() + { + _listener = new ActivityListener + { + ShouldListenTo = static _ => true, + Sample = static (ref ActivityCreationOptions _) => + ActivitySamplingResult.AllDataAndRecorded, + ActivityStopped = activity => _stopped.Add(CapturedActivity.From(activity)) + }; + ActivitySource.AddActivityListener(_listener); + } + + public IReadOnlyList All => _stopped.ToList(); + + public IReadOnlyList ForTrace(ActivityTraceId traceId) + => ForTraceHex(traceId.ToHexString()); + + public IReadOnlyList ForTraceHex(string traceIdHex) + => _stopped.Where(s => s.TraceId == traceIdHex).ToList(); + + /// + /// Drops spans captured so far. Use between independent runs that share one listener + /// so per-run counts are not cumulative. + /// + public void Clear() => _stopped.Clear(); + + public void Dispose() => _listener.Dispose(); +} diff --git a/tests/Lab/IntegrationTests/Infrastructure/Telemetry/LabTrace.cs b/tests/Lab/IntegrationTests/Infrastructure/Telemetry/LabTrace.cs new file mode 100644 index 0000000..3ea7783 --- /dev/null +++ b/tests/Lab/IntegrationTests/Infrastructure/Telemetry/LabTrace.cs @@ -0,0 +1,36 @@ +using System.Diagnostics; +using BuildingBlocks.Telemetry; + +namespace IntegrationTests.Infrastructure.Telemetry; + +/// +/// Generic W3C trace injection and common Activity source/tag predicates for Lab tests. +/// Callers interpret captured evidence; this type does not assert experiment semantics. +/// +public static class LabTrace +{ + public static (ActivityTraceId TraceId, ActivitySpanId SpanId) NewTraceParent() + => (ActivityTraceId.CreateRandom(), ActivitySpanId.CreateRandom()); + + public static string FormatTraceParent(ActivityTraceId traceId, ActivitySpanId spanId) + => $"00-{traceId}-{spanId}-01"; + + public static bool IsAspNetCore(CapturedActivity span) + => span.Source == "Microsoft.AspNetCore" + || HasTag(span, TelemetryComponentTags.TagName, TelemetryComponentTags.AspNetCore); + + public static bool IsMediator(CapturedActivity span) + => span.Source == "BuildingBlocks.Mediator" + || HasTag(span, TelemetryComponentTags.TagName, TelemetryComponentTags.Mediator); + + public static bool IsNpgsql(CapturedActivity span) + => span.Source == "Npgsql"; + + public static bool HasTag(CapturedActivity span, string key, string expected) + => span.Tags.TryGetValue(key, out var value) + && string.Equals(value, expected, StringComparison.Ordinal); + + public static bool HasTagContaining(CapturedActivity span, string key, string fragment) + => span.Tags.TryGetValue(key, out var value) + && value.Contains(fragment, StringComparison.OrdinalIgnoreCase); +} diff --git a/tests/Lab/IntegrationTests/IntegrationTests.csproj b/tests/Lab/IntegrationTests/IntegrationTests.csproj index 58e8555..ea91b40 100644 --- a/tests/Lab/IntegrationTests/IntegrationTests.csproj +++ b/tests/Lab/IntegrationTests/IntegrationTests.csproj @@ -23,6 +23,8 @@ + +