diff --git a/AgentExperience.NET.sln b/AgentExperience.NET.sln index dcffc84..f6fe66a 100644 --- a/AgentExperience.NET.sln +++ b/AgentExperience.NET.sln @@ -79,6 +79,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AgentExperience.Cli.Tests", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AgentExperience.Sample.Reflective", "samples\AgentExperience.Sample.Reflective\AgentExperience.Sample.Reflective.csproj", "{90ECF1CA-7772-4A1C-A5AD-33CBEC2B9984}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AgentExperience.ReflectionEval", "experiments\AgentExperience.ReflectionEval\AgentExperience.ReflectionEval.csproj", "{666F85C3-8232-4580-A2AD-109DDE355E07}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -485,6 +487,18 @@ Global {90ECF1CA-7772-4A1C-A5AD-33CBEC2B9984}.Release|x64.Build.0 = Release|Any CPU {90ECF1CA-7772-4A1C-A5AD-33CBEC2B9984}.Release|x86.ActiveCfg = Release|Any CPU {90ECF1CA-7772-4A1C-A5AD-33CBEC2B9984}.Release|x86.Build.0 = Release|Any CPU + {666F85C3-8232-4580-A2AD-109DDE355E07}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {666F85C3-8232-4580-A2AD-109DDE355E07}.Debug|Any CPU.Build.0 = Debug|Any CPU + {666F85C3-8232-4580-A2AD-109DDE355E07}.Debug|x64.ActiveCfg = Debug|Any CPU + {666F85C3-8232-4580-A2AD-109DDE355E07}.Debug|x64.Build.0 = Debug|Any CPU + {666F85C3-8232-4580-A2AD-109DDE355E07}.Debug|x86.ActiveCfg = Debug|Any CPU + {666F85C3-8232-4580-A2AD-109DDE355E07}.Debug|x86.Build.0 = Debug|Any CPU + {666F85C3-8232-4580-A2AD-109DDE355E07}.Release|Any CPU.ActiveCfg = Release|Any CPU + {666F85C3-8232-4580-A2AD-109DDE355E07}.Release|Any CPU.Build.0 = Release|Any CPU + {666F85C3-8232-4580-A2AD-109DDE355E07}.Release|x64.ActiveCfg = Release|Any CPU + {666F85C3-8232-4580-A2AD-109DDE355E07}.Release|x64.Build.0 = Release|Any CPU + {666F85C3-8232-4580-A2AD-109DDE355E07}.Release|x86.ActiveCfg = Release|Any CPU + {666F85C3-8232-4580-A2AD-109DDE355E07}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -523,5 +537,6 @@ Global {71300AC9-934C-4C03-B86E-A90CE6B698C6} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {B737918B-C0E0-4941-A968-F77BE825EEA5} = {0AB3BF05-4346-4AA6-1389-037BE0695223} {90ECF1CA-7772-4A1C-A5AD-33CBEC2B9984} = {5A1F3C7D-9B24-4E86-A0D1-7C3E5B9F2A48} + {666F85C3-8232-4580-A2AD-109DDE355E07} = {8E1D1CCC-9289-C849-48CA-D7E099D02B6F} EndGlobalSection EndGlobal diff --git a/CHANGELOG.md b/CHANGELOG.md index 5adf434..c748afa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ AgentExperience.NET is a **preview**. It is not production ready, and public API ### Added +- **Reflective execution: step events and a measuring harness.** `OnStep` reports each step of the loop (attempt, verification, + reflection, evaluation, replanning, retry authorized or denied, budget exhausted, stopped) as counts and enums, never text, and a + new `agentexperience.reflection.steps` counter counts them by kind. `experiments/AgentExperience.ReflectionEval` runs the four + arms (baseline, reflection, memory, both) and reports verified success, attempts, tool calls and latency. It uses a scripted + model, so it shows mechanics only. Additive, experimental. See [the guide](docs/guide/reflective-execution.md#steps). + - **`aexp init [--print] [--force] [--verify ]`** writes a commented `.aexp/config.json` from one command, run in the project root, with no agent named: the verifier detected when one kind of project is at the root (`dotnet test`, `npm test`, `pytest`, `go test ./...`; and now `make test`, for a `Makefile` with a `test:` target, diff --git a/README.md b/README.md index 8578dbb..60f5b10 100644 --- a/README.md +++ b/README.md @@ -272,7 +272,7 @@ An optional loop that evaluates a failed attempt, proposes a correction and retr and only when you authorize it. It is off by default and never overrides your own check: a model's "it worked" is advice, not verification. All attempts are one run, stored as one lesson. See [Reflective execution](docs/guide/reflective-execution.md) and the -[sample](samples/AgentExperience.Sample.Reflective/README.md). No benefit is claimed; measure it with the holdout. +[sample](samples/AgentExperience.Sample.Reflective/README.md). No benefit is claimed; measure it with the holdout. A [four-arm harness](experiments/AgentExperience.ReflectionEval/README.md) shows the mechanics against a scripted model. ## Seeing what it learned diff --git a/docs/architecture/reflective-execution-design.md b/docs/architecture/reflective-execution-design.md index 1f4e99d..d05f81b 100644 --- a/docs/architecture/reflective-execution-design.md +++ b/docs/architecture/reflective-execution-design.md @@ -141,6 +141,6 @@ has a result. - **Closing sooner needed one internal addition**, `OpenRunRegistry.CloseRunAsync`, which completes the run with the **last attempt's** status. Reusing the duration bound's close would have recorded the run as `Cancelled`, a wrong outcome. - **Telemetry** is four counters on their own experimental meter, outside `docs/telemetry-names.json` because that file is - shared with the Python and Go ports, which do not have this feature. `OnEvent` also reports each step. + shared with the Python and Go ports, which do not have this feature. `OnEvent` reports each execution and `OnStep` each step (nine kinds, plus the stop; no text). - **Tokens** are counted from the provider's reported usage only; a provider that reports none leaves `MaxTokens` unenforced. - **Python, Go, CLI and MCP are untouched**, as intended. diff --git a/docs/architecture/reflective-execution-report.md b/docs/architecture/reflective-execution-report.md new file mode 100644 index 0000000..5e73e69 --- /dev/null +++ b/docs/architecture/reflective-execution-report.md @@ -0,0 +1,95 @@ +# Reflective execution: engineering report + +Status: experimental, off by default. Shipped in `0.1.0-preview.19` (step events and the harness: the release after it). This is the +report the implementation prompt asked for. It states what is built, what was run and what was not. + +## Architecture implemented + +The loop wraps the built agent from outside (`UseReflectiveExecution`, called before `UseAgentExperience`). Every attempt is a captured +invocation of **one** `ExperienceRun` (`ContinuesRunId`, `ShouldCompleteRun`), finalized once through the host's own `Verify`. + +- **Core** (`AgentExperience.Core.ReflectiveExecution`, no MAF dependency): `ReflectionExecutionCoordinator` (the loop), + `IReflectionEvaluator`, `IReplanningStrategy`, `IRetryAuthorizer` (default denies), `ReflectionExecutionOptions` (disabled by default), + `ReflectionStep` (structured, text-free steps), `ReflectionStopReason`, evaluation validation. +- **MAF** (`AgentExperience.MicrosoftAgentFramework.ReflectiveExecution`): `ReflectiveExecutionAgent` (the wrapper), + `AddReflectiveExecution` / `UseReflectiveExecution`, `DeterministicReflectionEvaluator`, `ChatClientReflectionEvaluator`, + counters on their own meter. +- **Strategies:** failure-driven and deterministic (rules), requirement-driven, model-assisted (`IChatClient`), composite; a custom + `IReflectionEvaluator` or `IReplanningStrategy` replaces any of them. +- **Not added:** `IReflectionStrategy`, `IReflectionExecutionCoordinator` and `ReflectionDecision` (the prompt's proposed names): composition of + an evaluator and a replanning strategy covered the need without a larger hierarchy. `RequireVerifiedSuccess` is not an option because + only the host's check can end a loop as verified, always. + +Design and decisions: [reflective-execution-design.md](reflective-execution-design.md). Usage and limits: [the guide](../guide/reflective-execution.md). + +## Files changed + +Added: `src/AgentExperience.Core/ReflectiveExecution/*`, `src/AgentExperience.MicrosoftAgentFramework/ReflectiveExecution/*`, +`samples/AgentExperience.Sample.Reflective/`, `experiments/AgentExperience.ReflectionEval/`, the guide, the design and this report, +tests under `tests/*/ReflectiveExecution/` and `tests/AgentExperience.Sample.QuickStart.Tests/Reflective/`. +Modified (additive): `OpenRunRegistry` (an internal `CloseRunAsync` that closes with the last attempt's real status), +`AgentExperienceRegistration` and the builder extension (a compose hook), the public API baselines, README, guide index, +known-limits pointer, telemetry note, CHANGELOG. + +## Public API + +New, all experimental: `AddReflectiveExecution`, `UseReflectiveExecution`, `ReflectiveExecutionOptions` (`Limits`, `Evaluator`, `Replanning`, +`Authorizer`, `TrustedCheck`, `OnEvent`, `OnStep`), the Core types above. No existing signature, default, stored format or migration changed; +the API baselines grew and lost nothing. + +## Verification + +Commands run on the final tree, with the real results: + +| Command | Result | +| --- | --- | +| `dotnet build` | 0 errors | +| Abstractions / Core / MAF / InMemory / SQLite / CLI / ReuseBaseline tests | 117 / 1415 / 1478 / 292 / 247 / 102 / 170 passed | +| QuickStart (carries the sample's tests) / LiveReuse / CiTriage | 16 / 208 / 75 passed | +| CompatibilityProof / EndToEnd / RetrievalQuality / Contract (non-PostgreSQL) | 15 / 29 / 24 / 72 passed | +| `AgentExperience.Release.Tests` | 198 of 200 passed; the 2 failures (`PackageValidationBaselineTests`, `CompatibilityPinAgreementTests`) are caused by stale `.claude/worktrees` copies on the author's machine and fail on a clean `main` too | +| PostgreSQL-backed suites | not run locally; run by CI on every pull request, and they passed on the pull requests that carried this work | + +The prompt's 17 cases map to tests as follows. Unauthorized tool actions: the loop adds no tools, and a test asserts the agent's tools and task +are unchanged; **blocking** a tool call is the existing approval boundary's job, not this library's. + +| # | Case | Where | +| --- | --- | --- | +| 1-2 | success without reflection; failure corrected | `ReflectionExecutionCoordinatorTests`, `ReflectiveExecutionTests` | +| 3 | invalid correction | evaluation validator tests; prompt-injected evaluator test | +| 4 | retry exhaustion | coordinator tests; `ReflectionStepTests` | +| 5-6 | cancellation; provider failure or timeout | coordinator tests | +| 7 | verifier rejects a model's claim | coordinator and MAF tests | +| 8 | unauthorized tool actions | tools unchanged (see above) | +| 9 | tenant isolation | MAF tests | +| 10-11 | multi-attempt persistence; unverified is not promoted | MAF tests | +| 12 | lesson exposure and provenance | MAF tests (second task is handed the lesson) | +| 13 | repeated strategy | coordinator tests | +| 14 | interaction with failure recall | MAF tests | +| 15 | disabled is a pass-through | coordinator and MAF tests | +| 16 | non-idempotent operations | side-effect policy tests | +| 17 | sanitization and prompt injection | validator, strategies and MAF tests | + +## Security + +The loop cannot change identity or scope, add or approve tools, run a command, skip `Verify`, or write a record. A passing `TrustedCheck` is +the only way to stop as `VerifiedSuccess`; an evaluator's `Satisfied` is `EvaluatorSatisfied` (not verified) or `NotRetryable` against a failing +check. Retries need `IRetryAuthorizer` (default denies) and a side-effect policy (default never retries semantically). Evaluator and +replanning output is validated, bounded and screened; the correction note reaches the model as labelled, untrusted reference text. +Steps and counters carry counts and enums only. + +## Limitations + +- **No benefit demonstrated.** The harness and the sample use a scripted model; they show mechanics, not improvement. A real-model, + pre-registered run is described in the harness README and has not been done. +- The evaluator sees a bounded summary of a **finished** attempt, not plans or intermediate results. Replanning is a note the model may or may not follow. +- The correction note is model-derived text in the conversation; screening is not a boundary (KL-18). +- All attempts share one session. Streaming is not retried. Tokens are counted only when the provider reports them. +- `TrustedCheck` is the host's statement and is separate from `Verify`; if they disagree the run can stop "verified" and be stored as failed. +- Historical-experience reuse is not a step of its own; use the existing injection telemetry and the run id. +- Python, Go, the CLI and MCP are unchanged. + +## Further improvements + +Essential before any claim of benefit: a pre-registered real-model comparison. Optional: plan-level critique before acting, streaming support, +an evaluator that can request authorized read-only diagnostics, and a Python/Go port if there is demand. diff --git a/docs/guide/reflective-execution.md b/docs/guide/reflective-execution.md index 60375a9..55683a5 100644 --- a/docs/guide/reflective-execution.md +++ b/docs/guide/reflective-execution.md @@ -173,6 +173,7 @@ An evaluator saying "fine" against a failing trusted check ends as `NotRetryable | `Authorizer` | deny all | Called before every retry | | `TrustedCheck` | none | Your check, after each attempt: `true` passed, `false` failed, `null` no verdict | | `OnEvent` | none | One event per execution: counts and enums, no prompt, argument or model text | +| `OnStep` | none | One event per step of the loop, with the run ID once known: see [Steps](#steps) | | `Replanning` | evaluator's proposal | Builds the correction text from an evaluation | A bound that capture would enforce first (attempts, duration) is refused when the agent is built, with a message naming it. @@ -206,7 +207,7 @@ competing recovery path. ## Telemetry -Four counters on the meter `AgentExperience.MicrosoftAgentFramework.ReflectiveExecution`, one tag, enums only: +Five counters on the meter `AgentExperience.MicrosoftAgentFramework.ReflectiveExecution`, one tag each, enums only: | Counter | Counts | Evidence? | | --- | --- | --- | @@ -214,11 +215,34 @@ Four counters on the meter `AgentExperience.MicrosoftAgentFramework.ReflectiveEx | `agentexperience.reflection.retries` | executions in which a retry occurred | no | | `agentexperience.reflection.retries.improved` | executions in which a retry did better than the attempt before | no | | `agentexperience.reflection.retries.verified` | executions in which a retry ended with your own check passing | **the only one** | +| `agentexperience.reflection.steps` (tag `agentexperience.reflection.step`) | steps of the loop, by kind | n/a | No prompt, tool argument, error text, model text or identity is ever a tag. `OnEvent` carries the same facts with the run ID. The meter is deliberately not in `docs/telemetry-names.json`: the Python and Go ports have no reflective execution, so no port has to match these names. +### Steps + +`OnStep` (on `ReflectiveExecutionOptions`, or `OnStep` on `Limits` in Core) reports each step as it happens. A step is a kind, +an attempt number and, where it applies, a trusted-check verdict, an advisory assessment or a stop reason. It has no text field, +and a test keeps it that way, so it is safe to log. An observer that throws is ignored: watching the loop never changes it. + +| Kind | When | +| --- | --- | +| `ExecutionStarted` | the loop begins | +| `AttemptCompleted` | an attempt ends, whatever its result | +| `VerificationReceived` | your trusted check answered (`TrustedCheckPassed` is `null` when there is none) | +| `ReflectionRequested` | the evaluator is about to be asked (never for a first attempt that passed, or for a transient technical retry) | +| `EvaluationCompleted` | a valid evaluation came back (`Assessment` is advisory) | +| `ReplanningPerformed` | a correction was built and accepted | +| `RetryAuthorized` / `RetryDenied` | your `IRetryAuthorizer` answered | +| `BudgetExhausted` | attempts, cycles, duration, tool calls or tokens ended the loop (`StopReason` says which) | +| `ExecutionStopped` | always last; `StopReason` says why | + +**Historical experience reused** has no step of its own, on purpose. Injection and recall already report it through the +library's existing injection telemetry and `OnContextInjected`; a second report here would be a second source of truth. Use the +run ID on the steps to join them. + ## Public API compatibility Purely additive: new types in `AgentExperience.Core.ReflectiveExecution` and `AgentExperience.MicrosoftAgentFramework.ReflectiveExecution`, @@ -238,8 +262,12 @@ change in a later preview. With reflective execution unregistered or disabled, b - **Tokens are counted only when the provider reports usage.** A provider that does not makes `MaxTokens` inert. - **The meter is separate and experimental.** See [Telemetry](#telemetry); its names are not in the cross-language names file. - **Only `RunAsync`.** Streaming is not retried. -- **No benefit is demonstrated.** The sample's four-arm comparison uses a scripted model that obeys what it is handed. It - proves the mechanics, not an improvement. +- **No benefit is demonstrated.** The four-arm comparison (`experiments/AgentExperience.ReflectionEval`) uses a scripted model + that obeys what it is handed. It proves the mechanics, not an improvement, and it reports no tokens or cost. +- **Evaluation is of a finished attempt.** The evaluator sees a bounded summary (error classes, failed tool names), not the + plan or intermediate results, and never raw tool arguments or output. A host that wants to critique a plan before acting + needs its own step. +- **Replanning is advice.** The correction is a note the model reads; whether it changes strategy is up to the model. - **Python, Go, the CLI and the MCP server are unchanged.** See also the [design assessment](../architecture/reflective-execution-design.md). diff --git a/experiments/AgentExperience.ReflectionEval/AgentExperience.ReflectionEval.csproj b/experiments/AgentExperience.ReflectionEval/AgentExperience.ReflectionEval.csproj new file mode 100644 index 0000000..5610e69 --- /dev/null +++ b/experiments/AgentExperience.ReflectionEval/AgentExperience.ReflectionEval.csproj @@ -0,0 +1,21 @@ + + + + + Exe + AgentExperience.ReflectionEval + false + $(NoWarn);CS1591 + + + + + + + + + + + diff --git a/experiments/AgentExperience.ReflectionEval/Program.cs b/experiments/AgentExperience.ReflectionEval/Program.cs new file mode 100644 index 0000000..24a4eba --- /dev/null +++ b/experiments/AgentExperience.ReflectionEval/Program.cs @@ -0,0 +1,21 @@ +using AgentExperience.Sample.Reflective; + +// Runs the four arms on the same deterministic tasks and prints a Markdown table. Pass a path to also write it there. +var writer = new StringWriter(); +var arms = await ReflectiveDemo.CompareArmsAsync(TextWriter.Null); + +writer.WriteLine("| Arm | Verified success | Attempts | Tool calls | Model calls | Elapsed |"); +writer.WriteLine("| --- | --- | --- | --- | --- | --- |"); +foreach (var arm in arms) +{ + writer.WriteLine($"| {arm.Arm} | {arm.Verified}/{arm.Tasks} | {arm.Attempts} | {arm.ToolCalls} | {arm.ModelCalls} | {arm.Elapsed.TotalMilliseconds:F0} ms |"); +} + +writer.WriteLine(); +writer.WriteLine("Scripted model: it applies what it is handed. This shows the mechanics (a retry needs reflection, a stored lesson removes the need for one). It says nothing about whether reflection or memory helps a real model, and it reports no tokens or cost."); + +Console.Write(writer); +if (args.Length > 0) +{ + await File.WriteAllTextAsync(args[0], writer.ToString()); +} diff --git a/experiments/AgentExperience.ReflectionEval/README.md b/experiments/AgentExperience.ReflectionEval/README.md new file mode 100644 index 0000000..e532a89 --- /dev/null +++ b/experiments/AgentExperience.ReflectionEval/README.md @@ -0,0 +1,31 @@ +# Reflective execution: the four-arm harness + +Compares, on the same tasks: **baseline** (neither), **reflection only**, **memory only** (a stored lesson, no retry) and +**reflection + memory**. It reports externally verified success (the host's own build check, never a model's word), +attempts, tool calls, model calls and elapsed time. Tokens and cost are not reported: the scripted model has no usage to +report, and the harness does not invent it. + +```bash +dotnet run --project experiments/AgentExperience.ReflectionEval # print the table +dotnet run --project experiments/AgentExperience.ReflectionEval -- out.md # also write it +``` + +## What this does and does not show + +It runs against a **scripted model that obeys what it is handed**, so the result is fixed by construction: it shows the +mechanics (a retry happens only with reflection; a stored lesson removes the need for one; verified success is decided by +the build, not the model). **It is not evidence that reflective execution or memory improves a real model.** No +improvement is claimed. + +## A real-model run (not done) + +To find out whether reflection helps, run it the way this repository runs its other real-model experiments, and report it +with the same intervals: + +1. Fix before running: the task set (real repositories judged by their own tests), the model, the budgets + (`MaxAttempts`, `MaxReflectionCycles`), the number of runs per arm, and the single comparison that counts (reflection + versus baseline, verified success). +2. Replace `ScriptedModel` in the sample with an `IChatClient` for the chosen model; keep the host's own test run as + `TrustedCheck` and `Verify`. +3. Run all four arms on the same tasks; record verified success, attempts, tool calls, tokens, cost and latency. +4. Report Wilson and Newcombe intervals as the holdout does, including a null or negative result. diff --git a/experiments/AgentExperience.ReflectionEval/packages.lock.json b/experiments/AgentExperience.ReflectionEval/packages.lock.json new file mode 100644 index 0000000..4ec54d8 --- /dev/null +++ b/experiments/AgentExperience.ReflectionEval/packages.lock.json @@ -0,0 +1,257 @@ +{ + "version": 1, + "dependencies": { + "net10.0": { + "Microsoft.Extensions.DependencyInjection": { + "type": "Direct", + "requested": "[10.0.12, 10.0.12]", + "resolved": "10.0.12", + "contentHash": "lXyK2O5GoYvfxW8eCFcD16JFbcoSTM1sJkAM0UHS1jZyl9NYMW64Tqm6OQFT0IDBjZi+xHt95/Zg+nxZhGFhZg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.12" + } + }, + "Google.Protobuf": { + "type": "Transitive", + "resolved": "3.30.2", + "contentHash": "Y2aOVLIt75yeeEWigg9V9YnjsEm53sADtLGq0gLhwaXpk3iu8tYSoauolyhenagA2sWno2TQ2WujI0HQd6s1Vw==" + }, + "Microsoft.Agents.AI": { + "type": "Transitive", + "resolved": "1.22.0", + "contentHash": "YdQJbL47n8PJmTGCD6DzapD2TQiCfI1f/669ECcHsKw1po3WhTq2lZoO3X7PJ6sZSGE/I2RReJ/hTGwtBZ7WTA==", + "dependencies": { + "Microsoft.Agents.AI.Abstractions": "1.22.0", + "Microsoft.Extensions.AI": "10.10.0", + "Microsoft.Extensions.AI.Abstractions": "10.10.0", + "Microsoft.Extensions.AI.Evaluation": "10.10.0", + "Microsoft.Extensions.Compliance.Abstractions": "10.10.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.12", + "Microsoft.Extensions.FileSystemGlobbing": "10.0.12", + "Microsoft.Extensions.Logging.Abstractions": "10.0.12", + "Microsoft.Extensions.VectorData.Abstractions": "10.10.0", + "Microsoft.ML.Tokenizers": "2.0.0" + } + }, + "Microsoft.Agents.AI.Abstractions": { + "type": "Transitive", + "resolved": "1.22.0", + "contentHash": "ZgDJUP+wuo1k2rMXg7wtKGbx0A0zo/q0k2juoKQ2EIbdhH8gdSwwNjQ9Zew3u3UmBuRWgH/4EMsCQrc7vx2x1g==", + "dependencies": { + "Microsoft.Extensions.AI.Abstractions": "10.10.0" + } + }, + "Microsoft.Extensions.AI": { + "type": "Transitive", + "resolved": "10.10.0", + "contentHash": "XKvtDUD3KySkvA5UpS+UF67M5rcqiXjAQchTZZ26WnavVKXxwlUTVdK/Ju0tc/25+16AVUdNl8cnVVGX/tx03A==", + "dependencies": { + "Microsoft.Extensions.AI.Abstractions": "10.10.0", + "Microsoft.Extensions.Caching.Abstractions": "10.0.12", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.12", + "Microsoft.Extensions.Logging.Abstractions": "10.0.12", + "System.Numerics.Tensors": "10.0.12" + } + }, + "Microsoft.Extensions.AI.Abstractions": { + "type": "Transitive", + "resolved": "10.10.0", + "contentHash": "frBHj2ckpevA+KdDLikxv8oxGPrJiyTvaiblYp+vtVEt0q6J73XP5m2SICm86nFK6rl+L1GCKxIy4S5Lb3p2kQ==" + }, + "Microsoft.Extensions.AI.Evaluation": { + "type": "Transitive", + "resolved": "10.10.0", + "contentHash": "YXdqOSHl8X1/Vkmuu6QoV330FQ7aiYtNDqIUIiICSDd0HMx0No9D/ZWo0ocOPXd9bk00YQQfyK5HKcgRnRmQbQ==", + "dependencies": { + "Microsoft.Extensions.AI.Abstractions": "10.10.0" + } + }, + "Microsoft.Extensions.Caching.Abstractions": { + "type": "Transitive", + "resolved": "10.0.12", + "contentHash": "sI3A6MUAyZwccLnwq0cXuRmC3WBtXieimBWafXSfhRJ4jnM2uGVXrqImDDVyW27u4vVaTOGlJxNJd8d1FGvCNQ==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.12" + } + }, + "Microsoft.Extensions.Compliance.Abstractions": { + "type": "Transitive", + "resolved": "10.10.0", + "contentHash": "Sh3ujDRJpCMNY4x+43ljCROt4qBOoWfsL8l816RPYZS1S+Tw277r5002rbB15Sh6hFcCT8mGYBO7NtpfSYrQ0Q==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.12", + "Microsoft.Extensions.ObjectPool": "10.0.12" + } + }, + "Microsoft.Extensions.Compliance.Redaction": { + "type": "Transitive", + "resolved": "10.10.0", + "contentHash": "RW0HuSIl5CH/SSLa31MIKc6LPmy3DjtJmf/5LjTTKMmwBeNex6ZnF1Q46vzklAPSQ2BDu/MHL/6G5CBoyWLQAA==", + "dependencies": { + "Microsoft.Extensions.Compliance.Abstractions": "10.10.0", + "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.12" + } + }, + "Microsoft.Extensions.Configuration": { + "type": "Transitive", + "resolved": "10.0.12", + "contentHash": "e3IPP32CRNL031VZJAUTlCTG0YN7WFh4mN3fsSHTDQCJB+3+f0jGycv4fXk3rrftaY3B85XrQaj7sRthrOsavg==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.12", + "Microsoft.Extensions.Primitives": "10.0.12" + } + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "10.0.12", + "contentHash": "8xaGcvS/qZ1otoxPQCEJkNva389CVL/plNcvIETZhQTETYdRkYDPEYhUMoAGONo4FU45ufdfE0j29AfWVVj0wA==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.12" + } + }, + "Microsoft.Extensions.Configuration.Binder": { + "type": "Transitive", + "resolved": "10.0.12", + "contentHash": "dAgIf1TOr8KLs+aBRIbXUZBjHoSH4rDG8+XkX/Q6AZwkQdMA0+yPDKTHsieeXdZDfpOpZVHzOnuAb5Z2nX3KsA==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.12", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.12" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "10.0.12", + "contentHash": "9/qymSh7hVDMGTGwrLz8MRp5zRyXy9adGDOs4HwRdnLil3oZGYuWeZjbmHgCQ9BL1qBroVfgUK3U/nb61617Cw==" + }, + "Microsoft.Extensions.Diagnostics.Abstractions": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "mQiTzAj7PIJ2A9YXR5QhgulS1fTWhmQc3ckd1Mrf3hKW07d03fBDqx8vVaFw+cRTebDOeB6pNqdWdnRxsi1hBA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.3", + "Microsoft.Extensions.Options": "10.0.3" + } + }, + "Microsoft.Extensions.FileProviders.Abstractions": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "4TD9AXDRsipTmaemwnjt/DM5Ri0de2JzHQhvZ4woBTjUtL4XrPNsMrOk5oiLJAx1gTrE6pOIhxv+lEde5F6CZA==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.3" + } + }, + "Microsoft.Extensions.FileSystemGlobbing": { + "type": "Transitive", + "resolved": "10.0.12", + "contentHash": "45cV+UeI0chPAIXrx9Wv7K4PJXuuIR2pnOur5MOAo6kOFclBHSLLDrvGAxMfWRMtUvpv/kQZkxlgaUN56jwzSQ==" + }, + "Microsoft.Extensions.Hosting.Abstractions": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "GdMpC10Jf6poxSvUJ4lgYpJ5F/kJeaAoJmrPufjBoPYyCTKKY5Dyl0rZA+LBNvFqTq1cZa/lhlptlUhNvU6xrg==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.3", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.3", + "Microsoft.Extensions.Diagnostics.Abstractions": "10.0.3", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.3", + "Microsoft.Extensions.Logging.Abstractions": "10.0.3" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Transitive", + "resolved": "10.0.12", + "contentHash": "+24lC4plfbEDNfLAdTV/SWKS7dW+16X4HdydO3R++134kSNTzcbYA4KpR1Hdh6uWisB8Za3AzwyOn+K+NxWIug==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.12" + } + }, + "Microsoft.Extensions.ObjectPool": { + "type": "Transitive", + "resolved": "10.0.12", + "contentHash": "opBULzn9F7ZOL8EoFuqM2QjI6x9AG04DFNdnU2ky/kI4aUzbiXEfuy9jvCuu3D/ieccuDHTlalyC62esPD8wGg==" + }, + "Microsoft.Extensions.Options": { + "type": "Transitive", + "resolved": "10.0.12", + "contentHash": "TDYD33TSRpXKZWlmTXNlj5kCihxatmv2Ec1u6C+bMYLphCS7PoSLE9Pjd/nunDoE7yETk+LLKjVJX78HYtWjpA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.12", + "Microsoft.Extensions.Primitives": "10.0.12" + } + }, + "Microsoft.Extensions.Options.ConfigurationExtensions": { + "type": "Transitive", + "resolved": "10.0.12", + "contentHash": "rqpu4qj5WE9x1IHGXSIgHBKi7IUlQaHyp4aXCYIanG2OghlUMFZpZTgExaXwcvmLAJHsxKQWMPpc7D2WIbCVtA==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.12", + "Microsoft.Extensions.Configuration.Binder": "10.0.12", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.12", + "Microsoft.Extensions.Options": "10.0.12", + "Microsoft.Extensions.Primitives": "10.0.12" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "10.0.12", + "contentHash": "dYfCLR52UA+3DL7C4I/pvSaRPkNqxrUAQmbFL2u0zvYKKzqgrFCJl08Df+F1aYc8leu9JvpC9bsURUdpExcBXQ==" + }, + "Microsoft.Extensions.VectorData.Abstractions": { + "type": "Transitive", + "resolved": "10.10.0", + "contentHash": "bZdR7QbdLLqmc9xPJIiD5U6H678xyv8eMATjQ0SmH7OZyi+/12G/nFAGCE6DNG22q76YNwkXo+V7GMyUckjjgw==", + "dependencies": { + "Microsoft.Extensions.AI.Abstractions": "10.10.0" + } + }, + "Microsoft.ML.Tokenizers": { + "type": "Transitive", + "resolved": "2.0.0", + "contentHash": "+b8lT4cLLO/sBR2hjvE/qG6qrZG15h7/PBvnIrzTh4xDaAxdHUY6449rC+1pHzQUsBiCHZVbj+VMn+xS0sL7TA==", + "dependencies": { + "Google.Protobuf": "3.30.2" + } + }, + "System.Numerics.Tensors": { + "type": "Transitive", + "resolved": "10.0.12", + "contentHash": "17brFrnQHV/XMMazXNqFq4GkvzWy8yzDPo5Gb8TT8zJtMFbUqDycOCyPCp9F+eMc4L75kcCKRZqaeBHMwLCL9Q==" + }, + "agentexperience.abstractions": { + "type": "Project" + }, + "agentexperience.core": { + "type": "Project", + "dependencies": { + "AgentExperience.Abstractions": "[0.1.0-preview.19, )", + "Microsoft.Extensions.Compliance.Redaction": "[10.10.0, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.12, )" + } + }, + "agentexperience.microsoftagentframework": { + "type": "Project", + "dependencies": { + "AgentExperience.Core": "[0.1.0-preview.19, )", + "Microsoft.Agents.AI": "[1.22.0, 2.0.0)", + "Microsoft.Extensions.Hosting.Abstractions": "[10.0.3, )" + } + }, + "agentexperience.sample.reflective": { + "type": "Project", + "dependencies": { + "AgentExperience.MicrosoftAgentFramework": "[0.1.0-preview.19, )", + "AgentExperience.Storage.InMemory": "[0.1.0-preview.19, )", + "Microsoft.Extensions.DependencyInjection": "[10.0.12, 10.0.12]" + } + }, + "agentexperience.storage.inmemory": { + "type": "Project", + "dependencies": { + "AgentExperience.Abstractions": "[0.1.0-preview.19, )", + "Microsoft.Extensions.DependencyInjection.Abstractions": "[10.0.12, )", + "Microsoft.Extensions.Hosting.Abstractions": "[10.0.3, )" + } + } + } + } +} \ No newline at end of file diff --git a/samples/AgentExperience.Sample.Reflective/ReflectiveDemo.cs b/samples/AgentExperience.Sample.Reflective/ReflectiveDemo.cs index b10877b..526eca9 100644 --- a/samples/AgentExperience.Sample.Reflective/ReflectiveDemo.cs +++ b/samples/AgentExperience.Sample.Reflective/ReflectiveDemo.cs @@ -51,7 +51,11 @@ public static async Task> RunAsync(IChatClient model, Tex /// How many of the tasks ended with the host's build check green. /// How many tasks. /// The model calls used in total. - public sealed record ArmResult(string Arm, int Verified, int Tasks, int ModelCalls); + /// The attempts of the agent across the tasks (a task without reflection is one attempt). + /// The tool calls across the tasks. + /// The wall-clock time the arm's tasks took. Tiny here: the model is scripted. + /// Tokens and cost are not measured: the scripted model reports no usage, and the harness does not invent it. + public sealed record ArmResult(string Arm, int Verified, int Tasks, int ModelCalls, int Attempts = 0, int ToolCalls = 0, TimeSpan Elapsed = default); /// /// The four arms on the same deterministic tasks: neither, reflection only, memory only, both. This shows the @@ -66,6 +70,9 @@ public static async Task> CompareArmsAsync(TextWriter o { var verified = 0; var calls = 0; + var attempts = 0; + var toolCalls = 0; + var clock = System.Diagnostics.Stopwatch.StartNew(); if (memory) { using var model = new ScriptedModel(); @@ -73,13 +80,17 @@ public static async Task> CompareArmsAsync(TextWriter o await session.RunTaskAsync(tasks[0]); // the teaching run, always reflective: it is what stores the lesson session.Reflection = reflection; var before = model.Calls; + clock.Restart(); foreach (var task in tasks.Skip(1)) { - verified += (await session.RunTaskAsync(task)).BuildGreen ? 1 : 0; + var run = await session.RunTaskAsync(task); + verified += run.BuildGreen ? 1 : 0; + attempts += run.Reflection?.Attempts ?? 1; + toolCalls += run.FixesTried.Count; } calls = model.Calls - before; - results.Add(new ArmResult(name, verified, tasks.Length - 1, calls)); + results.Add(new ArmResult(name, verified, tasks.Length - 1, calls, attempts, toolCalls, clock.Elapsed)); } else { @@ -88,14 +99,18 @@ public static async Task> CompareArmsAsync(TextWriter o // A fresh memory per task: nothing carries over. using var model = new ScriptedModel(); await using var session = new Session(model, reflection); - verified += (await session.RunTaskAsync(task)).BuildGreen ? 1 : 0; + var run = await session.RunTaskAsync(task); + verified += run.BuildGreen ? 1 : 0; + attempts += run.Reflection?.Attempts ?? 1; + toolCalls += run.FixesTried.Count; calls += model.Calls; } - results.Add(new ArmResult(name, verified, tasks.Length - 1, calls)); + results.Add(new ArmResult(name, verified, tasks.Length - 1, calls, attempts, toolCalls, clock.Elapsed)); } - await output.WriteLineAsync($"{name,-22} verified {results[^1].Verified}/{results[^1].Tasks}, model calls {results[^1].ModelCalls}"); + var r = results[^1]; + await output.WriteLineAsync($"{name,-22} verified {r.Verified}/{r.Tasks}, attempts {r.Attempts}, tool calls {r.ToolCalls}, model calls {r.ModelCalls}, {r.Elapsed.TotalMilliseconds:F0} ms"); } return results; diff --git a/src/AgentExperience.Core/ReflectiveExecution/ReflectionExecutionCoordinator.cs b/src/AgentExperience.Core/ReflectiveExecution/ReflectionExecutionCoordinator.cs index 6836f5c..2c55459 100644 --- a/src/AgentExperience.Core/ReflectiveExecution/ReflectionExecutionCoordinator.cs +++ b/src/AgentExperience.Core/ReflectiveExecution/ReflectionExecutionCoordinator.cs @@ -79,6 +79,7 @@ public async ValueTask ExecuteAsync( long tokens = 0; var toolCalls = 0; var stop = ReflectionStopReason.AttemptsExhausted; + Emit(new ReflectionStep(ReflectionStepKind.ExecutionStarted, 0)); for (var n = 1; ; n++) { @@ -87,6 +88,8 @@ public async ValueTask ExecuteAsync( attempts.Add(outcome); toolCalls += outcome.ToolCalls ?? 0; tokens += outcome.TokensUsed ?? 0; + Emit(new ReflectionStep(ReflectionStepKind.AttemptCompleted, n)); + if (_options.Enabled) { Emit(new ReflectionStep(ReflectionStepKind.VerificationReceived, n, TrustedCheckPassed: outcome.TrustedCheckPassed)); } if (!_options.Enabled) { stop = ReflectionStopReason.Disabled; break; } if (outcome.TrustedCheckPassed == true) { stop = ReflectionStopReason.VerifiedSuccess; break; } @@ -111,10 +114,12 @@ public async ValueTask ExecuteAsync( else { if (_evaluator is null) { stop = ReflectionStopReason.NotRetryable; break; } + Emit(new ReflectionStep(ReflectionStepKind.ReflectionRequested, n)); var evaluated = await EvaluateAsync(context, cancellationToken).ConfigureAwait(false); if (evaluated is null) { stop = ReflectionStopReason.EvaluationFailed; break; } evaluation = evaluated; evaluations.Add(evaluation); + Emit(new ReflectionStep(ReflectionStepKind.EvaluationCompleted, n, Assessment: evaluation.Assessment)); if (evaluation.Assessment == ReflectionAssessment.Unrecoverable) { stop = ReflectionStopReason.Unrecoverable; break; } if (evaluation.Assessment == ReflectionAssessment.Satisfied) @@ -131,9 +136,22 @@ public async ValueTask ExecuteAsync( if (!seen.Add(Normalize(planned.Note))) { stop = ReflectionStopReason.RepeatedStrategy; break; } note = planned.Note; tried.Add(note); + Emit(new ReflectionStep(ReflectionStepKind.ReplanningPerformed, n)); } - if (!await AuthorizeAsync(context, evaluation, cancellationToken).ConfigureAwait(false)) { stop = ReflectionStopReason.RetryDenied; break; } + if (!await AuthorizeAsync(context, evaluation, cancellationToken).ConfigureAwait(false)) + { + Emit(new ReflectionStep(ReflectionStepKind.RetryDenied, n)); + stop = ReflectionStopReason.RetryDenied; + break; + } + + Emit(new ReflectionStep(ReflectionStepKind.RetryAuthorized, n)); + } + + if (stop is ReflectionStopReason.AttemptsExhausted or ReflectionStopReason.CyclesExhausted or ReflectionStopReason.DurationExhausted or ReflectionStopReason.BudgetExhausted) + { + Emit(new ReflectionStep(ReflectionStepKind.BudgetExhausted, attempts.Count, StopReason: stop)); } if (onStopped is not null) @@ -141,6 +159,8 @@ public async ValueTask ExecuteAsync( await onStopped(stop).ConfigureAwait(false); } + Emit(new ReflectionStep(ReflectionStepKind.ExecutionStopped, attempts.Count, StopReason: stop)); + return new ReflectionExecutionResult( stop, attempts, @@ -150,6 +170,18 @@ public async ValueTask ExecuteAsync( attempts.Count > 1 && attempts[^1].TrustedCheckPassed == true); } + private void Emit(ReflectionStep step) + { + try + { + _options.OnStep?.Invoke(step); + } + catch (Exception) + { + // Observing the loop never changes it. + } + } + private static async ValueTask RunAttemptAsync(ReflectionAttemptDelegate attempt, ReflectionAttemptInput input, CancellationToken ct) { try diff --git a/src/AgentExperience.Core/ReflectiveExecution/ReflectionExecutionOptions.cs b/src/AgentExperience.Core/ReflectiveExecution/ReflectionExecutionOptions.cs index 9cf1cc5..0593372 100644 --- a/src/AgentExperience.Core/ReflectiveExecution/ReflectionExecutionOptions.cs +++ b/src/AgentExperience.Core/ReflectiveExecution/ReflectionExecutionOptions.cs @@ -36,6 +36,12 @@ public sealed record ReflectionExecutionOptions /// The time source for . public TimeProvider TimeProvider { get; init; } = TimeProvider.System; + /// + /// Receives each as it happens. Counts and enums only. An exception from it is swallowed: + /// observing the loop never changes it. + /// + public Action? OnStep { get; init; } + /// Throws when a limit is out of range. /// A bound is not positive where it must be. public void Validate() diff --git a/src/AgentExperience.Core/ReflectiveExecution/ReflectionStep.cs b/src/AgentExperience.Core/ReflectiveExecution/ReflectionStep.cs new file mode 100644 index 0000000..240d3a3 --- /dev/null +++ b/src/AgentExperience.Core/ReflectiveExecution/ReflectionStep.cs @@ -0,0 +1,51 @@ +namespace AgentExperience.Core.ReflectiveExecution; + +/// What a reports. Enums and numbers only, never text. +public enum ReflectionStepKind +{ + /// The loop began. + ExecutionStarted, + + /// An attempt finished, whatever its result. + AttemptCompleted, + + /// The host's trusted check answered (or there is none: is ). + VerificationReceived, + + /// The evaluator is about to be asked. + ReflectionRequested, + + /// The evaluator returned a valid evaluation (). + EvaluationCompleted, + + /// A correction was built and accepted for the next attempt. + ReplanningPerformed, + + /// The host's allowed the retry. + RetryAuthorized, + + /// The host's refused (or failed to answer). + RetryDenied, + + /// A bound ended the loop: attempts, cycles, duration, tool calls or tokens. + BudgetExhausted, + + /// The loop ended; says why. Always the last step. + ExecutionStopped, +} + +/// +/// One structured step of a reflective execution, for logs and traces. It carries counts and enums only: no task text, +/// result, tool argument, error text, evaluator text or correction note ever appears in it, so it is safe to record. +/// +/// What happened. +/// The attempt it belongs to (1 for the first); 0 for . +/// For : the host's verdict, or for none. +/// For : the evaluator's advisory assessment. +/// For and : why the loop ended. +public sealed record ReflectionStep( + ReflectionStepKind Kind, + int AttemptNumber, + bool? TrustedCheckPassed = null, + ReflectionAssessment? Assessment = null, + ReflectionStopReason? StopReason = null); diff --git a/src/AgentExperience.MicrosoftAgentFramework/ReflectiveExecution/ReflectiveExecutionAgent.cs b/src/AgentExperience.MicrosoftAgentFramework/ReflectiveExecution/ReflectiveExecutionAgent.cs index c620016..29c150e 100644 --- a/src/AgentExperience.MicrosoftAgentFramework/ReflectiveExecution/ReflectiveExecutionAgent.cs +++ b/src/AgentExperience.MicrosoftAgentFramework/ReflectiveExecution/ReflectiveExecutionAgent.cs @@ -48,7 +48,16 @@ protected override async Task RunCoreAsync( Exception? lastError = null; Guid? runId = null; var lastCompleted = false; - var coordinator = new ReflectionExecutionCoordinator(settings.Limits, settings.Evaluator, settings.Authorizer, settings.Replanning); + var limits = settings.Limits with + { + OnStep = step => + { + ReflectiveExecutionTelemetry.RecordStep(step); + settings.Limits.OnStep?.Invoke(step); + settings.OnStep?.Invoke(new ReflectiveStepEvent(runId, step)); + }, + }; + var coordinator = new ReflectionExecutionCoordinator(limits, settings.Evaluator, settings.Authorizer, settings.Replanning); async ValueTask Attempt(ReflectionAttemptInput input, CancellationToken token) { diff --git a/src/AgentExperience.MicrosoftAgentFramework/ReflectiveExecution/ReflectiveExecutionOptions.cs b/src/AgentExperience.MicrosoftAgentFramework/ReflectiveExecution/ReflectiveExecutionOptions.cs index d3d5ac7..e18bc09 100644 --- a/src/AgentExperience.MicrosoftAgentFramework/ReflectiveExecution/ReflectiveExecutionOptions.cs +++ b/src/AgentExperience.MicrosoftAgentFramework/ReflectiveExecution/ReflectiveExecutionOptions.cs @@ -55,6 +55,13 @@ public sealed class ReflectiveExecutionOptions /// Called for each stop with the reason and counters; carries no prompt, tool argument or model text. public Action? OnEvent { get; set; } + + /// + /// Receives each step of the loop (attempt, verification, reflection, evaluation, replanning, retry authorized or + /// denied, budget exhausted, stopped) with the run id once it is known. Counts and enums only; an exception from it is + /// swallowed. on still works and is called too. + /// + public Action? OnStep { get; set; } } /// What the reflective execution did, as counts and enums only. @@ -71,3 +78,8 @@ public sealed record ReflectiveExecutionEvent( bool RetryHappened, bool RetryImprovedResult, bool RetryReachedVerifiedSuccess); + +/// One step of a reflective execution, with the run it belongs to. +/// The one run all attempts belong to, or before the first attempt records it. +/// The step: counts and enums only, never text. +public sealed record ReflectiveStepEvent(Guid? RunId, ReflectionStep Step); diff --git a/src/AgentExperience.MicrosoftAgentFramework/ReflectiveExecution/ReflectiveExecutionTelemetry.cs b/src/AgentExperience.MicrosoftAgentFramework/ReflectiveExecution/ReflectiveExecutionTelemetry.cs index 4151ebd..830805e 100644 --- a/src/AgentExperience.MicrosoftAgentFramework/ReflectiveExecution/ReflectiveExecutionTelemetry.cs +++ b/src/AgentExperience.MicrosoftAgentFramework/ReflectiveExecution/ReflectiveExecutionTelemetry.cs @@ -28,6 +28,12 @@ internal static class ReflectiveExecutionTelemetry private static readonly Counter Verified = Meter.CreateCounter( "agentexperience.reflection.retries.verified", "{execution}", "Executions in which a retry ended with the host's own check passing. The only one of the three that is evidence."); + private static readonly Counter Steps = Meter.CreateCounter( + "agentexperience.reflection.steps", "{step}", "Steps of reflective executions, tagged with the kind of step."); + + internal static void RecordStep(ReflectionStep step) => + Steps.Add(1, new KeyValuePair("agentexperience.reflection.step", step.Kind.ToString())); + internal static void Record(ReflectionExecutionResult result) { Executions.Add(1, new KeyValuePair("agentexperience.reflection.stop_reason", result.StopReason.ToString())); diff --git a/tests/AgentExperience.Core.Tests/ReflectiveExecution/ReflectionStepTests.cs b/tests/AgentExperience.Core.Tests/ReflectiveExecution/ReflectionStepTests.cs new file mode 100644 index 0000000..a429f20 --- /dev/null +++ b/tests/AgentExperience.Core.Tests/ReflectiveExecution/ReflectionStepTests.cs @@ -0,0 +1,107 @@ +using AgentExperience.Core.ReflectiveExecution; + +namespace AgentExperience.Core.Tests.ReflectiveExecution; + +/// The structured steps of the loop: the order, what they carry, and that observing never changes the loop. +public class ReflectionStepTests +{ + private static ReflectionAttemptOutcome Fail() => + new(false, "exit 1", null, [new ReflectionFailedCall("build", "exit 1")], 1, null, false); + + private static ReflectionAttemptOutcome Pass() => new(true, null, "ok", [], 1, null, true); + + private static ReflectionEvaluation Fix(string strategy) => + new(ReflectionAssessment.NeedsCorrection, [new ReflectionIssue(ReflectionIssueCategory.ToolFailure, "build failed", "exit 1")], strategy, true, 0.8); + + private sealed class Eval(Func f) : IReflectionEvaluator + { + public ValueTask EvaluateAsync(ReflectionAttemptContext c, CancellationToken ct) => ValueTask.FromResult(f()); + } + + private sealed class Gate(bool allow) : IRetryAuthorizer + { + public ValueTask AuthorizeAsync(ReflectionAttemptContext c, ReflectionEvaluation e, CancellationToken ct) => ValueTask.FromResult(allow); + } + + private static ReflectionAttemptDelegate Script(params ReflectionAttemptOutcome[] outcomes) + { + var i = 0; + return (_, _) => ValueTask.FromResult(outcomes[Math.Min(i++, outcomes.Length - 1)]); + } + + private static ReflectionExecutionOptions Options(List steps, int maxAttempts = 3) => + new() { Enabled = true, SideEffects = SideEffectPolicy.Idempotent, MaxAttempts = maxAttempts, MaxReflectionCycles = maxAttempts - 1, OnStep = steps.Add }; + + [Fact] + public async Task A_corrected_run_reports_every_step_in_order() + { + var steps = new List(); + await new ReflectionExecutionCoordinator(Options(steps), new Eval(() => Fix("pin the version")), new Gate(true)).ExecuteAsync("t", Script(Fail(), Pass())); + + Assert.Equal( + [ + ReflectionStepKind.ExecutionStarted, + ReflectionStepKind.AttemptCompleted, ReflectionStepKind.VerificationReceived, + ReflectionStepKind.ReflectionRequested, ReflectionStepKind.EvaluationCompleted, ReflectionStepKind.ReplanningPerformed, ReflectionStepKind.RetryAuthorized, + ReflectionStepKind.AttemptCompleted, ReflectionStepKind.VerificationReceived, + ReflectionStepKind.ExecutionStopped, + ], + steps.Select(s => s.Kind)); + Assert.Equal(false, steps.First(s => s.Kind == ReflectionStepKind.VerificationReceived).TrustedCheckPassed); + Assert.Equal(true, steps.Last(s => s.Kind == ReflectionStepKind.VerificationReceived).TrustedCheckPassed); + Assert.Equal(ReflectionAssessment.NeedsCorrection, steps.Single(s => s.Kind == ReflectionStepKind.EvaluationCompleted).Assessment); + Assert.Equal(ReflectionStopReason.VerifiedSuccess, steps[^1].StopReason); + } + + [Fact] + public async Task A_denied_retry_is_reported_as_denied_and_never_as_authorized() + { + var steps = new List(); + await new ReflectionExecutionCoordinator(Options(steps), new Eval(() => Fix("a")), new Gate(false)).ExecuteAsync("t", Script(Fail(), Pass())); + + Assert.Contains(steps, s => s.Kind == ReflectionStepKind.RetryDenied); + Assert.DoesNotContain(steps, s => s.Kind == ReflectionStepKind.RetryAuthorized); + Assert.Equal(ReflectionStopReason.RetryDenied, steps[^1].StopReason); + } + + [Fact] + public async Task Running_out_of_attempts_reports_the_budget_before_the_stop() + { + var steps = new List(); + var n = 0; + await new ReflectionExecutionCoordinator(Options(steps, maxAttempts: 2), new Eval(() => Fix("fix " + n++)), new Gate(true)).ExecuteAsync("t", Script(Fail())); + + var kinds = steps.Select(s => s.Kind).ToList(); + Assert.Equal(ReflectionStepKind.BudgetExhausted, kinds[^2]); + Assert.Equal(ReflectionStepKind.ExecutionStopped, kinds[^1]); + Assert.Equal(steps[^1].StopReason, steps[^2].StopReason); + } + + [Fact] + public async Task A_first_attempt_that_passes_asks_for_no_reflection() + { + var steps = new List(); + await new ReflectionExecutionCoordinator(Options(steps), new Eval(() => throw new InvalidOperationException()), new Gate(true)).ExecuteAsync("t", Script(Pass())); + + Assert.DoesNotContain(steps, s => s.Kind is ReflectionStepKind.ReflectionRequested or ReflectionStepKind.EvaluationCompleted); + Assert.Equal(ReflectionStopReason.VerifiedSuccess, steps[^1].StopReason); + } + + [Fact] + public async Task An_observer_that_throws_changes_nothing() + { + var options = new ReflectionExecutionOptions { Enabled = true, SideEffects = SideEffectPolicy.Idempotent, OnStep = _ => throw new InvalidOperationException("observer") }; + var result = await new ReflectionExecutionCoordinator(options, new Eval(() => Fix("a")), new Gate(true)).ExecuteAsync("t", Script(Fail(), Pass())); + + Assert.Equal(ReflectionStopReason.VerifiedSuccess, result.StopReason); + Assert.Equal(2, result.Attempts.Count); + } + + [Fact] + public void A_step_carries_no_text() + { + // The contract is enums, numbers and a nullable flag. A string property here would be a way for text to leak. + var types = typeof(ReflectionStep).GetProperties().Select(p => p.PropertyType).ToList(); + Assert.DoesNotContain(typeof(string), types); + } +} diff --git a/tests/AgentExperience.MicrosoftAgentFramework.Tests/ReflectiveExecution/ReflectiveExecutionTests.cs b/tests/AgentExperience.MicrosoftAgentFramework.Tests/ReflectiveExecution/ReflectiveExecutionTests.cs index 5373662..3db609c 100644 --- a/tests/AgentExperience.MicrosoftAgentFramework.Tests/ReflectiveExecution/ReflectiveExecutionTests.cs +++ b/tests/AgentExperience.MicrosoftAgentFramework.Tests/ReflectiveExecution/ReflectiveExecutionTests.cs @@ -405,6 +405,42 @@ public async Task Counters_report_the_execution_and_separate_a_retry_from_a_veri } } + [Fact] + public async Task Steps_reach_the_host_with_the_one_run_id_and_are_counted_by_kind() + { + var kindsCounted = new List(); + using var listener = new MeterListener(); + listener.InstrumentPublished = (instrument, l) => + { + if (instrument.Meter.Name == "AgentExperience.MicrosoftAgentFramework.ReflectiveExecution" && instrument.Name == "agentexperience.reflection.steps") { l.EnableMeasurementEvents(instrument); } + }; + listener.SetMeasurementEventCallback((_, _, tags, _) => + { + foreach (var tag in tags) { if (tag.Key == "agentexperience.reflection.step") { lock (kindsCounted) { kindsCounted.Add(tag.Value?.ToString() ?? string.Empty); } } } + }); + listener.Start(); + + var steps = new List(); + var h = new Harness(); + await using var services = h.Build(Enabled(options => options.OnStep = steps.Add)); + await h.Agent(services).RunAsync(TaskText); + + var kinds = steps.Select(s => s.Step.Kind).ToList(); + Assert.Equal(ReflectionStepKind.ExecutionStarted, kinds[0]); + Assert.Equal(ReflectionStepKind.ExecutionStopped, kinds[^1]); + Assert.Contains(ReflectionStepKind.ReplanningPerformed, kinds); + Assert.Contains(ReflectionStepKind.RetryAuthorized, kinds); + + // Every step after the first attempt is tied to the same single run. + var runIds = steps.Where(s => s.RunId is not null).Select(s => s.RunId).Distinct().ToList(); + Assert.Single(runIds); + + lock (kindsCounted) + { + Assert.Equal(kinds.Select(k => k.ToString()).Order().ToList(), kindsCounted.Order().ToList()); + } + } + private sealed class InjectedEvaluatorModel : IChatClient { public Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) => diff --git a/tests/AgentExperience.Release.Tests/PublicApi/AgentExperience.Core.verified.txt b/tests/AgentExperience.Release.Tests/PublicApi/AgentExperience.Core.verified.txt index 2fd2fe0..d5072bc 100644 --- a/tests/AgentExperience.Release.Tests/PublicApi/AgentExperience.Core.verified.txt +++ b/tests/AgentExperience.Release.Tests/PublicApi/AgentExperience.Core.verified.txt @@ -990,6 +990,7 @@ namespace AgentExperience.Core.ReflectiveExecution public int MaxReflectionCycles { get; init; } public long? MaxTokens { get; init; } public int? MaxToolCalls { get; init; } + public System.Action? OnStep { get; init; } public AgentExperience.Core.ReflectiveExecution.SideEffectPolicy SideEffects { get; init; } public System.TimeProvider TimeProvider { get; init; } public void Validate() { } @@ -1033,6 +1034,28 @@ namespace AgentExperience.Core.ReflectiveExecution public System.Func IsMet { get; init; } public string Name { get; init; } } + public sealed class ReflectionStep : System.IEquatable + { + public ReflectionStep(AgentExperience.Core.ReflectiveExecution.ReflectionStepKind Kind, int AttemptNumber, bool? TrustedCheckPassed = default, AgentExperience.Core.ReflectiveExecution.ReflectionAssessment? Assessment = default, AgentExperience.Core.ReflectiveExecution.ReflectionStopReason? StopReason = default) { } + public AgentExperience.Core.ReflectiveExecution.ReflectionAssessment? Assessment { get; init; } + public int AttemptNumber { get; init; } + public AgentExperience.Core.ReflectiveExecution.ReflectionStepKind Kind { get; init; } + public AgentExperience.Core.ReflectiveExecution.ReflectionStopReason? StopReason { get; init; } + public bool? TrustedCheckPassed { get; init; } + } + public enum ReflectionStepKind + { + ExecutionStarted = 0, + AttemptCompleted = 1, + VerificationReceived = 2, + ReflectionRequested = 3, + EvaluationCompleted = 4, + ReplanningPerformed = 5, + RetryAuthorized = 6, + RetryDenied = 7, + BudgetExhausted = 8, + ExecutionStopped = 9, + } public enum ReflectionStopReason { VerifiedSuccess = 0, diff --git a/tests/AgentExperience.Release.Tests/PublicApi/AgentExperience.MicrosoftAgentFramework.verified.txt b/tests/AgentExperience.Release.Tests/PublicApi/AgentExperience.MicrosoftAgentFramework.verified.txt index cdb30d0..4013f49 100644 --- a/tests/AgentExperience.Release.Tests/PublicApi/AgentExperience.MicrosoftAgentFramework.verified.txt +++ b/tests/AgentExperience.Release.Tests/PublicApi/AgentExperience.MicrosoftAgentFramework.verified.txt @@ -756,9 +756,16 @@ namespace AgentExperience.MicrosoftAgentFramework.ReflectiveExecution public AgentExperience.Core.ReflectiveExecution.IReflectionEvaluator? Evaluator { get; set; } public AgentExperience.Core.ReflectiveExecution.ReflectionExecutionOptions Limits { get; set; } public System.Action? OnEvent { get; set; } + public System.Action? OnStep { get; set; } public AgentExperience.Core.ReflectiveExecution.IReplanningStrategy? Replanning { get; set; } public System.Func>? TrustedCheck { get; set; } } + public sealed class ReflectiveStepEvent : System.IEquatable + { + public ReflectiveStepEvent(System.Guid? RunId, AgentExperience.Core.ReflectiveExecution.ReflectionStep Step) { } + public System.Guid? RunId { get; init; } + public AgentExperience.Core.ReflectiveExecution.ReflectionStep Step { get; init; } + } } namespace Microsoft.Extensions.DependencyInjection {