Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions AgentExperience.NET.sln
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AgentExperience.Cli.Tests",
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AgentExperience.Sample.Reflective", "samples\AgentExperience.Sample.Reflective\AgentExperience.Sample.Reflective.csproj", "{90ECF1CA-7772-4A1C-A5AD-33CBEC2B9984}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AgentExperience.ReflectionEval", "experiments\AgentExperience.ReflectionEval\AgentExperience.ReflectionEval.csproj", "{666F85C3-8232-4580-A2AD-109DDE355E07}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -485,6 +487,18 @@ Global
{90ECF1CA-7772-4A1C-A5AD-33CBEC2B9984}.Release|x64.Build.0 = Release|Any CPU
{90ECF1CA-7772-4A1C-A5AD-33CBEC2B9984}.Release|x86.ActiveCfg = Release|Any CPU
{90ECF1CA-7772-4A1C-A5AD-33CBEC2B9984}.Release|x86.Build.0 = Release|Any CPU
{666F85C3-8232-4580-A2AD-109DDE355E07}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{666F85C3-8232-4580-A2AD-109DDE355E07}.Debug|Any CPU.Build.0 = Debug|Any CPU
{666F85C3-8232-4580-A2AD-109DDE355E07}.Debug|x64.ActiveCfg = Debug|Any CPU
{666F85C3-8232-4580-A2AD-109DDE355E07}.Debug|x64.Build.0 = Debug|Any CPU
{666F85C3-8232-4580-A2AD-109DDE355E07}.Debug|x86.ActiveCfg = Debug|Any CPU
{666F85C3-8232-4580-A2AD-109DDE355E07}.Debug|x86.Build.0 = Debug|Any CPU
{666F85C3-8232-4580-A2AD-109DDE355E07}.Release|Any CPU.ActiveCfg = Release|Any CPU
{666F85C3-8232-4580-A2AD-109DDE355E07}.Release|Any CPU.Build.0 = Release|Any CPU
{666F85C3-8232-4580-A2AD-109DDE355E07}.Release|x64.ActiveCfg = Release|Any CPU
{666F85C3-8232-4580-A2AD-109DDE355E07}.Release|x64.Build.0 = Release|Any CPU
{666F85C3-8232-4580-A2AD-109DDE355E07}.Release|x86.ActiveCfg = Release|Any CPU
{666F85C3-8232-4580-A2AD-109DDE355E07}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -523,5 +537,6 @@ Global
{71300AC9-934C-4C03-B86E-A90CE6B698C6} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
{B737918B-C0E0-4941-A968-F77BE825EEA5} = {0AB3BF05-4346-4AA6-1389-037BE0695223}
{90ECF1CA-7772-4A1C-A5AD-33CBEC2B9984} = {5A1F3C7D-9B24-4E86-A0D1-7C3E5B9F2A48}
{666F85C3-8232-4580-A2AD-109DDE355E07} = {8E1D1CCC-9289-C849-48CA-D7E099D02B6F}
EndGlobalSection
EndGlobal
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,12 @@ AgentExperience.NET is a **preview**. It is not production ready, and public API

### Added

- **Reflective execution: step events and a measuring harness.** `OnStep` reports each step of the loop (attempt, verification,
reflection, evaluation, replanning, retry authorized or denied, budget exhausted, stopped) as counts and enums, never text, and a
new `agentexperience.reflection.steps` counter counts them by kind. `experiments/AgentExperience.ReflectionEval` runs the four
arms (baseline, reflection, memory, both) and reports verified success, attempts, tool calls and latency. It uses a scripted
model, so it shows mechanics only. Additive, experimental. See [the guide](docs/guide/reflective-execution.md#steps).

- **`aexp init [--print] [--force] [--verify <json array>]`** writes a commented `.aexp/config.json` from one command,
run in the project root, with no agent named: the verifier detected when one kind of project is at the root
(`dotnet test`, `npm test`, `pytest`, `go test ./...`; and now `make test`, for a `Makefile` with a `test:` target,
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ An optional loop that evaluates a failed attempt, proposes a correction and retr
and only when you authorize it. It is off by default and never overrides your own check: a model's "it worked" is advice, not
verification. All attempts are one run, stored as one lesson. See
[Reflective execution](docs/guide/reflective-execution.md) and the
[sample](samples/AgentExperience.Sample.Reflective/README.md). No benefit is claimed; measure it with the holdout.
[sample](samples/AgentExperience.Sample.Reflective/README.md). No benefit is claimed; measure it with the holdout. A [four-arm harness](experiments/AgentExperience.ReflectionEval/README.md) shows the mechanics against a scripted model.

## Seeing what it learned

Expand Down
2 changes: 1 addition & 1 deletion docs/architecture/reflective-execution-design.md
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,6 @@ has a result.
- **Closing sooner needed one internal addition**, `OpenRunRegistry.CloseRunAsync`, which completes the run with the **last
attempt's** status. Reusing the duration bound's close would have recorded the run as `Cancelled`, a wrong outcome.
- **Telemetry** is four counters on their own experimental meter, outside `docs/telemetry-names.json` because that file is
shared with the Python and Go ports, which do not have this feature. `OnEvent` also reports each step.
shared with the Python and Go ports, which do not have this feature. `OnEvent` reports each execution and `OnStep` each step (nine kinds, plus the stop; no text).
- **Tokens** are counted from the provider's reported usage only; a provider that reports none leaves `MaxTokens` unenforced.
- **Python, Go, CLI and MCP are untouched**, as intended.
95 changes: 95 additions & 0 deletions docs/architecture/reflective-execution-report.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
# Reflective execution: engineering report

Status: experimental, off by default. Shipped in `0.1.0-preview.19` (step events and the harness: the release after it). This is the
report the implementation prompt asked for. It states what is built, what was run and what was not.

## Architecture implemented

The loop wraps the built agent from outside (`UseReflectiveExecution`, called before `UseAgentExperience`). Every attempt is a captured
invocation of **one** `ExperienceRun` (`ContinuesRunId`, `ShouldCompleteRun`), finalized once through the host's own `Verify`.

- **Core** (`AgentExperience.Core.ReflectiveExecution`, no MAF dependency): `ReflectionExecutionCoordinator` (the loop),
`IReflectionEvaluator`, `IReplanningStrategy`, `IRetryAuthorizer` (default denies), `ReflectionExecutionOptions` (disabled by default),
`ReflectionStep` (structured, text-free steps), `ReflectionStopReason`, evaluation validation.
- **MAF** (`AgentExperience.MicrosoftAgentFramework.ReflectiveExecution`): `ReflectiveExecutionAgent` (the wrapper),
`AddReflectiveExecution` / `UseReflectiveExecution`, `DeterministicReflectionEvaluator`, `ChatClientReflectionEvaluator`,
counters on their own meter.
- **Strategies:** failure-driven and deterministic (rules), requirement-driven, model-assisted (`IChatClient`), composite; a custom
`IReflectionEvaluator` or `IReplanningStrategy` replaces any of them.
- **Not added:** `IReflectionStrategy`, `IReflectionExecutionCoordinator` and `ReflectionDecision` (the prompt's proposed names): composition of
an evaluator and a replanning strategy covered the need without a larger hierarchy. `RequireVerifiedSuccess` is not an option because
only the host's check can end a loop as verified, always.

Design and decisions: [reflective-execution-design.md](reflective-execution-design.md). Usage and limits: [the guide](../guide/reflective-execution.md).

## Files changed

Added: `src/AgentExperience.Core/ReflectiveExecution/*`, `src/AgentExperience.MicrosoftAgentFramework/ReflectiveExecution/*`,
`samples/AgentExperience.Sample.Reflective/`, `experiments/AgentExperience.ReflectionEval/`, the guide, the design and this report,
tests under `tests/*/ReflectiveExecution/` and `tests/AgentExperience.Sample.QuickStart.Tests/Reflective/`.
Modified (additive): `OpenRunRegistry` (an internal `CloseRunAsync` that closes with the last attempt's real status),
`AgentExperienceRegistration` and the builder extension (a compose hook), the public API baselines, README, guide index,
known-limits pointer, telemetry note, CHANGELOG.

## Public API

New, all experimental: `AddReflectiveExecution`, `UseReflectiveExecution`, `ReflectiveExecutionOptions` (`Limits`, `Evaluator`, `Replanning`,
`Authorizer`, `TrustedCheck`, `OnEvent`, `OnStep`), the Core types above. No existing signature, default, stored format or migration changed;
the API baselines grew and lost nothing.

## Verification

Commands run on the final tree, with the real results:

| Command | Result |
| --- | --- |
| `dotnet build` | 0 errors |
| Abstractions / Core / MAF / InMemory / SQLite / CLI / ReuseBaseline tests | 117 / 1415 / 1478 / 292 / 247 / 102 / 170 passed |
| QuickStart (carries the sample's tests) / LiveReuse / CiTriage | 16 / 208 / 75 passed |
| CompatibilityProof / EndToEnd / RetrievalQuality / Contract (non-PostgreSQL) | 15 / 29 / 24 / 72 passed |
| `AgentExperience.Release.Tests` | 198 of 200 passed; the 2 failures (`PackageValidationBaselineTests`, `CompatibilityPinAgreementTests`) are caused by stale `.claude/worktrees` copies on the author's machine and fail on a clean `main` too |
| PostgreSQL-backed suites | not run locally; run by CI on every pull request, and they passed on the pull requests that carried this work |

The prompt's 17 cases map to tests as follows. Unauthorized tool actions: the loop adds no tools, and a test asserts the agent's tools and task
are unchanged; **blocking** a tool call is the existing approval boundary's job, not this library's.

| # | Case | Where |
| --- | --- | --- |
| 1-2 | success without reflection; failure corrected | `ReflectionExecutionCoordinatorTests`, `ReflectiveExecutionTests` |
| 3 | invalid correction | evaluation validator tests; prompt-injected evaluator test |
| 4 | retry exhaustion | coordinator tests; `ReflectionStepTests` |
| 5-6 | cancellation; provider failure or timeout | coordinator tests |
| 7 | verifier rejects a model's claim | coordinator and MAF tests |
| 8 | unauthorized tool actions | tools unchanged (see above) |
| 9 | tenant isolation | MAF tests |
| 10-11 | multi-attempt persistence; unverified is not promoted | MAF tests |
| 12 | lesson exposure and provenance | MAF tests (second task is handed the lesson) |
| 13 | repeated strategy | coordinator tests |
| 14 | interaction with failure recall | MAF tests |
| 15 | disabled is a pass-through | coordinator and MAF tests |
| 16 | non-idempotent operations | side-effect policy tests |
| 17 | sanitization and prompt injection | validator, strategies and MAF tests |

## Security

The loop cannot change identity or scope, add or approve tools, run a command, skip `Verify`, or write a record. A passing `TrustedCheck` is
the only way to stop as `VerifiedSuccess`; an evaluator's `Satisfied` is `EvaluatorSatisfied` (not verified) or `NotRetryable` against a failing
check. Retries need `IRetryAuthorizer` (default denies) and a side-effect policy (default never retries semantically). Evaluator and
replanning output is validated, bounded and screened; the correction note reaches the model as labelled, untrusted reference text.
Steps and counters carry counts and enums only.

## Limitations

- **No benefit demonstrated.** The harness and the sample use a scripted model; they show mechanics, not improvement. A real-model,
pre-registered run is described in the harness README and has not been done.
- The evaluator sees a bounded summary of a **finished** attempt, not plans or intermediate results. Replanning is a note the model may or may not follow.
- The correction note is model-derived text in the conversation; screening is not a boundary (KL-18).
- All attempts share one session. Streaming is not retried. Tokens are counted only when the provider reports them.
- `TrustedCheck` is the host's statement and is separate from `Verify`; if they disagree the run can stop "verified" and be stored as failed.
- Historical-experience reuse is not a step of its own; use the existing injection telemetry and the run id.
- Python, Go, the CLI and MCP are unchanged.

## Further improvements

Essential before any claim of benefit: a pre-registered real-model comparison. Optional: plan-level critique before acting, streaming support,
an evaluator that can request authorized read-only diagnostics, and a Python/Go port if there is demand.
34 changes: 31 additions & 3 deletions docs/guide/reflective-execution.md
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ An evaluator saying "fine" against a failing trusted check ends as `NotRetryable
| `Authorizer` | deny all | Called before every retry |
| `TrustedCheck` | none | Your check, after each attempt: `true` passed, `false` failed, `null` no verdict |
| `OnEvent` | none | One event per execution: counts and enums, no prompt, argument or model text |
| `OnStep` | none | One event per step of the loop, with the run ID once known: see [Steps](#steps) |
| `Replanning` | evaluator's proposal | Builds the correction text from an evaluation |

A bound that capture would enforce first (attempts, duration) is refused when the agent is built, with a message naming it.
Expand Down Expand Up @@ -206,19 +207,42 @@ competing recovery path.

## Telemetry

Four counters on the meter `AgentExperience.MicrosoftAgentFramework.ReflectiveExecution`, one tag, enums only:
Five counters on the meter `AgentExperience.MicrosoftAgentFramework.ReflectiveExecution`, one tag each, enums only:

| Counter | Counts | Evidence? |
| --- | --- | --- |
| `agentexperience.reflection.executions` (tag `agentexperience.reflection.stop_reason`) | executions, by why they stopped | n/a |
| `agentexperience.reflection.retries` | executions in which a retry occurred | no |
| `agentexperience.reflection.retries.improved` | executions in which a retry did better than the attempt before | no |
| `agentexperience.reflection.retries.verified` | executions in which a retry ended with your own check passing | **the only one** |
| `agentexperience.reflection.steps` (tag `agentexperience.reflection.step`) | steps of the loop, by kind | n/a |

No prompt, tool argument, error text, model text or identity is ever a tag. `OnEvent` carries the same facts with the run ID.
The meter is deliberately not in `docs/telemetry-names.json`: the Python and Go ports have no reflective execution, so no
port has to match these names.

### Steps

`OnStep` (on `ReflectiveExecutionOptions`, or `OnStep` on `Limits` in Core) reports each step as it happens. A step is a kind,
an attempt number and, where it applies, a trusted-check verdict, an advisory assessment or a stop reason. It has no text field,
and a test keeps it that way, so it is safe to log. An observer that throws is ignored: watching the loop never changes it.

| Kind | When |
| --- | --- |
| `ExecutionStarted` | the loop begins |
| `AttemptCompleted` | an attempt ends, whatever its result |
| `VerificationReceived` | your trusted check answered (`TrustedCheckPassed` is `null` when there is none) |
| `ReflectionRequested` | the evaluator is about to be asked (never for a first attempt that passed, or for a transient technical retry) |
| `EvaluationCompleted` | a valid evaluation came back (`Assessment` is advisory) |
| `ReplanningPerformed` | a correction was built and accepted |
| `RetryAuthorized` / `RetryDenied` | your `IRetryAuthorizer` answered |
| `BudgetExhausted` | attempts, cycles, duration, tool calls or tokens ended the loop (`StopReason` says which) |
| `ExecutionStopped` | always last; `StopReason` says why |

**Historical experience reused** has no step of its own, on purpose. Injection and recall already report it through the
library's existing injection telemetry and `OnContextInjected`; a second report here would be a second source of truth. Use the
run ID on the steps to join them.

## Public API compatibility

Purely additive: new types in `AgentExperience.Core.ReflectiveExecution` and `AgentExperience.MicrosoftAgentFramework.ReflectiveExecution`,
Expand All @@ -238,8 +262,12 @@ change in a later preview. With reflective execution unregistered or disabled, b
- **Tokens are counted only when the provider reports usage.** A provider that does not makes `MaxTokens` inert.
- **The meter is separate and experimental.** See [Telemetry](#telemetry); its names are not in the cross-language names file.
- **Only `RunAsync`.** Streaming is not retried.
- **No benefit is demonstrated.** The sample's four-arm comparison uses a scripted model that obeys what it is handed. It
proves the mechanics, not an improvement.
- **No benefit is demonstrated.** The four-arm comparison (`experiments/AgentExperience.ReflectionEval`) uses a scripted model
that obeys what it is handed. It proves the mechanics, not an improvement, and it reports no tokens or cost.
- **Evaluation is of a finished attempt.** The evaluator sees a bounded summary (error classes, failed tool names), not the
plan or intermediate results, and never raw tool arguments or output. A host that wants to critique a plan before acting
needs its own step.
- **Replanning is advice.** The correction is a note the model reads; whether it changes strategy is up to the model.
- **Python, Go, the CLI and the MCP server are unchanged.**

See also the [design assessment](../architecture/reflective-execution-design.md).
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<Project Sdk="Microsoft.NET.Sdk">

<!-- The four-arm reflective execution harness: baseline, reflection only, memory only, both, on the same tasks, printing
externally verified success, attempts, tool calls and latency. With the scripted model it shows MECHANICS only (see
README.md); a real-model run is separate and pre-registered. -->
<PropertyGroup>
<OutputType>Exe</OutputType>
<RootNamespace>AgentExperience.ReflectionEval</RootNamespace>
<IsPackable>false</IsPackable>
<NoWarn>$(NoWarn);CS1591</NoWarn>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="[10.0.12]" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\..\samples\AgentExperience.Sample.Reflective\AgentExperience.Sample.Reflective.csproj" />
</ItemGroup>

</Project>
21 changes: 21 additions & 0 deletions experiments/AgentExperience.ReflectionEval/Program.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
using AgentExperience.Sample.Reflective;

// Runs the four arms on the same deterministic tasks and prints a Markdown table. Pass a path to also write it there.
var writer = new StringWriter();
var arms = await ReflectiveDemo.CompareArmsAsync(TextWriter.Null);

writer.WriteLine("| Arm | Verified success | Attempts | Tool calls | Model calls | Elapsed |");
writer.WriteLine("| --- | --- | --- | --- | --- | --- |");
foreach (var arm in arms)
{
writer.WriteLine($"| {arm.Arm} | {arm.Verified}/{arm.Tasks} | {arm.Attempts} | {arm.ToolCalls} | {arm.ModelCalls} | {arm.Elapsed.TotalMilliseconds:F0} ms |");
}

writer.WriteLine();
writer.WriteLine("Scripted model: it applies what it is handed. This shows the mechanics (a retry needs reflection, a stored lesson removes the need for one). It says nothing about whether reflection or memory helps a real model, and it reports no tokens or cost.");

Console.Write(writer);
if (args.Length > 0)
{
await File.WriteAllTextAsync(args[0], writer.ToString());
}
Loading
Loading