Add pressure-based durable history retention and metrics

dotnet/src/Microsoft.Agents.AI.DurableTask/AgentEntity.cs

@@ -33,7 +33,7 @@ internal class AgentEntity(IServiceProvider services, CancellationToken cancella
 
     protected override DurableAgentState InitializeState(TaskEntityOperation entityOperation)
     {
-        return this._options.EnableMailboxWrites &&
+        return this.MailboxWritesEnabled &&
             entityOperation.Name is nameof(Run) or nameof(RunAgentAsync)
             ? new DurableAgentState
             {
@@ -98,16 +98,33 @@ public async Task<AgentResponse> Run(RunRequest request)
             // without comparing request content, so callers must not reuse it for another request.
             // Surface failures before optional migration so failed delivery never writes state.
             AgentResponse committedResponse = existingOutcome.GetResponse(correlationId);
-            if (this._options.EnableMailboxWrites &&
+            bool legacyMailboxMigrationRequested =
+                this.MailboxWritesEnabled &&
+                this.State.SchemaVersion != DurableAgentState.RevisedSchemaVersion;
+            bool migrationAuthorized =
+                legacyMailboxMigrationRequested &&
+                this._options.AuthorizeLegacyMigration?.Invoke(this.State) == true;
+            if (this._options.HistoryRetentionMode == DurableAgentHistoryRetentionMode.Auto &&
                 this.State.SchemaVersion != DurableAgentState.RevisedSchemaVersion &&
-                this._options.AuthorizeLegacyMigration?.Invoke(this.State) == true &&
+                !migrationAuthorized)
+            {
+                throw new DurableAgentStateCorruptionException(
+                    "Automatic history retention requires schema 2 mailbox state. Legacy terminal transcript " +
+                    "entries must be converted from independently authoritative complete history before delivery.");
+            }
+
+            if (legacyMailboxMigrationRequested &&
+                migrationAuthorized &&
                 existingOutcome.Kind != DurableAgentRunOutcomeKind.CompletedResultUnavailable)
             {
                 // Legacy evidence is converted without constructing or invoking the agent.
                 DurableAgentState migrated = DurableAgentStateOutcomeResolver.PrepareRevisedWorkingState(
                     this.State, hasAuthoritativeLegacyHistory: true);
-                ValidateForCommit(migrated);
-                this.State = migrated;
+                this.ApplyRetentionAndCommit(
+                    migrated,
+                    sessionId,
+                    logger,
+                    deletionCheckExpiration: null);
             }
 
             return committedResponse;
@@ -120,12 +137,12 @@ public async Task<AgentResponse> Run(RunRequest request)
                 nameof(request));
         }
 
-        if (this._options.EnableMailboxWrites)
+        if (this.MailboxWritesEnabled)
         {
             DurableAgentStateContract.ValidateIdentifier(correlationId, "correlationId");
         }
 
-        if (!this._options.EnableMailboxWrites &&
+        if (!this.MailboxWritesEnabled &&
             this.State.SchemaVersion == DurableAgentState.RevisedSchemaVersion)
         {
             throw new InvalidOperationException("New mailbox requests require EnableMailboxWrites to be enabled.");
@@ -134,13 +151,22 @@ public async Task<AgentResponse> Run(RunRequest request)
         this._cancellationToken.ThrowIfCancellationRequested();
         // TaskEntity hydrates State with the backend-owned object. Mutate an independent copy so
         // an exception leaves the hydrated state unchanged.
-        bool migrateLegacy = this._options.EnableMailboxWrites &&
+        bool migrateLegacy = this.MailboxWritesEnabled &&
             this.State.SchemaVersion != DurableAgentState.RevisedSchemaVersion &&
             this._options.AuthorizeLegacyMigration?.Invoke(this.State) == true;
+        if (this._options.HistoryRetentionMode == DurableAgentHistoryRetentionMode.Auto &&
+            this.State.SchemaVersion != DurableAgentState.RevisedSchemaVersion &&
+            !migrateLegacy)
+        {
+            throw new DurableAgentStateCorruptionException(
+                "Automatic history retention requires schema 2 mailbox state. Legacy terminal transcript " +
+                "entries must be converted from independently authoritative complete history before execution.");
+        }
+
         DurableAgentState workingState = migrateLegacy
             ? DurableAgentStateOutcomeResolver.PrepareRevisedWorkingState(this.State, hasAuthoritativeLegacyHistory: true)
             : this.State.Clone();
-        if (this._options.EnableMailboxWrites &&
+        if (this.MailboxWritesEnabled &&
             workingState.SchemaVersion == DurableAgentState.RevisedSchemaVersion)
         {
             workingState.MailboxWritesAuthorized = true;
@@ -435,10 +461,13 @@ persistedHistoryBinding is not null ||
                     response.Usage?.TotalTokenCount);
             }
 
-            DateTime? deletionCheckExpiration = this.UpdateExpiration(workingState, sessionId, logger);
-            this._cancellationToken.ThrowIfCancellationRequested();
-            this.CommitWorkingState(workingState, sessionId, logger, deletionCheckExpiration);
-
+            DateTime? deletionCheckExpiration =
+                this.UpdateExpiration(workingState, sessionId, logger);
+            this.ApplyRetentionAndCommit(
+                workingState,
+                sessionId,
+                logger,
+                deletionCheckExpiration);
             return response;
         }
         catch (InvalidOperationException exception) when (
@@ -505,10 +534,36 @@ public void CheckAndDeleteIfExpired(AgentEntityDeletionCheck? scheduledCheck = n
             if (expirationTime.HasValue)
             {
                 logger.LogTTLExpirationTimeCleared(sessionId);
-                DurableAgentState workingState = this.State.Clone();
+                bool migrateLegacy =
+                    this._options.HistoryRetentionMode == DurableAgentHistoryRetentionMode.Auto &&
+                    this.State.SchemaVersion != DurableAgentState.RevisedSchemaVersion &&
+                    this._options.AuthorizeLegacyMigration?.Invoke(this.State) == true;
+                if (this._options.HistoryRetentionMode == DurableAgentHistoryRetentionMode.Auto &&
+                    this.State.SchemaVersion != DurableAgentState.RevisedSchemaVersion &&
+                    !migrateLegacy)
+                {
+                    throw new DurableAgentStateCorruptionException(
+                        "Automatic history retention requires schema 2 mailbox state. Legacy terminal transcript " +
+                        "entries must be converted from independently authoritative complete history before TTL mutation.");
+                }
+
+                DurableAgentState workingState = migrateLegacy
+                    ? DurableAgentStateOutcomeResolver.PrepareRevisedWorkingState(
+                        this.State,
+                        hasAuthoritativeLegacyHistory: true)
+                    : this.State.Clone();
+                if (this.MailboxWritesEnabled &&
+                    workingState.SchemaVersion == DurableAgentState.RevisedSchemaVersion)
+                {
+                    workingState.MailboxWritesAuthorized = true;
+                }
+
                 workingState.Data.ExpirationTimeUtc = null;
-                ValidateForCommit(workingState);
-                this.State = workingState;
+                this.ApplyRetentionAndCommit(
+                    workingState,
+                    sessionId,
+                    logger,
+                    deletionCheckExpiration: null);
             }
 
             return;
@@ -551,6 +606,10 @@ state.ExtensionData is null &&
             state.UnknownProperties is null;
     }
 
+    private bool MailboxWritesEnabled =>
+        this._options.EnableMailboxWrites ||
+        this._options.HistoryRetentionMode == DurableAgentHistoryRetentionMode.Auto;
+
     private static bool IsPostResponseServiceHistoryFailure(
         InvalidOperationException exception,
         ChatClientAgent? chatClientAgent)
@@ -716,13 +775,23 @@ ownership is DurableAgentHistoryOwnership.Entity or DurableAgentHistoryOwnership
                 : null;
     }
 
-    private void CommitWorkingState(
+    private void ApplyRetentionAndCommit(
         DurableAgentState workingState,
         AgentSessionId sessionId,
         ILogger logger,
         DateTime? deletionCheckExpiration)
     {
+        _ = DurableAgentStateRetention.Enforce(
+            workingState,
+            this._options.HistoryRetentionMode,
+            this._options.MaxStateBytes,
+            this._timeProvider.GetUtcNow(),
+            logger,
+            sessionId);
+
+        this._cancellationToken.ThrowIfCancellationRequested();
         ValidateForCommit(workingState);
+
         if (deletionCheckExpiration.HasValue)
         {
             // Pass the working-copy value explicitly: this.State still refers to the original state

dotnet/src/Microsoft.Agents.AI.DurableTask/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [Unreleased]
 
+- Added opt-in pressure-based durable transcript retention and low-cardinality operational metrics while protecting schema 2 mailbox and execution-control state.
 - Hardened durable-agent mailbox delivery, duplicate correlation handling, working-state rollback, and stale-safe TTL deletion scheduling; preserved historical message boundaries, opaque state profiles, and committed failure metadata; isolated untrusted response text from workflow controls ([#94](https://github.com/microsoft/agent-framework-durable-extension/pull/94))
 - Fail durable workflows with a `MaxSuperstepsExceededException` when they reach the configurable `MaxSupersteps` limit with work still queued, instead of returning a successful partial result ([#84](https://github.com/microsoft/agent-framework-durable-extension/pull/84))
 - Added passive .NET DTO, converter, validation, and source-generation support for the proposed durable agent state 2.0 contract ([tamirdresher/agent-framework-durable-extension#1](https://github.com/tamirdresher/agent-framework-durable-extension/pull/1))

dotnet/src/Microsoft.Agents.AI.DurableTask/DurableAgentHistoryRetentionMode.cs

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace Microsoft.Agents.AI.DurableTask;
+
+/// <summary>
+/// Controls how durable agent conversation state is retained.
+/// </summary>
+public enum DurableAgentHistoryRetentionMode
+{
+    /// <summary>
+    /// Never proactively removes conversation entries. Persistence can still fail when a backend or provider
+    /// state limit is reached.
+    /// </summary>
+    KeepAll,
+
+    /// <summary>
+    /// Removes the oldest eligible exchanges when serialized entity state reaches the configured high watermark.
+    /// </summary>
+    /// <remarks>
+    /// Only conversation transcript entries are eligible. Mailbox results, completion receipts, history
+    /// binding, provider continuation, TTL, and other execution-control state are protected. The newest
+    /// transcript exchange and system messages are never evicted; if protected state cannot fit below the
+    /// safe write threshold, the operation fails instead of persisting oversized state.
+    /// </remarks>
+    Auto,
+}