Carry a provider failure out of a container body

backend/src/CodeSpace.Core/Services/Workflows/Failures/CellFailureReader.cs

@@ -0,0 +1,51 @@
+using CodeSpace.Core.DependencyInjection;
+using CodeSpace.Core.Persistence.Db;
+using CodeSpace.Messages.Constants;
+using CodeSpace.Messages.Failures;
+using Microsoft.EntityFrameworkCore;
+
+namespace CodeSpace.Core.Services.Workflows.Failures;
+
+/// <summary>
+/// Reads the provider failure one cell recorded from the two places a node's failure is written — the cell's own <c>node.failed</c>
+/// record, then the agent run that worked in it — in the order, and with the parsers, <see cref="RunFailureReader"/> uses for the
+/// node that ended a run. So a failure inside a container reads exactly as it would have at the top level of a run. Never throws on a
+/// record it cannot read: an old row, or one written before failures were kept, simply has none.
+/// </summary>
+public sealed class CellFailureReader : ICellFailureReader, IScopedDependency
+{
+    private readonly CodeSpaceDbContext _db;
+
+    public CellFailureReader(CodeSpaceDbContext db) { _db = db; }
+
+    public async Task<PersistedFailure?> ReadAsync(Guid runId, Guid teamId, string nodeId, string iterationKey, CancellationToken cancellationToken)
+    {
+        var own = await ReadOwnRecordAsync(runId, nodeId, iterationKey, cancellationToken).ConfigureAwait(false);
+
+        return own ?? await ReadAgentRunAsync(runId, teamId, nodeId, iterationKey, cancellationToken).ConfigureAwait(false);
+    }
+
+    /// <summary>The newest <c>node.failed</c> record of the cell: a node that failed on one walk and again on the next says why it failed last.</summary>
+    private async Task<PersistedFailure?> ReadOwnRecordAsync(Guid runId, string nodeId, string iterationKey, CancellationToken cancellationToken)
+    {
+        var payload = await _db.WorkflowRunRecord.AsNoTracking()
+            .Where(r => r.RunId == runId && r.RecordType == WorkflowRunRecordTypes.NodeFailed && r.NodeId == nodeId && r.IterationKey == iterationKey)
+            .OrderByDescending(r => r.Sequence)
+            .Select(r => r.PayloadJson)
+            .FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
+
+        return PersistedFailureJson.Read(payload);
+    }
+
+    /// <summary>The last agent run of the cell only: an earlier attempt that met a disconnected credential and was respawned says nothing about why the cell ended.</summary>
+    private async Task<PersistedFailure?> ReadAgentRunAsync(Guid runId, Guid teamId, string nodeId, string iterationKey, CancellationToken cancellationToken)
+    {
+        var result = await _db.AgentRun.AsNoTracking()
+            .Where(a => a.WorkflowRunId == runId && a.TeamId == teamId && a.NodeId == nodeId && a.IterationKey == iterationKey && a.ResultJson != null)
+            .OrderByDescending(a => a.CreatedDate)
+            .Select(a => a.ResultJson)
+            .FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
+
+        return result is null ? null : RunFailureReader.ReadAgentResult(result);
+    }
+}

backend/src/CodeSpace.Core/Services/Workflows/Failures/ICellFailureReader.cs

@@ -0,0 +1,20 @@
+using CodeSpace.Messages.Failures;
+
+namespace CodeSpace.Core.Services.Workflows.Failures;
+
+/// <summary>
+/// Which provider failure one cell of a run — a node at an iteration key — is recorded as having failed on. The engine asks it when
+/// a container (a map, a loop) ends on the unhandled failure of a node in its body, so the container's own <c>node.failed</c> record
+/// can carry the failure of the cell that ended it: <see cref="IRunFailureReader"/> answers only for the node that ended the RUN,
+/// and for a run that a container ended that node is the container, which spawned no agent run and threw nothing itself. A rerun asks it
+/// too, for each failed cell it replays into its fork, so the fork's record of the cell carries what the original run's did.
+/// </summary>
+public interface ICellFailureReader
+{
+    /// <summary>
+    /// The failure the cell recorded, or null: it recorded none, or it did not fail on a provider connection. What the cell's own latest
+    /// <c>node.failed</c> record carries wins; failing that, the account of the agent run that last worked in that very cell — never another
+    /// iteration's, since a branch that failed on a sentence alone says nothing of its sibling's agent run.
+    /// </summary>
+    Task<PersistedFailure?> ReadAsync(Guid runId, Guid teamId, string nodeId, string iterationKey, CancellationToken cancellationToken);
+}

backend/src/CodeSpace.Core/Services/Workflows/Failures/RunFailureReader.cs

@@ -17,7 +17,9 @@ namespace CodeSpace.Core.Services.Workflows.Failures;
 /// <para>It answers for the node that ENDED the run, and for nothing earlier. A run walks past failures it handles (an error
 /// edge), so an older record that names a provider failure says only that the run once met one; the run's own error names the node
 /// that failed it (<c>Node 'x' failed.</c>), and that node's record decides — a failure carried there, else the agent run it spawned.
-/// A node that failed for another reason, or a run ended by the reconciler, answers null rather than the first failure found.</para>
+/// A node that failed for another reason, or a run ended by the reconciler, answers null rather than the first failure found.
+/// When the node that ended the run is a container (a map, a loop), its own record carries the failure of the cell that ended IT —
+/// the engine writes it there from <see cref="ICellFailureReader"/> — so this reads a container exactly as it reads any node.</para>
 /// </summary>
 public sealed class RunFailureReader : IRunFailureReader, IScopedDependency
 {

backend/src/CodeSpace.Core/Services/Workflows/Rerun/RerunCellSeeder.cs

@@ -2,9 +2,11 @@
 using CodeSpace.Core.DependencyInjection;
 using CodeSpace.Core.Persistence.Db;
 using CodeSpace.Core.Persistence.Entities;
+using CodeSpace.Core.Services.Workflows.Failures;
 using CodeSpace.Core.Services.Workflows.Lifecycle;
 using CodeSpace.Messages.Constants;
 using CodeSpace.Messages.Enums;
+using CodeSpace.Messages.Failures;
 using Microsoft.EntityFrameworkCore;
 
 namespace CodeSpace.Core.Services.Workflows.Rerun;
@@ -53,11 +55,13 @@ public sealed class RerunCellSeeder : IRerunCellSeeder, IScopedDependency
 
     private readonly CodeSpaceDbContext _db;
     private readonly IRunRecordLogger _recordLogger;
+    private readonly ICellFailureReader _cellFailureReader;
 
-    public RerunCellSeeder(CodeSpaceDbContext db, IRunRecordLogger recordLogger)
+    public RerunCellSeeder(CodeSpaceDbContext db, IRunRecordLogger recordLogger, ICellFailureReader cellFailureReader)
     {
         _db = db;
         _recordLogger = recordLogger;
+        _cellFailureReader = cellFailureReader;
     }
 
     public async Task SeedKeptCellsAsync(Guid originalRunId, Guid newRunId, IReadOnlySet<string> keptNodeIds, bool writeEmptySnapshotSentinel, CancellationToken cancellationToken)
@@ -129,8 +133,11 @@ private async Task ReEmitAsync(Guid newRunId, WorkflowRunNode cell, string itera
                 // A re-emitted Failure cell is one of: a top-level kept failure (error-edge-handled); a map sibling's
                 // continue-mode abandon row; or a map sibling's TERMINATE-mode failed row (both: a failed body node
                 // with no error edge) — all round-trip faithfully: RehydrateFromLedger / TrySettleBranch re-settle
-                // each from the node.failed record (the terminate arm replays it as a preserved failure, no re-run).
-                await _recordLogger.NodeFailedAsync(newRunId, cell.NodeId, iterationKey, cell.Error ?? "(failed)", TimeSpan.Zero, cancellationToken).ConfigureAwait(false);
+                // each from the node.failed record (the terminate arm replays it as a preserved failure, no re-run). The failure the
+                // original recorded for the cell rides along, so a map the fork fails on that preserved branch answers as the original's did.
+                var failure = await ReadOriginalFailureAsync(originalRunId, cell, cancellationToken).ConfigureAwait(false);
+
+                await _recordLogger.NodeFailedAsync(newRunId, cell.NodeId, iterationKey, cell.Error ?? "(failed)", TimeSpan.Zero, failure, cancellationToken).ConfigureAwait(false);
                 break;
 
             default:
@@ -142,6 +149,19 @@ private async Task ReEmitAsync(Guid newRunId, WorkflowRunNode cell, string itera
         }
     }
 
+    /// <summary>
+    /// What the original run says the cell failed on: its own <c>node.failed</c> record's failure, else the agent run that worked in it — read as the
+    /// engine reads any cell, so the account is the same wherever it is asked. The agent run belongs to the ORIGINAL run, so a fork that replays the
+    /// cell without carrying it would never find it: the cell's record on the fork is the only place the fork can hold it. Null when the cell recorded
+    /// none, so such a record is written exactly as it always was.
+    /// </summary>
+    private async Task<PersistedFailure?> ReadOriginalFailureAsync(Guid originalRunId, WorkflowRunNode cell, CancellationToken cancellationToken)
+    {
+        var teamId = await _db.WorkflowRun.AsNoTracking().Where(r => r.Id == originalRunId).Select(r => r.TeamId).SingleAsync(cancellationToken).ConfigureAwait(false);
+
+        return await _cellFailureReader.ReadAsync(originalRunId, teamId, cell.NodeId, cell.IterationKey, cancellationToken).ConfigureAwait(false);
+    }
+
     private static IReadOnlyDictionary<string, JsonElement> ParseOutputs(string json)
     {
         try