Trace a provider failure out of a sub-workflow child run

backend/src/CodeSpace.Core/Failures/ProviderFailureCapture.cs

@@ -32,6 +32,13 @@ public static class ProviderFailureCapture
         return null;
     }
 
+    /// <summary>
+    /// The failure a record already carries, kept only when it is one this capture would have written itself — a provider connection failure
+    /// (<see cref="FailureCodes.ProviderConnection"/>). For a node that read a failure back from where it happened (a sub-workflow's child run
+    /// recorded it) instead of catching an exception: the same filter, so what is carried onward is never wider than what is captured.
+    /// </summary>
+    public static PersistedFailure? OfRecord(PersistedFailure? recorded) => recorded is not null && FailureCodes.ProviderConnection.Contains(recorded.Code) ? recorded : null;
+
     private static PersistedFailure? FromAny(IEnumerable<Exception> candidates, Guid? repositoryId) =>
         candidates.Select(candidate => From(candidate, repositoryId)).FirstOrDefault(found => found is not null);
 

backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.Assemble.cs

@@ -55,6 +55,7 @@ private void StateFacts()
             RequestedOrigin = _ctx.Failure.Detail("requestedOrigin"),
             RedirectFix = RedirectFixFor(),
             ConnectedCredentials = _ctx.ConnectedCredentials,
+            Origin = _ctx.Origin,
             InstanceName = _ctx.Instance?.DisplayName,
             ProviderStatus = _ctx.Failure.Detail("providerStatus"),
             OAuthError = _ctx.Failure.Detail("oauthError"),
@@ -96,6 +97,7 @@ private void ChooseActions()
         Repositories = _ctx.Repositories.Select(repository => ToRepository(repository)).ToList(),
         ProviderInstance = _ctx.Instance is { } instance ? new RemedyProviderInstance { Id = instance.Id, DisplayName = instance.DisplayName, Provider = instance.Provider, BaseUrl = CredentialUrl.Masked(instance.BaseUrl)! } : null,
         RedirectsTo = _ctx.Failure.Detail("finalOrigin"),
+        Origin = _ctx.Origin,
         Actions = _ctx.Actions,
         AskPeople = await WhoToAskAsync(cancellationToken).ConfigureAwait(false)
     };

backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.Context.cs

@@ -22,6 +22,9 @@ private sealed class RemedyContext
         /// <summary>False for a refused delivery: there is no run to run again.</summary>
         public required bool OffersRerun { get; init; }
 
+        /// <summary>The sub-workflow run the failure happened in, when it was carried up to this run from one that is the team's; null otherwise.</summary>
+        public RemedyOrigin? Origin { get; init; }
+
         public List<RepositoryRow> Repositories { get; set; } = new();
         public Dictionary<Guid, CredentialRow> Credentials { get; set; } = new();
 
@@ -46,6 +49,9 @@ private sealed class RemedyContext
         public IReadOnlyList<RemedyAction> Actions { get; set; } = Array.Empty<RemedyAction>();
     }
 
+    /// <summary>The run a carried failure happened in — as the remedy names it, the repositories it was launched against and, when it was kept, when it recorded the failure.</summary>
+    private sealed record OriginRun(RemedyOrigin Origin, IReadOnlyList<Guid> ScopeRepositoryIds, DateTimeOffset? FailedAt);
+
     private sealed record RepositoryRow(Guid Id, string FullPath, RepositoryStatus Status, Guid? CredentialId, Guid ProviderInstanceId);
 
     /// <summary>A credential narrowed to what a remedy reads — deliberately NOT the entity, which carries the sealed token.</summary>

backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.Load.cs

@@ -2,6 +2,7 @@
 using CodeSpace.Core.Authorization;
 using CodeSpace.Core.Persistence.Entities;
 using CodeSpace.Messages.Authorization;
+using CodeSpace.Messages.Dtos.Credentials;
 using CodeSpace.Messages.Enums;
 using CodeSpace.Messages.Failures;
 using Microsoft.EntityFrameworkCore;
@@ -13,6 +14,23 @@ public sealed partial class ProviderRemedyService
     /// <summary>The most repositories one remedy names. A run launched against dozens still gets a card a person can read; the count it states is of what it names.</summary>
     private const int MaxRepositories = 10;
 
+    /// <summary>
+    /// The run a carried failure happened in, read under the asker's team: its workflow's name, the node that ended it and the repositories it was launched
+    /// against. A run that is not the team's, is gone, or is not an authored workflow's names nothing — a pointer in a record is never trusted to be the team's
+    /// own, so it is checked where it is read.
+    /// </summary>
+    private async Task<OriginRun?> ReadOriginAsync(FailureOrigin? origin, Guid teamId, CancellationToken cancellationToken)
+    {
+        if (origin is null) return null;
+
+        var run = await _db.WorkflowRun.AsNoTracking()
+            .Where(r => r.Id == origin.RunId && r.TeamId == teamId && r.WorkflowId != null)
+            .Select(r => new { WorkflowId = r.WorkflowId!.Value, Name = r.Workflow!.Name, r.ScopeRepositoryIds })
+            .FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
+
+        return run is null ? null : new OriginRun(new RemedyOrigin { RunId = origin.RunId, WorkflowId = run.WorkflowId, WorkflowName = run.Name, NodeId = origin.NodeId }, run.ScopeRepositoryIds, origin.At);
+    }
+
     /// <summary>The repositories the failure was about: the one it names, else the ones the run was launched against, else — when it names a credential and nothing else — whatever still leans on that credential.</summary>
     private async Task LoadRepositoriesAsync(CancellationToken cancellationToken)
     {

backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.cs

@@ -18,6 +18,11 @@ namespace CodeSpace.Core.Services.Credentials.Remedy;
 ///
 /// <para>Read-only. Nothing here changes a credential, a repository or a run: every action the remedy offers is a request the
 /// client makes of the endpoint that already does it, under that endpoint's own authorization.</para>
+///
+/// <para>A failure a <c>flow.subworkflow</c> node carried up from its child run is the child's remedy for the parent's page: the same code, credential,
+/// repositories, resolution (measured from when the child recorded the failure, not from when the parent was resumed) and actions, for the caller's standing in
+/// the PARENT's team (which is the child's, by construction), with the run it happened in named (<see cref="ProviderRemedy.Origin"/>) and Re-run still the run
+/// the page is for. A child that is not this team's run is never named, and changes nothing else about the remedy either.</para>
 /// </summary>
 public sealed partial class ProviderRemedyService : IProviderRemedyService, IScopedDependency
 {
@@ -42,7 +47,11 @@ public ProviderRemedyService(CodeSpaceDbContext db, ICurrentUser currentUser, Te
 
         if (recorded is null) return null;
 
-        _ctx = new RemedyContext { TeamId = teamId, Code = recorded.Failure.Code, Failure = recorded.Failure, At = recorded.At, ScopeRepositoryIds = recorded.ScopeRepositoryIds, OffersRerun = true };
+        // A failure carried up from a sub-workflow's child run is the child's remedy: traced through the repositories the CHILD was launched against, and judged
+        // "fixed since" from when the CHILD recorded it — as its own page does — because the parent's record is only as old as the last sibling it waited for.
+        var origin = await ReadOriginAsync(recorded.Failure.Origin, teamId, cancellationToken).ConfigureAwait(false);
+
+        _ctx = new RemedyContext { TeamId = teamId, Code = recorded.Failure.Code, Failure = recorded.Failure, At = origin?.FailedAt ?? recorded.At, ScopeRepositoryIds = origin?.ScopeRepositoryIds ?? recorded.ScopeRepositoryIds, Origin = origin?.Origin, OffersRerun = true };
 
         return await BuildAsync(cancellationToken).ConfigureAwait(false);
     }

backend/src/CodeSpace.Core/Services/Credentials/Remedy/RemedyCopy.cs

@@ -1,4 +1,5 @@
 using CodeSpace.Messages.Constants;
+using CodeSpace.Messages.Dtos.Credentials;
 using CodeSpace.Messages.Failures;
 
 namespace CodeSpace.Core.Services.Credentials.Remedy;
@@ -27,7 +28,14 @@ public static class RemedyCopy
         _ => "A provider connection needs attention"
     });
 
-    public static string Cause(string code, RemedyFacts facts) => Capitalise(facts.Resolved ? ResolvedCause(code, facts) : code switch
+    /// <summary>What happened, in sentences that name the credential and the repository — led, for a failure carried up from a sub-workflow's child run, by the sentence that says which run it happened in.</summary>
+    public static string Cause(string code, RemedyFacts facts) => facts.Origin is { } origin ? $"{Where(origin)} {CauseOf(code, facts)}" : CauseOf(code, facts);
+
+    /// <summary>Where a failure carried up from a child run happened: the sub-workflow's name, the run (the first eight characters of its id, as the run pages show it) and the node that ended it.</summary>
+    private static string Where(RemedyOrigin origin) =>
+        $"This failed in the sub-workflow “{origin.WorkflowName}” (run {origin.RunId.ToString()[..8]}){(origin.NodeId is { Length: > 0 } node ? $", at node '{node}'" : "")}.";
+
+    private static string CauseOf(string code, RemedyFacts facts) => Capitalise(facts.Resolved ? ResolvedCause(code, facts) : code switch
     {
         FailureCodes.CredentialDisconnected => Disconnected(facts),
         FailureCodes.CredentialNeedsReconnect => NeedsReconnect(facts),

backend/src/CodeSpace.Core/Services/Credentials/Remedy/RemedyFacts.cs

@@ -1,3 +1,4 @@
+using CodeSpace.Messages.Dtos.Credentials;
 using CodeSpace.Messages.Enums;
 
 namespace CodeSpace.Core.Services.Credentials.Remedy;
@@ -64,6 +65,9 @@ public sealed record RemedyFacts
     /// <summary>For a refused trigger delivery that could not read an author's standing: which of <see cref="CodeSpace.Messages.Constants.StandingLookupCauses"/> failed. Null for a row written before it was kept, which the sentence then words as either.</summary>
     public string? Lookup { get; init; }
 
+    /// <summary>The sub-workflow run the failure happened in, when it was carried up to the run the remedy is for; the sentence says where it happened. Null for a failure in the run itself.</summary>
+    public RemedyOrigin? Origin { get; init; }
+
     /// <summary>What the failure names works now.</summary>
     public bool Resolved { get; init; }
 

backend/src/CodeSpace.Core/Services/Workflows/Engine/SubworkflowService.cs

@@ -3,9 +3,11 @@
 using CodeSpace.Core.Persistence.Db;
 using CodeSpace.Core.Persistence.Entities;
 using CodeSpace.Core.Services.Workflows.Dispatch;
+using CodeSpace.Core.Services.Workflows.Failures;
 using CodeSpace.Core.Services.Workflows.RunSources;
 using CodeSpace.Messages.Constants;
 using CodeSpace.Messages.Enums;
+using CodeSpace.Messages.Failures;
 using CodeSpace.Messages.Workflows;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
@@ -37,7 +39,8 @@ public interface ISubworkflowService
     /// When a child run reaches a terminal state, resume the parent parked on it: find the parent's pending
     /// <c>Subworkflow</c> wait whose <c>Token</c> equals this child's id (which also distinguishes a sub-workflow child
     /// from a replay child — a replay's "parent" holds no such wait), then resolve it with the child's status + outputs +
-    /// error so the node can map them, via the wait-for-all completion path. Returns true iff THIS call resolved the wait
+    /// error — and, for a child that ended on a provider failure a person can repair, that failure and the run it happened in — so the
+    /// node can map them, via the wait-for-all completion path. Returns true iff THIS call resolved the wait
     /// (false when there is no such pending wait — not a sub-workflow child, or a concurrent resume / this child's own
     /// happy-path resume already won). Idempotent (the wait CAS) and best-effort (a failure is logged, never propagated —
     /// the child already completed cleanly). The ONE resume path shared by the engine's on-completion call AND the
@@ -55,14 +58,16 @@ public sealed class SubworkflowService : ISubworkflowService, IScopedDependency
     private readonly IRunStarter _runStarter;
     private readonly IWorkflowRunDispatcher _dispatcher;
     private readonly IWorkflowResumeService _resumeService;
+    private readonly IRunFailureReader _failures;
     private readonly ILogger<SubworkflowService> _logger;
 
-    public SubworkflowService(CodeSpaceDbContext db, IRunStarter runStarter, IWorkflowRunDispatcher dispatcher, IWorkflowResumeService resumeService, ILogger<SubworkflowService> logger)
+    public SubworkflowService(CodeSpaceDbContext db, IRunStarter runStarter, IWorkflowRunDispatcher dispatcher, IWorkflowResumeService resumeService, IRunFailureReader failures, ILogger<SubworkflowService> logger)
     {
         _db = db;
         _runStarter = runStarter;
         _dispatcher = dispatcher;
         _resumeService = resumeService;
+        _failures = failures;
         _logger = logger;
     }
 
@@ -123,12 +128,8 @@ public async Task<bool> ResumeParentIfChildTerminalAsync(WorkflowRun child, Work
 
             if (waitId is null) return false;   // not a sub-workflow child (e.g. a replay), or its wait already resolved
 
-            var payload = JsonSerializer.Serialize(new
-            {
-                status = status.ToString(),
-                outputs = JsonSafeParseObject(child.OutputsJson),
-                error,
-            });
+            var failure = status == WorkflowRunStatus.Failure ? await ReadChildFailureAsync(child, cancellationToken).ConfigureAwait(false) : null;
+            var payload = BuildResumePayload(child, status, error, failure);
 
             // Resolve ONLY this child's own wait — never the sibling waits a parallel fan-out of children holds — so each
             // parent subworkflow node resumes with its own child's outputs. The wait-COMPLETION path's CAS makes a
@@ -144,6 +145,39 @@ public async Task<bool> ResumeParentIfChildTerminalAsync(WorkflowRun child, Work
         }
     }
 
+    /// <summary>
+    /// The provider failure the child's run ended on, read by the reader the child's own run page asks (so the parent can never be told a different
+    /// failure than the child shows), with the run it happened in. A failure the child itself carried up from a child of its own keeps the origin it
+    /// already has — run, node and the time that run recorded it — so one that passed through nested sub-workflows names the deepest run that met it, and
+    /// is judged from when it did. Null when the child ended on anything else, and
+    /// when the record cannot be read: resuming the parent never waits on it, and a parent resumed without it fails with the sentence it always had.
+    /// </summary>
+    private async Task<PersistedFailure?> ReadChildFailureAsync(WorkflowRun child, CancellationToken cancellationToken)
+    {
+        try
+        {
+            var recorded = await _failures.ReadAsync(child.Id, child.TeamId, cancellationToken).ConfigureAwait(false);
+
+            return recorded is null ? null : recorded.Failure with { Origin = recorded.Failure.Origin ?? new FailureOrigin { RunId = child.Id, NodeId = recorded.NodeId, At = recorded.At } };
+        }
+        catch (Exception ex) when (ex is not OperationCanceledException)
+        {
+            _logger.LogWarning(ex, "Could not read the failure sub-workflow child {ChildRunId} ended on; its parent resumes with the sentence alone", child.Id);
+
+            return null;
+        }
+    }
+
+    /// <summary>The resume payload the sub-workflow node maps: status, outputs and error as it always carried, plus the child's failure under the key a ledger record keeps one under — absent unless there is one, so a payload without a failure is byte-identical to what it was.</summary>
+    private static string BuildResumePayload(WorkflowRun child, WorkflowRunStatus status, string? error, PersistedFailure? failure)
+    {
+        var payload = new Dictionary<string, object?> { ["status"] = status.ToString(), ["outputs"] = JsonSafeParseObject(child.OutputsJson), ["error"] = error };
+
+        if (failure is not null) payload[PersistedFailureJson.PayloadKey] = PersistedFailureJson.ToElement(failure);
+
+        return JsonSerializer.Serialize(payload);
+    }
+
     /// <summary>Parse a run's OutputsJson into a JsonElement object for the resume payload; an empty object on any trouble.</summary>
     private static JsonElement JsonSafeParseObject(string? json)
     {

backend/src/CodeSpace.Core/Services/Workflows/Engine/WorkflowEngine.cs

@@ -3423,14 +3423,14 @@ private async Task<NodeRunOutcome> ExecuteNodeAsync(WorkflowRun run, NodeDefinit
                 }
                 catch (SubworkflowStartException ex)
                 {
-                    return await FinalizeFailureAsync(exec, ex.Message, thrown: null, cancellationToken).ConfigureAwait(false);
+                    return await FinalizeFailureAsync(exec, ex.Message, thrown: null, carried: null, cancellationToken).ConfigureAwait(false);
                 }
                 catch (AgentRunAdmissionException ex)
                 {
                     // The fail-closed admission cap refused this agent run at staging — a clean node failure (like a
                     // failed sub-workflow start), so the agent.run node routes to its error edge and, inside a map
                     // branch, the map's continue-on-error / terminate policy applies. NOT a crash, NOT a leaked wait.
-                    return await FinalizeFailureAsync(exec, ex.Message, thrown: null, cancellationToken).ConfigureAwait(false);
+                    return await FinalizeFailureAsync(exec, ex.Message, thrown: null, carried: null, cancellationToken).ConfigureAwait(false);
                 }
             }
 
@@ -3450,7 +3450,7 @@ private async Task<NodeRunOutcome> ExecuteNodeAsync(WorkflowRun run, NodeDefinit
             retryable = retryable && (result?.Retryable ?? true);
 
             if (!retryable || attempt >= plan.MaxAttempts)
-                return await FinalizeFailureAsync(exec, lastError, thrown ?? result?.Cause, cancellationToken).ConfigureAwait(false);
+                return await FinalizeFailureAsync(exec, lastError, thrown ?? result?.Cause, result?.CarriedFailure, cancellationToken).ConfigureAwait(false);
 
             await LogRetryAndWaitAsync(exec, attempt, plan, lastError, attemptStartedAt, retryAfter, cancellationToken).ConfigureAwait(false);
 
@@ -3468,7 +3468,7 @@ private async Task<NodeRunOutcome> ExecuteNodeAsync(WorkflowRun run, NodeDefinit
         }
 
         // Defensive: the loop always returns on its final attempt.
-        return await FinalizeFailureAsync(exec, lastError, thrown: null, cancellationToken).ConfigureAwait(false);
+        return await FinalizeFailureAsync(exec, lastError, thrown: null, carried: null, cancellationToken).ConfigureAwait(false);
     }
 
     /// <summary>
@@ -3622,9 +3622,10 @@ private async Task<NodeRunOutcome> FinalizeNonFailureAsync(NodeExecution exec, N
     /// once the walker merges it. The scope write happens in <see cref="MergeNodeOutcome"/>; it's
     /// harmless when the node has no error edge — the run fails and scope is dropped.
     /// </summary>
-    private async Task<NodeRunOutcome> FinalizeFailureAsync(NodeExecution exec, string error, Exception? thrown, CancellationToken cancellationToken)
+    private async Task<NodeRunOutcome> FinalizeFailureAsync(NodeExecution exec, string error, Exception? thrown, PersistedFailure? carried, CancellationToken cancellationToken)
     {
-        var failure = ProviderFailureCapture.From(thrown, RepositoryActedOn(exec));
+        // What an exception declares wins; failing that, the failure the node itself read back from where it happened (a sub-workflow's child run).
+        var failure = ProviderFailureCapture.From(thrown, RepositoryActedOn(exec)) ?? carried;
 
         await _recordLogger.NodeFailedAsync(exec.Run.Id, exec.Node.Id, exec.IterationKey, RedactForPersistence(exec.Scope, error), DateTimeOffset.UtcNow - exec.StartedAt, failure, cancellationToken).ConfigureAwait(false);
         return new NodeRunOutcome(NodeStatus.Failure, BuildErrorOutput(error, exec.Node.Id), null, null);