diff --git a/backend/src/CodeSpace.Core/Failures/ProviderFailureCapture.cs b/backend/src/CodeSpace.Core/Failures/ProviderFailureCapture.cs index 4ae4bf710..dbbac81f0 100644 --- a/backend/src/CodeSpace.Core/Failures/ProviderFailureCapture.cs +++ b/backend/src/CodeSpace.Core/Failures/ProviderFailureCapture.cs @@ -32,6 +32,13 @@ public static class ProviderFailureCapture return null; } + /// + /// The failure a record already carries, kept only when it is one this capture would have written itself — a provider connection failure + /// (). For a node that read a failure back from where it happened (a sub-workflow's child run + /// recorded it) instead of catching an exception: the same filter, so what is carried onward is never wider than what is captured. + /// + public static PersistedFailure? OfRecord(PersistedFailure? recorded) => recorded is not null && FailureCodes.ProviderConnection.Contains(recorded.Code) ? recorded : null; + private static PersistedFailure? FromAny(IEnumerable candidates, Guid? repositoryId) => candidates.Select(candidate => From(candidate, repositoryId)).FirstOrDefault(found => found is not null); diff --git a/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.Assemble.cs b/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.Assemble.cs index c8d28364c..df5d489b9 100644 --- a/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.Assemble.cs +++ b/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.Assemble.cs @@ -55,6 +55,7 @@ private void StateFacts() RequestedOrigin = _ctx.Failure.Detail("requestedOrigin"), RedirectFix = RedirectFixFor(), ConnectedCredentials = _ctx.ConnectedCredentials, + Origin = _ctx.Origin, InstanceName = _ctx.Instance?.DisplayName, ProviderStatus = _ctx.Failure.Detail("providerStatus"), OAuthError = _ctx.Failure.Detail("oauthError"), @@ -96,6 +97,7 @@ private void ChooseActions() Repositories = _ctx.Repositories.Select(repository => ToRepository(repository)).ToList(), ProviderInstance = _ctx.Instance is { } instance ? new RemedyProviderInstance { Id = instance.Id, DisplayName = instance.DisplayName, Provider = instance.Provider, BaseUrl = CredentialUrl.Masked(instance.BaseUrl)! } : null, RedirectsTo = _ctx.Failure.Detail("finalOrigin"), + Origin = _ctx.Origin, Actions = _ctx.Actions, AskPeople = await WhoToAskAsync(cancellationToken).ConfigureAwait(false) }; diff --git a/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.Context.cs b/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.Context.cs index 7e6d1a681..0573f7638 100644 --- a/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.Context.cs +++ b/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.Context.cs @@ -22,6 +22,9 @@ private sealed class RemedyContext /// False for a refused delivery: there is no run to run again. public required bool OffersRerun { get; init; } + /// The sub-workflow run the failure happened in, when it was carried up to this run from one that is the team's; null otherwise. + public RemedyOrigin? Origin { get; init; } + public List Repositories { get; set; } = new(); public Dictionary Credentials { get; set; } = new(); @@ -46,6 +49,9 @@ private sealed class RemedyContext public IReadOnlyList Actions { get; set; } = Array.Empty(); } + /// The run a carried failure happened in — as the remedy names it, the repositories it was launched against and, when it was kept, when it recorded the failure. + private sealed record OriginRun(RemedyOrigin Origin, IReadOnlyList ScopeRepositoryIds, DateTimeOffset? FailedAt); + private sealed record RepositoryRow(Guid Id, string FullPath, RepositoryStatus Status, Guid? CredentialId, Guid ProviderInstanceId); /// A credential narrowed to what a remedy reads — deliberately NOT the entity, which carries the sealed token. diff --git a/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.Load.cs b/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.Load.cs index 7ee3d3cb8..6d9ff0b43 100644 --- a/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.Load.cs +++ b/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.Load.cs @@ -2,6 +2,7 @@ using CodeSpace.Core.Authorization; using CodeSpace.Core.Persistence.Entities; using CodeSpace.Messages.Authorization; +using CodeSpace.Messages.Dtos.Credentials; using CodeSpace.Messages.Enums; using CodeSpace.Messages.Failures; using Microsoft.EntityFrameworkCore; @@ -13,6 +14,23 @@ public sealed partial class ProviderRemedyService /// The most repositories one remedy names. A run launched against dozens still gets a card a person can read; the count it states is of what it names. private const int MaxRepositories = 10; + /// + /// The run a carried failure happened in, read under the asker's team: its workflow's name, the node that ended it and the repositories it was launched + /// against. A run that is not the team's, is gone, or is not an authored workflow's names nothing — a pointer in a record is never trusted to be the team's + /// own, so it is checked where it is read. + /// + private async Task ReadOriginAsync(FailureOrigin? origin, Guid teamId, CancellationToken cancellationToken) + { + if (origin is null) return null; + + var run = await _db.WorkflowRun.AsNoTracking() + .Where(r => r.Id == origin.RunId && r.TeamId == teamId && r.WorkflowId != null) + .Select(r => new { WorkflowId = r.WorkflowId!.Value, Name = r.Workflow!.Name, r.ScopeRepositoryIds }) + .FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + + return run is null ? null : new OriginRun(new RemedyOrigin { RunId = origin.RunId, WorkflowId = run.WorkflowId, WorkflowName = run.Name, NodeId = origin.NodeId }, run.ScopeRepositoryIds, origin.At); + } + /// The repositories the failure was about: the one it names, else the ones the run was launched against, else — when it names a credential and nothing else — whatever still leans on that credential. private async Task LoadRepositoriesAsync(CancellationToken cancellationToken) { diff --git a/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.cs b/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.cs index 9cdd3787f..e87fd9dde 100644 --- a/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.cs +++ b/backend/src/CodeSpace.Core/Services/Credentials/Remedy/ProviderRemedyService.cs @@ -18,6 +18,11 @@ namespace CodeSpace.Core.Services.Credentials.Remedy; /// /// Read-only. Nothing here changes a credential, a repository or a run: every action the remedy offers is a request the /// client makes of the endpoint that already does it, under that endpoint's own authorization. +/// +/// A failure a flow.subworkflow node carried up from its child run is the child's remedy for the parent's page: the same code, credential, +/// repositories, resolution (measured from when the child recorded the failure, not from when the parent was resumed) and actions, for the caller's standing in +/// the PARENT's team (which is the child's, by construction), with the run it happened in named () and Re-run still the run +/// the page is for. A child that is not this team's run is never named, and changes nothing else about the remedy either. /// public sealed partial class ProviderRemedyService : IProviderRemedyService, IScopedDependency { @@ -42,7 +47,11 @@ public ProviderRemedyService(CodeSpaceDbContext db, ICurrentUser currentUser, Te if (recorded is null) return null; - _ctx = new RemedyContext { TeamId = teamId, Code = recorded.Failure.Code, Failure = recorded.Failure, At = recorded.At, ScopeRepositoryIds = recorded.ScopeRepositoryIds, OffersRerun = true }; + // A failure carried up from a sub-workflow's child run is the child's remedy: traced through the repositories the CHILD was launched against, and judged + // "fixed since" from when the CHILD recorded it — as its own page does — because the parent's record is only as old as the last sibling it waited for. + var origin = await ReadOriginAsync(recorded.Failure.Origin, teamId, cancellationToken).ConfigureAwait(false); + + _ctx = new RemedyContext { TeamId = teamId, Code = recorded.Failure.Code, Failure = recorded.Failure, At = origin?.FailedAt ?? recorded.At, ScopeRepositoryIds = origin?.ScopeRepositoryIds ?? recorded.ScopeRepositoryIds, Origin = origin?.Origin, OffersRerun = true }; return await BuildAsync(cancellationToken).ConfigureAwait(false); } diff --git a/backend/src/CodeSpace.Core/Services/Credentials/Remedy/RemedyCopy.cs b/backend/src/CodeSpace.Core/Services/Credentials/Remedy/RemedyCopy.cs index 7f2a430bc..f65191b10 100644 --- a/backend/src/CodeSpace.Core/Services/Credentials/Remedy/RemedyCopy.cs +++ b/backend/src/CodeSpace.Core/Services/Credentials/Remedy/RemedyCopy.cs @@ -1,4 +1,5 @@ using CodeSpace.Messages.Constants; +using CodeSpace.Messages.Dtos.Credentials; using CodeSpace.Messages.Failures; namespace CodeSpace.Core.Services.Credentials.Remedy; @@ -27,7 +28,14 @@ public static class RemedyCopy _ => "A provider connection needs attention" }); - public static string Cause(string code, RemedyFacts facts) => Capitalise(facts.Resolved ? ResolvedCause(code, facts) : code switch + /// What happened, in sentences that name the credential and the repository — led, for a failure carried up from a sub-workflow's child run, by the sentence that says which run it happened in. + public static string Cause(string code, RemedyFacts facts) => facts.Origin is { } origin ? $"{Where(origin)} {CauseOf(code, facts)}" : CauseOf(code, facts); + + /// Where a failure carried up from a child run happened: the sub-workflow's name, the run (the first eight characters of its id, as the run pages show it) and the node that ended it. + private static string Where(RemedyOrigin origin) => + $"This failed in the sub-workflow “{origin.WorkflowName}” (run {origin.RunId.ToString()[..8]}){(origin.NodeId is { Length: > 0 } node ? $", at node '{node}'" : "")}."; + + private static string CauseOf(string code, RemedyFacts facts) => Capitalise(facts.Resolved ? ResolvedCause(code, facts) : code switch { FailureCodes.CredentialDisconnected => Disconnected(facts), FailureCodes.CredentialNeedsReconnect => NeedsReconnect(facts), diff --git a/backend/src/CodeSpace.Core/Services/Credentials/Remedy/RemedyFacts.cs b/backend/src/CodeSpace.Core/Services/Credentials/Remedy/RemedyFacts.cs index b1fdebbe1..1e8c02712 100644 --- a/backend/src/CodeSpace.Core/Services/Credentials/Remedy/RemedyFacts.cs +++ b/backend/src/CodeSpace.Core/Services/Credentials/Remedy/RemedyFacts.cs @@ -1,3 +1,4 @@ +using CodeSpace.Messages.Dtos.Credentials; using CodeSpace.Messages.Enums; namespace CodeSpace.Core.Services.Credentials.Remedy; @@ -64,6 +65,9 @@ public sealed record RemedyFacts /// For a refused trigger delivery that could not read an author's standing: which of failed. Null for a row written before it was kept, which the sentence then words as either. public string? Lookup { get; init; } + /// The sub-workflow run the failure happened in, when it was carried up to the run the remedy is for; the sentence says where it happened. Null for a failure in the run itself. + public RemedyOrigin? Origin { get; init; } + /// What the failure names works now. public bool Resolved { get; init; } diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Engine/SubworkflowService.cs b/backend/src/CodeSpace.Core/Services/Workflows/Engine/SubworkflowService.cs index d47dcf6e0..2342a2709 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Engine/SubworkflowService.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Engine/SubworkflowService.cs @@ -3,9 +3,11 @@ using CodeSpace.Core.Persistence.Db; using CodeSpace.Core.Persistence.Entities; using CodeSpace.Core.Services.Workflows.Dispatch; +using CodeSpace.Core.Services.Workflows.Failures; using CodeSpace.Core.Services.Workflows.RunSources; using CodeSpace.Messages.Constants; using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Failures; using CodeSpace.Messages.Workflows; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; @@ -37,7 +39,8 @@ public interface ISubworkflowService /// When a child run reaches a terminal state, resume the parent parked on it: find the parent's pending /// Subworkflow wait whose Token equals this child's id (which also distinguishes a sub-workflow child /// from a replay child — a replay's "parent" holds no such wait), then resolve it with the child's status + outputs + - /// error so the node can map them, via the wait-for-all completion path. Returns true iff THIS call resolved the wait + /// error — and, for a child that ended on a provider failure a person can repair, that failure and the run it happened in — so the + /// node can map them, via the wait-for-all completion path. Returns true iff THIS call resolved the wait /// (false when there is no such pending wait — not a sub-workflow child, or a concurrent resume / this child's own /// happy-path resume already won). Idempotent (the wait CAS) and best-effort (a failure is logged, never propagated — /// the child already completed cleanly). The ONE resume path shared by the engine's on-completion call AND the @@ -55,14 +58,16 @@ public sealed class SubworkflowService : ISubworkflowService, IScopedDependency private readonly IRunStarter _runStarter; private readonly IWorkflowRunDispatcher _dispatcher; private readonly IWorkflowResumeService _resumeService; + private readonly IRunFailureReader _failures; private readonly ILogger _logger; - public SubworkflowService(CodeSpaceDbContext db, IRunStarter runStarter, IWorkflowRunDispatcher dispatcher, IWorkflowResumeService resumeService, ILogger logger) + public SubworkflowService(CodeSpaceDbContext db, IRunStarter runStarter, IWorkflowRunDispatcher dispatcher, IWorkflowResumeService resumeService, IRunFailureReader failures, ILogger logger) { _db = db; _runStarter = runStarter; _dispatcher = dispatcher; _resumeService = resumeService; + _failures = failures; _logger = logger; } @@ -123,12 +128,8 @@ public async Task ResumeParentIfChildTerminalAsync(WorkflowRun child, Work if (waitId is null) return false; // not a sub-workflow child (e.g. a replay), or its wait already resolved - var payload = JsonSerializer.Serialize(new - { - status = status.ToString(), - outputs = JsonSafeParseObject(child.OutputsJson), - error, - }); + var failure = status == WorkflowRunStatus.Failure ? await ReadChildFailureAsync(child, cancellationToken).ConfigureAwait(false) : null; + var payload = BuildResumePayload(child, status, error, failure); // Resolve ONLY this child's own wait — never the sibling waits a parallel fan-out of children holds — so each // parent subworkflow node resumes with its own child's outputs. The wait-COMPLETION path's CAS makes a @@ -144,6 +145,39 @@ public async Task ResumeParentIfChildTerminalAsync(WorkflowRun child, Work } } + /// + /// The provider failure the child's run ended on, read by the reader the child's own run page asks (so the parent can never be told a different + /// failure than the child shows), with the run it happened in. A failure the child itself carried up from a child of its own keeps the origin it + /// already has — run, node and the time that run recorded it — so one that passed through nested sub-workflows names the deepest run that met it, and + /// is judged from when it did. Null when the child ended on anything else, and + /// when the record cannot be read: resuming the parent never waits on it, and a parent resumed without it fails with the sentence it always had. + /// + private async Task ReadChildFailureAsync(WorkflowRun child, CancellationToken cancellationToken) + { + try + { + var recorded = await _failures.ReadAsync(child.Id, child.TeamId, cancellationToken).ConfigureAwait(false); + + return recorded is null ? null : recorded.Failure with { Origin = recorded.Failure.Origin ?? new FailureOrigin { RunId = child.Id, NodeId = recorded.NodeId, At = recorded.At } }; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Could not read the failure sub-workflow child {ChildRunId} ended on; its parent resumes with the sentence alone", child.Id); + + return null; + } + } + + /// The resume payload the sub-workflow node maps: status, outputs and error as it always carried, plus the child's failure under the key a ledger record keeps one under — absent unless there is one, so a payload without a failure is byte-identical to what it was. + private static string BuildResumePayload(WorkflowRun child, WorkflowRunStatus status, string? error, PersistedFailure? failure) + { + var payload = new Dictionary { ["status"] = status.ToString(), ["outputs"] = JsonSafeParseObject(child.OutputsJson), ["error"] = error }; + + if (failure is not null) payload[PersistedFailureJson.PayloadKey] = PersistedFailureJson.ToElement(failure); + + return JsonSerializer.Serialize(payload); + } + /// Parse a run's OutputsJson into a JsonElement object for the resume payload; an empty object on any trouble. private static JsonElement JsonSafeParseObject(string? json) { diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Engine/WorkflowEngine.cs b/backend/src/CodeSpace.Core/Services/Workflows/Engine/WorkflowEngine.cs index 64da768d4..f9815f9a3 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Engine/WorkflowEngine.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Engine/WorkflowEngine.cs @@ -3423,14 +3423,14 @@ private async Task ExecuteNodeAsync(WorkflowRun run, NodeDefinit } catch (SubworkflowStartException ex) { - return await FinalizeFailureAsync(exec, ex.Message, thrown: null, cancellationToken).ConfigureAwait(false); + return await FinalizeFailureAsync(exec, ex.Message, thrown: null, carried: null, cancellationToken).ConfigureAwait(false); } catch (AgentRunAdmissionException ex) { // The fail-closed admission cap refused this agent run at staging — a clean node failure (like a // failed sub-workflow start), so the agent.run node routes to its error edge and, inside a map // branch, the map's continue-on-error / terminate policy applies. NOT a crash, NOT a leaked wait. - return await FinalizeFailureAsync(exec, ex.Message, thrown: null, cancellationToken).ConfigureAwait(false); + return await FinalizeFailureAsync(exec, ex.Message, thrown: null, carried: null, cancellationToken).ConfigureAwait(false); } } @@ -3450,7 +3450,7 @@ private async Task ExecuteNodeAsync(WorkflowRun run, NodeDefinit retryable = retryable && (result?.Retryable ?? true); if (!retryable || attempt >= plan.MaxAttempts) - return await FinalizeFailureAsync(exec, lastError, thrown ?? result?.Cause, cancellationToken).ConfigureAwait(false); + return await FinalizeFailureAsync(exec, lastError, thrown ?? result?.Cause, result?.CarriedFailure, cancellationToken).ConfigureAwait(false); await LogRetryAndWaitAsync(exec, attempt, plan, lastError, attemptStartedAt, retryAfter, cancellationToken).ConfigureAwait(false); @@ -3468,7 +3468,7 @@ private async Task ExecuteNodeAsync(WorkflowRun run, NodeDefinit } // Defensive: the loop always returns on its final attempt. - return await FinalizeFailureAsync(exec, lastError, thrown: null, cancellationToken).ConfigureAwait(false); + return await FinalizeFailureAsync(exec, lastError, thrown: null, carried: null, cancellationToken).ConfigureAwait(false); } /// @@ -3622,9 +3622,10 @@ private async Task FinalizeNonFailureAsync(NodeExecution exec, N /// once the walker merges it. The scope write happens in ; it's /// harmless when the node has no error edge — the run fails and scope is dropped. /// - private async Task FinalizeFailureAsync(NodeExecution exec, string error, Exception? thrown, CancellationToken cancellationToken) + private async Task FinalizeFailureAsync(NodeExecution exec, string error, Exception? thrown, PersistedFailure? carried, CancellationToken cancellationToken) { - var failure = ProviderFailureCapture.From(thrown, RepositoryActedOn(exec)); + // What an exception declares wins; failing that, the failure the node itself read back from where it happened (a sub-workflow's child run). + var failure = ProviderFailureCapture.From(thrown, RepositoryActedOn(exec)) ?? carried; await _recordLogger.NodeFailedAsync(exec.Run.Id, exec.Node.Id, exec.IterationKey, RedactForPersistence(exec.Scope, error), DateTimeOffset.UtcNow - exec.StartedAt, failure, cancellationToken).ConfigureAwait(false); return new NodeRunOutcome(NodeStatus.Failure, BuildErrorOutput(error, exec.Node.Id), null, null); diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Failures/IRunFailureReader.cs b/backend/src/CodeSpace.Core/Services/Workflows/Failures/IRunFailureReader.cs index e44377cbd..304c2d188 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Failures/IRunFailureReader.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Failures/IRunFailureReader.cs @@ -11,7 +11,8 @@ public interface IRunFailureReader /// /// The failure the run ended on, or null: it recorded none, it is not the team's, or it did not fail on a provider connection. /// For a failed run the node's or run's own record wins and the last agent run's result is the fallback; for a run that did - /// not fail, only a push the remote refused counts — the run stayed whole, and its branch was not published. + /// not fail, only a push the remote refused counts — the run stayed whole, and its branch was not published. The result names the node that ended the + /// run (), which is how a parent says where in a child run the failure happened. /// Task ReadAsync(Guid runId, Guid teamId, CancellationToken cancellationToken); } diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Failures/RunFailureReader.cs b/backend/src/CodeSpace.Core/Services/Workflows/Failures/RunFailureReader.cs index e1130ecf4..8625e6d8a 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Failures/RunFailureReader.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Failures/RunFailureReader.cs @@ -19,7 +19,10 @@ namespace CodeSpace.Core.Services.Workflows.Failures; /// that failed it (Node 'x' failed.), and that node's record decides — a failure carried there, else the agent run it spawned. /// A node that failed for another reason, or a run ended by the reconciler, answers null rather than the first failure found. /// When the node that ended the run is a container (a map, a loop), its own record carries the failure of the cell that ended IT — -/// the engine writes it there from — so this reads a container exactly as it reads any node. +/// the engine writes it there from — so this reads a container exactly as it reads any node. +/// A flow.subworkflow node whose child run ended on a provider failure fails carrying THAT failure — read from the child by this same reader when the +/// child finished, with the child run it happened in as its — so a parent run reads the same way too, and a chain of +/// nested sub-workflows reads to the deepest. /// public sealed class RunFailureReader : IRunFailureReader, IScopedDependency { @@ -40,7 +43,7 @@ public sealed class RunFailureReader : IRunFailureReader, IScopedDependency ? await FromWhereTheRunEndedAsync(runId, teamId, run.Error, cancellationToken).ConfigureAwait(false) : await FromLatestAgentRunAsync(runId, teamId, nodeId: null, publishRefusalOnly: true, cancellationToken).ConfigureAwait(false); - return found is null ? null : new RecordedFailure { Failure = found.Value.Failure, At = found.Value.At, ScopeRepositoryIds = run.ScopeRepositoryIds }; + return found is null ? null : new RecordedFailure { Failure = found.Value.Failure, At = found.Value.At, NodeId = found.Value.NodeId, ScopeRepositoryIds = run.ScopeRepositoryIds }; } private async Task LoadRunAsync(Guid runId, Guid teamId, CancellationToken cancellationToken) => @@ -50,11 +53,11 @@ await _db.WorkflowRun.AsNoTracking() .SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false); /// The failure the node that ended the run recorded; failing that, its agent run's own account. Where the ledger names no node that ended the run, nothing speaks for it — except a ledger with no node failure at all, which leaves the agent run as the only witness. - private async Task<(PersistedFailure Failure, DateTimeOffset At)?> FromWhereTheRunEndedAsync(Guid runId, Guid teamId, string? runError, CancellationToken cancellationToken) + private async Task<(PersistedFailure Failure, DateTimeOffset At, string? NodeId)?> FromWhereTheRunEndedAsync(Guid runId, Guid teamId, string? runError, CancellationToken cancellationToken) { var ended = await ReadLedgerEndAsync(runId, runError, cancellationToken).ConfigureAwait(false); - if (ended.Failure is { } failure) return (failure, ended.At); + if (ended.Failure is { } failure) return (failure, ended.At, ended.NodeId); return ended.NodeFailuresRecorded && ended.NodeId is null ? null : await FromLatestAgentRunAsync(runId, teamId, ended.NodeId, publishRefusalOnly: false, cancellationToken).ConfigureAwait(false); } @@ -101,12 +104,12 @@ private async Task ReadLedgerEndAsync(Guid runId, string? runError, C /// and was respawned says nothing about why the run ended. narrows it to the one failure a /// run that finished can carry — its branch was refused by the remote. /// - private async Task<(PersistedFailure Failure, DateTimeOffset At)?> FromLatestAgentRunAsync(Guid runId, Guid teamId, string? nodeId, bool publishRefusalOnly, CancellationToken cancellationToken) + private async Task<(PersistedFailure Failure, DateTimeOffset At, string? NodeId)?> FromLatestAgentRunAsync(Guid runId, Guid teamId, string? nodeId, bool publishRefusalOnly, CancellationToken cancellationToken) { var latest = await _db.AgentRun.AsNoTracking() .Where(a => a.WorkflowRunId == runId && a.TeamId == teamId && a.ResultJson != null && (nodeId == null || a.NodeId == nodeId)) .OrderByDescending(a => a.CreatedDate) - .Select(a => new { a.ResultJson, At = a.CompletedAt ?? a.LastModifiedDate }) + .Select(a => new { a.ResultJson, a.NodeId, At = a.CompletedAt ?? a.LastModifiedDate }) .FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); if (latest is null) return null; @@ -115,7 +118,7 @@ private async Task ReadLedgerEndAsync(Guid runId, string? runError, C if (failure is null || (publishRefusalOnly && failure.Code != FailureCodes.GitCredentialRejected)) return null; - return (failure, latest.At); + return (failure, latest.At, latest.NodeId); } /// The failure an agent run's stored result names: its own failure, else a push refused for the credential — the single-repo result's, or a multi-repo result's first refused repository. diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/FlowSubworkflowNode.cs b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/FlowSubworkflowNode.cs index d8d5567f0..7ccbc21d3 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/FlowSubworkflowNode.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/Builtin/FlowSubworkflowNode.cs @@ -1,6 +1,9 @@ using System.Text.Json; +using CodeSpace.Core.Failures; +using CodeSpace.Core.Services.Workflows.Failures; using CodeSpace.Messages.Constants; using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Failures; namespace CodeSpace.Core.Services.Workflows.Nodes.Builtin; @@ -13,7 +16,10 @@ namespace CodeSpace.Core.Services.Workflows.Nodes.Builtin; /// node's inputs as the child's payload) and parks the parent. When the child reaches a /// terminal state the engine resumes the parent with the child's outcome, which the node maps: /// child Success → this node's outputs ARE the child's outputs; child Failure → this node fails, -/// composing with the node's retry policy + error branch like any other failure. +/// composing with the node's retry policy + error branch like any other failure. A child that +/// ended on a provider failure a person can repair (a disconnected credential, a revoked token) reaches +/// the node on the same payload, and the node fails carrying it, so the parent's own record — and the +/// remedy built from it — says what is wrong and which child run it happened in. /// /// The node itself is pure — it never touches the DB. The engine owns the cross-run /// orchestration (staging, the suspend wait, the deferred dispatch, the completion → resume hook), @@ -69,7 +75,7 @@ public sealed class FlowSubworkflowNode : INodeRuntime public Task RunAsync(NodeRunContext context, CancellationToken cancellationToken) { - // Resumed: the child finished. ResumePayload = { status, outputs, error } (stamped by the + // Resumed: the child finished. ResumePayload = { status, outputs, error, failure? } (stamped by the // engine's completion hook). Map it onto this node's result. if (context.ResumePayload.HasValue) { @@ -78,7 +84,10 @@ public Task RunAsync(NodeRunContext context, CancellationToken cance return Task.FromResult(NodeResult.Ok(ReadObjectBag(payload, "outputs"))); var error = ReadString(payload, "error"); - return Task.FromResult(NodeResult.Fail($"Sub-workflow run failed: {(string.IsNullOrEmpty(error) ? "see the child run" : error)}")); + var sentence = $"Sub-workflow run failed: {(string.IsNullOrEmpty(error) ? "see the child run" : error)}"; + + // The sentence is the node's own and stays as it was; the provider failure the child ended on, when it recorded one, rides beside it. + return Task.FromResult(ReadChildFailure(payload) is { } failure ? NodeResult.Fail(sentence, failure) : NodeResult.Fail(sentence)); } // First pass: validate config + suspend with the spec. The engine stages + dispatches the @@ -103,6 +112,10 @@ public Task RunAsync(NodeRunContext context, CancellationToken cance })); } + /// The provider failure the child's run ended on, as the engine's completion hook read it off the child's own record — absent for a child that failed for any other reason, and for a payload stamped before children carried theirs. + private static PersistedFailure? ReadChildFailure(JsonElement payload) => + payload.ValueKind == JsonValueKind.Object && payload.TryGetProperty(PersistedFailureJson.PayloadKey, out var failure) ? ProviderFailureCapture.OfRecord(PersistedFailureJson.ReadElement(failure)) : null; + private static string ReadString(JsonElement bag, string key) => bag.ValueKind == JsonValueKind.Object && bag.TryGetProperty(key, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() ?? "" diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/NodeResult.cs b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/NodeResult.cs index f39ba7645..4e29b77e6 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Nodes/NodeResult.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Nodes/NodeResult.cs @@ -1,5 +1,6 @@ using System.Text.Json; using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Failures; namespace CodeSpace.Core.Services.Workflows.Nodes; @@ -31,6 +32,14 @@ public sealed record NodeResult /// public Exception? Cause { get; init; } + /// + /// A provider failure the node already holds in its recorded form, when what it failed on happened somewhere else and was read back — a + /// sub-workflow's child run ended on it — so there is no exception to hand over as . The engine writes it beside the + /// sentence exactly as it writes the failure it reads off an exception, and an exception's own declared failure wins over it. Like + /// it never decides a retry. + /// + public PersistedFailure? CarriedFailure { get; init; } + /// Set when Status == Suspended. Opaque token the resume mechanism feeds back. public SuspensionToken? SuspendUntil { get; init; } @@ -55,6 +64,9 @@ public static NodeResult Route(IReadOnlyList handles, IReadOnlyDictionar /// A failure worded by the node, keeping the exception it was worded from () so what it declared can be recorded. public static NodeResult Fail(string error, Exception cause) => new() { Status = NodeStatus.Failure, Error = error, Cause = cause }; + /// A failure worded by the node, keeping the failure it was read back as () so what was recorded where it happened is recorded here too. + public static NodeResult Fail(string error, PersistedFailure carried) => new() { Status = NodeStatus.Failure, Error = error, CarriedFailure = carried }; + /// A failure with an explicit retry verdict — false marks a deterministic outcome the retry policy must not re-run. public static NodeResult Fail(string error, bool retryable) => new() { Status = NodeStatus.Failure, Error = error, Retryable = retryable }; diff --git a/backend/src/CodeSpace.Messages/Dtos/Credentials/ProviderRemedy.cs b/backend/src/CodeSpace.Messages/Dtos/Credentials/ProviderRemedy.cs index e4c7a194d..609296cc2 100644 --- a/backend/src/CodeSpace.Messages/Dtos/Credentials/ProviderRemedy.cs +++ b/backend/src/CodeSpace.Messages/Dtos/Credentials/ProviderRemedy.cs @@ -40,6 +40,9 @@ public sealed record ProviderRemedy /// For provider_base_url_redirects: the origin the provider answered with, what its base URL would be edited to. Null for every other failure. public string? RedirectsTo { get; init; } + /// Where the failure happened when it was not in the run this remedy is for: the sub-workflow run that met it, which this run's flow.subworkflow node ran. Null for a failure that happened in the run itself — and for one whose sub-workflow run is not the team's or is gone, which is never named. + public RemedyOrigin? Origin { get; init; } + /// What can be done, best first, each saying whether the caller may do it and, when not, why. Never empty. public required IReadOnlyList Actions { get; init; } @@ -47,6 +50,21 @@ public sealed record ProviderRemedy public IReadOnlyList AskPeople { get; init; } = Array.Empty(); } +/// The sub-workflow run a failure happened in, as the run page names it: which run (to link to), which workflow it ran and the node that ended it. Re-running is still the PARENT's — this only says where to look. +public sealed record RemedyOrigin +{ + /// The sub-workflow run — the deepest one, when sub-workflows are nested. + public required Guid RunId { get; init; } + + public required Guid WorkflowId { get; init; } + + /// The name of the workflow that run executed, read under the asker's team. + public required string WorkflowName { get; init; } + + /// The node that ended that run; null when the run's own record carried the failure. + public string? NodeId { get; init; } +} + /// The kinds of fix the remedy offers. Serialized as the lower-case names a client branches on, so a rename is a breaking change. [JsonConverter(typeof(JsonStringEnumConverter))] public enum RemedyActionKind diff --git a/backend/src/CodeSpace.Messages/Failures/FailureOrigin.cs b/backend/src/CodeSpace.Messages/Failures/FailureOrigin.cs new file mode 100644 index 000000000..212dff862 --- /dev/null +++ b/backend/src/CodeSpace.Messages/Failures/FailureOrigin.cs @@ -0,0 +1,25 @@ +namespace CodeSpace.Messages.Failures; + +/// +/// Where a failure happened when it did not happen in the run whose record carries it: the sub-workflow run that met the provider +/// failure, and the node that ended it. A flow.subworkflow node fails with the failure its child run ended on, so the parent's +/// own record can say what is wrong; this is the pointer that says which child run that was, so the parent's page can name it and link to it. +/// +/// The DEEPEST run that met the failure, never an intermediate one: a failure that travelled through nested sub-workflows is +/// carried by every level with the origin it was first stamped with. Only identifiers and a time are kept — the run, the node and when +/// that run recorded the failure — because what the run is called and what it was launched against are read live, under the asker's team, when the remedy is built. +/// +/// The time matters because the parent that carries the failure is resumed only once every sub-workflow it is waiting on has finished, so its own +/// record can be much later than the failure; whether the credential has been fixed since is measured from the failure, not from the resume. +/// +public sealed record FailureOrigin +{ + /// The run whose own record names the failure — the child, or the deepest child of a chain of them. + public required Guid RunId { get; init; } + + /// The node that ended that run. Null where the run's own record carried the failure (a run that could not start) and no node did. + public string? NodeId { get; init; } + + /// When that run recorded the failure. Null in a record written before it was kept, which is then judged from the time of the record that carries it, as it always was. + public DateTimeOffset? At { get; init; } +} diff --git a/backend/src/CodeSpace.Messages/Failures/PersistedFailure.cs b/backend/src/CodeSpace.Messages/Failures/PersistedFailure.cs index 722b7c34a..ac6c251be 100644 --- a/backend/src/CodeSpace.Messages/Failures/PersistedFailure.cs +++ b/backend/src/CodeSpace.Messages/Failures/PersistedFailure.cs @@ -25,6 +25,9 @@ public sealed record PersistedFailure /// The repository the failing step was acting on, when the step names one — the subject a remedy re-links. A failure that carries a credential of its own does not need it, and one that carries none (a provider's 401) cannot be traced to a credential without it. public Guid? RepositoryId { get; init; } + /// Where the failure happened when it was not in the run that records it — a sub-workflow's run, carried up to the parent node that ran it (). Absent for a failure that happened in the run itself, and for every record written before sub-workflows carried their failures, which then reads exactly as it did. + public FailureOrigin? Origin { get; init; } + public string? Detail(string key) => Details is not null && Details.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value) ? value : null; public Guid? DetailGuid(string key) => Guid.TryParse(Detail(key), out var id) && id != Guid.Empty ? id : null; diff --git a/backend/src/CodeSpace.Messages/Failures/RecordedFailure.cs b/backend/src/CodeSpace.Messages/Failures/RecordedFailure.cs index 95548bbbe..9b845f141 100644 --- a/backend/src/CodeSpace.Messages/Failures/RecordedFailure.cs +++ b/backend/src/CodeSpace.Messages/Failures/RecordedFailure.cs @@ -13,6 +13,9 @@ public sealed record RecordedFailure /// When the record naming the failure was written — the node's failing, or the agent run's ending. public required DateTimeOffset At { get; init; } + /// The node that ended the run and recorded the failure — the one the run's own error names. Null where the run's record itself carried the failure (a run that could not start) or an agent run was the only witness and named no node. + public string? NodeId { get; init; } + /// The repositories the run was launched against; empty for a run launched with none (an authored workflow). public IReadOnlyList ScopeRepositoryIds { get; init; } = Array.Empty(); } diff --git a/backend/tests/CodeSpace.E2ETests/Credentials/ProviderFailureInASubWorkflowHttpE2ETests.cs b/backend/tests/CodeSpace.E2ETests/Credentials/ProviderFailureInASubWorkflowHttpE2ETests.cs new file mode 100644 index 000000000..825ec4caf --- /dev/null +++ b/backend/tests/CodeSpace.E2ETests/Credentials/ProviderFailureInASubWorkflowHttpE2ETests.cs @@ -0,0 +1,318 @@ +using System.Net; +using System.Text.Json; +using CodeSpace.Core.Failures; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Credentials; +using CodeSpace.Core.Services.Workflows.Lifecycle; +using CodeSpace.E2ETests.Infrastructure; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.Messages.Constants; +using CodeSpace.Messages.Credentials; +using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Exceptions; +using CodeSpace.Messages.Failures; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Shouldly; + +namespace CodeSpace.E2ETests.Credentials; + +/// +/// 🟢 High fidelity (HTTP surface) — a team authors a workflow that runs ANOTHER workflow as a sub-workflow, the child lists pull requests through a +/// GitLab connection whose OAuth grant is dead, the parent is run, and the PARENT's page is read the way the SPA reads it. Everything runs through the real +/// ASP.NET pipeline (JWT, the X-Team-Id scope, the controllers and their JSON options) over real Postgres: both workflows are saved with +/// POST /api/workflows, the parent is started with POST /api/workflows/{id}/run, and the real engine (the in-test deferred queue is the Hangfire +/// transport) parks the parent, runs the child against a loopback , resumes the parent with the child's outcome and fails +/// it — then GET /api/workflows/runs/{id} answers as an Owner, a Member and a Viewer. +/// +/// The parent's page says what the child's does — the credential, the repository, each role's buttons — and names the child run it happened in, which the +/// page links to. Re-run on the parent is the parent's: it replays the PARENT, which starts a fresh child. A parent recorded before children carried their +/// failures answers exactly as it did, and a pointer to a run that is not the team's is never named. +/// +[Trait("Category", "E2E")] +[Trait("Surface", "Http")] +public sealed class ProviderFailureInASubWorkflowHttpE2ETests : IClassFixture +{ + private const string ChildName = "Review the pull requests"; + + private readonly TaskLaunchApiFactory _factory; + private readonly TenantHarness _tenants; + + public ProviderFailureInASubWorkflowHttpE2ETests(TaskLaunchApiFactory factory) + { + _factory = factory; + _tenants = new TenantHarness(factory); + } + + [Fact] + public async Task A_parent_run_whose_child_ended_on_a_dead_connection_carries_the_childs_remedy_naming_the_child_run_for_an_owner_a_member_and_a_viewer() + { + using var gitlab = new GitLabDoorkeeperStub(); // never heard of the pair the row holds: the refresh is answered invalid_grant + var world = await SeedWorldAsync(gitlab); + var parentRunId = await RunParentAsync(world); + var childRunId = await ChildRunOfAsync(world, parentRunId); + + foreach (var (asker, label, reconnect, relink) in new[] { (world.Owner, "owner", true, true), (world.Member, "member", false, false), (world.Viewer, "viewer", false, false) }) + { + var response = await _tenants.SendAsync(HttpMethod.Get, $"/api/workflows/runs/{parentRunId}", asker, world.Team); + var body = await response.Content.ReadAsStringAsync(); + + response.StatusCode.ShouldBe(HttpStatusCode.OK, $"{label}: {body}"); + + var run = JsonDocument.Parse(body).RootElement; + run.GetProperty("status").GetString().ShouldBe("Failure", $"{label}: fixture check: the child's failure ended the parent"); + run.GetProperty("error").GetString().ShouldBe("Node 'sub' failed.", $"{label}: the sentence is untouched"); + + var remedy = run.GetProperty("remedy"); + remedy.GetProperty("code").GetString().ShouldBe("credential_needs_reconnect", label); + remedy.GetProperty("resolved").GetBoolean().ShouldBeFalse(label); + remedy.GetProperty("title").GetString().ShouldBe("GitLab no longer accepts the credential this run used", label); + remedy.GetProperty("cause").GetString().ShouldStartWith($"This failed in the sub-workflow “{ChildName}” (run {childRunId.ToString()[..8]}), at node 'list'. ", Case.Sensitive, label); + remedy.GetProperty("credential").GetProperty("id").GetGuid().ShouldBe(world.CredentialId, label); + remedy.GetProperty("repositories")[0].GetProperty("fullPath").GetString().ShouldBe("acme/api", label); + + var origin = remedy.GetProperty("origin"); + origin.GetProperty("runId").GetGuid().ShouldBe(childRunId, $"{label}: the child run the page links to"); + origin.GetProperty("workflowId").GetGuid().ShouldBe(world.ChildWorkflowId, label); + origin.GetProperty("workflowName").GetString().ShouldBe(ChildName, label); + origin.GetProperty("nodeId").GetString().ShouldBe("list", label); + + var actions = remedy.GetProperty("actions").EnumerateArray().ToDictionary(a => a.GetProperty("kind").GetString()!, a => a); + actions["reconnect"].GetProperty("enabled").GetBoolean().ShouldBe(reconnect, $"{label} reconnect"); + actions["relink"].GetProperty("enabled").GetBoolean().ShouldBe(relink, $"{label} relink"); + actions["open_providers"].GetProperty("enabled").GetBoolean().ShouldBeTrue(label); + actions["rerun"].GetProperty("enabled").GetBoolean().ShouldBeFalse($"{label}: nothing is fixed yet"); + } + + var lean = await _tenants.SendAsync(HttpMethod.Get, $"/api/workflows/runs/{parentRunId}/remedy", world.Viewer, world.Team); + + lean.StatusCode.ShouldBe(HttpStatusCode.OK, "the room reads the same remedy without the run detail around it"); + JsonDocument.Parse(await lean.Content.ReadAsStringAsync()).RootElement.GetProperty("origin").GetProperty("runId").GetGuid().ShouldBe(childRunId); + + // The child's own page says what happened there, and has nowhere else to point. + var child = JsonDocument.Parse(await (await _tenants.SendAsync(HttpMethod.Get, $"/api/workflows/runs/{childRunId}", world.Owner, world.Team)).Content.ReadAsStringAsync()).RootElement; + var childRemedy = child.GetProperty("remedy"); + + childRemedy.GetProperty("code").GetString().ShouldBe("credential_needs_reconnect"); + (!childRemedy.TryGetProperty("origin", out var childOrigin) || childOrigin.ValueKind == JsonValueKind.Null).ShouldBeTrue("the failure happened in the child run itself"); + childRemedy.GetProperty("cause").GetString().ShouldNotContain("sub-workflow", Case.Insensitive); + + // The wire contract: `origin` is the only member this adds. Every member a client that predates it reads is still there, so an older SPA (and a mixed fleet's older API) loses nothing. + var members = childRemedy.EnumerateObject().Select(p => p.Name).ToList(); + + members.ShouldBeSubsetOf(new[] { "code", "title", "cause", "resolved", "credential", "repositories", "providerInstance", "redirectsTo", "origin", "actions", "askPeople" }, "no member beyond the one this adds"); + members.ShouldContain("code"); + members.ShouldContain("title"); + members.ShouldContain("cause"); + members.ShouldContain("resolved"); + members.ShouldContain("credential"); + members.ShouldContain("repositories"); + members.ShouldContain("actions"); + members.ShouldContain("askPeople"); + } + + [Fact] + public async Task Re_running_the_parent_replays_the_parent_and_starts_a_fresh_child_while_the_original_child_is_left_as_it_was() + { + using var gitlab = new GitLabDoorkeeperStub(); + var world = await SeedWorldAsync(gitlab); + var parentRunId = await RunParentAsync(world); + var firstChildRunId = await ChildRunOfAsync(world, parentRunId); + + var replay = await _tenants.SendAsync(HttpMethod.Post, $"/api/workflows/runs/{parentRunId}/replay", world.Owner, world.Team); + var replayRunId = Guid.Parse(await ReadStringAsync(replay, "runId")); + + await _factory.JobClient.DrainAsync(); + + replayRunId.ShouldNotBe(parentRunId, "the Re-run is a fresh run of the parent"); + + var replayed = JsonDocument.Parse(await (await _tenants.SendAsync(HttpMethod.Get, $"/api/workflows/runs/{replayRunId}", world.Owner, world.Team)).Content.ReadAsStringAsync()).RootElement; + var freshChildRunId = replayed.GetProperty("nodes").EnumerateArray().Single(n => n.GetProperty("nodeId").GetString() == "sub").GetProperty("childRunId").GetGuid(); + + freshChildRunId.ShouldNotBe(firstChildRunId, "the parent started a FRESH child, never reusing the failed one"); + replayed.GetProperty("status").GetString().ShouldBe("Failure", "the connection is still dead, so the fresh child fails on it again"); + replayed.GetProperty("remedy").GetProperty("origin").GetProperty("runId").GetGuid().ShouldBe(freshChildRunId, "and the replayed parent's page names the run it happened in this time"); + + var original = JsonDocument.Parse(await (await _tenants.SendAsync(HttpMethod.Get, $"/api/workflows/runs/{parentRunId}", world.Owner, world.Team)).Content.ReadAsStringAsync()).RootElement; + + original.GetProperty("remedy").GetProperty("origin").GetProperty("runId").GetGuid().ShouldBe(firstChildRunId, "the original run still points at the child it had"); + } + + [Fact] + public async Task A_parent_recorded_before_children_carried_their_failures_answers_exactly_as_it_did() + { + var world = await SeedWorldAsync(gitlab: null); + var parentRunId = await SeedFailedParentAsync(world); + + using (var scope = _factory.Services.CreateScope()) + { + var logger = scope.ServiceProvider.GetRequiredService(); + + await logger.NodeStartedAsync(parentRunId, "sub", string.Empty, new Dictionary(), new Dictionary(), CancellationToken.None); + await logger.NodeFailedAsync(parentRunId, "sub", string.Empty, "Sub-workflow run failed: Node 'list' failed.", TimeSpan.FromMilliseconds(9), CancellationToken.None); + } + + var detail = await _tenants.SendAsync(HttpMethod.Get, $"/api/workflows/runs/{parentRunId}", world.Owner, world.Team); + var run = JsonDocument.Parse(await detail.Content.ReadAsStringAsync()).RootElement; + + detail.StatusCode.ShouldBe(HttpStatusCode.OK); + run.GetProperty("error").GetString().ShouldBe("Node 'sub' failed."); + (!run.TryGetProperty("remedy", out var remedy) || remedy.ValueKind == JsonValueKind.Null).ShouldBeTrue("the node recorded no failure, so nothing claims a remedy"); + (await _tenants.SendAsync(HttpMethod.Get, $"/api/workflows/runs/{parentRunId}/remedy", world.Owner, world.Team)).StatusCode.ShouldBe(HttpStatusCode.NoContent); + } + + [Fact] + public async Task A_pointer_to_a_run_that_is_not_the_teams_is_never_named_on_the_wire() + { + var world = await SeedWorldAsync(gitlab: null); + var elsewhere = await SeedWorldAsync(gitlab: null, childName: "THE-OTHER-TEAMS-SECRET-WORKFLOW"); + var foreignRunId = Guid.Parse(await ReadStringAsync(await _tenants.SendAsync(HttpMethod.Post, $"/api/workflows/{elsewhere.ChildWorkflowId}/run", elsewhere.Owner, elsewhere.Team, new { }), "runId")); + var parentRunId = await SeedFailedParentAsync(world); + + await _factory.JobClient.DrainAsync(); + + using (var scope = _factory.Services.CreateScope()) + { + var logger = scope.ServiceProvider.GetRequiredService(); + var failure = ProviderFailureCapture.From(new CredentialNeedsReconnectException(world.CredentialId, "GitLab · NACHO.L", "invalid_grant"), world.RepositoryId)! with { Origin = new FailureOrigin { RunId = foreignRunId, NodeId = "list" } }; + + await logger.NodeStartedAsync(parentRunId, "sub", string.Empty, new Dictionary(), new Dictionary(), CancellationToken.None); + await logger.NodeFailedAsync(parentRunId, "sub", string.Empty, "Sub-workflow run failed: Node 'list' failed.", TimeSpan.FromMilliseconds(9), failure, CancellationToken.None); + } + + var detail = await _tenants.SendAsync(HttpMethod.Get, $"/api/workflows/runs/{parentRunId}", world.Owner, world.Team); + var body = await detail.Content.ReadAsStringAsync(); + + detail.StatusCode.ShouldBe(HttpStatusCode.OK, body); + body.ShouldNotContain("THE-OTHER-TEAMS-SECRET-WORKFLOW"); + body.ShouldNotContain(foreignRunId.ToString()); + + var remedy = JsonDocument.Parse(body).RootElement.GetProperty("remedy"); + + remedy.GetProperty("credential").GetProperty("id").GetGuid().ShouldBe(world.CredentialId, "the remedy is still the parent's own"); + (!remedy.TryGetProperty("origin", out var origin) || origin.ValueKind == JsonValueKind.Null).ShouldBeTrue("an origin that is not this team's run names nothing"); + + // And the other team cannot read this team's parent at all. + (await _tenants.SendAsync(HttpMethod.Get, $"/api/workflows/runs/{parentRunId}/remedy", elsewhere.Owner, elsewhere.Team)).StatusCode.ShouldBeOneOf(HttpStatusCode.NoContent, HttpStatusCode.NotFound); + } + + // ─── Seeds ───────────────────────────────────────────────────────────────── + + private sealed record World(Guid Team, Guid Owner, Guid Member, Guid Viewer, Guid CredentialId, Guid RepositoryId, Guid ChildWorkflowId, Guid ParentWorkflowId); + + /// A team of an Owner, a Member and a Viewer; one GitLab connection — an OAuth grant due for renewal that the stub never issued, belonging to the Owner as a connect leaves it — and one repository running through it; a child workflow that lists its pull requests and a parent that runs it. + private async Task SeedWorldAsync(GitLabDoorkeeperStub? gitlab, string childName = ChildName) + { + var owner = await _tenants.AccountAsync(); + var member = await _tenants.AccountAsync(); + var viewer = await _tenants.AccountAsync(); + var team = await _tenants.TeamAsync((owner, TeamRole.Owner), (member, TeamRole.Member), (viewer, TeamRole.Viewer)); + var baseUrl = gitlab?.BaseUrl ?? $"http://127.0.0.1:1/{Guid.NewGuid():N}"; + var instanceId = Guid.NewGuid(); + var credentialId = Guid.NewGuid(); + var repositoryId = Guid.NewGuid(); + + using (var scope = _factory.Services.CreateScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + var encryptor = scope.ServiceProvider.GetRequiredService(); + var serializer = scope.ServiceProvider.GetRequiredService(); + var payload = new OAuthPayload { AccessToken = "at-0", RefreshToken = "rt-0", ExpiresAt = DateTimeOffset.UtcNow.AddMinutes(-1) }; + + db.ProviderInstance.Add(new ProviderInstance { Id = instanceId, TeamId = team, Provider = ProviderKind.GitLab, DisplayName = "gitlab", BaseUrl = baseUrl, ApiUrl = baseUrl, OauthClientId = GitLabDoorkeeperStub.DefaultClientId, OauthClientSecretEnc = encryptor.Encrypt(GitLabDoorkeeperStub.DefaultClientSecret) }); + db.Credential.Add(new Credential { Id = credentialId, TeamId = team, ProviderInstanceId = instanceId, AuthType = AuthType.OAuth, Ownership = CredentialOwnership.Personal, OwnerUserId = owner, DisplayName = "GitLab · NACHO.L", EncryptedPayload = encryptor.Encrypt(serializer.Serialize(payload)), ExpiresDate = payload.ExpiresAt, Status = CredentialStatus.Active }); + db.Repository.Add(new Repository { Id = repositoryId, TeamId = team, ProviderInstanceId = instanceId, CredentialId = credentialId, ExternalId = Random.Shared.Next(100_000, 999_999).ToString(), NamespacePath = "acme", Name = "api", FullPath = "acme/api", DefaultBranch = "main", Visibility = RepositoryVisibility.Private, WebUrl = $"{baseUrl}/acme/api", CloneUrlHttps = $"{baseUrl}/acme/api.git", Status = RepositoryStatus.Active }); + await db.SaveChangesAsync(); + } + + var childWorkflowId = Guid.Parse(await ReadStringAsync(await _tenants.SendAsync(HttpMethod.Post, "/api/workflows", owner, team, ChildBody(repositoryId, childName)), "id")); + var parentWorkflowId = Guid.Parse(await ReadStringAsync(await _tenants.SendAsync(HttpMethod.Post, "/api/workflows", owner, team, ParentBody(childWorkflowId)), "id")); + + return new World(team, owner, member, viewer, credentialId, repositoryId, childWorkflowId, parentWorkflowId); + } + + /// Starts the parent with POST /api/workflows/{id}/run and lets the engine walk it: parent parks, child fails on the dead connection, the child's completion resumes the parent, which fails. + private async Task RunParentAsync(World world) + { + var runId = Guid.Parse(await ReadStringAsync(await _tenants.SendAsync(HttpMethod.Post, $"/api/workflows/{world.ParentWorkflowId}/run", world.Owner, world.Team, new { }), "runId")); + + await _factory.JobClient.DrainAsync(); + + return runId; + } + + private async Task ChildRunOfAsync(World world, Guid parentRunId) + { + var detail = JsonDocument.Parse(await (await _tenants.SendAsync(HttpMethod.Get, $"/api/workflows/runs/{parentRunId}", world.Owner, world.Team)).Content.ReadAsStringAsync()).RootElement; + + return detail.GetProperty("nodes").EnumerateArray().Single(n => n.GetProperty("nodeId").GetString() == "sub").GetProperty("childRunId").GetGuid(); + } + + /// A run of the team's that ended on the sub node, written the way the engine ends one — for a run whose ledger a test writes by hand. + private async Task SeedFailedParentAsync(World world) + { + var runId = Guid.NewGuid(); + var requestId = Guid.NewGuid(); + var now = DateTimeOffset.UtcNow; + + await _tenants.DbAsync(async db => + { + db.WorkflowRunRequest.Add(new WorkflowRunRequest { Id = requestId, TeamId = world.Team, WorkflowId = null, SourceType = WorkflowRunSourceTypes.Snapshot, ActorType = "user", ActorId = SystemUsers.SeederId, NormalizedPayloadJson = "{}", Status = WorkflowRunRequestStatus.Consumed, ReceivedAt = now, VerifiedAt = now, NormalizedAt = now }); + db.WorkflowRun.Add(new WorkflowRun { Id = runId, WorkflowId = null, WorkflowVersion = null, TeamId = world.Team, RunRequestId = requestId, SourceType = WorkflowRunSourceTypes.Snapshot, Status = WorkflowRunStatus.Failure, Error = "Node 'sub' failed.", ScopeRepositoryIds = [], ScopeProjectIds = [], CreatedDate = now, CreatedBy = SystemUsers.SeederId, LastModifiedBy = SystemUsers.SeederId }); + await db.SaveChangesAsync(); + return 0; + }); + + return runId; + } + + // start → list (git.list_prs) → end. No error edge: the dead connection ends the run. + private static object ChildBody(Guid repositoryId, string name) => new + { + name, + definition = new + { + schemaVersion = 1, + nodes = new object[] + { + new { id = "start", typeKey = "trigger.manual", config = new { }, inputs = new { } }, + new { id = "list", typeKey = "git.list_prs", config = new { }, inputs = new { repositoryId } }, + new { id = "end", typeKey = "builtin.terminal", config = new { }, inputs = new { } }, + }, + edges = new object[] { new { from = "start", to = "list" }, new { from = "list", to = "end" } }, + }, + activations = Array.Empty(), + enabled = true, + }; + + // start → sub (flow.subworkflow: the child) → end. + private static object ParentBody(Guid childWorkflowId) => new + { + name = $"parent-{Guid.NewGuid():N}"[..16], + definition = new + { + schemaVersion = 1, + nodes = new object[] + { + new { id = "start", typeKey = "trigger.manual", config = new { }, inputs = new { } }, + new { id = "sub", typeKey = "flow.subworkflow", config = new { workflowId = childWorkflowId }, inputs = new { inputs = new { } } }, + new { id = "end", typeKey = "builtin.terminal", config = new { }, inputs = new { } }, + }, + edges = new object[] { new { from = "start", to = "sub" }, new { from = "sub", to = "end" } }, + }, + activations = Array.Empty(), + enabled = true, + }; + + private static async Task ReadStringAsync(HttpResponseMessage response, string property) + { + var text = await response.Content.ReadAsStringAsync(); + + response.IsSuccessStatusCode.ShouldBeTrue($"{response.RequestMessage?.Method} {response.RequestMessage?.RequestUri?.AbsolutePath} answered {(int)response.StatusCode}: {text}"); + + var value = JsonDocument.Parse(text).RootElement.GetProperty(property); + + return value.ValueKind == JsonValueKind.String ? value.GetString()! : value.GetRawText(); + } +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Credentials/ProviderFailureThroughTheEngineFlowTests.Rerun.cs b/backend/tests/CodeSpace.IntegrationTests/Credentials/ProviderFailureThroughTheEngineFlowTests.Rerun.cs index 51c8fd429..7d53cb13e 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Credentials/ProviderFailureThroughTheEngineFlowTests.Rerun.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Credentials/ProviderFailureThroughTheEngineFlowTests.Rerun.cs @@ -7,6 +7,7 @@ using CodeSpace.Messages.Enums; using CodeSpace.Messages.Exceptions; using CodeSpace.Messages.Failures; +using Microsoft.EntityFrameworkCore; using Shouldly; namespace CodeSpace.IntegrationTests.Credentials; @@ -64,6 +65,49 @@ public async Task A_map_branch_rerun_fork_that_fails_on_a_replayed_failed_branch remedy.Credential!.Id.ShouldBe(first.CredentialId); } + [Fact] + public async Task A_map_branch_rerun_fork_that_fails_on_a_replayed_sub_workflow_branch_names_the_child_run_the_original_branch_ran() + { + // The replayed branch's sub-workflow node recorded the failure its child ended on, origin and all; the seeder carries it onto the fork's record of the + // cell, so the fork's map answers for it as the original's did — and says it happened in the ORIGINAL branch's child, not in the fresh branch's. + using var gitlab = new GitLabDoorkeeperStub(); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: true, existingTeam: team); + await RecordThePersonItWasConnectedForAsync(seed.CredentialId, team.UserId); + var child = await CreateNamedWorkflowAsync(team, "Review", ListThenEndDefinition(seed.RepositoryId)); + var workflowId = await CreateWorkflowAsync(team, SubWorkflowParent(child, SubWorkflowSite.InMap)); + var originalRunId = await WorkflowsTestSeed.SeedAdmittedManualRunAsync(_fixture, workflowId, team.TeamId, payloadJson: JsonSerializer.Serialize(new { things = new[] { Guid.NewGuid(), Guid.NewGuid() } })); + + await RunEngineAsync(originalRunId); // both branches park on a child of their own + var originalChildren = new[] { await ChildOfBranchAsync(originalRunId, "map#0"), await ChildOfBranchAsync(originalRunId, "map#1") }; + + originalChildren.Distinct().Count().ShouldBe(2, "fixture check: each branch ran a child of its own"); + + foreach (var childRunId in originalChildren) await RunEngineAsync(childRunId); + + await RunEngineAsync(originalRunId); + await AssertRunStatusAsync(originalRunId, WorkflowRunStatus.Failure, "fixture check: the children's failure ended the map"); + + (await RemedyForAsync(team.UserId, team.TeamId, originalRunId)).Origin!.RunId.ShouldBe(originalChildren[0], "fixture check: the original answers for element 0, the first branch to fail by element order"); + + var forkRunId = await RerunBranchAsync(team, originalRunId, branchIndex: 1); + + await RunEngineAsync(forkRunId); // branch 0 is replayed failed; branch 1 runs fresh and parks on a fresh child + var freshChild = await ChildOfBranchAsync(forkRunId, "map#1"); + + freshChild.ShouldNotBe(originalChildren[1], "fixture check: the fresh branch ran a fresh child"); + + await RunEngineAsync(freshChild); + await RunEngineAsync(forkRunId); + await AssertRunStatusAsync(forkRunId, WorkflowRunStatus.Failure, "fixture check: element 0 is replayed failed, so the map fails again"); + + var remedy = await RemedyForAsync(team.UserId, team.TeamId, forkRunId); + + remedy.Origin!.RunId.ShouldBe(originalChildren[0], "the replayed branch's child — the run the failure happened in — not the fresh branch's"); + remedy.Origin.WorkflowName.ShouldBe("Review"); + remedy.Credential!.Id.ShouldBe(seed.CredentialId); + } + // ─── Helpers ─────────────────────────────────────────────────────────────── /// @@ -91,4 +135,15 @@ private async Task RerunBranchAsync((Guid TeamId, Guid UserId) team, Guid return await scope.Resolve().RerunMapBranchAsync(originalRunId, "map", branchIndex, team.TeamId, team.UserId, operationId: null, CancellationToken.None); } + + /// The child run a map branch's sub-workflow node parked on — the wait's token is the child run id. + private async Task ChildOfBranchAsync(Guid parentRunId, string iterationKey) + { + using var scope = _fixture.BeginScope(); + var token = await scope.Resolve().WorkflowRunWait.AsNoTracking() + .Where(w => w.RunId == parentRunId && w.NodeId == "sub" && w.IterationKey == iterationKey && w.WaitKind == CodeSpace.Messages.Constants.WorkflowWaitKinds.Subworkflow) + .Select(w => w.Token).SingleAsync(); + + return Guid.Parse(token); + } } diff --git a/backend/tests/CodeSpace.IntegrationTests/Credentials/ProviderFailureThroughTheEngineFlowTests.SubWorkflows.cs b/backend/tests/CodeSpace.IntegrationTests/Credentials/ProviderFailureThroughTheEngineFlowTests.SubWorkflows.cs new file mode 100644 index 000000000..caf72053c --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Credentials/ProviderFailureThroughTheEngineFlowTests.SubWorkflows.cs @@ -0,0 +1,704 @@ +using System.Text.Json; +using Autofac; +using CodeSpace.Core.Failures; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Services.Workflows.Failures; +using CodeSpace.Core.Services.Workflows.Lifecycle; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.IntegrationTests.Workflows.Infrastructure; +using CodeSpace.Messages.Commands.Workflows; +using CodeSpace.Messages.Constants; +using CodeSpace.Messages.Dtos.Credentials; +using CodeSpace.Messages.Dtos.Workflows; +using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Exceptions; +using CodeSpace.Messages.Failures; +using MediatR; +using Microsoft.EntityFrameworkCore; +using Shouldly; + +namespace CodeSpace.IntegrationTests.Credentials; + +/// +/// 🟢 High fidelity — a provider connection that fails inside a flow.subworkflow CHILD run and so ends the PARENT run. The real engine walks +/// the parent, which parks on its flow.subworkflow node while a real child run executes a real git.list_prs against the real GitLab +/// provider, credential resolver and OAuth refresher over real Postgres; the one fake is GitLab (). What is pinned +/// is that the failure is read back where the child recorded it, by the reader the child's own page asks, and carried up: the resume payload, the +/// parent node's own node.failed record and — through a map or a loop the node sits in, or a chain of nested sub-workflows — the record of +/// whatever ended the parent, so the parent's remedy is the one the child's would be, naming the child run it happened in. +/// +/// The negatives that keep it honest: a child that failed for any other reason, a child that handled the failure, and a child that handled it +/// and then failed on something else all leave the parent with the sentence it always had; an origin that is not the team's run is never named; and +/// what the page offers is the same remedy the child's page offers, because it is the same reader and the same builder. +/// +public partial class ProviderFailureThroughTheEngineFlowTests +{ + public enum SubWorkflowSite { TopLevel, InMap, InLoop } + + // The sentence the parent's node wrote before failures were carried: "Sub-workflow run failed: " and what the child's run said. + private const string ChildSentence = "Sub-workflow run failed: Node 'list' failed."; + + [Theory] + [InlineData(SubWorkflowSite.TopLevel)] + [InlineData(SubWorkflowSite.InMap)] + [InlineData(SubWorkflowSite.InLoop)] + public async Task A_provider_failure_that_ends_a_child_run_gives_the_parent_the_childs_remedy_naming_the_child_run(SubWorkflowSite site) + { + using var gitlab = new GitLabDoorkeeperStub(); // never heard of the pair the row holds: the refresh is answered invalid_grant + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: true, existingTeam: team); + await RecordThePersonItWasConnectedForAsync(seed.CredentialId, team.UserId); + var child = await CreateNamedWorkflowAsync(team, "Review the pull requests", ListThenEndDefinition(seed.RepositoryId)); + var parentRunId = await StartRunOfAsync(team, SubWorkflowParent(child, site)); + + var chain = await WalkThroughChildRunsAsync(parentRunId); + + chain.Count.ShouldBe(2, "fixture check: the parent ran one child"); + var childRunId = chain[1]; + + await AssertRunStatusAsync(childRunId, WorkflowRunStatus.Failure, "fixture check: the dead connection ended the child"); + await AssertRunStatusAsync(parentRunId, WorkflowRunStatus.Failure, "fixture check: the child's failure ended the parent"); + + var expectedOrigin = new FailureOrigin { RunId = childRunId, NodeId = "list", At = await ReadFailureTimeAsync(childRunId, team.TeamId) }; + var records = await ReadFailedRecordsAsync(parentRunId); + var subRecord = records.Single(r => r.NodeId == "sub"); + + subRecord.Error.ShouldBe(ChildSentence, "the sentence the node always wrote is untouched"); + subRecord.Failure.ShouldNotBeNull("the parent's node carries the failure the child ended on"); + subRecord.Failure.Code.ShouldBe(FailureCodes.CredentialNeedsReconnect); + subRecord.Failure.DetailGuid("credentialId").ShouldBe(seed.CredentialId); + subRecord.Failure.RepositoryId.ShouldBe(seed.RepositoryId, "the repository the child's node declared it acted on travels with it"); + subRecord.Failure.Origin.ShouldBe(expectedOrigin, "and the child run it happened in, the node that ended that run and when the child recorded it — the time the child's own page judges it from"); + + foreach (var container in records.Where(r => r.NodeId is "map" or "loop")) + container.Failure.ShouldNotBeNull($"{container.NodeId}: the container the node sat in carries the failure of the cell that ended it, origin and all").Origin.ShouldBe(expectedOrigin); + + (await ReadRunErrorAsync(parentRunId)).ShouldBe(site switch { SubWorkflowSite.TopLevel => "Node 'sub' failed.", SubWorkflowSite.InMap => "Node 'map' failed.", _ => "Node 'loop' failed." }, "the run's own sentence is untouched"); + + var remedy = await RemedyForAsync(team.UserId, team.TeamId, parentRunId); + + remedy.Code.ShouldBe(FailureCodes.CredentialNeedsReconnect); + remedy.Resolved.ShouldBeFalse(); + remedy.Origin.ShouldNotBeNull("the remedy says where it happened"); + remedy.Origin.RunId.ShouldBe(childRunId); + remedy.Origin.WorkflowId.ShouldBe(child.Id); + remedy.Origin.WorkflowName.ShouldBe("Review the pull requests"); + remedy.Origin.NodeId.ShouldBe("list"); + remedy.Cause.ShouldContain($"sub-workflow “Review the pull requests” (run {childRunId.ToString()[..8]})", Case.Sensitive, "the words name it too"); + remedy.Cause.ShouldContain("“GitLab · NACHO.L”", Case.Sensitive, "and still say which credential"); + remedy.Credential!.Id.ShouldBe(seed.CredentialId); + remedy.Repositories.ShouldHaveSingleItem().FullPath.ShouldBe("acme/api"); + remedy.Actions.Select(a => (a.Kind, a.Enabled)).ShouldBe(new[] + { + (RemedyActionKind.Reconnect, true), (RemedyActionKind.Relink, true), (RemedyActionKind.OpenProviders, true), (RemedyActionKind.Rerun, false) + }, "an Owner may do everything but run it again — that waits for the fix, and it runs the PARENT again, which starts a fresh child"); + } + + [Fact] + public async Task The_parents_remedy_is_the_remedy_the_childs_own_page_gives_and_the_childs_names_no_origin() + { + using var gitlab = new GitLabDoorkeeperStub(); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: true, existingTeam: team); + await RecordThePersonItWasConnectedForAsync(seed.CredentialId, team.UserId); + var child = await CreateNamedWorkflowAsync(team, "Review", ListThenEndDefinition(seed.RepositoryId)); + var chain = await WalkThroughChildRunsAsync(await StartRunOfAsync(team, SubWorkflowParent(child, SubWorkflowSite.TopLevel))); + + var parentRemedy = await RemedyForAsync(team.UserId, team.TeamId, chain[0]); + var childRemedy = await RemedyForAsync(team.UserId, team.TeamId, chain[1]); + + childRemedy.Origin.ShouldBeNull("the child's failure happened in the child: there is nowhere else to point"); + parentRemedy.Origin.ShouldNotBeNull(); + parentRemedy.Code.ShouldBe(childRemedy.Code); + parentRemedy.Title.ShouldBe(childRemedy.Title); + parentRemedy.Resolved.ShouldBe(childRemedy.Resolved); + parentRemedy.Credential!.Id.ShouldBe(childRemedy.Credential!.Id); + parentRemedy.Repositories.Select(r => (r.Id, r.CredentialWorks)).ShouldBe(childRemedy.Repositories.Select(r => (r.Id, r.CredentialWorks))); + parentRemedy.Actions.Select(a => (a.Kind, a.Label, a.Enabled)).ShouldBe(childRemedy.Actions.Select(a => (a.Kind, a.Label, a.Enabled))); + parentRemedy.Cause.ShouldEndWith(childRemedy.Cause, Case.Sensitive, "the parent's words are the child's, after the sentence that says where it happened"); + + // Fixed through the credential the failure named: both pages read the repair the same way, and the parent's leads with running it again. + var replacement = await AddLiveCredentialAsync(seed); + await BindAsync(seed.RepositoryId, replacement); + + var parentAfter = await RemedyForAsync(team.UserId, team.TeamId, chain[0]); + var childAfter = await RemedyForAsync(team.UserId, team.TeamId, chain[1]); + + parentAfter.Resolved.ShouldBeTrue("what failed in the child works again"); + childAfter.Resolved.ShouldBeTrue(); + parentAfter.Actions[0].Kind.ShouldBe(RemedyActionKind.Rerun, "running the parent again leads once the fix shows"); + parentAfter.Actions[0].Enabled.ShouldBeTrue(); + parentAfter.Origin!.RunId.ShouldBe(chain[1], "it is still said where it happened"); + } + + [Fact] + public async Task Nested_sub_workflows_carry_the_deepest_childs_failure_to_every_parent_above_it() + { + using var gitlab = new GitLabDoorkeeperStub(); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: true, existingTeam: team); + await RecordThePersonItWasConnectedForAsync(seed.CredentialId, team.UserId); + var leaf = await CreateNamedWorkflowAsync(team, "Leaf: list the pull requests", ListThenEndDefinition(seed.RepositoryId)); + var middle = await CreateNamedWorkflowAsync(team, "Middle", SubWorkflowParent(leaf, SubWorkflowSite.TopLevel)); + var topRunId = await StartRunOfAsync(team, SubWorkflowParent(middle, SubWorkflowSite.TopLevel)); + + var chain = await WalkThroughChildRunsAsync(topRunId); + + chain.Count.ShouldBe(3, "fixture check: top → middle → leaf"); + foreach (var run in chain) await AssertRunStatusAsync(run, WorkflowRunStatus.Failure, $"fixture check: the failure ended every level ({run})"); + + var (topRun, middleRun, leafRun) = (chain[0], chain[1], chain[2]); + + foreach (var run in new[] { topRun, middleRun }) + { + var sub = (await ReadFailedRecordsAsync(run)).Single(r => r.NodeId == "sub"); + + sub.Error.ShouldBe(run == topRun ? "Sub-workflow run failed: Node 'sub' failed." : ChildSentence, "each level's own sentence is untouched"); + sub.Failure.ShouldNotBeNull(); + sub.Failure.Origin.ShouldBe(new FailureOrigin { RunId = leafRun, NodeId = "list", At = await ReadFailureTimeAsync(leafRun, team.TeamId) }, "the deepest run that met it, however many levels it passed through — and when IT recorded it, not when each level resumed"); + sub.Failure.DetailGuid("credentialId").ShouldBe(seed.CredentialId); + + var remedy = await RemedyForAsync(team.UserId, team.TeamId, run); + + remedy.Origin!.RunId.ShouldBe(leafRun); + remedy.Origin.WorkflowName.ShouldBe("Leaf: list the pull requests"); + remedy.Origin.NodeId.ShouldBe("list"); + remedy.Credential!.Id.ShouldBe(seed.CredentialId); + } + + (await RemedyForAsync(team.UserId, team.TeamId, leafRun)).Origin.ShouldBeNull("the run that met it is the origin of it, not a parent of anything"); + } + + [Fact] + public async Task The_failure_of_the_last_attempt_is_the_one_a_retried_sub_workflow_node_carries() + { + using var gitlab = new GitLabDoorkeeperStub(); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: true, existingTeam: team); + var child = await CreateNamedWorkflowAsync(team, "Review", ListThenEndDefinition(seed.RepositoryId)); + var plain = SubWorkflowParent(child, SubWorkflowSite.TopLevel); + var parentRunId = await StartRunOfAsync(team, plain with { Nodes = plain.Nodes.Select(n => n.Id == "sub" ? n with { Retry = new RetryPolicy { MaxAttempts = 2, BackoffSeconds = 0 } } : n).ToList() }); + + await RunEngineAsync(parentRunId); + var firstChild = await ChildRunIdAsync(parentRunId); + await RunEngineAsync(firstChild); + await RunEngineAsync(parentRunId); // the first child failed: the node retries, which stages a fresh child and parks again + await AssertRunStatusAsync(parentRunId, WorkflowRunStatus.Suspended, "fixture check: the retry parked the parent on a second child"); + + var secondChild = (await ChildRunIdsAsync(parentRunId)).Single(id => id != firstChild); + + await RunEngineAsync(secondChild); + await RunEngineAsync(parentRunId); + await AssertRunStatusAsync(parentRunId, WorkflowRunStatus.Failure, "fixture check: the second child failed too, so the node is out of attempts"); + + var sub = (await ReadFailedRecordsAsync(parentRunId)).Single(r => r.NodeId == "sub"); + + sub.Failure.ShouldNotBeNull("the retried node ends carrying the failure"); + sub.Failure.Origin!.RunId.ShouldBe(secondChild, "of the attempt that ended it, never of the first one that was retried"); + } + + [Fact] + public async Task A_child_that_failed_on_something_else_leaves_the_parent_with_the_sentence_alone() + { + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var child = await CreateNamedWorkflowAsync(team, "Flaky child", FlakyThenEndDefinition(Guid.NewGuid().ToString("N"))); + var chain = await WalkThroughChildRunsAsync(await StartRunOfAsync(team, SubWorkflowParent(child, SubWorkflowSite.TopLevel))); + + await AssertRunStatusAsync(chain[1], WorkflowRunStatus.Failure, "fixture check: the child failed"); + await AssertRunStatusAsync(chain[0], WorkflowRunStatus.Failure, "fixture check: so did the parent"); + + var sub = (await ReadFailedRecordsAsync(chain[0])).Single(r => r.NodeId == "sub"); + + sub.Error.ShouldBe("Sub-workflow run failed: Node 'flaky' failed."); + sub.Failure.ShouldBeNull("nothing a person can repair outside the run was recorded in the child, so nothing is carried"); + (await ReadResumePayloadAsync(chain[0])).TryGetProperty("failure", out _).ShouldBeFalse("and the resume payload is the shape it always was"); + using var scope = _fixture.BeginScope(); + + (await scope.Resolve().ReadAsync(chain[0], team.TeamId, CancellationToken.None)).ShouldBeNull(); + (await RemedyForAsync(team.UserId, team.TeamId, chain[0], allowNone: true)).ShouldBeNull("no card blames a credential for a parent whose child failed for another reason"); + } + + [Fact] + public async Task A_child_whose_node_worded_its_failure_alone_is_traced_through_its_own_agent_run_exactly_as_the_childs_own_page_does() + { + // The child's page answers from the agent run that worked in the node that ended it when that node recorded no failure of its own. The parent is told + // by the same reader, so the two can never name different failures — nor can the parent miss one the child's page shows. + using var gitlab = new GitLabDoorkeeperStub(); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: false, existingTeam: team); + await DisconnectAsync(seed.CredentialId); + var child = await CreateNamedWorkflowAsync(team, "Runs an agent", FlakyThenEndDefinition(Guid.NewGuid().ToString("N"))); + var parentRunId = await StartRunOfAsync(team, SubWorkflowParent(child, SubWorkflowSite.TopLevel)); + + await RunEngineAsync(parentRunId); + var childRunId = await ChildRunIdAsync(parentRunId); + + await SeedAgentRunAsync(team.TeamId, childRunId, "flaky", string.Empty, ProviderFailureCapture.From(new CredentialDisconnectedException(seed.CredentialId, "GitLab · NACHO.L"), seed.RepositoryId)!); + await RunEngineAsync(childRunId); + await RunEngineAsync(parentRunId); + + await AssertRunStatusAsync(parentRunId, WorkflowRunStatus.Failure, "fixture check: the child's failure ended the parent"); + (await ReadFailedRecordsAsync(childRunId)).Single(r => r.NodeId == "flaky").Failure.ShouldBeNull("fixture check: the child's node worded its failure alone"); + + var childRemedy = await RemedyForAsync(team.UserId, team.TeamId, childRunId); + var parentRemedy = await RemedyForAsync(team.UserId, team.TeamId, parentRunId); + + childRemedy.Code.ShouldBe(FailureCodes.CredentialDisconnected, "fixture check: the child's own page is traced through its agent run"); + parentRemedy.Code.ShouldBe(FailureCodes.CredentialDisconnected); + parentRemedy.Credential!.Id.ShouldBe(seed.CredentialId); + parentRemedy.Origin!.RunId.ShouldBe(childRunId); + parentRemedy.Origin.NodeId.ShouldBe("flaky", "the node that ended the child, which the agent run worked in"); + (await ReadFailedRecordsAsync(parentRunId)).Single(r => r.NodeId == "sub").Failure!.Code.ShouldBe(FailureCodes.CredentialDisconnected, "and the parent's own record carries it"); + } + + [Fact] + public async Task A_provider_failure_the_child_handled_leaves_a_successful_parent_and_no_remedy() + { + using var gitlab = new GitLabDoorkeeperStub(); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: false, existingTeam: team); + await DisconnectAsync(seed.CredentialId); + var child = await CreateNamedWorkflowAsync(team, "Handles it", ListWithErrorEdgeDefinition(seed.RepositoryId)); + var chain = await WalkThroughChildRunsAsync(await StartRunOfAsync(team, SubWorkflowParent(child, SubWorkflowSite.TopLevel))); + + await AssertRunStatusAsync(chain[1], WorkflowRunStatus.Success, "fixture check: the child's error edge handled the failure"); + await AssertRunStatusAsync(chain[0], WorkflowRunStatus.Success, "fixture check: so the parent went on"); + PersistedFailureJson.Read((await ReadFailedPayloadsAsync(chain[1])).Single()).ShouldNotBeNull("fixture check: the child's own record still says what happened"); + (await ReadFailedRecordsAsync(chain[0])).ShouldBeEmpty("the parent's node did not fail"); + (await RemedyForAsync(team.UserId, team.TeamId, chain[0], allowNone: true)).ShouldBeNull(); + } + + [Fact] + public async Task A_provider_failure_the_child_handled_and_then_failed_on_something_else_carries_nothing_to_the_parent() + { + using var gitlab = new GitLabDoorkeeperStub(); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: false, existingTeam: team); + await DisconnectAsync(seed.CredentialId); + var child = await CreateNamedWorkflowAsync(team, "Handles then fails", ListHandledThenFlakyDefinition(seed.RepositoryId, Guid.NewGuid().ToString("N"))); + var chain = await WalkThroughChildRunsAsync(await StartRunOfAsync(team, SubWorkflowParent(child, SubWorkflowSite.TopLevel))); + + await AssertRunStatusAsync(chain[0], WorkflowRunStatus.Failure, "fixture check: the child ended on the node that failed after the handled failure"); + PersistedFailureJson.Read((await ReadFailedPayloadsAsync(chain[1])).First()).ShouldNotBeNull("fixture check: the OLD failure is on the child's ledger"); + (await ReadFailedRecordsAsync(chain[0])).Single(r => r.NodeId == "sub").Failure.ShouldBeNull("the child's decisive failure is the one that ended it, not one it handled earlier"); + (await RemedyForAsync(team.UserId, team.TeamId, chain[0], allowNone: true)).ShouldBeNull(); + } + + [Fact] + public async Task A_failure_that_names_no_repository_is_traced_through_the_childs_scope_exactly_as_the_childs_own_page_does() + { + await using var gitlab = LoopbackResponder.Start(_ => "HTTP/1.1 401 Unauthorized\r\nContent-Type: application/json\r\nContent-Length: 30\r\nConnection: close\r\n\r\n{\"message\":\"401 Unauthorized\"}"); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, $"http://127.0.0.1:{gitlab.Port}", $"glpat-{Guid.NewGuid():N}", existingTeam: team); + // git.create_issue declares no repository input and a 401 names no credential: only the run's own launch scope could trace it. + var child = await CreateNamedWorkflowAsync(team, "Files an issue", NodeThenEndDefinition("git.create_issue", seed.RepositoryId, """{"title":"t"}""")); + var parentRunId = await StartRunOfAsync(team, SubWorkflowParent(child, SubWorkflowSite.TopLevel)); + + await SetScopeAsync(parentRunId, seed.RepositoryId); // the parent was launched against the repository; the child, started by a node, was not + + var chain = await WalkThroughChildRunsAsync(parentRunId); + + var childRemedy = await RemedyForAsync(team.UserId, team.TeamId, chain[1]); + var parentRemedy = await RemedyForAsync(team.UserId, team.TeamId, chain[0]); + + childRemedy.Repositories.ShouldBeEmpty("fixture check: the child's page traces nothing — the 401 names no repository and the child was launched against none"); + parentRemedy.Repositories.ShouldBeEmpty("the parent's is the child's remedy: not the parent's own launch scope read onto a failure that happened elsewhere"); + parentRemedy.Credential.ShouldBe(childRemedy.Credential); + parentRemedy.Actions.Select(a => (a.Kind, a.Enabled)).ShouldBe(childRemedy.Actions.Select(a => (a.Kind, a.Enabled))); + + // Launched against the repository, the child's own page traces it — and so does the parent's, since it is the child's remedy. + await SetScopeAsync(chain[1], seed.RepositoryId); + + (await RemedyForAsync(team.UserId, team.TeamId, chain[1])).Repositories.ShouldHaveSingleItem().Id.ShouldBe(seed.RepositoryId); + (await RemedyForAsync(team.UserId, team.TeamId, chain[0])).Repositories.ShouldHaveSingleItem().Id.ShouldBe(seed.RepositoryId); + } + + [Fact] + public async Task A_fan_out_parent_judges_the_fix_from_when_its_child_failed_not_from_when_the_slowest_sibling_let_it_resume() + { + // The parent is resumed only once EVERY sub-workflow it waits on has finished, so its own failure record can be written long after the child's. A 401 on a + // token that looks fine is shown fixed only by something that validated the credential SINCE the failure — and "since" is the child's failure, which is what the + // child's own page measures. Here the person replaces the token between the child's failure and the slow sibling's end; both pages must say it is fixed. + using var gitlab = new GitLabDoorkeeperStub(); + var token = $"glpat-{Guid.NewGuid():N}"; + gitlab.AddAccessToken(token, "nacho"); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, token, existingTeam: team); + gitlab.RevokeAccessToken(token); + var failing = await CreateNamedWorkflowAsync(team, "Failing child", ListThenEndDefinition(seed.RepositoryId)); + var slow = await CreateNamedWorkflowAsync(team, "Slow sibling", new WorkflowDefinition { SchemaVersion = 1, Nodes = [Node("start", "trigger.manual", null), Node("end", "builtin.terminal", null)], Edges = [Edge("start", "end")] }); + var parentRunId = await StartRunOfAsync(team, FanOutParent(failing, slow)); + + await RunEngineAsync(parentRunId); // parks on both children at once + var failingRunId = await ChildRunOfNodeAsync(parentRunId, "subA"); + var slowRunId = await ChildRunOfNodeAsync(parentRunId, "subB"); + + await RunEngineAsync(failingRunId); + await AssertRunStatusAsync(failingRunId, WorkflowRunStatus.Failure, "fixture check: the revoked token ended the first child"); + await AssertRunStatusAsync(parentRunId, WorkflowRunStatus.Suspended, "fixture check: the parent is still parked on the slow sibling — it has not been told yet"); + + var childFailedAt = await ReadFailureTimeAsync(failingRunId, team.TeamId); + var before = await RemedyForAsync(team.UserId, team.TeamId, failingRunId); + + before.Code.ShouldBe(FailureCodes.ProviderUnauthorized); + before.Resolved.ShouldBeFalse("fixture check: nothing validated the token since it failed"); + + await Task.Delay(200); + var replacedAt = DateTimeOffset.UtcNow; + await GitLabCredentialSeed.SetLastValidatedAsync(_fixture, seed.CredentialId, replacedAt); // the person replaced the token, from the child's page + await Task.Delay(200); + + await RunEngineAsync(slowRunId); + await RunEngineAsync(parentRunId); // the last wait resolved: the parent walks, and fails on the first child's failure + await AssertRunStatusAsync(parentRunId, WorkflowRunStatus.Failure, "fixture check: the child's failure ended the parent"); + + (await ReadFailureTimeAsync(parentRunId, team.TeamId)).ShouldBeGreaterThan(replacedAt, "fixture check: the parent's own record is LATER than the fix — judged from it, the fix would read as older than the failure"); + + var parentRemedy = await RemedyForAsync(team.UserId, team.TeamId, parentRunId); + var childRemedy = await RemedyForAsync(team.UserId, team.TeamId, failingRunId); + + childRemedy.Resolved.ShouldBeTrue("the child's page: validated after it failed"); + parentRemedy.Origin!.RunId.ShouldBe(failingRunId); + parentRemedy.Resolved.ShouldBeTrue("the parent's is the child's remedy, judged from when the child failed"); + parentRemedy.Actions.Select(a => (a.Kind, a.Enabled)).ShouldBe(childRemedy.Actions.Select(a => (a.Kind, a.Enabled))); + parentRemedy.Actions[0].Kind.ShouldBe(RemedyActionKind.Rerun, "so running the parent again leads, exactly as it does on the child's page"); + parentRemedy.Repositories.Select(r => r.CredentialWorks).ShouldBe(childRemedy.Repositories.Select(r => r.CredentialWorks)); + (await ReadFailedRecordsAsync(parentRunId)).Single(r => r.NodeId == "subA").Failure!.Origin!.At.ShouldBe(childFailedAt); + } + + [Theory] + [InlineData(-120, true)] // the child failed two minutes before the parent recorded it; the token was validated a minute before that record — AFTER the child failed: fixed + [InlineData(-30, false)] // the child failed half a minute before the record, AFTER the last validation: nothing has fixed it since + [InlineData(null, false)] // a record that kept no time of its own (an older writer's) is judged from the record that carries it, as it always was + public async Task A_carried_failure_is_judged_from_the_time_it_kept_and_from_its_records_own_when_it_kept_none(int? failedSecondsBeforeTheRecord, bool fixedSince) + { + using var gitlab = new GitLabDoorkeeperStub(); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, $"glpat-{Guid.NewGuid():N}", existingTeam: team); + var child = await CreateNamedWorkflowAsync(team, "Review", ListThenEndDefinition(seed.RepositoryId)); + var childRunId = await WorkflowsTestSeed.SeedManualRunAsync(_fixture, child.Id, team.TeamId); + var parentRunId = await StartRunOfAsync(team, SubWorkflowParent(child, SubWorkflowSite.TopLevel)); + DateTimeOffset? failedAt = failedSecondsBeforeTheRecord is { } seconds ? DateTimeOffset.UtcNow.AddSeconds(seconds) : null; + + await FailRunAsync(parentRunId, "Node 'sub' failed."); + await RecordFailureAsync(parentRunId, "sub", ChildSentence, new PersistedFailure { Code = FailureCodes.ProviderUnauthorized, RepositoryId = seed.RepositoryId, Origin = new FailureOrigin { RunId = childRunId, NodeId = "list", At = failedAt } }); + await GitLabCredentialSeed.SetLastValidatedAsync(_fixture, seed.CredentialId, DateTimeOffset.UtcNow.AddSeconds(-60)); + + var remedy = await RemedyForAsync(team.UserId, team.TeamId, parentRunId); + + remedy.Origin!.RunId.ShouldBe(childRunId, "fixture check: the origin was read live and named — only then is its time the one to judge from"); + remedy.Resolved.ShouldBe(fixedSince); + } + + [Fact] + public async Task An_origin_that_is_not_this_teams_run_is_never_named_and_never_changes_the_remedy() + { + using var gitlab = new GitLabDoorkeeperStub(); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var other = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: false, existingTeam: team); + await DisconnectAsync(seed.CredentialId); + var foreign = await CreateNamedWorkflowAsync(other, "THE-OTHER-TEAMS-SECRET-WORKFLOW", ListThenEndDefinition(seed.RepositoryId)); + var foreignRunId = await WorkflowsTestSeed.SeedManualRunAsync(_fixture, foreign.Id, other.TeamId); + var ours = await CreateNamedWorkflowAsync(team, "Ours", ListThenEndDefinition(seed.RepositoryId)); + var details = new Dictionary { ["credentialId"] = seed.CredentialId.ToString() }; + + var forged = await StartRunOfAsync(team, SubWorkflowParent(ours, SubWorkflowSite.TopLevel)); + await FailRunAsync(forged, "Node 'sub' failed."); + await RecordFailureAsync(forged, "sub", ChildSentence, new PersistedFailure { Code = FailureCodes.CredentialDisconnected, Details = details, RepositoryId = seed.RepositoryId, Origin = new FailureOrigin { RunId = foreignRunId, NodeId = "list" } }); + + var remedy = await RemedyForAsync(team.UserId, team.TeamId, forged); + + remedy.Origin.ShouldBeNull("a run of another team is not named, nor linked to"); + JsonSerializer.Serialize(remedy).ShouldNotContain("THE-OTHER-TEAMS-SECRET-WORKFLOW"); + JsonSerializer.Serialize(remedy).ShouldNotContain(foreignRunId.ToString()); + remedy.Cause.ShouldNotContain("sub-workflow", Case.Insensitive); + remedy.Credential!.Id.ShouldBe(seed.CredentialId, "the remedy is still the parent's own, read under the parent's team"); + + // A run that is nobody's — gone, or never there — is not named either. + var missing = await StartRunOfAsync(team, SubWorkflowParent(ours, SubWorkflowSite.TopLevel)); + await FailRunAsync(missing, "Node 'sub' failed."); + await RecordFailureAsync(missing, "sub", ChildSentence, new PersistedFailure { Code = FailureCodes.CredentialDisconnected, Details = details, Origin = new FailureOrigin { RunId = Guid.NewGuid(), NodeId = "list" } }); + + (await RemedyForAsync(team.UserId, team.TeamId, missing)).Origin.ShouldBeNull(); + } + + [Fact] + public async Task A_parent_recorded_before_sub_workflows_carried_their_failures_answers_exactly_as_it_did() + { + using var gitlab = new GitLabDoorkeeperStub(); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: false, existingTeam: team); + await DisconnectAsync(seed.CredentialId); + var child = await CreateNamedWorkflowAsync(team, "Review", ListThenEndDefinition(seed.RepositoryId)); + var parentRunId = await StartRunOfAsync(team, SubWorkflowParent(child, SubWorkflowSite.TopLevel)); + + // What the engine wrote before: the sub node failed on the child's sentence alone, and the child's own run held the failure. + await FailRunAsync(parentRunId, "Node 'sub' failed."); + await RecordFailureAsync(parentRunId, "sub", ChildSentence, failure: null); + + using var scope = _fixture.BeginScope(); + + (await scope.Resolve().ReadAsync(parentRunId, team.TeamId, CancellationToken.None)).ShouldBeNull(); + (await RemedyForAsync(team.UserId, team.TeamId, parentRunId, allowNone: true)).ShouldBeNull("the row has no failure to build a remedy from, exactly as it never did"); + } + + [Fact] + public async Task The_resume_payload_carries_the_childs_failure_beside_the_status_outputs_and_error_it_always_carried() + { + using var gitlab = new GitLabDoorkeeperStub(); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: true, existingTeam: team); + var child = await CreateNamedWorkflowAsync(team, "Review", ListThenEndDefinition(seed.RepositoryId)); + var chain = await WalkThroughChildRunsAsync(await StartRunOfAsync(team, SubWorkflowParent(child, SubWorkflowSite.TopLevel))); + + var payload = await ReadResumePayloadAsync(chain[0]); + + payload.GetProperty("status").GetString().ShouldBe("Failure"); + payload.GetProperty("error").GetString().ShouldBe("Node 'list' failed."); + payload.GetProperty("outputs").ValueKind.ShouldBe(JsonValueKind.Object); + payload.GetProperty("failure").GetProperty("code").GetString().ShouldBe(FailureCodes.CredentialNeedsReconnect); + payload.GetProperty("failure").GetProperty("origin").GetProperty("runId").GetGuid().ShouldBe(chain[1]); + payload.GetProperty("failure").GetProperty("origin").GetProperty("nodeId").GetString().ShouldBe("list"); + payload.GetProperty("failure").GetProperty("origin").GetProperty("at").GetDateTimeOffset().ShouldBe(await ReadFailureTimeAsync(chain[1], team.TeamId), "the child's own failure time, for a parent resumed long after it"); + payload.GetRawText().ShouldNotContain("at-0", Case.Sensitive, "ids, names and codes travel — never a token"); + payload.GetRawText().ShouldNotContain("rt-0", Case.Sensitive); + } + + [Fact] + public async Task A_child_whose_failure_cannot_be_read_still_resumes_its_parent_with_the_sentence_alone() + { + // Reading what the child ended on is a courtesy to the parent's page. If the read itself fails, the parent must still be resumed — a parent left parked + // because its child's ledger could not be read would be a run stuck forever on something that was only ever there to explain it. + using var gitlab = new GitLabDoorkeeperStub(); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: true, existingTeam: team); + var child = await CreateNamedWorkflowAsync(team, "Review", ListThenEndDefinition(seed.RepositoryId)); + var parentRunId = await StartRunOfAsync(team, SubWorkflowParent(child, SubWorkflowSite.TopLevel)); + + await RunEngineAsync(parentRunId); // the parent parks on its child + var childRunId = await ChildRunIdAsync(parentRunId); + + using (var scope = _fixture.BeginScope()) + { + var db = scope.Resolve(); + var failed = await db.WorkflowRun.SingleAsync(r => r.Id == childRunId); + + failed.Status = WorkflowRunStatus.Failure; + failed.Error = "Node 'list' failed."; + failed.CompletedAt = DateTimeOffset.UtcNow; + await db.SaveChangesAsync(); + } + + using (var scope = _fixture.BeginScope()) + { + var db = scope.Resolve(); + var service = new CodeSpace.Core.Services.Workflows.Engine.SubworkflowService(db, scope.Resolve(), scope.Resolve(), scope.Resolve(), new UnreadableFailures(), Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + var resumed = await service.ResumeParentIfChildTerminalAsync(await db.WorkflowRun.AsNoTracking().SingleAsync(r => r.Id == childRunId), WorkflowRunStatus.Failure, "Node 'list' failed.", CancellationToken.None); + + resumed.ShouldBeTrue("the parent was resumed although the child's failure could not be read"); + } + + var payload = await ReadResumePayloadAsync(parentRunId); + + payload.GetProperty("status").GetString().ShouldBe("Failure"); + payload.GetProperty("error").GetString().ShouldBe("Node 'list' failed."); + payload.TryGetProperty("failure", out _).ShouldBeFalse("it carries the sentence alone"); + + await RunEngineAsync(parentRunId); + + (await ReadFailedRecordsAsync(parentRunId)).Single(r => r.NodeId == "sub").Failure.ShouldBeNull("and the parent fails as it always did"); + } + + [Fact] + public async Task A_child_that_was_cancelled_carries_nothing_even_when_an_agent_run_of_it_met_a_refused_push() + { + // The reader answers for a run that did not fail with one thing — a push the remote refused, which the run survived. A cancelled child did not end on + // that: the parent is told it was cancelled, and a card sending someone to reconnect a credential for it would blame the wrong thing. + using var gitlab = new GitLabDoorkeeperStub(); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: false, existingTeam: team); + var child = await CreateNamedWorkflowAsync(team, "Review", ListThenEndDefinition(seed.RepositoryId)); + var parentRunId = await StartRunOfAsync(team, SubWorkflowParent(child, SubWorkflowSite.TopLevel)); + + await RunEngineAsync(parentRunId); // the parent parks on its child + var childRunId = await ChildRunIdAsync(parentRunId); + + await SeedAgentRunAsync(team.TeamId, childRunId, "review", string.Empty, new PersistedFailure { Code = FailureCodes.GitCredentialRejected, RepositoryId = seed.RepositoryId }); + + using (var scope = _fixture.BeginScope()) + { + var db = scope.Resolve(); + var cancelled = await db.WorkflowRun.SingleAsync(r => r.Id == childRunId); + + cancelled.Status = WorkflowRunStatus.Cancelled; + cancelled.CompletedAt = DateTimeOffset.UtcNow; + await db.SaveChangesAsync(); + + (await scope.Resolve().ReadAsync(childRunId, team.TeamId, CancellationToken.None))!.Failure.Code.ShouldBe(FailureCodes.GitCredentialRejected, "fixture check: the reader WOULD answer the refused push for a run that did not fail"); + } + + using (var scope = _fixture.BeginScope()) + { + var db = scope.Resolve(); + + (await scope.Resolve().ResumeParentIfChildTerminalAsync(await db.WorkflowRun.AsNoTracking().SingleAsync(r => r.Id == childRunId), WorkflowRunStatus.Cancelled, null, CancellationToken.None)).ShouldBeTrue(); + } + + var cancelledPayload = await ReadResumePayloadAsync(parentRunId); + + cancelledPayload.TryGetProperty("failure", out _).ShouldBeFalse($"a cancelled child is a cancel, whatever its agent runs met: {cancelledPayload.GetRawText()}"); + + await RunEngineAsync(parentRunId); + + var sub = (await ReadFailedRecordsAsync(parentRunId)).Single(r => r.NodeId == "sub"); + + sub.Error.ShouldBe("Sub-workflow run failed: see the child run"); + sub.Failure.ShouldBeNull(); + } + + /// A failure reader whose read always fails — the database hiccup the courtesy must survive. + private sealed class UnreadableFailures : IRunFailureReader + { + public Task ReadAsync(Guid runId, Guid teamId, CancellationToken cancellationToken) => throw new InvalidOperationException("the child's ledger could not be read"); + } + + // ─── Helpers ─────────────────────────────────────────────────────────────── + + private sealed record NamedWorkflow(Guid Id, string Name); + + private async Task CreateNamedWorkflowAsync((Guid TeamId, Guid UserId) team, string name, WorkflowDefinition definition) + { + using var scope = _fixture.BeginScopeAs(team.UserId, team.TeamId, Roles.Admin); + var id = await scope.Resolve().Send(new CreateWorkflowCommand { Name = name, Description = null, Definition = definition, Activations = new List(), Enabled = true }); + + return new NamedWorkflow(id, name); + } + + /// The workflow that is the PARENT: it runs as a sub-workflow, on its own, inside a map's body, or inside a loop's body. + private static WorkflowDefinition SubWorkflowParent(NamedWorkflow child, SubWorkflowSite site) + { + var sub = Node("sub", "flow.subworkflow", site == SubWorkflowSite.TopLevel ? null : site == SubWorkflowSite.InMap ? "map" : "loop", config: $$"""{ "workflowId": "{{child.Id}}" }""", inputs: """{ "inputs": {} }"""); + + return site switch + { + SubWorkflowSite.TopLevel => new WorkflowDefinition { SchemaVersion = 1, Nodes = [Node("start", "trigger.manual", null), sub, Node("end", "builtin.terminal", null)], Edges = [Edge("start", "sub"), Edge("sub", "end")] }, + SubWorkflowSite.InMap => new WorkflowDefinition { SchemaVersion = 1, Nodes = [Node("start", "trigger.manual", null), MapNode("map", null), Node("ms", "flow.map_start", "map"), sub, Node("end", "builtin.terminal", null)], Edges = [Edge("start", "map"), Edge("map", "end"), Edge("ms", "sub")] }, + _ => new WorkflowDefinition { SchemaVersion = 1, Nodes = [Node("start", "trigger.manual", null), LoopNode("loop", null), Node("ls", "flow.loop_start", "loop"), sub, Node("end", "builtin.terminal", null)], Edges = [Edge("start", "loop"), Edge("loop", "end"), Edge("ls", "sub")] }, + }; + } + + /// The parent that runs two children at once and waits for both: at subA, at subB. + private static WorkflowDefinition FanOutParent(NamedWorkflow first, NamedWorkflow second) => new() + { + SchemaVersion = 1, + Nodes = + [ + Node("start", "trigger.manual", null), + Node("subA", "flow.subworkflow", null, config: $$"""{ "workflowId": "{{first.Id}}" }""", inputs: """{ "inputs": {} }"""), + Node("subB", "flow.subworkflow", null, config: $$"""{ "workflowId": "{{second.Id}}" }""", inputs: """{ "inputs": {} }"""), + Node("end", "builtin.terminal", null) + ], + Edges = [Edge("start", "subA"), Edge("start", "subB"), Edge("subA", "end"), Edge("subB", "end")], + }; + + /// The child run a parent's sub-workflow node is parked on — the token of its pending wait. + private async Task ChildRunOfNodeAsync(Guid parentRunId, string nodeId) + { + using var scope = _fixture.BeginScope(); + var token = await scope.Resolve().WorkflowRunWait.AsNoTracking().Where(w => w.RunId == parentRunId && w.NodeId == nodeId && w.WaitKind == WorkflowWaitKinds.Subworkflow).Select(w => w.Token).SingleAsync(); + + return Guid.Parse(token); + } + + /// Saves as a workflow of the team and seeds a manual run of it — one element for a map to run a branch over. + private async Task StartRunOfAsync((Guid TeamId, Guid UserId) team, WorkflowDefinition definition) + { + var workflow = await CreateNamedWorkflowAsync(team, "Parent " + Guid.NewGuid().ToString("N")[..6], definition); + + return await WorkflowsTestSeed.SeedManualRunAsync(_fixture, workflow.Id, team.TeamId, payloadJson: JsonSerializer.Serialize(new { things = new[] { Guid.NewGuid() } })); + } + + /// + /// Walks a run and every child run it parks on, to the end: the run until it parks on a child, the child the same way, and the run again once the child's + /// completion has resumed it. Returns the runs in the order they started — the run first, then the child, then the child's child. + /// + private async Task> WalkThroughChildRunsAsync(Guid runId) + { + var chain = new List { runId }; + + await RunEngineAsync(runId); + + if (await ChildRunIdsAsync(runId) is not [var childId]) return chain; + + chain.AddRange(await WalkThroughChildRunsAsync(childId)); + await RunEngineAsync(runId); + + return chain; + } + + private async Task> ChildRunIdsAsync(Guid parentRunId) + { + using var scope = _fixture.BeginScope(); + + return await scope.Resolve().WorkflowRun.AsNoTracking().Where(r => r.ParentRunId == parentRunId && r.SourceType == WorkflowRunSourceTypes.ChildWorkflow).OrderBy(r => r.CreatedDate).Select(r => r.Id).ToListAsync(); + } + + private async Task ChildRunIdAsync(Guid parentRunId) => (await ChildRunIdsAsync(parentRunId)).Single(); + + /// When the run's own record of its failure was written — what its page judges "fixed since" from. + private async Task ReadFailureTimeAsync(Guid runId, Guid teamId) + { + using var scope = _fixture.BeginScope(); + + return (await scope.Resolve().ReadAsync(runId, teamId, CancellationToken.None))!.At; + } + + /// What the parent's sub-workflow wait was resolved with — the payload the child's completion stamped, which the node read when it resumed. + private async Task ReadResumePayloadAsync(Guid parentRunId) + { + using var scope = _fixture.BeginScope(); + var payload = await scope.Resolve().WorkflowRunWait.AsNoTracking().Where(w => w.RunId == parentRunId && w.WaitKind == WorkflowWaitKinds.Subworkflow && w.Status == WorkflowWaitStatuses.Resolved).Select(w => w.PayloadJson).SingleAsync(); + + return JsonDocument.Parse(payload!).RootElement.Clone(); + } + + private async Task> ReadFailedPayloadsAsync(Guid runId) + { + using var scope = _fixture.BeginScope(); + + return await scope.Resolve().WorkflowRunRecord.AsNoTracking().Where(r => r.RunId == runId && r.RecordType == WorkflowRunRecordTypes.NodeFailed).OrderBy(r => r.Sequence).Select(r => r.PayloadJson).ToListAsync(); + } + + private async Task SetScopeAsync(Guid runId, Guid repositoryId) + { + using var scope = _fixture.BeginScope(); + var db = scope.Resolve(); + var run = await db.WorkflowRun.SingleAsync(r => r.Id == runId); + + run.ScopeRepositoryIds = [repositoryId]; + await db.SaveChangesAsync(); + } + + /// Ends a run on a sentence, as the engine would have — for a run whose ledger a test writes by hand. + private async Task FailRunAsync(Guid runId, string error) + { + using var scope = _fixture.BeginScope(); + var db = scope.Resolve(); + var run = await db.WorkflowRun.SingleAsync(r => r.Id == runId); + + run.Status = WorkflowRunStatus.Failure; + run.Error = error; + run.CompletedAt = DateTimeOffset.UtcNow; + await db.SaveChangesAsync(); + } + + private async Task RecordFailureAsync(Guid runId, string nodeId, string sentence, PersistedFailure? failure) + { + using var scope = _fixture.BeginScope(); + var logger = scope.Resolve(); + + await logger.NodeStartedAsync(runId, nodeId, string.Empty, new Dictionary(), new Dictionary(), CancellationToken.None); + await logger.NodeFailedAsync(runId, nodeId, string.Empty, sentence, TimeSpan.FromMilliseconds(5), failure, CancellationToken.None); + } +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/StuckRunReconcilerFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/StuckRunReconcilerFlowTests.cs index 1a412e4bd..ed476301d 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/StuckRunReconcilerFlowTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/StuckRunReconcilerFlowTests.cs @@ -928,6 +928,65 @@ public async Task A_stranded_subworkflow_parent_with_a_terminal_child_is_recover payload.GetProperty("outputs").GetProperty("result").GetString().ShouldBe("ok", "the child's REAL outputs ride to the parent — the same mapping as the happy path"); } + [Theory] + [InlineData(WorkflowRunStatus.Failure, true)] + [InlineData(WorkflowRunStatus.Failure, false)] + [InlineData(WorkflowRunStatus.Cancelled, true)] + public async Task A_stranded_subworkflow_parent_whose_child_ended_on_a_provider_failure_is_recovered_carrying_it_only_when_the_child_failed(WorkflowRunStatus childStatus, bool failedOnAProviderConnection) + { + // The reconciler re-fires the SAME resume the engine's completion hook runs, so a parent recovered after a lost hook learns what the child + // ended on exactly as one resumed in time does — read from the child's own ledger by the reader its own page asks. A child that was CANCELLED + // did not fail on anything: whatever its ledger holds, it is a cancel the parent is told about, never a connection to repair. + var (teamId, userId) = await WorkflowsTestSeed.SeedTeamAsync(_fixture); + var workflowId = await CreateWorkflowAsync(teamId, userId); + var credentialId = Guid.NewGuid(); + + var parentRunId = await StageStuckRunAsync(workflowId, teamId, status: WorkflowRunStatus.Suspended, createdAgo: TimeSpan.FromMinutes(10), backdateLastModified: true); + var childRunId = await SeedChildRunAsync(teamId, parentRunId, childStatus, outputsJson: "{}"); + await EndChildOnAsync(childRunId, "list", failedOnAProviderConnection ? CodeSpace.Core.Failures.ProviderFailureCapture.From(new CodeSpace.Messages.Exceptions.CredentialDisconnectedException(credentialId, "GitLab · NACHO.L"), Guid.NewGuid()) : null); + await SeedSubworkflowWaitAsync(parentRunId, childRunId); + + await ReconcileAsync(); + + using var scope = _fixture.BeginScope(); + var wait = await scope.Resolve().WorkflowRunWait.AsNoTracking().SingleAsync(w => w.RunId == parentRunId && w.WaitKind == WorkflowWaitKinds.Subworkflow); + + wait.Status.ShouldBe(WorkflowWaitStatuses.Resolved, "the re-fire resolved the wait"); + + var payload = JsonDocument.Parse(wait.PayloadJson!).RootElement; + + payload.GetProperty("status").GetString().ShouldBe(childStatus.ToString(), "the child's REAL status — nothing faked"); + payload.GetProperty("error").GetString().ShouldBe("Node 'list' failed."); + + if (childStatus != WorkflowRunStatus.Failure || !failedOnAProviderConnection) + { + payload.TryGetProperty("failure", out _).ShouldBeFalse("a child that failed on a sentence alone — or was cancelled — leaves the payload the shape it always was"); + return; + } + + payload.GetProperty("failure").GetProperty("code").GetString().ShouldBe("credential_disconnected"); + payload.GetProperty("failure").GetProperty("details").GetProperty("credentialId").GetGuid().ShouldBe(credentialId); + payload.GetProperty("failure").GetProperty("origin").GetProperty("runId").GetGuid().ShouldBe(childRunId, "the child run it happened in"); + payload.GetProperty("failure").GetProperty("origin").GetProperty("nodeId").GetString().ShouldBe("list"); + } + + /// Ends a seeded child run the way the engine does: the node's node.failed record (carrying the failure it was thrown with, when it was a provider connection) and the run's own sentence naming that node. + private async Task EndChildOnAsync(Guid childRunId, string nodeId, CodeSpace.Messages.Failures.PersistedFailure? failure) + { + using var scope = _fixture.BeginScope(); + var db = scope.Resolve(); + var logger = scope.Resolve(); + + await logger.NodeStartedAsync(childRunId, nodeId, string.Empty, new Dictionary(), new Dictionary(), CancellationToken.None); + await logger.NodeFailedAsync(childRunId, nodeId, string.Empty, "the node's own sentence", TimeSpan.FromMilliseconds(5), failure, CancellationToken.None); + + var child = await db.WorkflowRun.SingleAsync(r => r.Id == childRunId); + + child.Error = $"Node '{nodeId}' failed."; + child.CompletedAt = DateTimeOffset.UtcNow; + await db.SaveChangesAsync(); + } + [Fact] public async Task A_subworkflow_parent_whose_child_is_still_running_is_left_alone() { diff --git a/backend/tests/CodeSpace.UnitTests/Credentials/Remedy/RemedyCopyTests.cs b/backend/tests/CodeSpace.UnitTests/Credentials/Remedy/RemedyCopyTests.cs index 8c7f1053e..ee5036e1d 100644 --- a/backend/tests/CodeSpace.UnitTests/Credentials/Remedy/RemedyCopyTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Credentials/Remedy/RemedyCopyTests.cs @@ -1,5 +1,6 @@ using CodeSpace.Core.Services.Credentials.Remedy; using CodeSpace.Messages.Constants; +using CodeSpace.Messages.Dtos.Credentials; using CodeSpace.Messages.Failures; using Shouldly; @@ -128,4 +129,42 @@ public void Nothing_a_remedy_says_carries_a_token() RemedyCopy.Title(code, Facts()).ShouldNotBeNullOrWhiteSpace(); } } + + private static RemedyOrigin Origin(string? node = "list") => new() { RunId = Guid.Parse("1a2b3c4d-0000-0000-0000-000000000000"), WorkflowId = Guid.NewGuid(), WorkflowName = "Review the pull requests", NodeId = node }; + + [Fact] + public void A_failure_carried_up_from_a_child_run_names_the_run_it_happened_in_before_saying_what_is_wrong() + { + var cause = RemedyCopy.Cause(FailureCodes.CredentialNeedsReconnect, Facts() with { Origin = Origin() }); + + cause.ShouldStartWith("This failed in the sub-workflow “Review the pull requests” (run 1a2b3c4d), at node 'list'. "); + cause.ShouldContain("refused to renew the sign-in behind “GitLab · NACHO.L”"); + } + + [Fact] + public void A_child_run_that_names_no_node_is_named_without_one() + { + RemedyCopy.Cause(FailureCodes.CredentialDisconnected, Facts() with { Origin = Origin(node: null) }).ShouldStartWith("This failed in the sub-workflow “Review the pull requests” (run 1a2b3c4d). "); + } + + [Fact] + public void The_origin_leads_every_cause_and_changes_nothing_else_the_remedy_says() + { + foreach (var code in FailureCodes.ProviderConnection) + foreach (var resolved in new[] { false, true }) + { + var plain = Facts() with { Resolved = resolved }; + var carried = plain with { Origin = Origin() }; + + RemedyCopy.Cause(code, carried).ShouldEndWith(RemedyCopy.Cause(code, plain), Case.Sensitive, $"{code}/{resolved}: the child's words are the parent's, after the sentence that says where"); + RemedyCopy.Title(code, carried).ShouldBe(RemedyCopy.Title(code, plain), $"{code}/{resolved}: the title is the failure's, wherever it happened"); + } + } + + [Fact] + public void A_failure_that_happened_in_the_run_itself_reads_exactly_as_it_did() + { + RemedyCopy.Cause(FailureCodes.CredentialDisconnected, Facts()).ShouldNotContain("sub-workflow", Case.Insensitive); + RemedyCopy.Cause(FailureCodes.CredentialDisconnected, Facts() with { Origin = null }).ShouldBe(RemedyCopy.Cause(FailureCodes.CredentialDisconnected, Facts())); + } } diff --git a/backend/tests/CodeSpace.UnitTests/Failures/PersistedFailureJsonTests.cs b/backend/tests/CodeSpace.UnitTests/Failures/PersistedFailureJsonTests.cs index 94d738846..1f22c7987 100644 --- a/backend/tests/CodeSpace.UnitTests/Failures/PersistedFailureJsonTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Failures/PersistedFailureJsonTests.cs @@ -78,6 +78,106 @@ public void A_blank_or_malformed_detail_reads_as_absent() failure.DetailGuid("empty").ShouldBeNull(); } + [Fact] + public void Where_a_failure_happened_survives_the_trip_and_is_stored_beside_what_it_names() + { + var run = Guid.NewGuid(); + var failure = new PersistedFailure { Code = FailureCodes.ProviderUnauthorized, RepositoryId = Repository, Origin = new FailureOrigin { RunId = run, NodeId = "list" } }; + + PersistedFailureJson.ToElement(failure).GetRawText().ShouldBe($$$"""{"code":"provider_unauthorized","repositoryId":"{{{Repository}}}","origin":{"runId":"{{{run}}}","nodeId":"list"}}"""); + + var read = PersistedFailureJson.Read(PersistedFailureJson.AsPayload(failure)); + + read.ShouldNotBeNull(); + read.Origin.ShouldBe(new FailureOrigin { RunId = run, NodeId = "list" }); + read.RepositoryId.ShouldBe(Repository); + } + + [Fact] + public void When_the_failure_happened_in_that_run_is_stored_with_the_origin_and_survives_the_trip() + { + var run = Guid.NewGuid(); + var at = new DateTimeOffset(2026, 10, 19, 4, 29, 49, TimeSpan.Zero); + var failure = new PersistedFailure { Code = FailureCodes.ProviderUnauthorized, Origin = new FailureOrigin { RunId = run, NodeId = "list", At = at } }; + + PersistedFailureJson.ToElement(failure).GetRawText().ShouldBe($$$"""{"code":"provider_unauthorized","origin":{"runId":"{{{run}}}","nodeId":"list","at":"2026-10-19T04:29:49+00:00"}}"""); + PersistedFailureJson.Read(PersistedFailureJson.AsPayload(failure))!.Origin.ShouldBe(new FailureOrigin { RunId = run, NodeId = "list", At = at }); + } + + [Fact] + public void An_origin_written_before_it_kept_its_time_reads_with_none_and_is_stored_exactly_as_it_was() + { + var before = $$$"""{"code":"provider_unauthorized","origin":{"runId":"{{{Guid.NewGuid()}}}","nodeId":"list"}}"""; + var read = PersistedFailureJson.ReadElement(JsonDocument.Parse(before).RootElement)!; + + read.Origin!.At.ShouldBeNull("the record that carries it is then the only time there is"); + PersistedFailureJson.ToElement(read).GetRawText().ShouldBe(before, "an absent time is not written as null"); + } + + [Fact] + public void A_failure_with_an_origin_that_names_no_node_keeps_the_run_alone() + { + var run = Guid.NewGuid(); + var json = PersistedFailureJson.ToElement(new PersistedFailure { Code = FailureCodes.CredentialDisconnected, Origin = new FailureOrigin { RunId = run } }).GetRawText(); + + json.ShouldBe($$$"""{"code":"credential_disconnected","origin":{"runId":"{{{run}}}"}}""", "an absent node is not written as null"); + PersistedFailureJson.ReadElement(JsonDocument.Parse(json).RootElement)!.Origin!.NodeId.ShouldBeNull(); + } + + [Fact] + public void A_failure_written_before_origins_reads_with_none_and_is_stored_exactly_as_it_was() + { + var before = $$$"""{"code":"credential_disconnected","details":{"credentialId":"{{{Credential}}}"},"repositoryId":"{{{Repository}}}"}"""; + var read = PersistedFailureJson.ReadElement(JsonDocument.Parse(before).RootElement)!; + + read.Origin.ShouldBeNull(); + PersistedFailureJson.ToElement(read).GetRawText().ShouldBe(before, "reading an old row and writing it back adds nothing"); + } + + [Fact] + public void A_member_a_later_build_adds_to_a_failure_is_ignored_not_an_error() + { + // What an older reader meets after a rollback (and what this reader will meet from the next additive field): unknown members are skipped, + // so the failure — and the remedy built from it — still reads, minus only what the older reader never knew to look for. + var written = $$$$"""{"error":"x","failure":{"code":"credential_disconnected","origin":{"runId":"{{{{Guid.NewGuid()}}}}","nodeId":"n","someLaterMember":[1,2]},"aMemberFromTheFuture":{"a":1}}}"""; + + PersistedFailureJson.Read(written)!.Code.ShouldBe(FailureCodes.CredentialDisconnected); + } + + /// The failure exactly as it was before origins: a build rolled back to it (or a worker not yet upgraded) deserializes records with this shape. + private sealed record PreviousShape(string Code, Dictionary? Details, Guid? RepositoryId); + + /// The failure as it was when an origin named only the run and the node: a build rolled back to that one reads a record that now also carries the time. + private sealed record PreviousOriginShape(string Code, PreviousOrigin? Origin); + + private sealed record PreviousOrigin(Guid RunId, string? NodeId); + + [Fact] + public void A_build_that_predates_the_origins_time_reads_what_is_written_now_minus_the_time() + { + var run = Guid.NewGuid(); + var written = PersistedFailureJson.ToElement(new PersistedFailure { Code = FailureCodes.ProviderUnauthorized, Origin = new FailureOrigin { RunId = run, NodeId = "list", At = DateTimeOffset.UtcNow } }); + + var old = written.Deserialize(new JsonSerializerOptions(JsonSerializerDefaults.Web))!; + + old.Code.ShouldBe(FailureCodes.ProviderUnauthorized); + old.Origin.ShouldBe(new PreviousOrigin(run, "list"), "the older reader still names the run and the node; the time it never knew is skipped"); + } + + [Fact] + public void A_build_that_predates_origins_reads_what_is_written_now_minus_the_origin() + { + var run = Guid.NewGuid(); + var written = PersistedFailureJson.ToElement(new PersistedFailure { Code = FailureCodes.CredentialNeedsReconnect, Details = new Dictionary { ["credentialId"] = Credential.ToString() }, RepositoryId = Repository, Origin = new FailureOrigin { RunId = run, NodeId = "list", At = DateTimeOffset.UtcNow } }); + + // The same serializer defaults the ledger reads with: unknown members are skipped, so the older build still has the failure — and the remedy built from it. + var old = written.Deserialize(new JsonSerializerOptions(JsonSerializerDefaults.Web))!; + + old.Code.ShouldBe(FailureCodes.CredentialNeedsReconnect); + old.Details!["credentialId"].ShouldBe(Credential.ToString()); + old.RepositoryId.ShouldBe(Repository); + } + [Fact] public void An_agent_result_names_its_failure_or_a_refused_push() { diff --git a/backend/tests/CodeSpace.UnitTests/Failures/ProviderFailureCaptureTests.cs b/backend/tests/CodeSpace.UnitTests/Failures/ProviderFailureCaptureTests.cs index cee64f217..2be9c9256 100644 --- a/backend/tests/CodeSpace.UnitTests/Failures/ProviderFailureCaptureTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Failures/ProviderFailureCaptureTests.cs @@ -132,4 +132,28 @@ public void Nothing_thrown_is_nothing_captured() { ProviderFailureCapture.From(null, Repository).ShouldBeNull(); } + + [Theory] + [InlineData(FailureCodes.CredentialDisconnected)] + [InlineData(FailureCodes.CredentialNeedsReconnect)] + [InlineData(FailureCodes.ProviderUnauthorized)] + [InlineData(FailureCodes.GitCredentialRejected)] + [InlineData(FailureCodes.ProviderBaseUrlRedirects)] + [InlineData(FailureCodes.OAuthExchangeFailed)] + public void A_recorded_provider_connection_failure_is_kept_exactly_as_recorded(string code) + { + var recorded = new PersistedFailure { Code = code, RepositoryId = Repository, Origin = new FailureOrigin { RunId = Guid.NewGuid(), NodeId = "list" } }; + + ProviderFailureCapture.OfRecord(recorded).ShouldBeSameAs(recorded); + } + + [Theory] + [InlineData(FailureCodes.Internal)] + [InlineData(FailureCodes.Forbidden)] + [InlineData("not_a_code")] + public void A_recorded_failure_that_is_not_a_provider_connection_is_not_kept_and_nothing_is_nothing(string code) + { + ProviderFailureCapture.OfRecord(new PersistedFailure { Code = code }).ShouldBeNull("what is carried onward is never wider than what is captured"); + ProviderFailureCapture.OfRecord(null).ShouldBeNull(); + } } diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/FlowSubworkflowNodeFailureTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/FlowSubworkflowNodeFailureTests.cs new file mode 100644 index 000000000..005f563fb --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Workflows/FlowSubworkflowNodeFailureTests.cs @@ -0,0 +1,134 @@ +using System.Text.Json; +using CodeSpace.Core.Services.Workflows.Failures; +using CodeSpace.Core.Services.Workflows.Nodes; +using CodeSpace.Core.Services.Workflows.Nodes.Builtin; +using CodeSpace.Core.Services.Workflows.Runtime; +using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Failures; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; + +namespace CodeSpace.UnitTests.Workflows; + +/// +/// What a flow.subworkflow node does with the payload the child's completion resumes it with: a success maps the child's outputs, a failure +/// is the node's own sentence — unchanged — and, when the child ended on a provider failure a person can repair, that failure beside it. A payload +/// stamped before children carried theirs, one carrying nothing, and one carrying anything that is not a provider connection failure all fail the +/// node with the sentence alone, and nothing here ever throws: a payload that cannot be read must never fail the run worse than the child already did. +/// +[Trait("Category", "Unit")] +public class FlowSubworkflowNodeFailureTests +{ + private static readonly Guid ChildRun = Guid.NewGuid(); + private static readonly Guid Credential = Guid.NewGuid(); + + private static Task ResumeWithAsync(string payload) => new FlowSubworkflowNode().RunAsync(new NodeRunContext + { + NodeId = "sub", + Config = new Dictionary { ["workflowId"] = JsonSerializer.SerializeToElement(Guid.NewGuid()) }, + Inputs = new Dictionary(), + RawConfig = JsonDocument.Parse("{}").RootElement.Clone(), + RawInputs = JsonDocument.Parse("{}").RootElement.Clone(), + Scope = new NodeRunScope { Trigger = new Dictionary() }, + Logger = NullLogger.Instance, + Observability = NodeObservability.NoOp, + ResumePayload = JsonDocument.Parse(payload).RootElement.Clone(), + }, CancellationToken.None); + + private static string FailedPayload(string failureMember = "") => $$$"""{"status":"Failure","outputs":{},"error":"Node 'list' failed."{{{failureMember}}}}"""; + + private static string Failure(string code = FailureCodes.CredentialNeedsReconnect) => + $$$""","failure":{"code":"{{{code}}}","details":{"credentialId":"{{{Credential}}}"},"repositoryId":"{{{Guid.NewGuid()}}}","origin":{"runId":"{{{ChildRun}}}","nodeId":"list"}}"""; + + [Fact] + public async Task A_child_that_ended_on_a_provider_failure_fails_the_node_with_its_own_sentence_and_the_failure_beside_it() + { + var result = await ResumeWithAsync(FailedPayload(Failure())); + + result.Status.ShouldBe(NodeStatus.Failure); + result.Error.ShouldBe("Sub-workflow run failed: Node 'list' failed.", "the sentence is the node's own and is exactly what it always was"); + result.CarriedFailure.ShouldNotBeNull(); + result.CarriedFailure.Code.ShouldBe(FailureCodes.CredentialNeedsReconnect); + result.CarriedFailure.DetailGuid("credentialId").ShouldBe(Credential); + result.CarriedFailure.Origin.ShouldBe(new FailureOrigin { RunId = ChildRun, NodeId = "list" }); + result.Retryable.ShouldBeTrue("carrying a failure never changes whether the node retries"); + result.Cause.ShouldBeNull("there is no exception: the failure was read back, not thrown"); + } + + [Fact] + public async Task The_time_the_child_recorded_the_failure_rides_through_the_node_untouched() + { + var at = new DateTimeOffset(2026, 10, 19, 4, 29, 49, TimeSpan.Zero); + var failure = $$$""","failure":{"code":"provider_unauthorized","origin":{"runId":"{{{ChildRun}}}","nodeId":"list","at":"2026-10-19T04:29:49+00:00"}}"""; + + (await ResumeWithAsync(FailedPayload(failure))).CarriedFailure!.Origin.ShouldBe(new FailureOrigin { RunId = ChildRun, NodeId = "list", At = at }, "when the child failed, not when the node woke up"); + } + + [Theory] + [InlineData("")] // a payload stamped before children carried their failures + [InlineData(",\"failure\":null")] + [InlineData(",\"failure\":\"a string\"")] + [InlineData(",\"failure\":{}")] + [InlineData(",\"failure\":{\"code\":\"\"}")] + [InlineData(",\"failure\":{\"code\":\"credential_disconnected\",\"origin\":\"not an object\"}")] // the whole failure is unreadable, so none is carried — never a throw + public async Task A_payload_with_no_readable_failure_fails_the_node_with_the_sentence_alone(string failureMember) + { + var result = await ResumeWithAsync(FailedPayload(failureMember)); + + result.Status.ShouldBe(NodeStatus.Failure); + result.Error.ShouldBe("Sub-workflow run failed: Node 'list' failed."); + result.CarriedFailure.ShouldBeNull(); + } + + [Theory] + [InlineData(FailureCodes.Internal)] + [InlineData(FailureCodes.RunBudgetExhausted)] + [InlineData("a_code_nobody_declared")] + public async Task A_failure_that_is_not_a_provider_connection_is_not_carried(string code) + { + var result = await ResumeWithAsync(FailedPayload(Failure(code))); + + result.Status.ShouldBe(NodeStatus.Failure); + result.CarriedFailure.ShouldBeNull("only the failures a person can repair outside the run are written down, wherever they are read back"); + } + + [Theory] + [InlineData(FailureCodes.CredentialDisconnected)] + [InlineData(FailureCodes.CredentialNeedsReconnect)] + [InlineData(FailureCodes.ProviderUnauthorized)] + [InlineData(FailureCodes.GitCredentialRejected)] + [InlineData(FailureCodes.ProviderBaseUrlRedirects)] + [InlineData(FailureCodes.OAuthExchangeFailed)] + public async Task Every_provider_connection_failure_is_carried(string code) + { + (await ResumeWithAsync(FailedPayload(Failure(code)))).CarriedFailure!.Code.ShouldBe(code); + } + + [Fact] + public async Task A_child_without_an_error_sentence_still_fails_the_node_with_the_sentence_it_always_had() + { + var result = await ResumeWithAsync("""{"status":"Cancelled","outputs":{},"error":null}"""); + + result.Status.ShouldBe(NodeStatus.Failure); + result.Error.ShouldBe("Sub-workflow run failed: see the child run"); + result.CarriedFailure.ShouldBeNull(); + } + + [Fact] + public async Task A_child_that_succeeded_maps_its_outputs_and_carries_nothing_even_if_a_failure_rides_the_payload() + { + var result = await ResumeWithAsync($$$"""{"status":"Success","outputs":{"result":"ok"},"error":null{{{Failure()}}}}"""); + + result.Status.ShouldBe(NodeStatus.Success); + result.Outputs["result"].GetString().ShouldBe("ok"); + result.CarriedFailure.ShouldBeNull(); + } + + [Fact] + public void The_failure_a_payload_carries_is_the_shape_a_ledger_record_keeps_it_in() + { + // The completion hook writes it with PersistedFailureJson and the node reads it with PersistedFailureJson: the same key, the same object. + PersistedFailureJson.PayloadKey.ShouldBe("failure"); + PersistedFailureJson.Read(FailedPayload(Failure()))!.Origin!.RunId.ShouldBe(ChildRun); + } +} diff --git a/backend/tests/TESTING.md b/backend/tests/TESTING.md index edeb5e4f7..8abe5beca 100644 --- a/backend/tests/TESTING.md +++ b/backend/tests/TESTING.md @@ -89,6 +89,23 @@ Provider-credential tests share two more pieces, both in `CodeSpace.IntegrationT again on the replayed failed branch answers with the original run's remedy, never the fresh branch's (`ProviderFailureThroughTheEngineFlowTests.Rerun.cs`, and over HTTP in `ProviderFailureInAContainerHttpE2ETests`, which also pins what the run detail and the lean remedy read answer, plus that a run recorded before containers carried the failure answers exactly as it did). +- A provider failure inside a `flow.subworkflow` CHILD run that ends the PARENT is read back where the child recorded it — by the same + `IRunFailureReader` the child's own page asks — and carried up: the engine's completion hook (`ISubworkflowService.ResumeParentIfChildTerminalAsync`, the + one resume path the reconciler's stranded-parent backstop re-fires too) stamps it on the resume payload, the node fails with its own sentence untouched and + the failure beside it (`NodeResult.CarriedFailure`), and the parent's `node.failed` — and, through a map or loop the node sits in, the container's — carries + it with a `FailureOrigin` naming the deepest child run that met it, the node that ended that run and when that run recorded it. `ProviderFailureThroughTheEngineFlowTests.SubWorkflows.cs` + drives the real engine parent → child (top level, in a map, in a loop, and three levels deep) against the stub and pins: the parent's remedy is the + child's (same code, credential, repositories, buttons, resolution — compared with the child's own remedy) plus the origin and a sentence that names it; a + failure that names no repository is traced through the CHILD's launch scope, never the parent's; a retried node carries the last attempt's child; and the + negatives — a child that failed on something else, handled the failure, or handled it and failed on something else carries nothing — plus an origin that is + not the team's run (another team's, or none) is never named and never changes the remedy; a fan-out parent (two children at once, one slow) whose token is replaced between the + child's failure and the barrier's release reads "fixed" exactly as the child's page does, because the fix is measured from when the CHILD recorded the failure + (`FailureOrigin.At`), not from the parent's later record — and a record that kept no time falls back to the carrying record's own; a child whose node worded + its failure alone is traced through its own agent run, as the child's page is; a cancelled child carries nothing even when an agent run of it met a refused push; and a map-branch rerun fork that fails on a + replayed sub-workflow branch names the child the ORIGINAL branch ran (`….Rerun.cs`). `StuckRunReconcilerFlowTests` pins the backstop path; + `ProviderFailureInASubWorkflowHttpE2ETests` pins the wire shape for an Owner, a Member and a Viewer, that Re-run on the parent replays the PARENT and + starts a fresh child, and that a parent recorded before the field answers as it did. The frontend's `ProviderRemedyCard.test.tsx` / `RunRemedyCard.test.tsx` + pin the "In sub-workflow … (run …)" line, its link, and that Re-run replays the run the page is on. - `RefreshPodHarness` — pods as real OS processes (`CodeSpace.StorageTestWorker`'s `oauth-refresh-pod` mode) running the production refresher, or the pinned previous build's (`StorageTestWorker/PreviousBuild`), against the fixture's Postgres. One process shares one refresh semaphore, so only separate processes exercise the cross-pod advisory lock. diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index f96156cee..2eaeed5f0 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -126,6 +126,20 @@ export interface RemedyProviderInstance { baseUrl: string; } +/** + * The sub-workflow run a failure happened in, when the run the remedy is for ran it as a step and failed on what it ended on. Only says where + * to look: Re-run is still the run the card is on. Absent for a failure that happened in the run itself — and for a run written before + * sub-workflows carried their failures, which reads exactly as it did. + */ +export interface RemedyOrigin { + /** The sub-workflow run — the deepest one, when sub-workflows are nested. */ + runId: string; + workflowId: string; + workflowName: string; + /** The node that ended that run. */ + nodeId?: string | null; +} + export interface RemedyPerson { userId: string; name: string; @@ -149,6 +163,8 @@ export interface ProviderRemedy { providerInstance?: RemedyProviderInstance | null; /** For a redirecting provider: the origin it answered with — what its base URL would be edited to. */ redirectsTo?: string | null; + /** The sub-workflow run the failure happened in, when it was carried up to this run from one. */ + origin?: RemedyOrigin | null; actions: RemedyAction[]; askPeople: RemedyPerson[]; } diff --git a/frontend/src/components/remedy/ProviderRemedyCard.test.tsx b/frontend/src/components/remedy/ProviderRemedyCard.test.tsx index 0c2e3b42b..6bb219218 100644 --- a/frontend/src/components/remedy/ProviderRemedyCard.test.tsx +++ b/frontend/src/components/remedy/ProviderRemedyCard.test.tsx @@ -33,7 +33,7 @@ const remedy = (over: Partial = {}): ProviderRemedy => ({ type Call = { method: string; url: string; body: unknown }; -function renderCard(r: ProviderRemedy, props: { onRerun?: () => void; rawDetail?: string } = {}, answers: Record = {}) { +function renderCard(r: ProviderRemedy, props: { onRerun?: () => void; rawDetail?: string; onOpenRun?: (runId: string) => void } = {}, answers: Record = {}) { localStorage.setItem("codespace.jwt", "test-jwt"); localStorage.setItem("codespace.activeTeamId", team.id); @@ -107,6 +107,66 @@ describe("ProviderRemedyCard — what it says", () => { }); }); +describe("ProviderRemedyCard — a failure that happened in a sub-workflow", () => { + const origin = { runId: "1a2b3c4d-0000-4000-8000-000000000000", workflowId: "w-child", workflowName: "Review the pull requests", nodeId: "list" }; + const carried = (over: Partial = {}) => remedy({ + cause: "This failed in the sub-workflow “Review the pull requests” (run 1a2b3c4d), at node 'list'. GitLab refused to renew the sign-in behind “GitLab · nacho”.", + origin, + ...over, + }); + + it("names the sub-workflow and its run, and opens that run from the link", () => { + const onOpenRun = vi.fn(); + + renderCard(carried(), { onRerun: vi.fn(), onOpenRun }); + + const link = screen.getByRole("button", { name: "In sub-workflow Review the pull requests (run 1a2b3c4d)" }); + + expect(link.getAttribute("title")).toBe("Open the sub-workflow run"); + + fireEvent.click(link); + + expect(onOpenRun).toHaveBeenCalledTimes(1); + expect(onOpenRun).toHaveBeenCalledWith(origin.runId); + }); + + it("names the run as text, with no link, where no run can be opened", () => { + renderCard(carried()); + + expect(screen.getByText("In sub-workflow Review the pull requests (run 1a2b3c4d)")).toBeTruthy(); + expect(screen.queryByRole("button", { name: /In sub-workflow/ })).toBeNull(); + }); + + it("keeps the credential, the repository and every action exactly as the backend sent them, and Re-run is the card's own", () => { + const onRerun = vi.fn(); + const onOpenRun = vi.fn(); + + renderCard(carried({ resolved: true, actions: [action("rerun"), action("reconnect"), action("open_providers")] }), { onRerun, onOpenRun }); + + expect(screen.getByText("GitLab · nacho")).toBeTruthy(); + expect(screen.getByText("acme/api")).toBeTruthy(); + expect(screen.getAllByRole("button").map((b) => b.textContent?.trim())).toEqual(["In sub-workflow Review the pull requests (run 1a2b3c4d)", "Re-run", "Reconnect GitLab", "Open Providers"]); // the line that says where sits above the actions, which are the backend's list as sent + + fireEvent.click(screen.getByRole("button", { name: "Re-run" })); + + expect(onRerun).toHaveBeenCalledTimes(1); + expect(onOpenRun).not.toHaveBeenCalled(); // re-running is the parent's: it never opens the child + }); + + it.each([undefined, null])("says nothing about a sub-workflow when the failure happened in the run itself (origin %s)", (value) => { + renderCard(remedy({ origin: value }), { onOpenRun: vi.fn() }); + + expect(screen.queryByText(/sub-workflow/i)).toBeNull(); + expect(screen.getAllByRole("button").map((b) => b.textContent?.trim())).toEqual(["Reconnect GitLab", "Re-link repository", "Open Providers"]); + }); + + it("names a run whose workflow has no node recorded exactly the same way", () => { + renderCard(carried({ origin: { ...origin, nodeId: null } })); + + expect(screen.getByText("In sub-workflow Review the pull requests (run 1a2b3c4d)")).toBeTruthy(); + }); +}); + describe("ProviderRemedyCard — a failure that is nobody's credential to repair", () => { it("says what the provider did and offers only what the backend offered: no repair button for a renewal that was not answered", () => { renderCard(remedy({ diff --git a/frontend/src/components/remedy/ProviderRemedyCard.tsx b/frontend/src/components/remedy/ProviderRemedyCard.tsx index ad67d9607..6d8d920f1 100644 --- a/frontend/src/components/remedy/ProviderRemedyCard.tsx +++ b/frontend/src/components/remedy/ProviderRemedyCard.tsx @@ -1,7 +1,7 @@ import { useState } from "react"; import { Ic } from "@/_imported/ai-code-space/icons"; -import type { ProviderRemedy, RemedyAction, RemedyActionKind } from "@/api/types"; +import type { ProviderRemedy, RemedyAction, RemedyActionKind, RemedyOrigin } from "@/api/types"; import { useProviderFixes } from "./useProviderFixes"; @@ -14,6 +14,8 @@ interface ProviderRemedyCardProps { rawDetail?: string | null; /** Called after a fix landed — the caller re-reads whatever it shows. */ onFixed?: () => void; + /** Opens another run: the sub-workflow run a failure happened in. Absent where no run viewer is mounted, and the run is then named without a link. */ + onOpenRun?: (runId: string) => void; } /** @@ -25,8 +27,11 @@ interface ProviderRemedyCardProps { * * Once what failed works again the card says so and puts Re-run first — the repair is done, the run only needs another go. Until then * there is no Re-run to press: nothing about the run changes until the connection does. + * + * A failure a sub-workflow node carried up from its child run is the child's remedy on the parent's page, plus a line naming the child run it + * happened in (a link, where a run can be opened). Re-run stays the parent's: it starts the parent again, which starts a fresh child. */ -export function ProviderRemedyCard({ remedy, onRerun, rerunning = false, rawDetail = null, onFixed }: ProviderRemedyCardProps) { +export function ProviderRemedyCard({ remedy, onRerun, rerunning = false, rawDetail = null, onFixed, onOpenRun }: ProviderRemedyCardProps) { const fix = useProviderFixes({ onFixed }); const [showRaw, setShowRaw] = useState(false); const credential = remedy.credential ?? null; @@ -75,8 +80,9 @@ export function ProviderRemedyCard({ remedy, onRerun, rerunning = false, rawDeta
{remedy.title}

{remedy.cause}

- {(credential || remedy.repositories.length > 0) && ( + {(remedy.origin || credential || remedy.repositories.length > 0) && (
+ {remedy.origin && } {credential && ( {credential.displayName} @@ -120,6 +126,21 @@ export function ProviderRemedyCard({ remedy, onRerun, rerunning = false, rawDeta ); } +/** Where the failure happened when it was not in this run: the sub-workflow and its run, the way run pages show a run (its first eight characters) — a link where the run can be opened. */ +function OriginSubject({ origin, onOpenRun }: { origin: RemedyOrigin; onOpenRun?: (runId: string) => void }) { + const label = `In sub-workflow ${origin.workflowName} (run ${origin.runId.slice(0, 8)})`; + + return onOpenRun ? ( + + ) : ( + + {label} + + ); +} + function actionIcon(kind: RemedyActionKind) { switch (kind) { case "relink": return ; diff --git a/frontend/src/components/remedy/RunRemedyCard.test.tsx b/frontend/src/components/remedy/RunRemedyCard.test.tsx new file mode 100644 index 000000000..dfc0c85ef --- /dev/null +++ b/frontend/src/components/remedy/RunRemedyCard.test.tsx @@ -0,0 +1,109 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { ProviderRemedy } from "@/api/types"; +import { DialogProvider } from "@/components/dialog"; +import { RunOpenContext } from "@/components/workflows/runOpenContext"; + +import { RunRemedyCard } from "./RunRemedyCard"; + +/** + * The remediation card for a run page. For a failure carried up from a sub-workflow's child run the page the card is on is the PARENT's: the + * remedy is read for the parent's run id, the link opens the child run the failure happened in, and Re-run replays the PARENT — which starts a + * fresh child — never the child the failure happened in. + */ +const PARENT = "11111111-1111-4111-8111-111111111111"; +const CHILD = "1a2b3c4d-0000-4000-8000-000000000000"; +const REPLAY = "99999999-9999-4999-8999-999999999999"; + +const remedy: ProviderRemedy = { + code: "credential_needs_reconnect", + title: "GitLab no longer accepts the credential this run used", + cause: "This failed in the sub-workflow “Review the pull requests” (run 1a2b3c4d), at node 'list'. GitLab refused to renew the sign-in behind “GitLab · nacho”.", + resolved: true, + credential: { id: "c1", displayName: "GitLab · nacho", ownership: "Personal", ownerUserId: "u1", ownerName: "Nacho", authType: "OAuth", status: "Active", health: "active", retired: false, providerInstanceId: "pi1" }, + repositories: [{ id: "r1", fullPath: "acme/api", status: "Active", credentialId: "c1", credentialWorks: true }], + providerInstance: null, + origin: { runId: CHILD, workflowId: "w-child", workflowName: "Review the pull requests", nodeId: "list" }, + actions: [{ kind: "rerun", label: "Re-run", enabled: true }, { kind: "open_providers", label: "Open Providers", enabled: true }], + askPeople: [], +}; + +type Call = { method: string; url: string }; + +function renderCard(props: { onOpenRun?: (runId: string) => void; context?: ((runId: string) => void) | null } = {}) { + localStorage.setItem("codespace.jwt", "test-jwt"); + localStorage.setItem("codespace.activeTeamId", "t1"); + + const calls: Call[] = []; + + vi.stubGlobal("fetch", vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = typeof input === "string" ? input : input.toString(); + const method = init?.method ?? "GET"; + + calls.push({ method, url }); + + const body = url.endsWith(`/api/workflows/runs/${PARENT}/remedy`) ? remedy : url.endsWith(`/api/workflows/runs/${PARENT}/replay`) ? { runId: REPLAY } : undefined; + + return new Response(body === undefined ? "" : JSON.stringify(body), { status: body === undefined ? 404 : 200, headers: { "Content-Type": "application/json" } }); + })); + + render( + + + + + + + , + ); + + return calls; +} + +afterEach(() => { localStorage.clear(); vi.unstubAllGlobals(); vi.restoreAllMocks(); }); + +describe("RunRemedyCard — a failure carried up from a sub-workflow's child run", () => { + it("reads the remedy for the run the page is on and links to the child run it happened in", async () => { + const onOpenRun = vi.fn(); + const calls = renderCard({ onOpenRun }); + + fireEvent.click(await screen.findByRole("button", { name: "In sub-workflow Review the pull requests (run 1a2b3c4d)" })); + + expect(onOpenRun).toHaveBeenCalledWith(CHILD); + expect(calls.filter((c) => c.url.includes("/remedy")).map((c) => c.url)).toEqual([expect.stringContaining(`/api/workflows/runs/${PARENT}/remedy`)]); + }); + + it("opens the child through the run viewer's context when the page was not handed a handler", async () => { + const context = vi.fn(); + + renderCard({ context }); + + fireEvent.click(await screen.findByRole("button", { name: /In sub-workflow Review the pull requests/ })); + + expect(context).toHaveBeenCalledWith(CHILD); + }); + + it("names the run without a link where no run can be opened", async () => { + renderCard(); + + expect(await screen.findByText("In sub-workflow Review the pull requests (run 1a2b3c4d)")).toBeTruthy(); + expect(screen.queryByRole("button", { name: /In sub-workflow/ })).toBeNull(); + }); + + it("re-runs the PARENT — the run the page is on — and opens the run that starts, never the child the failure happened in", async () => { + const onOpenRun = vi.fn(); + const calls = renderCard({ onOpenRun }); + + fireEvent.click(await screen.findByRole("button", { name: "Re-run" })); + + await waitFor(() => expect(onOpenRun).toHaveBeenCalledWith(REPLAY)); + + const replays = calls.filter((c) => c.method === "POST"); + + expect(replays.map((c) => c.url)).toEqual([expect.stringContaining(`/api/workflows/runs/${PARENT}/replay`)]); + expect(calls.some((c) => c.url.includes(CHILD))).toBe(false); // nothing is asked of, or done to, the child run + expect(onOpenRun).not.toHaveBeenCalledWith(CHILD); + }); +}); diff --git a/frontend/src/components/remedy/RunRemedyCard.tsx b/frontend/src/components/remedy/RunRemedyCard.tsx index 993174522..6f3e9b6a5 100644 --- a/frontend/src/components/remedy/RunRemedyCard.tsx +++ b/frontend/src/components/remedy/RunRemedyCard.tsx @@ -11,7 +11,8 @@ import { ProviderRemedyCard } from "./ProviderRemedyCard"; * read still loading, or it failed). The answer is read fresh and re-read after any fix, so the card turns to "run it again" the * moment the connection works. * - * Re-run replays the run and opens the new one, the same Re-run the turn's own footer offers. + * Re-run replays the run and opens the new one, the same Re-run the turn's own footer offers. For a failure carried up from a sub-workflow's child + * run it is still THIS run that is replayed — the parent, which starts a fresh child — and the card links to the child run the failure happened in. */ export function RunRemedyCard({ runId, rawDetail, onOpenRun }: { runId: string; rawDetail?: string | null; onOpenRun?: (runId: string) => void }) { const remedy = useRunRemedy(runId); @@ -26,5 +27,5 @@ export function RunRemedyCard({ runId, rawDetail, onOpenRun }: { runId: string; openRun?.(result.runId); }; - return void remedy.refetch()} />; + return void remedy.refetch()} onOpenRun={openRun ?? undefined} />; } diff --git a/frontend/src/styles/ai-code-space.css b/frontend/src/styles/ai-code-space.css index 49b83d6b1..971073718 100644 --- a/frontend/src/styles/ai-code-space.css +++ b/frontend/src/styles/ai-code-space.css @@ -6547,6 +6547,8 @@ body.chat-resizing .acs-root .app { transition: none; } .rmd-subject { display: inline-flex; align-items: center; gap: 5px; max-width: 100%; padding: 2px 9px; border: 1px solid var(--line); border-radius: 999px; background: #fff; font-size: 11.5px; line-height: 1.6; color: var(--ink-2); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .rmd-subject[data-works="true"] { border-color: var(--rm-good-line); color: var(--rm-good); } .rmd-subject .cn-cred-owner { margin-left: 2px; } +button.rmd-subject { font: inherit; font-size: 11.5px; line-height: 1.6; color: var(--accent); cursor: pointer; transition: background .12s, border-color .12s; } +button.rmd-subject:hover { background: var(--panel-2); border-color: var(--line-2); } .rmd-ask { margin: 10px 0 0; font-size: 12px; line-height: 1.55; color: var(--ink-2); } .rmd-ask-list { margin: 3px 0 5px; padding-left: 18px; } .rmd-ask-list strong { font-weight: 600; color: var(--ink); }