Guide the fix for a provider failure where it shows up

backend/src/CodeSpace.Api/Controllers/WorkflowRunsController.cs

@@ -104,6 +104,17 @@ public async Task<IActionResult> Get([FromRoute] string idOrNumber, Cancellation
         return result == null ? NotFound() : Ok(result);
     }
 
+    /// <summary>
+    /// What is wrong with the provider connection a run failed on, and what the CALLER can do about it. The run page asks this for a
+    /// failed turn instead of a whole run detail; 200 with <c>null</c> when the run recorded no failure a person can repair. Team-scoped.
+    /// </summary>
+    [HttpGet("{runId:guid}/remedy")]
+    public async Task<IActionResult> GetRemedy([FromRoute] Guid runId, CancellationToken cancellationToken)
+    {
+        var result = await _mediator.Send(new GetWorkflowRunRemedyQuery { RunId = runId }, cancellationToken).ConfigureAwait(false);
+        return Ok(result);
+    }
+
     /// <summary>
     /// Bounded, metadata-only canvas view. It exposes exact cell coordinates and a narrow frozen topology, never run
     /// outputs, node inputs/outputs, wait payloads, artifact references, prompts or node configuration.

backend/src/CodeSpace.Core/Failures/ProviderFailureCapture.cs

@@ -0,0 +1,49 @@
+using CodeSpace.Messages.Failures;
+
+namespace CodeSpace.Core.Failures;
+
+/// <summary>
+/// Reads the failure a thrown exception DECLARED and, when it is one a person can repair outside the run, turns it into the
+/// <see cref="PersistedFailure"/> a run's record keeps. Pure and static for the reason <see cref="FailureClassifier"/> is: it is
+/// asked from the engine's node runner, from the agent executor's catch and from the bootstrap failure path, none of which should
+/// need a container on a failure path.
+///
+/// <para>It asks the exception CHAIN, not only the top: a provider call that failed inside a retry, a task or an aggregate reaches
+/// the engine wrapped, and the credential that was disconnected is still the reason. The first declared provider failure
+/// outward-in wins; a wrapper that declares a different failure of its own is believed over what it wraps, as everywhere the
+/// taxonomy is consulted.</para>
+/// </summary>
+public static class ProviderFailureCapture
+{
+    /// <summary>
+    /// The provider-connection failure <paramref name="thrown"/> carries, or null for anything else — including a declared failure
+    /// that is not about a provider connection, which stays the sentence it was. <paramref name="repositoryId"/> is the repository
+    /// the failing step was acting on, when the step names one.
+    /// </summary>
+    public static PersistedFailure? From(Exception? thrown, Guid? repositoryId)
+    {
+        for (var failure = thrown; failure is not null; failure = failure.InnerException)
+        {
+            if (failure is AggregateException aggregate && FromAny(aggregate.InnerExceptions, repositoryId) is { } inner) return inner;
+
+            if (failure is IFailure declared && FailureCodes.ProviderConnection.Contains(declared.Code)) return Describe(declared, repositoryId);
+        }
+
+        return null;
+    }
+
+    private static PersistedFailure? FromAny(IEnumerable<Exception> candidates, Guid? repositoryId) =>
+        candidates.Select(candidate => From(candidate, repositoryId)).FirstOrDefault(found => found is not null);
+
+    /// <summary>The repository the failure names itself wins over the one the step was acting on: it is the more specific fact.</summary>
+    private static PersistedFailure Describe(IFailure declared, Guid? repositoryId)
+    {
+        var details = PersistedFailure.DetailsAsText(declared.Details);
+        var own = details is not null && details.TryGetValue(RepositoryIdDetail, out var text) && Guid.TryParse(text, out var id) ? id : (Guid?)null;
+
+        return new PersistedFailure { Code = declared.Code, Details = details, RepositoryId = own ?? repositoryId };
+    }
+
+    /// <summary>The key a failure's details name its repository under.</summary>
+    public const string RepositoryIdDetail = "repositoryId";
+}

backend/src/CodeSpace.Core/Handlers/QueryHandlers/Repositories/ListRepositoryRejectedDeliveriesQueryHandler.cs

@@ -1,3 +1,5 @@
+using CodeSpace.Core.Services.Credentials.Remedy;
+using CodeSpace.Core.Services.Identity;
 using CodeSpace.Core.Services.Webhooks;
 using CodeSpace.Messages.Dtos.Repositories;
 using CodeSpace.Messages.Queries.Repositories;
@@ -8,9 +10,21 @@ namespace CodeSpace.Core.Handlers.QueryHandlers.Repositories;
 public sealed class ListRepositoryRejectedDeliveriesQueryHandler : IRequestHandler<ListRepositoryRejectedDeliveriesQuery, RepositoryRejectedDeliveries>
 {
     private readonly IRejectedDeliveryReader _reader;
+    private readonly IProviderRemedyService _remedies;
+    private readonly ICurrentTeam _currentTeam;
 
-    public ListRepositoryRejectedDeliveriesQueryHandler(IRejectedDeliveryReader reader) { _reader = reader; }
+    public ListRepositoryRejectedDeliveriesQueryHandler(IRejectedDeliveryReader reader, IProviderRemedyService remedies, ICurrentTeam currentTeam)
+    {
+        _reader = reader;
+        _remedies = remedies;
+        _currentTeam = currentTeam;
+    }
 
-    public async Task<RepositoryRejectedDeliveries> Handle(ListRepositoryRejectedDeliveriesQuery request, CancellationToken cancellationToken) =>
-        await _reader.ListForRepositoryAsync(request.RepositoryId, cancellationToken).ConfigureAwait(false);
+    public async Task<RepositoryRejectedDeliveries> Handle(ListRepositoryRejectedDeliveriesQuery request, CancellationToken cancellationToken)
+    {
+        var refused = await _reader.ListForRepositoryAsync(request.RepositoryId, cancellationToken).ConfigureAwait(false);
+
+        // The refusals that blame the repository's credential carry what to do about it, for whoever is asking.
+        return refused with { Remedy = await _remedies.GetForRefusedDeliveriesAsync(request.RepositoryId, refused.Deliveries, _currentTeam.Id!.Value, cancellationToken).ConfigureAwait(false) };
+    }
 }

backend/src/CodeSpace.Core/Handlers/QueryHandlers/Workflows/GetWorkflowRunByRefQueryHandler.cs

@@ -1,3 +1,4 @@
+using CodeSpace.Core.Services.Credentials.Remedy;
 using CodeSpace.Core.Services.Identity;
 using CodeSpace.Core.Services.Workflows;
 using CodeSpace.Core.Services.Workflows.Artifacts;
@@ -16,12 +17,14 @@ public sealed class GetWorkflowRunByRefQueryHandler : IRequestHandler<GetWorkflo
     private readonly IWorkflowService _service;
     private readonly IRunNodeOutputInflater _inflater;
     private readonly ICurrentTeam _currentTeam;
+    private readonly IProviderRemedyService _remedies;
 
-    public GetWorkflowRunByRefQueryHandler(IWorkflowService service, IRunNodeOutputInflater inflater, ICurrentTeam currentTeam)
+    public GetWorkflowRunByRefQueryHandler(IWorkflowService service, IRunNodeOutputInflater inflater, ICurrentTeam currentTeam, IProviderRemedyService remedies)
     {
         _service = service;
         _inflater = inflater;
         _currentTeam = currentTeam;
+        _remedies = remedies;
     }
 
     public async Task<WorkflowRunDetail?> Handle(GetWorkflowRunByRefQuery request, CancellationToken cancellationToken)
@@ -30,6 +33,11 @@ public GetWorkflowRunByRefQueryHandler(IWorkflowService service, IRunNodeOutputI
 
         var run = await _service.GetRunByRefAsync(request.IdOrNumber, teamId, cancellationToken).ConfigureAwait(false);
 
-        return run == null ? null : await _inflater.InflateAsync(run, teamId, cancellationToken).ConfigureAwait(false);
+        if (run == null) return null;
+
+        var detail = await _inflater.InflateAsync(run, teamId, cancellationToken).ConfigureAwait(false);
+
+        // What is wrong with a provider connection the run failed on, and what THIS caller can do about it — only a finished run has one.
+        return WorkflowRunState.IsTerminal(detail.Status) ? detail with { Remedy = await _remedies.GetForRunAsync(detail.Id, teamId, cancellationToken).ConfigureAwait(false) } : detail;
     }
 }

backend/src/CodeSpace.Core/Handlers/QueryHandlers/Workflows/GetWorkflowRunQueryHandler.cs

@@ -1,3 +1,4 @@
+using CodeSpace.Core.Services.Credentials.Remedy;
 using CodeSpace.Core.Services.Identity;
 using CodeSpace.Core.Services.Workflows;
 using CodeSpace.Core.Services.Workflows.Artifacts;
@@ -19,12 +20,14 @@ public sealed class GetWorkflowRunQueryHandler : IRequestHandler<GetWorkflowRunQ
     private readonly IWorkflowService _service;
     private readonly IRunNodeOutputInflater _inflater;
     private readonly ICurrentTeam _currentTeam;
+    private readonly IProviderRemedyService _remedies;
 
-    public GetWorkflowRunQueryHandler(IWorkflowService service, IRunNodeOutputInflater inflater, ICurrentTeam currentTeam)
+    public GetWorkflowRunQueryHandler(IWorkflowService service, IRunNodeOutputInflater inflater, ICurrentTeam currentTeam, IProviderRemedyService remedies)
     {
         _service = service;
         _inflater = inflater;
         _currentTeam = currentTeam;
+        _remedies = remedies;
     }
 
     public async Task<WorkflowRunDetail?> Handle(GetWorkflowRunQuery request, CancellationToken cancellationToken)
@@ -33,6 +36,11 @@ public GetWorkflowRunQueryHandler(IWorkflowService service, IRunNodeOutputInflat
 
         var run = await _service.GetRunAsync(request.RunId, teamId, cancellationToken).ConfigureAwait(false);
 
-        return run == null ? null : await _inflater.InflateAsync(run, teamId, cancellationToken).ConfigureAwait(false);
+        if (run == null) return null;
+
+        var detail = await _inflater.InflateAsync(run, teamId, cancellationToken).ConfigureAwait(false);
+
+        // What is wrong with a provider connection the run failed on, and what THIS caller can do about it — only a finished run has one.
+        return WorkflowRunState.IsTerminal(detail.Status) ? detail with { Remedy = await _remedies.GetForRunAsync(detail.Id, teamId, cancellationToken).ConfigureAwait(false) } : detail;
     }
 }

backend/src/CodeSpace.Core/Handlers/QueryHandlers/Workflows/GetWorkflowRunRemedyQueryHandler.cs

@@ -0,0 +1,23 @@
+using CodeSpace.Core.Services.Credentials.Remedy;
+using CodeSpace.Core.Services.Identity;
+using CodeSpace.Messages.Dtos.Credentials;
+using CodeSpace.Messages.Queries.Workflows;
+using MediatR;
+
+namespace CodeSpace.Core.Handlers.QueryHandlers.Workflows;
+
+/// <summary>The lean read behind the run page's remediation card: the remedy alone, for the asking user, without the run detail around it.</summary>
+public sealed class GetWorkflowRunRemedyQueryHandler : IRequestHandler<GetWorkflowRunRemedyQuery, ProviderRemedy?>
+{
+    private readonly IProviderRemedyService _remedies;
+    private readonly ICurrentTeam _currentTeam;
+
+    public GetWorkflowRunRemedyQueryHandler(IProviderRemedyService remedies, ICurrentTeam currentTeam)
+    {
+        _remedies = remedies;
+        _currentTeam = currentTeam;
+    }
+
+    public async Task<ProviderRemedy?> Handle(GetWorkflowRunRemedyQuery request, CancellationToken cancellationToken) =>
+        await _remedies.GetForRunAsync(request.RunId, _currentTeam.Id!.Value, cancellationToken).ConfigureAwait(false);
+}

backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs

@@ -4,6 +4,7 @@
 using System.Text.Json;
 using System.Net.Sockets;
 using CodeSpace.Core.DependencyInjection;
+using CodeSpace.Core.Failures;
 using CodeSpace.Core.Persistence.Db;
 using CodeSpace.Core.Persistence.Entities;
 using CodeSpace.Core.Services.Agents.AgentRunLogging;
@@ -326,6 +327,10 @@
         // for a re-attach is durability or a silent degrade.
         var brokeredHere = false;
 
+        // Hoisted for the executor-error arm, so a provider failure it lands can name the repository the run was working in:
+        // the failure carries the credential when it knows one, and a 401 does not.
+        Guid? taskRepositoryId = null;
+
         try
         {
             // Re-check the parent workflow run's status the instant after the Queued→Running claim wins, closing the TOCTOU
@@ -340,6 +345,8 @@
             var task = AgentTaskSecrets.Open(run, _sealer)
                        ?? throw new InvalidOperationException($"AgentRun {agentRunId} has an empty task envelope.");
 
+            taskRepositoryId = task.RepositoryId;
+
             // Reconcile the authored harness with the model's provider (from the pinned credential, or — for the
             // planner's loose model name — the pool row backing it) — if the pairing is impossible (e.g. an
             // Anthropic-provider model under a codex-cli default), repair to a harness that CAN drive it so the agent
@@ -813,7 +820,7 @@
             // A launch that got as far as reserving may already have billed a provider, so this terminal settles the
             // claim PESSIMISTICALLY (no observed cost ⇒ Indeterminate at the reserved amount) rather than leaving it
             // live. A failure is not evidence that nothing was spent.
-            await CompleteAndNotifyAsync(owner, run.TeamId, new AgentRunResult { Status = AgentRunStatus.Failed, ExitReason = ExecutorExitReason(ex), Error = redactor.Redact(ex.Message) }, cancellationToken).ConfigureAwait(false);
+            await CompleteAndNotifyAsync(owner, run.TeamId, new AgentRunResult { Status = AgentRunStatus.Failed, ExitReason = ExecutorExitReason(ex), Error = redactor.Redact(ex.Message), Failure = ProviderFailureCapture.From(ex, taskRepositoryId) }, cancellationToken).ConfigureAwait(false);
         }
         finally
         {
@@ -2262,7 +2269,7 @@
             // token already redacted (the handle redacts it), so it's safe to persist onto the timeline.
             _logger.LogWarning(ex, "Agent run {RunId}: failed to push the produced branch ({Reason}); the run stays Succeeded with no branch output", runId, DescribePushFailure(ex));
             await AppendPushFailureWarningAsync(owner, ex.Message, cancellationToken).ConfigureAwait(false);
-            return result with { PublishError = ex.Message, PublishFailureCode = PublishFailureCodeOf(ex) };
+            return result with { PublishError = ex.Message, PublishFailureCode = PublishFailureCodeOf(ex), Failure = ProviderFailureCapture.From(ex, task.RepositoryId) };
         }
     }
 
@@ -5367,10 +5374,10 @@
     /// <summary>The same reconstruction from a payload that came from somewhere other than the row — an offloaded one fetched back out of the artifact store.</summary>
     private static AgentEvent ReplayedEvent(AgentEventKind kind, string? text, string? dataJson)
     {
         if (dataJson is not { Length: > 0 } json) return new AgentEvent { Kind = kind, Text = text };
 
         try { using var doc = JsonDocument.Parse(json); return new AgentEvent { Kind = kind, Text = text, Data = doc.RootElement.Clone() }; }
         catch (JsonException) { return new AgentEvent { Kind = kind, Text = text }; }
     }
 
     /// <summary>Ask the row, on a token of its own, whether the run actually reached a terminal state — the only honest answer to "did the landing take?" once an exception has been raised somewhere after the fenced write.</summary>

backend/src/CodeSpace.Core/Services/Agents/Workspace/GitCredentialRejectedException.cs

@@ -21,7 +21,24 @@ public sealed class GitCredentialRejectedException : Exception, IFailure
 
     public string Code => FailureCodes.GitCredentialRejected;
 
+    /// <summary>What a person acting on the failure needs to find the subject: the repository the push was for and, when the refusal came from a credential that is gone, that credential — both identifiers, never a token.</summary>
+    public IReadOnlyDictionary<string, object?>? Details => _details;
+
+    private readonly Dictionary<string, object?>? _details;
+
     public GitCredentialRejectedException(string message) : base(message) { }
 
-    public GitCredentialRejectedException(string message, Exception inner) : base(message, inner) { }
+    public GitCredentialRejectedException(string message, Guid? repositoryId) : base(message) { _details = DetailsOf(repositoryId, inner: null); }
+
+    public GitCredentialRejectedException(string message, Exception inner, Guid? repositoryId = null) : base(message, inner) { _details = DetailsOf(repositoryId, inner); }
+
+    private static Dictionary<string, object?>? DetailsOf(Guid? repositoryId, Exception? inner)
+    {
+        var details = new Dictionary<string, object?>();
+
+        if (repositoryId is { } repository) details["repositoryId"] = repository;
+        if (inner is IFailure { Details: { } innerDetails } && innerDetails.TryGetValue("credentialId", out var credentialId)) details["credentialId"] = credentialId;
+
+        return details.Count == 0 ? null : details;
+    }
 }

backend/src/CodeSpace.Core/Services/Agents/Workspace/Providers/LocalGitWorkspaceProvider.cs

@@ -973,11 +973,11 @@ private async Task UseCurrentCredentialAsync(MaterializedRepo repo, Cancellation
             }
             catch (CredentialDisconnectedException ex)
             {
-                throw new GitCredentialRejectedException($"The credential this repository pushes with was disconnected while the run worked, so its branch could not be published. {ReconnectRemedy} ({Redact(ex.Message, repo.Token)})", ex);
+                throw new GitCredentialRejectedException($"The credential this repository pushes with was disconnected while the run worked, so its branch could not be published. {ReconnectRemedy} ({Redact(ex.Message, repo.Token)})", ex, repo.RepositoryId);
             }
             catch (CredentialNeedsReconnectException ex)
             {
-                throw new GitCredentialRejectedException($"The credential this repository pushes with was refused by its provider while the run worked, so its branch could not be published. {ReconnectRemedy} ({Redact(ex.Message, repo.Token)})", ex);
+                throw new GitCredentialRejectedException($"The credential this repository pushes with was refused by its provider while the run worked, so its branch could not be published. {ReconnectRemedy} ({Redact(ex.Message, repo.Token)})", ex, repo.RepositoryId);
             }
             catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
             {
@@ -1036,7 +1036,7 @@ private async Task PushToRemoteOrThrowAsync(MaterializedRepo repo, string publis
         }
 
         private static GitCredentialRejectedException Rejected(MaterializedRepo repo, string remoteUrl, string detail, SandboxResult refusal) =>
-            new($"The remote refused the git credential for {remoteUrl}, {detail}, so the branch was not published. {ReconnectRemedy} git said: {Redact(Summarize(refusal.Stderr), repo.Token)}");
+            new($"The remote refused the git credential for {remoteUrl}, {detail}, so the branch was not published. {ReconnectRemedy} git said: {Redact(Summarize(refusal.Stderr), repo.Token)}", repo.RepositoryId);
 
         /// <summary>Commit everything staged under a fixed CodeSpace identity; returns false (no commit) when there was nothing to commit. The identity AND <c>commit.gpgsign=false</c> are set inline via <c>-c</c> so the clone's git config is never mutated — and the automated capture commit can never inherit a host/global <c>commit.gpgsign=true</c> that would make it block on a signing key the unattended agent does not have (which would fail the branch push → the produced branch is silently lost). An internal automation commit under a synthetic identity has no meaningful signature, so signing is always disabled here.</summary>
         private async Task<bool> CommitOrDetectEmptyAsync(MaterializedRepo repo, string branchName, CancellationToken cancellationToken)

backend/src/CodeSpace.Core/Services/Credentials/CredentialService.cs

@@ -99,15 +99,25 @@ private async Task<IReadOnlyList<CredentialSummary>> WithHealthAsync(List<Creden
     {
         var ids = summaries.Select(s => s.Id).ToList();
         var left = await DepartedCredentials.Of(_db).AsNoTracking().Where(c => c.TeamId == teamId && ids.Contains(c.Id)).Select(c => c.Id).ToListAsync(cancellationToken).ConfigureAwait(false);
+        var bound = await CountBoundRepositoriesAsync(teamId, ids, cancellationToken).ConfigureAwait(false);
         var now = DateTimeOffset.UtcNow;
 
         return summaries.Select(s => s with
         {
             Health = CredentialHealthDerivation.Derive(s.Status, s.AuthType, s.ExpiresDate, left.Contains(s.Id), now),
-            DaysToExpiry = CredentialHealthDerivation.DaysToExpiry(s.AuthType, s.ExpiresDate, now)
+            DaysToExpiry = CredentialHealthDerivation.DaysToExpiry(s.AuthType, s.ExpiresDate, now),
+            BoundRepositoryCount = bound.GetValueOrDefault(s.Id)
         }).ToList();
     }
 
+    /// <summary>How many live repositories each listed credential holds — one grouped read for the whole list, so the number beside a credential (what reconnecting or disconnecting it puts at stake) costs the list no more than one query.</summary>
+    private async Task<Dictionary<Guid, int>> CountBoundRepositoriesAsync(Guid teamId, List<Guid> credentialIds, CancellationToken cancellationToken) =>
+        await _db.Repository.AsNoTracking()
+            .Where(r => r.TeamId == teamId && r.DeletedDate == null && r.CredentialId != null && credentialIds.Contains(r.CredentialId.Value))
+            .GroupBy(r => r.CredentialId!.Value)
+            .Select(g => new { CredentialId = g.Key, Count = g.Count() })
+            .ToDictionaryAsync(g => g.CredentialId, g => g.Count, cancellationToken).ConfigureAwait(false);
+
     public async Task<CredentialUsage> GetUsageAsync(Guid credentialId, CancellationToken cancellationToken)
     {
         var credential = await LoadCredentialAsync(credentialId, cancellationToken).ConfigureAwait(false);