diff --git a/backend/src/CodeSpace.Core/Services/Agents/AgentAcceptanceContract.cs b/backend/src/CodeSpace.Core/Services/Agents/AgentAcceptanceContract.cs index a015e9afe..f28432aec 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/AgentAcceptanceContract.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/AgentAcceptanceContract.cs @@ -115,6 +115,16 @@ public static AgentRunResult FailClosed(AgentRunResult result, string? detail) = /// Whether is a VACUOUS pass () rather than an executed check's verdict — prefix-matched so a future variant that names WHICH expectation was vacuous still classifies. public static bool IsVacuousPass(string? detail) => detail is not null && detail.StartsWith("not-applicable", StringComparison.Ordinal); + /// + /// The of a contract that found nothing to grade because the run's own + /// push was refused for its credential ( = + /// git_credential_rejected) — where it used to read no-branch-or-repo, which sends a reader after the + /// agent's work when the work exists and the remote would not take it. Infra-classed (): + /// the check never ran, and no revise round or retry can change a credential the remote refuses. Pinned by a unit + /// test (Rule 8): the executor writes it and the infra classification reads it. + /// + public const string CredentialRejectedDetail = "credential-rejected"; + /// /// S2's vacuous pass: the contract's own OWNER declared no diff was expected, and none was produced — so the /// missing branch/repo is NOT a failure, it is the correctly-predicted outcome. Unlike @@ -190,6 +200,9 @@ or Eval.Benchmark.Graders.TestsPassGrader.ResourceExhaustedDetail // detail string (the no-progress evidence discount, the receipts, the decider's verdict line, // the model-escalation trigger) reach the same answer it does. || effective.StartsWith(Messages.Agents.SupervisorAgentResult.InfraExitDetailPrefix, StringComparison.Ordinal) + // The push the contract needed a branch from was refused for its credential: the check never ran, and + // unlike a bare no-branch-or-repo this says WHY, so it is infra whether or not work is visible. + || effective == CredentialRejectedDetail || (effective == "no-branch-or-repo" && workPresent)); } diff --git a/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs b/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs index 2b43f1040..1d32e9ef0 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/AgentRunExecutor.cs @@ -2222,10 +2222,13 @@ internal static string TruncatePatch(string patch, int maxChars) /// branch), and a RECLAIMED attempt pushes its own generation's ref — the pre-check below narrows the zombie /// window, the generation ref closes it at the remote. /// - /// Best-effort like : a is - /// SWALLOWED (a push hiccup — e.g. a read-only credential 403 — never flips the run's own status) but is - /// surfaced as a Warning event on the timeline (token already redacted in the message) so the operator sees - /// WHY no branch appeared. Cancellation still propagates (worker torn down). + /// Best-effort like : a — or a + /// , the typed failure the workspace provider raises when the remote refused + /// the credential even after it was resolved again — is SWALLOWED (a push hiccup — e.g. a read-only credential 403 — + /// never flips the run's own status) but is surfaced as a Warning event on the timeline (token already redacted in the + /// message) so the operator sees WHY no branch appeared. A credential refusal is additionally recorded on the result as + /// ; it is the only one of the two that is not retried first. + /// Cancellation still propagates (worker torn down). /// internal async Task PushProducedBranchIfEnabledAsync(AgentRunOwnerToken owner, AgentTask task, AgentRunResult result, IWorkspaceHandle? workspace, CancellationToken cancellationToken) { @@ -2253,23 +2256,29 @@ internal async Task PushProducedBranchIfEnabledAsync(AgentRunOwn return branch is null ? result : result with { ProducedBranch = branch, PushedCommitSha = pushHandle.LastPushedCommitSha() }; } - catch (WorkspaceException ex) + catch (Exception ex) when (ex is WorkspaceException or GitCredentialRejectedException) { // Best-effort: a push failure must never flip a Succeeded run to Failed. The exception message has the // token already redacted (the handle redacts it), so it's safe to persist onto the timeline. - _logger.LogWarning(ex, "Agent run {RunId}: failed to push the produced branch after {Attempts} attempt(s); the run stays Succeeded with no branch output", runId, PushMaxAttempts); + _logger.LogWarning(ex, "Agent run {RunId}: failed to push the produced branch ({Reason}); the run stays Succeeded with no branch output", runId, DescribePushFailure(ex)); await AppendPushFailureWarningAsync(owner, ex.Message, cancellationToken).ConfigureAwait(false); - return result with { PublishError = ex.Message }; + return result with { PublishError = ex.Message, PublishFailureCode = PublishFailureCodeOf(ex) }; } } - /// Bounded attempts for one branch push before giving up — a contract-bearing task forces the push opt-in (F4), so a single transient git failure (network blip, remote hiccup) would otherwise convert a fully correct run into AcceptanceFailed("no-branch-or-repo") with zero chance to recover. carries no transient/deterministic classification (a flat clone/push wrapper), so this retries EVERY push failure a fixed number of times rather than sniffing error text — the same "retry blind, bounded" posture the P0.3 agent-respawn fix already took for a transient-vs-deterministic distinction git itself doesn't expose. Pinned (Rule 8). + /// The stable code to record beside a push failure's message: the failure's own code when it is one a retry of the run cannot fix (a credential the remote refused), null for the generic push failure — whose message is all there is to say about it. + internal static string? PublishFailureCodeOf(Exception failure) => failure is GitCredentialRejectedException rejected ? rejected.Code : null; + + /// What the log line names as the reason a push ended: the typed code, or "after N attempt(s)" for the generic failure that spent the whole retry budget. + private static string DescribePushFailure(Exception failure) => PublishFailureCodeOf(failure) ?? $"after {PushMaxAttempts} attempt(s)"; + + /// Bounded attempts for one branch push before giving up — a contract-bearing task forces the push opt-in (F4), so a single transient git failure (network blip, remote hiccup) would otherwise convert a fully correct run into AcceptanceFailed("no-branch-or-repo") with zero chance to recover. carries no transient/deterministic classification (a flat clone/push wrapper), so this retries EVERY a fixed number of times rather than classifying it — the same "retry blind, bounded" posture the P0.3 agent-respawn fix already took for a transient-vs-deterministic distinction git itself doesn't expose. The one failure that IS classified is a remote refusing the credential: the workspace provider recognises it from git's stderr (LocalGitWorkspaceProvider.IsCredentialRefusal), tries a freshly resolved credential once, and raises a typed that is not retried here. Pinned (Rule 8). internal const int PushMaxAttempts = 3; - /// Fixed backoff between push attempts — short because this runs inside the agent's own bounded wall clock; a network blip clears in well under a second, and a deterministic failure (auth, permission) just burns 3 short waits (~1s total) before falling through to the existing best-effort no-branch-output path unchanged. Pinned (Rule 8). + /// Fixed backoff between push attempts — short because this runs inside the agent's own bounded wall clock; a network blip clears in well under a second, and a deterministic failure the provider does not classify (a permission 403, or a repository whose credential is not re-resolved) just burns 3 short waits (~1s total) before falling through to the existing best-effort no-branch-output path unchanged. A refused credential is typed and never waits: it is not retried. Pinned (Rule 8). internal static readonly TimeSpan PushRetryBackoff = TimeSpan.FromMilliseconds(500); - /// Retry a single push call up to times on , with between attempts. The FINAL attempt's exception propagates unchanged — callers keep their existing single-catch best-effort fallback (per-repo isolation for multi-repo, run-stays-Succeeded for single-repo). Internal so both push paths share one retry posture and a test can pin the attempt count directly. + /// Retry a single push call up to times on , with between attempts. The FINAL attempt's exception propagates unchanged — callers keep their existing single-catch best-effort fallback (per-repo isolation for multi-repo, run-stays-Succeeded for single-repo). A is not retried here: the workspace provider has already resolved the credential again and tried it, so presenting it a third time could only repeat a refusal (and each refused login counts against the account). Internal so both push paths share one retry posture and a test can pin the attempt count directly. internal async Task PushWithRetryAsync(Func> push, CancellationToken cancellationToken) { for (var attempt = 1; ; attempt++) @@ -2318,9 +2327,9 @@ private async Task PushRepositoryResultsAsync(RepositoryPushCont continue; } - var (pushed, error) = await PushOneRepoOrNullAsync(owner, repo.Alias, branchName, pushHandle, cancellationToken).ConfigureAwait(false); + var (pushed, error, failureCode) = await PushOneRepoOrNullAsync(owner, repo.Alias, branchName, pushHandle, cancellationToken).ConfigureAwait(false); - updated.Add(repo with { ProducedBranch = pushed, PublishError = error, PushedCommitSha = pushed is null ? null : pushHandle.LastPushedCommitSha(repo.Alias) }); + updated.Add(repo with { ProducedBranch = pushed, PublishError = error, PublishFailureCode = failureCode, PushedCommitSha = pushed is null ? null : pushHandle.LastPushedCommitSha(repo.Alias) }); if (repo.Alias == workspace.PrimaryAlias) primaryBranch = pushed; } @@ -2328,19 +2337,19 @@ private async Task PushRepositoryResultsAsync(RepositoryPushCont return result with { RepositoryResults = updated, ProducedBranch = primaryBranch }; } - /// Push one repo by alias, ISOLATING its failure: a — after retries — is logged + surfaced as a per-repo Warning on the timeline (token already redacted) and returned as the error half of the tuple, so a sibling repo's already-pushed branch is never discarded. Cancellation propagates. - private async Task<(string? Branch, string? Error)> PushOneRepoOrNullAsync(AgentRunOwnerToken owner, string alias, string branchName, IWorkspacePushHandle pushHandle, CancellationToken cancellationToken) + /// Push one repo by alias, ISOLATING its failure: a — after retries — or a is logged + surfaced as a per-repo Warning on the timeline (token already redacted) and returned as the error half of the tuple, so a sibling repo's already-pushed branch is never discarded. Cancellation propagates. + private async Task<(string? Branch, string? Error, string? FailureCode)> PushOneRepoOrNullAsync(AgentRunOwnerToken owner, string alias, string branchName, IWorkspacePushHandle pushHandle, CancellationToken cancellationToken) { var runId = owner.RunId; try { - return (await PushWithRetryAsync(async ct => { await _runs.AssertOwnershipAsync(owner, ct).ConfigureAwait(false); return await pushHandle.PushChangesAsync(alias, branchName, ct).ConfigureAwait(false); }, cancellationToken).ConfigureAwait(false), null); + return (await PushWithRetryAsync(async ct => { await _runs.AssertOwnershipAsync(owner, ct).ConfigureAwait(false); return await pushHandle.PushChangesAsync(alias, branchName, ct).ConfigureAwait(false); }, cancellationToken).ConfigureAwait(false), null, null); } - catch (WorkspaceException ex) + catch (Exception ex) when (ex is WorkspaceException or GitCredentialRejectedException) { - _logger.LogWarning(ex, "Agent run {RunId}: failed to push repo '{Alias}' after {Attempts} attempt(s); keeping the other repos' branches in the change set", runId, alias, PushMaxAttempts); + _logger.LogWarning(ex, "Agent run {RunId}: failed to push repo '{Alias}' ({Reason}); keeping the other repos' branches in the change set", runId, alias, DescribePushFailure(ex)); await AppendPushFailureWarningAsync(owner, $"[{alias}] {ex.Message}", cancellationToken).ConfigureAwait(false); - return (null, ex.Message); + return (null, ex.Message, PublishFailureCodeOf(ex)); } } @@ -3068,7 +3077,7 @@ private async Task GradeAgainstOracleAsync(AcceptanceInvocation } _logger.LogWarning("Agent run {RunId}: an acceptance contract is present but there is no produced branch or recorded patch to grade — failing closed", run.Id); - return AcceptanceFailed(result, "no-branch-or-repo"); + return AcceptanceFailed(result, NothingToGradeDetail(result)); } BenchmarkGrade grade; @@ -3168,7 +3177,7 @@ private async Task GradeMultiRepoAcceptanceAsync(AgentRun run, A } _logger.LogWarning("Agent run {RunId}: an acceptance contract is present but no repo produced a branch to grade — failing closed", run.Id); - return AcceptanceFailed(result, "no-branch-or-repo"); + return AcceptanceFailed(result, NothingToGradeDetail(result)); } using var scope = _scopeFactory.CreateScope(); @@ -3205,6 +3214,17 @@ private async Task GradeMultiRepoAcceptanceAsync(AgentRun run, A return FoldGrade(result, new BenchmarkGrade { Passed = true, Detail = "accepted", OracleNote = oracleNotes.Count == 0 ? null : string.Join("; ", oracleNotes) }); } + /// + /// Why an acceptance contract found nothing to grade: when + /// a push of the run's own was refused for its credential (on the run or on any of its repos), else the long-standing + /// no-branch-or-repo. A verdict that names the credential keeps the run's Failed status — the contract still was + /// not met — but stops reading as "the agent produced nothing" for work the remote would not take. + /// + private static string NothingToGradeDetail(AgentRunResult result) => + result.PublishFailureCode == FailureCodes.GitCredentialRejected || result.RepositoryResults.Any(repo => repo.PublishFailureCode == FailureCodes.GitCredentialRejected) + ? AgentAcceptanceContract.CredentialRejectedDetail + : "no-branch-or-repo"; + /// Whether carries a recorded patch this executor could grade with (S2) — a base to anchor the independent clone on, PLUS either an inline diff or an offloaded artifact reference. Absent any one of these there is genuinely nothing to apply. private static bool HasGradeablePatch(AgentRunResult result) => !string.IsNullOrEmpty(result.BaseSha) && (result.PatchArtifactId is not null || !string.IsNullOrEmpty(result.Patch)); diff --git a/backend/src/CodeSpace.Core/Services/Agents/Workspace/GitCredentialRejectedException.cs b/backend/src/CodeSpace.Core/Services/Agents/Workspace/GitCredentialRejectedException.cs new file mode 100644 index 000000000..1b730d41d --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Agents/Workspace/GitCredentialRejectedException.cs @@ -0,0 +1,27 @@ +using CodeSpace.Messages.Failures; + +namespace CodeSpace.Core.Services.Agents.Workspace; + +/// +/// Thrown when a push could not be authenticated and trying again with the same credential cannot help: the remote +/// refused the credential even after it was resolved again immediately before the push (a credential that was only stale +/// is replaced by that second resolve and never reaches this), or the credential was disconnected while the run worked. +/// A carries no such classification, which is why the push retry used to spend its whole +/// budget presenting a refused credential three times — and why the failure could only ever read as "git push failed". +/// The remedy is a credential that works: reconnect the provider, or re-link the repository to an active one. +/// +/// Deliberately NOT a : that type is the one the push retry retries, and this one is +/// the retry's stopping signal. The two catch sites that swallow a push failure into a recorded, run-preserving publish +/// error name both. The message is already redacted — the workspace provider redacts every credential it ever held before +/// it throws. +/// +public sealed class GitCredentialRejectedException : Exception, IFailure +{ + public FailureKind Kind => FailureKind.PreconditionRequired; + + public string Code => FailureCodes.GitCredentialRejected; + + public GitCredentialRejectedException(string message) : base(message) { } + + public GitCredentialRejectedException(string message, Exception inner) : base(message, inner) { } +} diff --git a/backend/src/CodeSpace.Core/Services/Agents/Workspace/IWorkspaceProvider.cs b/backend/src/CodeSpace.Core/Services/Agents/Workspace/IWorkspaceProvider.cs index 5c23e9827..07f079cb9 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Workspace/IWorkspaceProvider.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Workspace/IWorkspaceProvider.cs @@ -88,7 +88,9 @@ public interface IWorkspacePushHandle /// Commit everything in the PRIMARY clone and force-push it to on the origin /// remote, returning the pushed branch name on success. Returns null when the run made no changes /// (nothing to commit) or the clone carries no push credential (an anonymous clone) — neither is a - /// failure. Throws (with the token REDACTED) on a git failure. + /// failure. Throws (with the token REDACTED) on a git failure, or + /// when the remote refused the credential even after it was resolved + /// again — a failure no retry of the same push can fix. /// Task PushChangesAsync(string branchName, CancellationToken cancellationToken); @@ -96,7 +98,8 @@ public interface IWorkspacePushHandle /// Multi-repo PR3: commit + push the repo named (one of Repositories) to /// on ITS own origin. Same semantics as the primary-only overload, scoped to /// one repo — each repo pushes to its distinct remote, so the same branch name across the set is fine. - /// Throws for an unknown / read-only alias or a git failure. + /// Throws for an unknown / read-only alias or a git failure, and + /// as the primary-only overload does. /// Task PushChangesAsync(string alias, string branchName, CancellationToken cancellationToken); diff --git a/backend/src/CodeSpace.Core/Services/Agents/Workspace/IWorkspaceTokenSource.cs b/backend/src/CodeSpace.Core/Services/Agents/Workspace/IWorkspaceTokenSource.cs new file mode 100644 index 000000000..1289ab3af --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Agents/Workspace/IWorkspaceTokenSource.cs @@ -0,0 +1,22 @@ +namespace CodeSpace.Core.Services.Agents.Workspace; + +/// +/// Where a workspace gets the git credential it pushes with, at the moment it pushes. Narrow on purpose (Rule 7 / ISP): one +/// question, asked of a repository by id, so a host that keeps credentials some other way plugs in without touching the +/// provider, and the provider never learns how a token is refreshed. +/// +/// It exists because a credential outlives neither the run nor the process that read it: a GitLab OAuth access token +/// lives two hours and is revoked the moment ANY caller refreshes it, so the token a run was handed at provisioning can be +/// dead — expired, or rotated by a sibling run — by the time its last act, the push, runs. An implementation therefore +/// answers from the credential store as it stands now, never from a value it kept. +/// +public interface IWorkspaceTokenSource +{ + /// + /// The access token that authenticates 's remote right now, refreshed if its provider + /// says it is due; null when the repository has no credential bound (an anonymous remote). Resolved for + /// only, and fails closed like the provisioning resolve does: a repository that was deleted + /// or moved to another team, or a credential that was disconnected, throws instead of answering from a stale copy. + /// + Task ResolveAsync(Guid repositoryId, Guid teamId, CancellationToken cancellationToken); +} diff --git a/backend/src/CodeSpace.Core/Services/Agents/Workspace/Providers/LocalGitWorkspaceProvider.cs b/backend/src/CodeSpace.Core/Services/Agents/Workspace/Providers/LocalGitWorkspaceProvider.cs index 638954881..48db283ad 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Workspace/Providers/LocalGitWorkspaceProvider.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Workspace/Providers/LocalGitWorkspaceProvider.cs @@ -3,6 +3,7 @@ using CodeSpace.Core.Services.Agents.Sandbox.Isolation; using CodeSpace.Core.Services.Agents.Sandbox.Exceptions; using CodeSpace.Messages.Agents; +using CodeSpace.Messages.Exceptions; using Microsoft.Extensions.Logging; namespace CodeSpace.Core.Services.Agents.Workspace.Providers; @@ -20,6 +21,15 @@ namespace CodeSpace.Core.Services.Agents.Workspace.Providers; /// same URL after the clone stays as a belt), an operator's store or cache helper never keeps it, no trace2 /// target records it, and a redirect to another authority is never sent it. Any token text is still redacted from /// surfaced error output. +/// +/// Credential freshness: the token a clone was provisioned with is only a snapshot. When a request names its +/// repository () and an is present, every +/// command of the publish that reaches the remote — the LFS upload, the push, the readback — asks the source for the +/// credential immediately beforehand, so a run that outlived its access token (or whose token a sibling run rotated away) +/// still pushes with a live one. A push the remote refuses is tried once more with a freshly resolved credential when that +/// differs from the refused one; refused again — or resolved to the same one — it ends as a +/// instead of presenting the same refusal again. Without a repository id (a +/// hand-built request) or a source, the captured token is used exactly as before. /// public sealed class LocalGitWorkspaceProvider : IWorkspaceProvider, IWorkspaceJanitor, IWorkspacePathCapture, ISingletonDependency { @@ -49,15 +59,24 @@ public sealed class LocalGitWorkspaceProvider : IWorkspaceProvider, IWorkspaceJa private static readonly TimeSpan DefaultStaleThreshold = TimeSpan.FromHours(2); private readonly ISandboxRunnerRegistry _runners; + private readonly IWorkspaceTokenSource? _tokenSource; private readonly ILogger _logger; private readonly string _workspacesRoot; - public LocalGitWorkspaceProvider(ISandboxRunnerRegistry runners, ILogger logger) : this(runners, logger, WorkspacesRoot) { } + /// Without a token source: every push carries the token its clone was provisioned with — the behaviour before the credential was re-resolved at the push, kept for a host that has no credential store to ask. + public LocalGitWorkspaceProvider(ISandboxRunnerRegistry runners, ILogger logger) : this(runners, null, logger, WorkspacesRoot) { } + + /// The composition the container builds: pushes re-resolve their credential through whenever the request names its repository. + public LocalGitWorkspaceProvider(ISandboxRunnerRegistry runners, IWorkspaceTokenSource tokenSource, ILogger logger) : this(runners, tokenSource, logger, WorkspacesRoot) { } /// Provisions (and sweeps) under instead of the worker-wide , which every process on the host writes — so a test can own every directory it counts. - internal LocalGitWorkspaceProvider(ISandboxRunnerRegistry runners, ILogger logger, string workspacesRoot) + internal LocalGitWorkspaceProvider(ISandboxRunnerRegistry runners, ILogger logger, string workspacesRoot) : this(runners, null, logger, workspacesRoot) { } + + /// A test-owned workspaces root with a credential source of the test's own. + internal LocalGitWorkspaceProvider(ISandboxRunnerRegistry runners, IWorkspaceTokenSource? tokenSource, ILogger logger, string workspacesRoot) { _runners = runners; + _tokenSource = tokenSource; _logger = logger; _workspacesRoot = workspacesRoot; } @@ -99,7 +118,7 @@ public async Task PrepareAsync(WorkspaceProvisionRequest reque var primaryAlias = (request.Primary ?? throw new WorkspaceException("Workspace provision has no resolvable primary repository.")).Alias; var cwd = ResolveCwd(request.CwdMode, workspaceRoot, materialized, primaryAlias, single); - return new LocalWorkspaceHandle(workspaceRoot, cwd, materialized, primaryAlias, _runners.Resolve(Kind), _logger, _workspacesRoot); + return new LocalWorkspaceHandle(workspaceRoot, cwd, materialized, primaryAlias, new HandleDependencies(_runners.Resolve(Kind), _tokenSource, _logger, _workspacesRoot)); } catch { @@ -164,9 +183,10 @@ private async Task MaterializeAsync(WorkspaceRepositoryProvisi var baseSha = await ReadBaseShaAsync(context, cancellationToken).ConfigureAwait(false); - // Carry the SAME short-lived clone credential forward (in-memory only, never persisted / never in .git/config) so a - // later push carries it in the push's environment without a second auth round-trip. - return new MaterializedRepo(repo.Alias, directory, repo.Access, repo.CloneRequest.RepositoryUrl, repo.CloneRequest.TokenUsername, repo.CloneRequest.Token, baseSha, repo.CloneRequest.Ref) { ReadOnlyPaths = context.ReadOnlyPaths }; + // Carry the clone credential forward (in-memory only, never persisted / never in .git/config) as the one a later push + // starts from, with the repository it was resolved for: the handle re-resolves it right before each remote command + // when a token source is present, because a workspace outlives its token. + return new MaterializedRepo(repo.Alias, directory, repo.Access, repo.CloneRequest.RepositoryUrl, repo.CloneRequest.TokenUsername, repo.CloneRequest.Token, baseSha, repo.CloneRequest.Ref) { ReadOnlyPaths = context.ReadOnlyPaths, RepositoryId = repo.CloneRequest.RepositoryId, TeamId = repo.CloneRequest.TeamId }; } /// Where the harness runs: Auto → the primary repo's dir for one repo (the invariant), the workspace root for many; or the explicit mode. @@ -210,6 +230,13 @@ private sealed record MaterializedRepo(string Alias, string Directory, Workspace /// P3b-2: the remote-CONFIRMED tip of the last successful push (readback matched the local tip); null = unconfirmed/no push. public string? PushedCommitSha { get; set; } public IReadOnlyList ReadOnlyPaths { get; init; } = Array.Empty(); + + /// The repository (and its team) the clone credential was resolved for — what the token source is asked about before each remote command. Null → the captured token is the only one there is. + public Guid? RepositoryId { get; init; } + public Guid? TeamId { get; init; } + + /// The credential the next command that reaches the remote carries: the one the clone was provisioned with until a push resolves a fresher one. Replaced, never accumulated — a command only ever carries this one, so it is the only one an error can echo. + public string? Token { get; set; } = Token; } /// Record the cloned HEAD revision so can diff the agent's work against it — robust whether the agent commits or leaves changes uncommitted. @@ -629,6 +656,25 @@ internal static string Redact(string text, string? token) return encoded == token ? redacted : redacted.Replace(encoded, "***"); } + /// + /// What git, GitLab and our own credential helper write to stderr when the remote refuses the credential a command + /// carried: git's Authentication failed for (its answer to an HTTP 401 after it offered a credential), the body + /// GitLab sends with the 401 (HTTP Basic: Access denied), curl's returned error: 401, and the helper's own + /// refused this credential (what git-lfs is told after the remote's 401, where git itself says nothing). git exits + /// 128 for every fatal error and has no structured signal for this one, so stderr is all there is. Used only to decide + /// whether trying the same credential again can help; a miss falls back to the plain failure, a hit to the typed one. + /// A 403 is deliberately absent: the remote accepted the credential and refused the ACTION, which no new token fixes. + /// Pinned by a test against the helper's own text and against what real git printed (Rule 8 spirit). + /// + internal static readonly IReadOnlyList CredentialRefusalMarkers = new[] { "Authentication failed for", "HTTP Basic: Access denied", "returned error: 401", "refused this credential" }; + + /// Whether says the remote refused the credential (see ). Pure + internal so it is unit-pinned. + internal static bool IsCredentialRefusal(string? stderr) => + !string.IsNullOrEmpty(stderr) && CredentialRefusalMarkers.Any(marker => stderr.Contains(marker, StringComparison.OrdinalIgnoreCase)); + + /// What a workspace handle needs from the provider that built it, in one object so the handle's constructor stays within the five-parameter cap. + private sealed record HandleDependencies(ISandboxRunner Runner, IWorkspaceTokenSource? TokenSource, ILogger Logger, string PublishRoot); + private sealed class LocalWorkspaceHandle : IWorkspaceHandle, IWorkspacePushHandle { private readonly string _workspaceRoot; @@ -636,17 +682,19 @@ private sealed class LocalWorkspaceHandle : IWorkspaceHandle, IWorkspacePushHand private readonly IReadOnlyList _repos; private readonly MaterializedRepo _primary; private readonly ISandboxRunner _runner; + private readonly IWorkspaceTokenSource? _tokenSource; private readonly ILogger _logger; - public LocalWorkspaceHandle(string workspaceRoot, string cwd, IReadOnlyList repos, string primaryAlias, ISandboxRunner runner, ILogger logger, string publishRoot) + public LocalWorkspaceHandle(string workspaceRoot, string cwd, IReadOnlyList repos, string primaryAlias, HandleDependencies dependencies) { _workspaceRoot = workspaceRoot; - _publishRoot = publishRoot; + _publishRoot = dependencies.PublishRoot; Directory = cwd; _repos = repos; _primary = repos.First(r => r.Alias == primaryAlias); - _runner = runner; - _logger = logger; + _runner = dependencies.Runner; + _tokenSource = dependencies.TokenSource; + _logger = dependencies.Logger; } public string Directory { get; } @@ -764,25 +812,25 @@ private async Task CaptureRepoChangesAsync(MaterializedRepo re await ImportBundlesAsync(repo, publishDir, branchName, addsObjects, cancellationToken).ConfigureAwait(false); - // Carry the SAME clone credential in the environment of the commands that reach the remote (never an argv, - // never a remote, never .git/config). Plain --force, not --force-with-lease: an observe-then-lease would still - // admit a zombie whose observation is fresh at push time, so the zombie fence lives in the REF NAME instead - // (AgentRunExecutor.BuildBranchName is generation-specific — a superseded attempt cannot name the current - // attempt's ref), and a lease's no-remote-tracking-ref semantics vary by git version. Bounded timeout so a - // hung push can't delay completion. - var remote = TokenedGitCommand.RemoteFor(repo.RepositoryUrl, repo.TokenUsername, repo.Token); - + // Each command that reaches the remote carries the credential in its environment (never an argv, never a remote, + // never .git/config), resolved immediately beforehand when the repository names a token source — a run can + // outlive the access token it was provisioned with. Plain --force, not --force-with-lease: an + // observe-then-lease would still admit a zombie whose observation is fresh at push time, so the zombie fence + // lives in the REF NAME instead (AgentRunExecutor.BuildBranchName is generation-specific — a superseded + // attempt cannot name the current attempt's ref), and a lease's no-remote-tracking-ref semantics vary by git + // version. Bounded timeout so a hung push can't delay completion. + // // LFS blobs BEFORE the refs (git's own pre-push order), so the remote never holds a pointer whose object is // missing. The clean repo has no working-tree .lfsconfig (nothing is checked out), and the endpoint comes // from the explicit remote URL — a hostile committed .lfsconfig cannot redirect the upload. Lock verification // is off: against a remote without the locks API git-lfs would otherwise record lfs..locksverify in the // publish repo's .git/config. if (hasLfs) - await RunTokenedPublishGitOrThrowAsync(repo, publishDir, remote, new[] { "-c", "lfs.locksverify=false", "lfs", "push", remote.Url, branchName }, cancellationToken).ConfigureAwait(false); + await PushToRemoteOrThrowAsync(repo, publishDir, url => new[] { "-c", "lfs.locksverify=false", "lfs", "push", url, branchName }, cancellationToken).ConfigureAwait(false); - await RunTokenedPublishGitOrThrowAsync(repo, publishDir, remote, new[] { "push", "--force", remote.Url, $"{branchName}:{branchName}" }, cancellationToken).ConfigureAwait(false); + await PushToRemoteOrThrowAsync(repo, publishDir, url => new[] { "push", "--force", url, $"{branchName}:{branchName}" }, cancellationToken).ConfigureAwait(false); - repo.PushedCommitSha = await ReadBackPushedShaAsync(repo, publishDir, remote, branchName, cancellationToken).ConfigureAwait(false); + repo.PushedCommitSha = await ReadBackPushedShaAsync(repo, publishDir, branchName, cancellationToken).ConfigureAwait(false); return branchName; } @@ -842,15 +890,20 @@ private async Task ImportBundlesAsync(MaterializedRepo repo, string publishDir, /// /// P3b-2 provider readback: re-read the just-pushed branch FROM THE REMOTE (ls-remote) and confirm it /// equals the local tip — "arrival" becomes an observed remote fact instead of a push self-report. Best-effort - /// by design: an unreadable remote or a mismatched tip (raced) returns null with a warning — the push itself - /// already succeeded, so the branch stands; only the CONFIRMATION is withheld, never fabricated. + /// by design: an unreadable remote, a credential that could not be resolved for the read, or a mismatched tip + /// (raced) returns null with a warning — the push itself already succeeded, so the branch stands; only the + /// CONFIRMATION is withheld, never fabricated. /// - private async Task ReadBackPushedShaAsync(MaterializedRepo repo, string publishDir, TokenedGitCommand.Remote remote, string branchName, CancellationToken cancellationToken) + private async Task ReadBackPushedShaAsync(MaterializedRepo repo, string publishDir, string branchName, CancellationToken cancellationToken) { try { var localTip = (await RunPublishGitOrThrowAsync(repo, publishDir, new[] { "rev-parse", $"refs/heads/{branchName}" }, cancellationToken, network: false).ConfigureAwait(false)).Trim(); + var remote = await CurrentRemoteForReadbackAsync(repo, branchName, cancellationToken).ConfigureAwait(false); + + if (remote is null) return null; + var readback = await RunTokenedPublishGitAsync(repo, publishDir, remote, new[] { "ls-remote", remote.Url, $"refs/heads/{branchName}" }, cancellationToken).ConfigureAwait(false); if (readback.Status != SandboxStatus.Success || readback.ExitCode != 0) @@ -874,6 +927,113 @@ private async Task ImportBundlesAsync(MaterializedRepo repo, string publishDir, } } + /// The remote the readback reads, with the credential resolved for it now — or null, with the confirmation withheld, when none can be: the push already landed, so a credential that cannot be resolved a moment later costs only the confirmation. + private async Task CurrentRemoteForReadbackAsync(MaterializedRepo repo, string branchName, CancellationToken cancellationToken) + { + try + { + return await CurrentRemoteAsync(repo, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is WorkspaceException or GitCredentialRejectedException) + { + _logger.LogWarning(ex, "Push readback for '{Branch}' could not resolve a credential to read the remote with; the successful push is preserved and arrival stays unconfirmed", branchName); + return null; + } + } + + // ── The credential a remote command carries, resolved when the command runs ── + + /// True when this repo's credential is resolved afresh before each remote command: its request named the repository, and a token source is there to ask. + private bool ResolvesItsCredential(MaterializedRepo repo) => _tokenSource is not null && repo.RepositoryId.HasValue && repo.TeamId.HasValue; + + /// The remote the next command names, carrying the credential as it is right now (). + private async Task CurrentRemoteAsync(MaterializedRepo repo, CancellationToken cancellationToken) + { + await UseCurrentCredentialAsync(repo, cancellationToken).ConfigureAwait(false); + + return TokenedGitCommand.RemoteFor(repo.RepositoryUrl, repo.TokenUsername, repo.Token); + } + + /// + /// Ask the token source for the repository's credential and carry what it answers. A no-op for a repo that does not + /// resolve its credential, so a hand-built request keeps its captured token. A credential that was disconnected + /// while the run worked is a — resolving it again cannot help; any other + /// failure of the resolve (a store hiccup, a provider that did not answer) is a , + /// which the push retry rightly retries; and a repository that no longer has a credential has nothing to push with. + /// + private async Task UseCurrentCredentialAsync(MaterializedRepo repo, CancellationToken cancellationToken) + { + if (!ResolvesItsCredential(repo)) return; + + string? current; + + try + { + current = await _tokenSource!.ResolveAsync(repo.RepositoryId!.Value, repo.TeamId!.Value, cancellationToken).ConfigureAwait(false); + } + catch (CredentialDisconnectedException ex) + { + throw new GitCredentialRejectedException($"The credential this repository pushes with was disconnected while the run worked, so its branch could not be published. {ReconnectRemedy} ({Redact(ex.Message, repo.Token)})", ex); + } + catch (Exception ex) when (!cancellationToken.IsCancellationRequested) + { + // The run's own cancellation is the only thing allowed through unwrapped: a provider call that timed out + // surfaces as a TaskCanceledException too, and a timeout is a failure to resolve — retryable — not a cancelled run. + throw new WorkspaceException($"Could not resolve the git credential to push with: {Redact(ex.Message, repo.Token)}", ex); + } + + if (string.IsNullOrEmpty(current)) + throw new WorkspaceException("The repository has no credential bound any more, so there is nothing to push its branch with."); + + repo.Token = current; + } + + /// What an operator does about a credential the remote will not accept — stated once so every message carrying it says the same thing. + private const string ReconnectRemedy = "Reconnect the provider, or re-link the repository to an active credential, then re-run."; + + /// + /// Run one publish command that reaches the remote — the LFS upload, the push — with the credential resolved + /// immediately before it. builds the argv from the remote's URL. A repo that resolves its + /// credential gets one more try when the remote refuses it: the credential is resolved again, and only a DIFFERENT one + /// is worth presenting (a stale token is replaced by it; the same one would be refused the same way, and each + /// refused login counts against the account). Refused again, or resolved to the same one, it is a + /// . A repo that does not resolve its credential, or any other failure, + /// throws the plain it always did. + /// + private async Task PushToRemoteOrThrowAsync(MaterializedRepo repo, string publishDir, Func> args, CancellationToken cancellationToken) + { + var remote = await CurrentRemoteAsync(repo, cancellationToken).ConfigureAwait(false); + var first = await RunTokenedPublishGitAsync(repo, publishDir, remote, args(remote.Url), cancellationToken).ConfigureAwait(false); + + if (first.Status != SandboxStatus.Success && TokenedGitCommand.OldGitFailure(remote.IsTokened, first.Stderr) is { } oldGit) throw oldGit; + + if (first.Status == SandboxStatus.Success || !ResolvesItsCredential(repo) || !IsCredentialRefusal(first.Stderr)) + { + EnsureSuccess(repo, args(remote.Url), first, remote.IsTokened); + return; + } + + var again = await CurrentRemoteAsync(repo, cancellationToken).ConfigureAwait(false); + + if (again.Password == remote.Password) + { + _logger.LogWarning("The remote refused the git credential for repository {RepositoryId} and resolving it again returned the same one; not presenting it a second time", repo.RepositoryId); + throw Rejected(repo, remote.Url, "and resolving it again returned the same one", first); + } + + _logger.LogInformation("The remote refused the git credential for repository {RepositoryId}; retrying once with a freshly resolved one", repo.RepositoryId); + + var second = await RunTokenedPublishGitAsync(repo, publishDir, again, args(again.Url), cancellationToken).ConfigureAwait(false); + + if (second.Status != SandboxStatus.Success && IsCredentialRefusal(second.Stderr)) + throw Rejected(repo, again.Url, "and again after resolving a fresh one", second); + + EnsureSuccess(repo, args(again.Url), second, again.IsTokened); + } + + private static GitCredentialRejectedException Rejected(MaterializedRepo repo, string remoteUrl, string detail, SandboxResult refusal) => + new($"The remote refused the git credential for {remoteUrl}, {detail}, so the branch was not published. {ReconnectRemedy} git said: {Redact(Summarize(refusal.Stderr), repo.Token)}"); + /// Commit everything staged under a fixed CodeSpace identity; returns false (no commit) when there was nothing to commit. The identity AND commit.gpgsign=false are set inline via -c so the clone's git config is never mutated — and the automated capture commit can never inherit a host/global commit.gpgsign=true that would make it block on a signing key the unattended agent does not have (which would fail the branch push → the produced branch is silently lost). An internal automation commit under a synthetic identity has no meaningful signature, so signing is always disabled here. private async Task CommitOrDetectEmptyAsync(MaterializedRepo repo, string branchName, CancellationToken cancellationToken) { @@ -933,9 +1093,6 @@ private Task RunPublishGitOrThrowAsync(MaterializedRepo repo, string pub private Task RunPublishGitAsync(MaterializedRepo repo, string publishDir, IReadOnlyList args, CancellationToken cancellationToken, bool network, int timeoutSeconds) => ExecuteGitAsync(repo, args, PublishGitSpec(publishDir, args, network, timeoutSeconds), cancellationToken); - private Task RunTokenedPublishGitOrThrowAsync(MaterializedRepo repo, string publishDir, TokenedGitCommand.Remote remote, IReadOnlyList args, CancellationToken cancellationToken) => - EnsureSuccessAsync(repo, args, RunTokenedPublishGitAsync(repo, publishDir, remote, args, cancellationToken), remote.IsTokened); - /// Run a publish-repo command that reaches — the LFS upload, the push, the readback — as a , with the network and the push budget. private Task RunTokenedPublishGitAsync(MaterializedRepo repo, string publishDir, TokenedGitCommand.Remote remote, IReadOnlyList args, CancellationToken cancellationToken) => ExecuteGitAsync(repo, args, TokenedGitCommand.Spec(remote, PublishGitSpec(publishDir, args, network: true, PushTimeoutSeconds)), cancellationToken); @@ -1034,10 +1191,12 @@ private async Task ExecuteGitAsync(MaterializedRepo repo, IReadOn /// is whether the command ran against a remote that carried a credential (), /// the only commands an old git can fail for want of one: it is the remote's own answer, not the run's clone token, so the two cannot disagree. /// - private static async Task EnsureSuccessAsync(MaterializedRepo repo, IReadOnlyList args, Task run, bool tokened = false) - { - var result = await run.ConfigureAwait(false); + private static async Task EnsureSuccessAsync(MaterializedRepo repo, IReadOnlyList args, Task run, bool tokened = false) => + EnsureSuccess(repo, args, await run.ConfigureAwait(false), tokened); + /// The result of a git command already run: its stdout, or a redacted on a non-success status. + private static string EnsureSuccess(MaterializedRepo repo, IReadOnlyList args, SandboxResult result, bool tokened = false) + { if (result.Status != SandboxStatus.Success) throw TokenedGitCommand.OldGitFailure(tokened, result.Stderr) ?? new WorkspaceException($"git {string.Join(' ', RedactArgs(args, repo.Token))} failed (exit {result.ExitCode}): {Redact(result.Stderr.Trim(), repo.Token)}"); diff --git a/backend/src/CodeSpace.Core/Services/Agents/Workspace/RepositoryWorkspaceResolver.cs b/backend/src/CodeSpace.Core/Services/Agents/Workspace/RepositoryWorkspaceResolver.cs index 86e89f0cc..9c1a13f26 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Workspace/RepositoryWorkspaceResolver.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Workspace/RepositoryWorkspaceResolver.cs @@ -92,6 +92,11 @@ public RepositoryWorkspaceResolver(CodeSpaceDbContext db, IProviderAuthResolver DefaultRef = softFallback && requestedRef is not null && !string.Equals(requestedRef, repo.DefaultBranch, StringComparison.Ordinal) ? repo.DefaultBranch : null, Token = token, TokenUsername = token is null ? null : TokenUsernameFor(repo.ProviderInstance.Provider), + // The token is a snapshot of a credential that expires and is revoked when anyone refreshes it, and a + // workspace's last act (the push) can run hours after this resolve — so the request names the repository + // it was resolved for, and the provider asks the token source again right before each remote command. + RepositoryId = repositoryId, + TeamId = teamId, // S1: the immutable base pin — validated to a hex commit id (4-40 chars) so garbage can never reach the // git argv as a flag-shaped positional, then forced through a full clone + hard checkout by the provider. PinnedSha = ValidatePinnedSha(pinnedSha), diff --git a/backend/src/CodeSpace.Core/Services/Agents/Workspace/RepositoryWorkspaceTokenSource.cs b/backend/src/CodeSpace.Core/Services/Agents/Workspace/RepositoryWorkspaceTokenSource.cs new file mode 100644 index 000000000..ada9c8410 --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Agents/Workspace/RepositoryWorkspaceTokenSource.cs @@ -0,0 +1,35 @@ +using CodeSpace.Core.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; + +namespace CodeSpace.Core.Services.Agents.Workspace; + +/// +/// over the same resolve a run's workspace was provisioned with: it asks the scoped +/// for the repository again, so the repository is loaded team-scoped and non-deleted +/// with its CURRENT credential row, and the token comes through the provider auth layer — which refreshes an OAuth token +/// that is due, under the refresh lock, and reads a pair another run already rotated rather than refreshing it twice. +/// Nothing is cached here: the stored credential is the only truth, so there is nothing to invalidate after a refusal — asking +/// again IS the invalidation. +/// +/// A singleton (the workspace provider that asks is one) that opens one short DI scope per call, the way the +/// executor reaches its other scoped services: a workspace outlives any request scope, and a long-lived scope's DbContext +/// would serve the very credential row this exists to re-read. +/// +public sealed class RepositoryWorkspaceTokenSource : IWorkspaceTokenSource, ISingletonDependency +{ + private readonly IServiceScopeFactory _scopes; + + public RepositoryWorkspaceTokenSource(IServiceScopeFactory scopes) + { + _scopes = scopes; + } + + public async Task ResolveAsync(Guid repositoryId, Guid teamId, CancellationToken cancellationToken) + { + using var scope = _scopes.CreateScope(); + + var request = await scope.ServiceProvider.GetRequiredService().ResolveByRepositoryIdAsync(repositoryId, teamId, cancellationToken).ConfigureAwait(false); + + return request?.Token; + } +} diff --git a/backend/src/CodeSpace.Messages/Agents/AgentRunResult.cs b/backend/src/CodeSpace.Messages/Agents/AgentRunResult.cs index 0a1be9047..dac81e0ae 100644 --- a/backend/src/CodeSpace.Messages/Agents/AgentRunResult.cs +++ b/backend/src/CodeSpace.Messages/Agents/AgentRunResult.cs @@ -279,6 +279,10 @@ public sealed record AgentRunResult /// Set ONLY when a non-empty diff failed to push after retries — the publish-manifest's signal that being null was an ATTEMPT that failed, not a policy choice. Already-redacted (the workspace provider redacts before throwing). Null on success, on a policy skip, and on every non-publish outcome. public string? PublishError { get; init; } + /// The stable FailureCodes name of WHY the push named by failed, when the failure is one a retry of the same run cannot fix — today only git_credential_rejected (the remote refused the git credential even after it was resolved again; the remedy is to reconnect or re-link it). Null for a generic push failure, for success and for a policy skip, and omitted from the stored result then, so every earlier result reads and writes byte-identically. The run's is never changed by it: a rejected push is a recorded publish failure, not a failed run. + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? PublishFailureCode { get; init; } + /// Set ONLY when the publish guard chain (IPublishGuard) blocked the push — a BY-CHOICE skip (never both this and at once). The winning guard's human-readable reason, folded onto the publish-manifest row's Summary so a by-choice skip explains itself. public string? PublishSkipReason { get; init; } } @@ -341,6 +345,10 @@ public sealed record RepositoryRunResult /// Mirrors for this repo — set ONLY when this repo's non-empty diff failed to push after retries. public string? PublishError { get; init; } + /// Mirrors for this repo — the stable code of why THIS repo's push failed, when a retry cannot fix it. Null-omitted. + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? PublishFailureCode { get; init; } + /// Mirrors for this repo — set ONLY when the publish guard chain blocked THIS repo's push (a multi-repo run can mix policies across repos). public string? PublishSkipReason { get; init; } diff --git a/backend/src/CodeSpace.Messages/Agents/WorkspaceRequest.cs b/backend/src/CodeSpace.Messages/Agents/WorkspaceRequest.cs index 58739f833..e692e8de6 100644 --- a/backend/src/CodeSpace.Messages/Agents/WorkspaceRequest.cs +++ b/backend/src/CodeSpace.Messages/Agents/WorkspaceRequest.cs @@ -33,6 +33,18 @@ public sealed record WorkspaceRequest /// Basic-auth username paired with — provider-specific ("x-access-token", "oauth2"). Ignored when is null; defaults to "x-access-token". public string? TokenUsername { get; init; } + /// + /// The repository was resolved for, together with . An access token has a + /// lifetime (a GitLab OAuth one lasts two hours, and is revoked the moment any caller refreshes it), while a workspace + /// outlives it, so a provider that pushes at the END of a long run asks for the credential again by these two ids + /// instead of carrying across the run. Null (the default) → the run carries + /// and nothing else, exactly as before; set by the resolver that produced this request. + /// + public Guid? RepositoryId { get; init; } + + /// The team that owns ; read only together with it. Null whenever is. + public Guid? TeamId { get; init; } + /// Shallow-clone depth. 1 (default) fetches only the tip; 0 → a full clone. With set the clone STARTS at this depth and only deepens (fetch-by-sha, then unshallow) when the pin is not the fetched tip — the common pin-equals-tip launch stays shallow. public int Depth { get; init; } = 1; diff --git a/backend/src/CodeSpace.Messages/Failures/FailureCodes.cs b/backend/src/CodeSpace.Messages/Failures/FailureCodes.cs index eda5ff993..f83778220 100644 --- a/backend/src/CodeSpace.Messages/Failures/FailureCodes.cs +++ b/backend/src/CodeSpace.Messages/Failures/FailureCodes.cs @@ -55,6 +55,9 @@ public static class FailureCodes /// Something tried to authenticate through a credential that was disconnected — revoked, its token material destroyed. Remedy: reconnect the provider (bound repositories follow their owner's reconnect) or re-link to an active credential; a retry as-is reads the same dead row. public const string CredentialDisconnected = "credential_disconnected"; + /// The remote refused the git credential a push carried even after it was resolved again immediately before the push (or the credential was disconnected while the run worked), so the agent's branch was not published. Remedy: reconnect the provider, or re-link the repository to an active credential, then re-run — a retry as-is is refused the same way. + public const string GitCredentialRejected = "git_credential_rejected"; + /// An update moved a model credential's endpoint (scheme, host or port) without entering its key again. Remedy: send the key with the new endpoint; a stored key never follows a repoint on its own. public const string ModelEndpointKeyRequired = "model_endpoint_key_required"; diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentPublishCredentialFreshnessFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentPublishCredentialFreshnessFlowTests.cs new file mode 100644 index 000000000..d5b9842b4 --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentPublishCredentialFreshnessFlowTests.cs @@ -0,0 +1,432 @@ +using System.Reflection; +using System.Text.Json; +using System.Web; +using Autofac; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Agents.Sandbox; +using CodeSpace.Core.Services.Agents.Sandbox.Runners; +using CodeSpace.Core.Services.Agents.Workspace; +using CodeSpace.Core.Services.Agents.Workspace.Providers; +using CodeSpace.Core.Services.Credentials; +using CodeSpace.Core.Services.OAuth; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.IntegrationTests.OAuth; +using CodeSpace.IntegrationTests.Webhooks; +using CodeSpace.Messages.Agents; +using CodeSpace.Messages.Commands.Credentials; +using CodeSpace.Messages.Constants; +using CodeSpace.Messages.Credentials; +using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Failures; +using MediatR; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; + +namespace CodeSpace.IntegrationTests.Workflows; + +/// +/// HIGH fidelity (Rule 12): a run's last act — pushing the agent's branch — against a git remote that behaves like GitLab, +/// with the credential chain COMPOSED for real: the workspace resolver provisions the clone, the REAL +/// re-resolves the credential before the push through the real +/// provider auth resolver, OAuth strategy, refresher, Postgres advisory lock and payload writer, against a loopback token +/// endpoint that rotates single-use refresh tokens, over a real and real git. +/// The remote revokes the access token it issued the moment the token endpoint issues a new one — GitLab's behaviour, and +/// the reason a run that outlives its token, or whose token a sibling run refreshed, used to push with a dead one. +/// +/// Probe P1, composed: the clone is made with the access token the run was provisioned with; the agent works while the +/// stored pair ages into its refresh window; the push finds the token due, refreshes it, and lands the branch. On +/// the code before the token source existed the same shape ended in git push --force … failed (exit 128): fatal: +/// Authentication failed, which the executor swallowed into a Warning on a Succeeded run. +/// +/// A run's own clock is not what moves here: the stored pair is aged by rewriting its recorded expiry, which is what the +/// refresher reads, so no test waits for a token to age. Each test owns GUID-named rows and a temp tree, removes the tree on +/// every path, and skips on Windows or without git. +/// +[Collection(PostgresCollection.Name)] +[Trait("Category", "Integration")] +public sealed class AgentPublishCredentialFreshnessFlowTests +{ + private const string UsernameForGitLab = "oauth2"; + + private readonly PostgresFixture _fixture; + + public AgentPublishCredentialFreshnessFlowTests(PostgresFixture fixture) { _fixture = fixture; } + + [Fact] + public async Task A_token_that_ages_into_its_refresh_window_while_the_agent_works_is_refreshed_for_the_push() + { + if (!await CanRunGitAsync()) return; + + using var ctx = await FreshnessContext.StartAsync(_fixture, liveRefreshToken: "rt-0"); + await using var handle = await ctx.ProvisionAsync(); + ctx.ProvisionedToken.ShouldBe("at-0", "fixture check: the clone was made with the access token the run was provisioned with"); + + await ctx.AgeStoredPairAsync(secondsLeft: 30); // the agent has worked for most of the token's life + await ctx.AgentWritesAsync(handle.Directory); + + var branch = await ((IWorkspacePushHandle)handle).PushChangesAsync(ctx.BranchName, CancellationToken.None); + + branch.ShouldBe(ctx.BranchName); + (await ctx.RemoteHasBranchAsync()).ShouldBeTrue("the branch landed on the remote"); + ctx.Remote.RefusedAuthorizations.ShouldBeEmpty("the token the clone carried was never presented at the push — it was due for a refresh, so the push refreshed it first"); + ctx.Endpoint.Requests.ShouldBe(1, "exactly one refresh: the push's — the readback finds the new token live"); + (await ctx.StoredAsync()).RefreshToken.ShouldBe("rt-1", "the rotated pair is what the row holds, so the next run refreshes with a live refresh token"); + ((IWorkspacePushHandle)handle).LastPushedCommitSha().ShouldNotBeNull("the readback was authenticated with the refreshed token too"); + } + + [Fact] + public async Task A_token_another_run_rotated_while_this_one_worked_is_replaced_by_the_one_it_stored() + { + // GitLab revokes the old access token on ANY refresh, so a sibling run (or a webhook admission) that refreshed the + // same credential killed the token this run holds — long before its own expiry, with nothing for this run to notice. + if (!await CanRunGitAsync()) return; + + using var ctx = await FreshnessContext.StartAsync(_fixture, liveRefreshToken: "rt-0"); + await using var handle = await ctx.ProvisionAsync(); + + await ctx.AnotherRunRefreshesAsync(); + ctx.Endpoint.Requests.ShouldBe(1, "fixture check: the sibling's refresh reached the token endpoint"); + await ctx.AgentWritesAsync(handle.Directory); + + var branch = await ((IWorkspacePushHandle)handle).PushChangesAsync(ctx.BranchName, CancellationToken.None); + + branch.ShouldBe(ctx.BranchName); + ctx.Remote.RefusedAuthorizations.ShouldBeEmpty("the revoked token was not presented: the push read the pair the sibling stored"); + ctx.Endpoint.Requests.ShouldBe(1, "this run did not refresh again — the stored token was live"); + } + + [Fact] + public async Task A_personal_access_token_bound_repository_pushes_with_its_token_unchanged() + { + if (!await CanRunGitAsync()) return; + + using var ctx = await FreshnessContext.StartAsync(_fixture, liveRefreshToken: null, patToken: "glpat-fake-personal-token"); + await using var handle = await ctx.ProvisionAsync(); + ctx.ProvisionedToken.ShouldBe("glpat-fake-personal-token"); + + await ctx.AgentWritesAsync(handle.Directory); + var branch = await ((IWorkspacePushHandle)handle).PushChangesAsync(ctx.BranchName, CancellationToken.None); + + branch.ShouldBe(ctx.BranchName); + (await ctx.RemoteHasBranchAsync()).ShouldBeTrue(); + ctx.Remote.RefusedAuthorizations.ShouldBeEmpty(); + ctx.Endpoint.Requests.ShouldBe(0, "a static token has nothing to refresh"); + } + + [Fact] + public async Task A_credential_disconnected_while_the_agent_worked_is_a_typed_rejection_and_nothing_is_presented_to_the_remote() + { + if (!await CanRunGitAsync()) return; + + using var ctx = await FreshnessContext.StartAsync(_fixture, liveRefreshToken: "rt-0"); + await using var handle = await ctx.ProvisionAsync(); + + await ctx.DisconnectAsync(); + await ctx.AgentWritesAsync(handle.Directory); + + var failure = await Should.ThrowAsync(() => ((IWorkspacePushHandle)handle).PushChangesAsync(ctx.BranchName, CancellationToken.None)); + + failure.Code.ShouldBe(FailureCodes.GitCredentialRejected); + failure.Message.ShouldContain("was disconnected"); + failure.Message.ShouldContain("Reconnect the provider"); + ctx.Remote.RefusedAuthorizations.ShouldBeEmpty("a disconnected credential is never put to the remote"); + (await ctx.RemoteHasBranchAsync()).ShouldBeFalse(); + } + + [Fact] + public async Task A_repository_deleted_while_the_agent_worked_is_not_pushed_to() + { + // Fail closed, as the provisioning resolve does: an operator who removed the repository does not expect a run that + // started earlier to still publish to it through a credential that has not expired. + if (!await CanRunGitAsync()) return; + + using var ctx = await FreshnessContext.StartAsync(_fixture, liveRefreshToken: "rt-0"); + await using var handle = await ctx.ProvisionAsync(); + + await ctx.DeleteRepositoryAsync(); + await ctx.AgentWritesAsync(handle.Directory); + + var failure = await Should.ThrowAsync(() => ((IWorkspacePushHandle)handle).PushChangesAsync(ctx.BranchName, CancellationToken.None)); + + failure.Message.ShouldContain("not found for this team"); + ctx.Remote.RefusedAuthorizations.ShouldBeEmpty(); + (await ctx.RemoteHasBranchAsync()).ShouldBeFalse(); + } + + [Fact] + public async Task A_refresh_the_provider_refuses_is_a_workspace_failure_the_push_retry_can_see_not_a_raw_oauth_exception() + { + // The stored refresh token is one the provider no longer honours (rotated elsewhere, grant revoked). Turning that + // into a typed 'reconnect' is the refresher's own arc; what this pins is that the push path stays inside its own + // exception contract — a WorkspaceException — instead of letting an OAuth exception escape the publish and flip the + // run that produced the work. + if (!await CanRunGitAsync()) return; + + using var ctx = await FreshnessContext.StartAsync(_fixture, liveRefreshToken: "rt-held-by-someone-else"); + await using var handle = await ctx.ProvisionAsync(); + + await ctx.AgeStoredPairAsync(secondsLeft: 30); + await ctx.AgentWritesAsync(handle.Directory); + + var failure = await Should.ThrowAsync(() => ((IWorkspacePushHandle)handle).PushChangesAsync(ctx.BranchName, CancellationToken.None)); + + failure.Message.ShouldContain("Could not resolve the git credential to push with"); + failure.Message.ShouldContain("invalid_grant", Case.Insensitive); + failure.Message.ShouldNotContain("at-0", Case.Insensitive, "the token is never in the message"); + ctx.Remote.RefusedAuthorizations.ShouldBeEmpty(); + } + + [Fact] + public void The_container_hands_the_workspace_provider_the_token_source() + { + // The whole fix is inert if the container builds the provider through the constructor that has no token source: every + // push would carry the captured token again, with every other test here still green. Autofac picks the widest + // constructor it can satisfy, so this pins that it finds the one with the source. + using var scope = _fixture.BeginScope(); + + var provider = scope.Resolve().Resolve(SandboxKinds.Local).ShouldBeOfType(); + var source = typeof(LocalGitWorkspaceProvider).GetField("_tokenSource", BindingFlags.Instance | BindingFlags.NonPublic).ShouldNotBeNull().GetValue(provider); + + source.ShouldBeOfType("the registered provider must push with a freshly resolved credential"); + scope.Resolve().ShouldBeSameAs(source, "the provider and the container share one source"); + } + + // ── Plumbing ─────────────────────────────────────────────────────────────────── + + private static async Task CanRunGitAsync() + { + if (OperatingSystem.IsWindows()) return false; + + try { return (await new LocalProcessRunner().RunAsync(new SandboxSpec { Command = "git", Args = new[] { "--version" }, TimeoutSeconds = 15 }, CancellationToken.None)).Status == SandboxStatus.Success; } + catch { return false; } + } + + /// A hermetic git environment: no operator config, no prompt, a HOME of its own — the runner under test otherwise inherits the host's. + private sealed class HermeticRunner(string home) : ISandboxRunner + { + private readonly LocalProcessRunner _inner = new(); + + public string Kind => "local"; + + public Task RunAsync(SandboxSpec spec, CancellationToken cancellationToken) + { + var environment = new Dictionary(spec.Environment) { ["HOME"] = home, ["GIT_CONFIG_NOSYSTEM"] = "1", ["GIT_CONFIG_GLOBAL"] = Path.Combine(home, ".gitconfig"), ["GIT_TERMINAL_PROMPT"] = "0" }; + + return _inner.RunAsync(spec with { Environment = environment }, cancellationToken); + } + } + + /// + /// GitLab's token endpoint for grant_type=refresh_token: each refresh token is single-use — presenting the live one + /// rotates it (rt-N → rt-N+1, access token at-N+1, valid two hours) and revokes the previous access + /// token, which relays to the git remote; presenting any other is refused with GitLab's own + /// invalid_grant body. + /// + private sealed class RotatingEndpoint + { + private const string InvalidGrantBody = """{"error":"invalid_grant","error_description":"The provided authorization grant is invalid, expired, revoked, does not match the redirection URI used in the authorization request, or was issued to another client."}"""; + + private readonly object _gate = new(); + private readonly StubProviderHost _provider; + private readonly Action _onRotated; + private string? _liveRefreshToken; + + private RotatingEndpoint(StubProviderHost provider, string? liveRefreshToken, Action onRotated) + { + _provider = provider; + _liveRefreshToken = liveRefreshToken; + _onRotated = onRotated; + } + + public static RotatingEndpoint Serve(StubProviderHost provider, string? liveRefreshToken, Action onRotated) + { + var endpoint = new RotatingEndpoint(provider, liveRefreshToken, onRotated); + provider.Answer("POST", "/oauth/token", endpoint.Refresh); + return endpoint; + } + + /// Refresh requests that reached the endpoint, rotated or refused. + public int Requests => _provider.Requests.Count(r => r.PathAndQuery.StartsWith("/oauth/token", StringComparison.Ordinal)); + + private StubProviderHost.StubReply Refresh(StubProviderHost.RecordedRequest request) + { + var presented = HttpUtility.ParseQueryString(request.Body)["refresh_token"]; + + lock (_gate) + { + if (presented is null || presented != _liveRefreshToken) return new StubProviderHost.StubReply(400, InvalidGrantBody); + + var generation = int.Parse(_liveRefreshToken["rt-".Length..]) + 1; + _liveRefreshToken = $"rt-{generation}"; + _onRotated($"at-{generation}"); + + return new StubProviderHost.StubReply(200, JsonSerializer.Serialize(new { access_token = $"at-{generation}", token_type = "Bearer", expires_in = 7200, refresh_token = _liveRefreshToken, scope = "api" })); + } + } + } + + private sealed record Seeded(Guid UserId, Guid TeamId, Guid CredentialId, Guid RepositoryId); + + private sealed class FreshnessContext : IDisposable + { + private readonly PostgresFixture _fixture; + private readonly string _root = Path.Combine(Path.GetTempPath(), "cs-credfresh-" + Guid.NewGuid().ToString("N")); + private readonly StubProviderHost _provider = new(); + private readonly ILifetimeScope _sourceScope; + private Seeded _seed = null!; + + private FreshnessContext(PostgresFixture fixture, string? liveRefreshToken) + { + _fixture = fixture; + Directory.CreateDirectory(Path.Combine(_root, "home")); + Endpoint = RotatingEndpoint.Serve(_provider, liveRefreshToken, access => Remote.AcceptOnly(UsernameForGitLab, access)); + _sourceScope = fixture.BeginScope(); + + PushProvider = new LocalGitWorkspaceProvider( + new SandboxRunnerRegistry(new ISandboxRunner[] { new HermeticRunner(Path.Combine(_root, "home")) }), + new RepositoryWorkspaceTokenSource(_sourceScope.Resolve()), + NullLogger.Instance, + Path.Combine(_root, "workspaces")); + } + + /// Reads authenticate too, as a private GitLab project's do, so the clone itself proves the token it was given. + public GitPublishRemoteFixture Remote { get; } = new() { AuthenticateReads = true }; + public RotatingEndpoint Endpoint { get; } + public LocalGitWorkspaceProvider PushProvider { get; } + public string BranchName { get; } = "codespace/agent/" + Guid.NewGuid().ToString("N"); + + /// The access token the clone was provisioned with (the one the resolver handed the request). + public string? ProvisionedToken { get; private set; } + + public static async Task StartAsync(PostgresFixture fixture, string? liveRefreshToken, string? patToken = null) + { + var ctx = new FreshnessContext(fixture, liveRefreshToken); + await ctx.Remote.StartAsync(); + await ctx.SeedAsync(patToken); + ctx.Remote.AcceptOnly(UsernameForGitLab, patToken ?? "at-0"); // the token the run is provisioned with is the live one — until it is rotated + return ctx; + } + + /// The workspace the way the executor provisions it: the REAL resolver builds the request (token, repository id, team id), the provider clones it. + public async Task ProvisionAsync() + { + using var scope = _fixture.BeginScope(); + var request = await scope.Resolve().ResolveByRepositoryIdAsync(_seed.RepositoryId, _seed.TeamId, CancellationToken.None).ConfigureAwait(false); + + request.ShouldNotBeNull(); + request.RepositoryId.ShouldBe(_seed.RepositoryId, "fixture check: the resolver names the repository it resolved the token for"); + request.TeamId.ShouldBe(_seed.TeamId); + ProvisionedToken = request.Token; + + return await PushProvider.PrepareAsync(WorkspaceProvisionRequest.FromSingle(request with { Depth = 1 }), CancellationToken.None).ConfigureAwait(false); + } + + public Task AgentWritesAsync(string cloneDir) => File.WriteAllTextAsync(Path.Combine(cloneDir, "agent.txt"), "the agent's work\n"); + + public async Task RemoteHasBranchAsync() => + !string.IsNullOrWhiteSpace(await GitPublishRemoteFixture.GitAsync(_root, new[] { "--git-dir", Remote.Remote, "for-each-ref", $"refs/heads/{BranchName}" }).ConfigureAwait(false)); + + /// Time passes for the stored pair: the same tokens, now recorded as expiring in — inside the refresher's two-minute window. + public async Task AgeStoredPairAsync(int secondsLeft) + { + var stored = await StoredAsync().ConfigureAwait(false); + + await StoreAsync(new OAuthPayload { AccessToken = stored.AccessToken, RefreshToken = stored.RefreshToken, ExpiresAt = DateTimeOffset.UtcNow.AddSeconds(secondsLeft) }).ConfigureAwait(false); + } + + /// What a sibling run does on finding the token due: refresh against the provider, store the rotated pair. The provider has by then revoked the access token this run holds. + public async Task AnotherRunRefreshesAsync() + { + using var scope = _fixture.BeginScope(); + var credential = await scope.Resolve().Credential.AsNoTracking().Include(c => c.ProviderInstance).SingleAsync(c => c.Id == _seed.CredentialId).ConfigureAwait(false); + var current = (OAuthPayload)scope.Resolve().Resolve(credential); + var instance = credential.ProviderInstance; + + var response = await scope.Resolve().Get(ProviderKind.GitLab).RefreshAsync(new OAuthRefreshInput + { + Instance = instance, ClientId = instance.OauthClientId!, ClientSecret = scope.Resolve().Decrypt(instance.OauthClientSecretEnc!), RefreshToken = current.RefreshToken!, + }, CancellationToken.None).ConfigureAwait(false); + + await StoreAsync(new OAuthPayload { AccessToken = response.AccessToken, RefreshToken = response.RefreshToken, ExpiresAt = response.ExpiresAt }).ConfigureAwait(false); + } + + public async Task StoredAsync() + { + using var scope = _fixture.BeginScope(); + var credential = await scope.Resolve().Credential.AsNoTracking().SingleAsync(c => c.Id == _seed.CredentialId).ConfigureAwait(false); + + return (OAuthPayload)scope.Resolve().Resolve(credential); + } + + private async Task StoreAsync(OAuthPayload payload) + { + using var scope = _fixture.BeginScope(); + var credential = await scope.Resolve().Credential.AsNoTracking().SingleAsync(c => c.Id == _seed.CredentialId).ConfigureAwait(false); + + var stored = await scope.Resolve().UpdatePayloadAsync(credential, credential.EncryptedPayload, payload, CancellationToken.None).ConfigureAwait(false); + + stored.ShouldBeTrue("nothing else holds the refresh lock, so the row should still hold what it read"); + } + + /// The user disconnecting the credential: the real revoke command, as an admin of its team. Its own call to the provider's revoke endpoint goes to a recording client. + public async Task DisconnectAsync() + { + using var scope = _fixture.BeginScopeAs(_seed.UserId, _seed.TeamId, Roles.Admin); + using var revoke = scope.BeginLifetimeScope(b => b.RegisterInstance(new SingleClientRegistry(new StubOAuthClient(ProviderKind.GitLab, new OAuthTokenResponse { AccessToken = "unused" }))).SingleInstance()); + + await revoke.Resolve().Send(new RevokeCredentialCommand { CredentialId = _seed.CredentialId }).ConfigureAwait(false); + } + + public async Task DeleteRepositoryAsync() + { + using var scope = _fixture.BeginScope(); + + await scope.Resolve().Repository.Where(r => r.Id == _seed.RepositoryId).ExecuteUpdateAsync(s => s.SetProperty(r => r.DeletedDate, DateTimeOffset.UtcNow)).ConfigureAwait(false); + } + + private async Task SeedAsync(string? patToken) + { + using var scope = _fixture.BeginScope(); + var db = scope.Resolve(); + var encryptor = scope.Resolve(); + var serializer = scope.Resolve(); + + var suffix = Guid.NewGuid().ToString("N")[..8]; + var user = new User { Id = Guid.NewGuid(), Email = $"u-{suffix}@x", Name = "tester" }; + var team = new Team { Id = Guid.NewGuid(), Slug = $"t-{suffix}", Name = "Team" }; + var instance = new ProviderInstance { Id = Guid.NewGuid(), TeamId = team.Id, Provider = ProviderKind.GitLab, DisplayName = "gitlab", BaseUrl = _provider.BaseUrl, OauthClientId = "client", OauthClientSecretEnc = encryptor.Encrypt("secret") }; + + CredentialPayload payload = patToken is null + ? new OAuthPayload { AccessToken = "at-0", RefreshToken = "rt-0", ExpiresAt = DateTimeOffset.UtcNow.AddHours(2) } + : new PatPayload { Token = patToken }; + + var credential = new Credential { Id = Guid.NewGuid(), TeamId = team.Id, ProviderInstanceId = instance.Id, AuthType = payload.Type, DisplayName = "gitlab credential", EncryptedPayload = encryptor.Encrypt(serializer.Serialize(payload)), ExpiresDate = (payload as OAuthPayload)?.ExpiresAt, Status = CredentialStatus.Active }; + var repository = new Repository { Id = Guid.NewGuid(), TeamId = team.Id, ProviderInstanceId = instance.Id, CredentialId = credential.Id, ExternalId = suffix, NamespacePath = "acme", Name = "repo", FullPath = "acme/repo", DefaultBranch = "main", Visibility = RepositoryVisibility.Private, WebUrl = $"{_provider.BaseUrl}/acme/repo", CloneUrlHttps = Remote.Url, Status = RepositoryStatus.Active }; + + db.User.Add(user); + db.Team.Add(team); + db.ProviderInstance.Add(instance); + db.Credential.Add(credential); + db.Repository.Add(repository); + await db.SaveChangesAsync().ConfigureAwait(false); + + _seed = new Seeded(user.Id, team.Id, credential.Id, repository.Id); + } + + public void Dispose() + { + _sourceScope.Dispose(); + Remote.DisposeAsync().AsTask().GetAwaiter().GetResult(); + _provider.Dispose(); + try { Directory.Delete(_root, recursive: true); } catch { /* best-effort */ } + } + } + + private sealed class SingleClientRegistry(IOAuthClient client) : IOAuthClientRegistry + { + public IOAuthClient Get(ProviderKind kind) => client; + } +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentPublishCredentialRefusalFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentPublishCredentialRefusalFlowTests.cs new file mode 100644 index 000000000..c05adab9f --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentPublishCredentialRefusalFlowTests.cs @@ -0,0 +1,243 @@ +using CodeSpace.Core.Services.Agents.Sandbox; +using CodeSpace.Core.Services.Agents.Sandbox.Runners; +using CodeSpace.Core.Services.Agents.Workspace; +using CodeSpace.Core.Services.Agents.Workspace.Providers; +using CodeSpace.Messages.Agents; +using CodeSpace.Messages.Failures; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; + +namespace CodeSpace.IntegrationTests.Workflows; + +/// +/// HIGH fidelity (Rule 12): the REAL + real git pushing an agent's branch to a +/// real loopback smart-HTTP remote () that, like GitLab, revokes the access token it +/// issued the moment a new one exists and answers every request carrying the old one with a 401. Only the credential store +/// is replaced — by , which answers from a script so each test controls exactly which token +/// the push finds when it asks. (The real store, refresher and database are composed in +/// .) +/// +/// The shape is the one a long run produces: the clone is made with the token the run was provisioned with, the agent +/// works, the token is rotated away from under it, and the push — the run's last act — needs another. Each test asserts +/// from the REMOTE's side what it saw: which credentials it refused, and what landed. +/// +/// Each test owns a GUID-named branch and temp tree, removes them on every path, and skips on Windows or without git. +/// +[Trait("Category", "Integration")] +public sealed class AgentPublishCredentialRefusalFlowTests +{ + private const string Stale = "stale-access-token-aaaa"; + private const string Fresh = "fresh-access-token-bbbb"; + + [Fact] + public async Task A_push_carries_the_credential_resolved_at_the_push_not_the_one_the_clone_was_given() + { + if (OperatingSystem.IsWindows() || !await GitAvailableAsync()) return; + + using var ctx = new RefusalContext(tokens: new[] { Fresh }); + await ctx.StartAsync(); + await using var handle = await ctx.CloneWithAsync(Stale, repositoryNamed: true); + ctx.RotateRemoteTokenTo(Fresh); // the access token the clone carries is revoked while the agent works + + await ctx.AgentWritesAsync(handle.Directory); + var branch = await ((IWorkspacePushHandle)handle).PushChangesAsync(ctx.BranchName, CancellationToken.None); + + branch.ShouldBe(ctx.BranchName); + (await ctx.RemoteHasBranchAsync()).ShouldBeTrue("the branch landed on the remote"); + ctx.Remote.RefusedAuthorizations.ShouldBeEmpty("the revoked token was never presented to the remote at the push"); + ctx.TokenSource.Calls.ShouldBe(2, "the push and its readback each asked for the credential; no local command did"); + ((IWorkspacePushHandle)handle).LastPushedCommitSha().ShouldNotBeNull("the readback was authenticated with the fresh credential too"); + } + + [Fact] + public async Task Without_a_repository_id_the_push_carries_the_token_it_was_given_and_the_revoked_one_is_refused() + { + // The control: a hand-built request that names no repository keeps every earlier behaviour — the token captured at the + // clone is the one the push carries. On a remote that revoked it, THAT is the failure a long run used to end in. + if (OperatingSystem.IsWindows() || !await GitAvailableAsync()) return; + + using var ctx = new RefusalContext(tokens: new[] { Fresh }); + await ctx.StartAsync(); + await using var handle = await ctx.CloneWithAsync(Stale, repositoryNamed: false); + ctx.RotateRemoteTokenTo(Fresh); + + await ctx.AgentWritesAsync(handle.Directory); + var failure = await Should.ThrowAsync(() => ((IWorkspacePushHandle)handle).PushChangesAsync(ctx.BranchName, CancellationToken.None)); + + failure.Message.ShouldContain("Authentication failed", Case.Insensitive, "the remote's refusal of the revoked token, as git reports it"); + failure.Message.ShouldNotContain(Stale, Case.Insensitive, "the refused token never reaches the message"); + ctx.Remote.RefusedAuthorizations.ShouldBe(new[] { GitPublishRemoteFixture.BasicAuthorization("oauth2", Stale) }, "the remote saw the revoked token, once"); + ctx.TokenSource.Calls.ShouldBe(0, "a request that names no repository never asks for a credential"); + (await ctx.RemoteHasBranchAsync()).ShouldBeFalse(); + } + + [Fact] + public async Task A_refused_push_is_tried_once_more_with_the_freshly_resolved_credential() + { + // The store still answered with the stale token when the push asked (a rotation another run had not stored yet), the + // remote refused it, and the credential resolved again is a different one — which is what the second try presents. + if (OperatingSystem.IsWindows() || !await GitAvailableAsync()) return; + + using var ctx = new RefusalContext(tokens: new[] { Stale, Fresh, Fresh }); + await ctx.StartAsync(); + await using var handle = await ctx.CloneWithAsync(Stale, repositoryNamed: true); + ctx.RotateRemoteTokenTo(Fresh); + + await ctx.AgentWritesAsync(handle.Directory); + var branch = await ((IWorkspacePushHandle)handle).PushChangesAsync(ctx.BranchName, CancellationToken.None); + + branch.ShouldBe(ctx.BranchName); + (await ctx.RemoteHasBranchAsync()).ShouldBeTrue(); + ctx.Remote.RefusedAuthorizations.ShouldBe(new[] { GitPublishRemoteFixture.BasicAuthorization("oauth2", Stale) }, "the stale credential was refused once and never presented again"); + ctx.TokenSource.Calls.ShouldBe(3, "the push, the resolve for its one retry, and the readback"); + } + + [Fact] + public async Task A_credential_the_remote_refuses_even_after_a_fresh_resolve_is_a_typed_rejection_with_the_credentials_redacted() + { + if (OperatingSystem.IsWindows() || !await GitAvailableAsync()) return; + + using var ctx = new RefusalContext(tokens: new[] { Stale, "second-bad-token-cccc" }); + await ctx.StartAsync(); + await using var handle = await ctx.CloneWithAsync(Stale, repositoryNamed: true); + ctx.RotateRemoteTokenTo(Fresh); // neither of the two the store answers with is the live one + + await ctx.AgentWritesAsync(handle.Directory); + var failure = await Should.ThrowAsync(() => ((IWorkspacePushHandle)handle).PushChangesAsync(ctx.BranchName, CancellationToken.None)); + + failure.Code.ShouldBe(FailureCodes.GitCredentialRejected); + failure.Kind.ShouldBe(FailureKind.PreconditionRequired); + failure.Message.ShouldContain("Reconnect the provider", customMessage: "the message names what to do about it"); + failure.Message.ShouldContain(ctx.Remote.Url, customMessage: "the message names the remote, without userinfo"); + failure.Message.ShouldNotContain(Stale, Case.Insensitive); + failure.Message.ShouldNotContain("second-bad-token-cccc", Case.Insensitive); + ctx.Remote.RefusedAuthorizations.Count.ShouldBe(2, "each different credential was presented exactly once"); + ctx.TokenSource.Calls.ShouldBe(2, "the push and the one resolve for its retry; no readback of a push that never landed"); + (await ctx.RemoteHasBranchAsync()).ShouldBeFalse(); + } + + [Fact] + public async Task A_credential_that_resolves_to_the_same_refused_one_is_not_presented_a_second_time() + { + // GitLab counts every failed login against the account, and a retry of the same token is refused the same way. + if (OperatingSystem.IsWindows() || !await GitAvailableAsync()) return; + + using var ctx = new RefusalContext(tokens: new[] { Stale, Stale }); + await ctx.StartAsync(); + await using var handle = await ctx.CloneWithAsync(Stale, repositoryNamed: true); + ctx.RotateRemoteTokenTo(Fresh); + + await ctx.AgentWritesAsync(handle.Directory); + var failure = await Should.ThrowAsync(() => ((IWorkspacePushHandle)handle).PushChangesAsync(ctx.BranchName, CancellationToken.None)); + + failure.Message.ShouldContain("returned the same one"); + ctx.Remote.RefusedAuthorizations.ShouldBe(new[] { GitPublishRemoteFixture.BasicAuthorization("oauth2", Stale) }, "the refused token went out once, not once per attempt"); + ctx.TokenSource.Calls.ShouldBe(2); + } + + [Fact] + public async Task What_real_git_prints_for_a_refused_credential_is_what_the_classifier_reads() + { + // The fixture check for every scripted refusal in the unit tests: git's own words, from git, against a remote that + // answered 401 — never a string typed from memory. + if (OperatingSystem.IsWindows() || !await GitAvailableAsync()) return; + + using var ctx = new RefusalContext(tokens: Array.Empty()); + await ctx.StartAsync(); + await using var handle = await ctx.CloneWithAsync(Stale, repositoryNamed: false); + ctx.RotateRemoteTokenTo(Fresh); + + await ctx.AgentWritesAsync(handle.Directory); + var failure = await Should.ThrowAsync(() => ((IWorkspacePushHandle)handle).PushChangesAsync(ctx.BranchName, CancellationToken.None)); + + var stderr = failure.Message[(failure.Message.IndexOf("): ", StringComparison.Ordinal) + 3)..]; + Console.WriteLine($"[credential-refusal] git said: {stderr}"); + + LocalGitWorkspaceProvider.IsCredentialRefusal(stderr).ShouldBeTrue($"git's refusal text was not recognised: {stderr}"); + } + + // ── Plumbing ─────────────────────────────────────────────────────────────────── + + private static async Task GitAvailableAsync() + { + try { return (await new LocalProcessRunner().RunAsync(new SandboxSpec { Command = "git", Args = new[] { "--version" }, TimeoutSeconds = 15 }, CancellationToken.None)).Status == SandboxStatus.Success; } + catch { return false; } + } + + /// The credential store, from a script: answers each ask with the next token (repeating the last), and counts the asks. + private sealed class ScriptedTokenSource(IReadOnlyList tokens) : IWorkspaceTokenSource + { + private int _calls; + + public int Calls => Volatile.Read(ref _calls); + + public Task ResolveAsync(Guid repositoryId, Guid teamId, CancellationToken cancellationToken) + { + var index = Interlocked.Increment(ref _calls) - 1; + + return Task.FromResult(tokens.Count == 0 ? null : tokens[Math.Min(index, tokens.Count - 1)]); + } + } + + /// A hermetic git environment: no operator config, no prompt, a HOME of its own — the runner under test otherwise inherits the host's. + private sealed class HermeticRunner(string home) : ISandboxRunner + { + private readonly LocalProcessRunner _inner = new(); + + public string Kind => "local"; + + public Task RunAsync(SandboxSpec spec, CancellationToken cancellationToken) + { + var environment = new Dictionary(spec.Environment) { ["HOME"] = home, ["GIT_CONFIG_NOSYSTEM"] = "1", ["GIT_CONFIG_GLOBAL"] = Path.Combine(home, ".gitconfig"), ["GIT_TERMINAL_PROMPT"] = "0" }; + + return _inner.RunAsync(spec with { Environment = environment }, cancellationToken); + } + } + + private sealed class RefusalContext : IDisposable + { + private readonly string _root = Path.Combine(Path.GetTempPath(), "cs-credrefusal-" + Guid.NewGuid().ToString("N")); + private readonly Guid _repositoryId = Guid.NewGuid(); + private readonly Guid _teamId = Guid.NewGuid(); + + public RefusalContext(IReadOnlyList tokens) + { + Directory.CreateDirectory(Path.Combine(_root, "home")); + TokenSource = new ScriptedTokenSource(tokens); + Provider = new LocalGitWorkspaceProvider(new SandboxRunnerRegistry(new ISandboxRunner[] { new HermeticRunner(Path.Combine(_root, "home")) }), TokenSource, NullLogger.Instance, Path.Combine(_root, "workspaces")); + } + + /// Reads authenticate too, as a private GitLab project's do, so the clone itself proves the token it was given. + public GitPublishRemoteFixture Remote { get; } = new() { AuthenticateReads = true }; + public ScriptedTokenSource TokenSource { get; } + public LocalGitWorkspaceProvider Provider { get; } + public string BranchName { get; } = "codespace/agent/" + Guid.NewGuid().ToString("N"); + + public async Task StartAsync() + { + await Remote.StartAsync(); + Remote.AcceptOnly("oauth2", Stale); // the token the run is provisioned with is the live one — until it is rotated + } + + /// The provider issues a new access token and revokes the old: from here the remote refuses the token the clone carries. + public void RotateRemoteTokenTo(string token) => Remote.AcceptOnly("oauth2", token); + + public async Task CloneWithAsync(string token, bool repositoryNamed) => + await Provider.PrepareAsync(WorkspaceProvisionRequest.FromSingle(new WorkspaceRequest + { + RepositoryUrl = Remote.Url, Token = token, TokenUsername = "oauth2", Depth = 1, + RepositoryId = repositoryNamed ? _repositoryId : null, TeamId = repositoryNamed ? _teamId : null, + }), CancellationToken.None); + + public Task AgentWritesAsync(string cloneDir) => File.WriteAllTextAsync(Path.Combine(cloneDir, "agent.txt"), "the agent's work\n"); + + public async Task RemoteHasBranchAsync() => + !string.IsNullOrWhiteSpace(await GitPublishRemoteFixture.GitAsync(_root, new[] { "--git-dir", Remote.Remote, "for-each-ref", $"refs/heads/{BranchName}" })); + + public void Dispose() + { + Remote.DisposeAsync().AsTask().GetAwaiter().GetResult(); + try { Directory.Delete(_root, recursive: true); } catch { /* best-effort */ } + } + } +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunLongRunPublishFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunLongRunPublishFlowTests.cs new file mode 100644 index 000000000..6956a4ade --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunLongRunPublishFlowTests.cs @@ -0,0 +1,358 @@ +using System.Text.Json; +using System.Web; +using Autofac; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Agents; +using CodeSpace.Core.Services.Agents.ModelCredentials; +using CodeSpace.Core.Services.Agents.Sandbox; +using CodeSpace.Core.Services.Agents.Sandbox.Runners; +using CodeSpace.Core.Services.Agents.Workspace; +using CodeSpace.Core.Services.Agents.Workspace.Providers; +using CodeSpace.Core.Services.Credentials; +using CodeSpace.Core.Services.OAuth; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.IntegrationTests.Webhooks; +using CodeSpace.IntegrationTests.Workflows.Infrastructure; +using CodeSpace.Messages.Agents; +using CodeSpace.Messages.Credentials; +using CodeSpace.Messages.Enums; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; + +namespace CodeSpace.IntegrationTests.Workflows; + +/// +/// HIGH fidelity (Rule 12), end to end: the REAL — real Postgres, a real /bin/sh +/// harness, the real resolver → token source → provider auth → OAuth refresher chain, real git — runs a LONG run on a +/// GitLab OAuth-bound repository against a remote that revokes the access token it issued the moment a new one exists. +/// The harness holds the run open until the test has done, from outside, what a sibling run does to a credential in +/// production: refreshed it, so the token the run was provisioned with is dead by the time its last act — the push — runs. +/// +/// The run must publish its branch; and when no credential the store can offer is one the remote accepts, the run keeps +/// its Succeeded status — it did its work, and every caller that reads Succeeded must keep reading it — while the failure +/// is recorded as a typed, durable publish failure (code, message, manifest, timeline) instead of a Warning nobody reads. +/// +/// Only the workspace provider's git runner is isolated from the host's git config; everything else is the production +/// composition. Each test owns GUID-named rows, a marker directory and a temp tree, removes them on every path, and skips on +/// Windows or without git. +/// +[Collection(PostgresCollection.Name)] +[Trait("Category", "Integration")] +public sealed class AgentRunLongRunPublishFlowTests +{ + private readonly PostgresFixture _fixture; + + public AgentRunLongRunPublishFlowTests(PostgresFixture fixture) { _fixture = fixture; } + + [Fact] + public async Task A_run_that_outlived_its_access_token_still_publishes_its_branch() + { + if (!await CanRunGitAsync()) return; + + using var world = await LongRunWorld.StartAsync(_fixture); + + var runId = await world.RunUntilTheTokenIsRevokedAsync(refreshAsASibling: true); + + var run = await world.ReadRunAsync(runId); + run.Status.ShouldBe(AgentRunStatus.Succeeded); + + var result = JsonSerializer.Deserialize(run.ResultJson!, AgentJson.Options)!; + result.ProducedBranch.ShouldBe(AgentRunExecutor.BuildBranchName(runId), "the branch published, though the token the run was provisioned with was revoked before the push"); + result.PublishError.ShouldBeNull(); + result.PublishFailureCode.ShouldBeNull(); + result.PushedCommitSha.ShouldNotBeNull("the readback confirmed the remote tip with the fresh credential too"); + + (await world.RemoteHasBranchAsync(AgentRunExecutor.BuildBranchName(runId))).ShouldBeTrue("the remote carries the branch"); + world.Remote.RefusedAuthorizations.ShouldBeEmpty("the revoked token was never presented to the remote at the push"); + } + + [Fact] + public async Task A_run_whose_credential_the_remote_refuses_stays_succeeded_with_a_typed_recorded_publish_failure() + { + if (!await CanRunGitAsync()) return; + + using var world = await LongRunWorld.StartAsync(_fixture); + + // After the sibling's refresh the remote accepts a token the store has never held — as when the grant was revoked at + // the provider by an administrator: nothing the store can answer works, however many times it is asked. + var runId = await world.RunUntilTheTokenIsRevokedAsync(refreshAsASibling: false); + + var run = await world.ReadRunAsync(runId); + run.Status.ShouldBe(AgentRunStatus.Succeeded, "the run did its work; the publish is the side effect that failed — callers that read Succeeded keep reading it"); + + var result = JsonSerializer.Deserialize(run.ResultJson!, AgentJson.Options)!; + result.ProducedBranch.ShouldBeNull(); + result.PublishFailureCode.ShouldBe("git_credential_rejected", "recorded on the run, typed"); + result.PublishError.ShouldNotBeNullOrEmpty(); + result.PublishError.ShouldContain("Reconnect the provider", customMessage: "the recorded failure says what to do about it"); + result.PublishError.ShouldNotContain("at-0", Case.Insensitive, "no token reaches the record"); + result.ChangedFiles.ShouldContain("agent-change.txt", "the agent's work is still captured — the patch is what a reviewer can still use"); + + var manifest = await world.ReadPublishManifestAsync(runId); + manifest.PublishError.ShouldNotBeNullOrEmpty("the durable publish record carries the failure too"); + manifest.PublishError.ShouldContain("Reconnect the provider"); + manifest.Branch.ShouldBeNull(); + + (await world.ReadEventsAsync(runId)).ShouldContain(e => e.Kind == AgentEventKind.Warning && e.Text!.StartsWith("Could not push the agent's changes to a branch"), "the timeline says why no branch appeared"); + + (await world.RemoteHasBranchAsync(AgentRunExecutor.BuildBranchName(runId))).ShouldBeFalse(); + world.Remote.RefusedAuthorizations.Count.ShouldBe(1, "the credential the store offered was presented once and not again: a refusal resolved to the same credential is not retried"); + } + + // ── World ────────────────────────────────────────────────────────────────────── + + private static async Task CanRunGitAsync() + { + if (OperatingSystem.IsWindows()) return false; + + try { return (await new LocalProcessRunner().RunAsync(new SandboxSpec { Command = "git", Args = new[] { "--version" }, TimeoutSeconds = 15 }, CancellationToken.None)).Status == SandboxStatus.Success; } + catch { return false; } + } + + /// A hermetic git environment for the workspace provider: no operator config, no prompt, a HOME of its own. + private sealed class HermeticRunner(string home) : ISandboxRunner + { + private readonly LocalProcessRunner _inner = new(); + + public string Kind => "local"; + + public Task RunAsync(SandboxSpec spec, CancellationToken cancellationToken) + { + var environment = new Dictionary(spec.Environment) { ["HOME"] = home, ["GIT_CONFIG_NOSYSTEM"] = "1", ["GIT_CONFIG_GLOBAL"] = Path.Combine(home, ".gitconfig"), ["GIT_TERMINAL_PROMPT"] = "0" }; + + return _inner.RunAsync(spec with { Environment = environment }, cancellationToken); + } + } + + /// A shell harness that records when it started, does its edit, then holds the run open until released — a run long enough for the world to change under it. + private sealed class HoldingHarness(string startedMarker, string releaseMarker) : IAgentHarness + { + public string Kind => "scripted"; + public string Version => "test"; + public IReadOnlyList Models { get; } = new[] { "test-model" }; + + public SandboxSpec BuildInvocation(AgentTask task) => new() + { + Command = "/bin/sh", + Args = new[] { "-c", $"printf 'by the agent\\n' > agent-change.txt; : > '{startedMarker}'; while [ ! -f '{releaseMarker}' ]; do sleep 0.1; done; echo edited" }, + WorkingDirectory = task.WorkspaceDirectory, + TimeoutSeconds = task.TimeoutSeconds, + }; + + public IReadOnlyList ParseEvents(string rawLine) => + string.IsNullOrWhiteSpace(rawLine) ? Array.Empty() : new[] { new AgentEvent { Kind = AgentEventKind.AssistantMessage, Text = rawLine.Trim() } }; + + public IAgentEventFolder CreateFolder() => ScriptedFolders.Result(); + } + + private sealed class LongRunWorld : IDisposable + { + private const string InvalidGrantBody = """{"error":"invalid_grant","error_description":"The provided authorization grant is invalid, expired, revoked, does not match the redirection URI used in the authorization request, or was issued to another client."}"""; + + private readonly PostgresFixture _fixture; + private readonly string _root = Path.Combine(Path.GetTempPath(), "cs-longrun-" + Guid.NewGuid().ToString("N")); + private readonly StubProviderHost _provider = new(); + private readonly object _gate = new(); + private string _liveRefreshToken = "rt-0"; + private Guid _teamId; + private Guid _credentialId; + private Guid _repositoryId; + + private LongRunWorld(PostgresFixture fixture) + { + _fixture = fixture; + Directory.CreateDirectory(Path.Combine(_root, "home")); + Directory.CreateDirectory(Path.Combine(_root, "markers")); + } + + /// Reads authenticate too, as a private GitLab project's do, so the clone itself proves the token it was given. + public GitPublishRemoteFixture Remote { get; } = new() { AuthenticateReads = true }; + + public static async Task StartAsync(PostgresFixture fixture) + { + var world = new LongRunWorld(fixture); + await world.Remote.StartAsync(); + world.Remote.AcceptOnly("oauth2", "at-0"); // the token the run is provisioned with is the live one — until it is rotated + world._provider.Answer("POST", "/oauth/token", world.Refresh); + await world.SeedAsync(); + return world; + } + + /// + /// Run the executor against the world: the harness holds the run open once the workspace is cloned; the test then does + /// what happens to a long run's credential — a sibling refreshes it (the provider revokes the token this run holds and + /// the remote stops accepting it) — and releases the harness, so the push is the run's next act. With + /// false the rotated pair never reaches the store, so the only token it can offer + /// is one the remote has already refused. + /// + public async Task RunUntilTheTokenIsRevokedAsync(bool refreshAsASibling) + { + var runId = await CreateRunAsync(); + var started = Path.Combine(_root, "markers", "started"); + var release = Path.Combine(_root, "markers", "release"); + + var executing = Task.Run(() => ExecuteAsync(runId, new HoldingHarness(started, release))); + + await WaitForAsync(started, executing).ConfigureAwait(false); + + if (refreshAsASibling) await AnotherRunRefreshesAsync().ConfigureAwait(false); + else Remote.AcceptOnly("oauth2", "a-token-the-store-never-held"); + + await File.WriteAllTextAsync(release, "go").ConfigureAwait(false); + await executing.WaitAsync(TimeSpan.FromSeconds(120)).ConfigureAwait(false); + + return runId; + } + + public async Task ReadRunAsync(Guid runId) + { + using var scope = _fixture.BeginScope(); + + return await scope.Resolve().GetAsync(runId, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ReadPublishManifestAsync(Guid runId) + { + using var scope = _fixture.BeginScope(); + + return await scope.Resolve().PublishManifest.AsNoTracking().SingleAsync(m => m.AgentRunId == runId).ConfigureAwait(false); + } + + public async Task> ReadEventsAsync(Guid runId) + { + using var scope = _fixture.BeginScope(); + + return await scope.Resolve().GetEventsAsync(runId, _teamId, 0, CancellationToken.None).ConfigureAwait(false); + } + + public async Task RemoteHasBranchAsync(string branch) => + !string.IsNullOrWhiteSpace(await GitPublishRemoteFixture.GitAsync(_root, new[] { "--git-dir", Remote.Remote, "for-each-ref", $"refs/heads/{branch}" }).ConfigureAwait(false)); + + /// GitLab's token endpoint: the live refresh token rotates (rt-N → rt-N+1, access token at-N+1) and the access token it replaces is revoked — at the git remote too; any other refresh token is refused. + private StubProviderHost.StubReply Refresh(StubProviderHost.RecordedRequest request) + { + var presented = HttpUtility.ParseQueryString(request.Body)["refresh_token"]; + + lock (_gate) + { + if (presented is null || presented != _liveRefreshToken) return new StubProviderHost.StubReply(400, InvalidGrantBody); + + var generation = int.Parse(_liveRefreshToken["rt-".Length..]) + 1; + _liveRefreshToken = $"rt-{generation}"; + Remote.AcceptOnly("oauth2", $"at-{generation}"); + + return new StubProviderHost.StubReply(200, JsonSerializer.Serialize(new { access_token = $"at-{generation}", token_type = "Bearer", expires_in = 7200, refresh_token = _liveRefreshToken, scope = "api" })); + } + } + + private static async Task WaitForAsync(string marker, Task executing) + { + var deadline = DateTimeOffset.UtcNow.AddSeconds(60); + + while (!File.Exists(marker)) + { + if (executing.IsCompleted) await executing.ConfigureAwait(false); // surfaces why the executor ended before the harness started + + if (DateTimeOffset.UtcNow > deadline) throw new TimeoutException($"The harness never started within 60s — the run did not get past provisioning its workspace. Inspect the run's events and the clone under {Path.GetTempPath()}codespace-agent-workspaces."); + + await Task.Delay(50).ConfigureAwait(false); + } + } + + /// What a sibling run does on finding the token due: refresh against the provider and store the rotated pair. + private async Task AnotherRunRefreshesAsync() + { + using var scope = _fixture.BeginScope(); + var credential = await scope.Resolve().Credential.AsNoTracking().Include(c => c.ProviderInstance).SingleAsync(c => c.Id == _credentialId).ConfigureAwait(false); + var current = (OAuthPayload)scope.Resolve().Resolve(credential); + var instance = credential.ProviderInstance; + + var response = await scope.Resolve().Get(ProviderKind.GitLab).RefreshAsync(new OAuthRefreshInput + { + Instance = instance, ClientId = instance.OauthClientId!, ClientSecret = scope.Resolve().Decrypt(instance.OauthClientSecretEnc!), RefreshToken = current.RefreshToken!, + }, CancellationToken.None).ConfigureAwait(false); + + var stored = await scope.Resolve().UpdatePayloadAsync(credential, credential.EncryptedPayload, new OAuthPayload { AccessToken = response.AccessToken, RefreshToken = response.RefreshToken, ExpiresAt = response.ExpiresAt }, CancellationToken.None).ConfigureAwait(false); + + stored.ShouldBeTrue("nothing else holds the refresh lock, so the row should still hold what it read"); + } + + private async Task CreateRunAsync() + { + using var scope = await WorkflowsTestSeed.BeginSeedOperatorScopeAsync(_fixture, _teamId).ConfigureAwait(false); + var run = await scope.Resolve().CreateAsync(new AgentTask { Goal = "edit", Harness = "scripted", Model = "test-model", RepositoryId = _repositoryId }, _teamId, null, null, iterationKey: "", cancellationToken: CancellationToken.None).ConfigureAwait(false); + + return run.Id; + } + + /// The executor as production composes it — except the workspace provider's git runner, isolated from the host's git config. + private async Task ExecuteAsync(Guid runId, IAgentHarness harness) + { + using var scope = _fixture.BeginScope(); + + var provider = new LocalGitWorkspaceProvider( + new SandboxRunnerRegistry(new ISandboxRunner[] { new HermeticRunner(Path.Combine(_root, "home")) }), + scope.Resolve(), + NullLogger.Instance, + Path.Combine(_root, "workspaces")); + + var executor = new AgentRunExecutor( + scope.Resolve(), + new AgentHarnessRegistry(new[] { harness }), + new HarnessModelReconciler(new AgentHarnessRegistry(new[] { harness }), scope.Resolve(), scope.Resolve()), + scope.Resolve(), + scope.Resolve(), + scope.Resolve(), + new WorkspaceProviderRegistry(new IWorkspaceProvider[] { provider }), + scope.Resolve(), + scope.Resolve(), + scope.Resolve(), + scope.Resolve(), + scope.Resolve(), + scope.Resolve(), + scope.Resolve(), scope.Resolve(), scope.Resolve(), + scope.Resolve>(), + NullLogger.Instance); + + await executor.ExecuteAsync(runId, CancellationToken.None).ConfigureAwait(false); + } + + private async Task SeedAsync() + { + using var scope = _fixture.BeginScope(); + var db = scope.Resolve(); + var encryptor = scope.Resolve(); + var serializer = scope.Resolve(); + + var suffix = Guid.NewGuid().ToString("N")[..8]; + var userId = Guid.NewGuid(); + _teamId = Guid.NewGuid(); + db.User.Add(new User { Id = userId, Email = $"agent-{suffix}@test.local", Name = $"agent-{suffix}" }); + db.Team.Add(new Team { Id = _teamId, Slug = $"agent-{suffix}", Name = "Agent Team", Kind = TeamKind.Workspace }); + db.TeamMembership.Add(new TeamMembership { Id = Guid.NewGuid(), TeamId = _teamId, UserId = userId, Role = TeamRole.Owner }); + + var instance = new ProviderInstance { Id = Guid.NewGuid(), TeamId = _teamId, Provider = ProviderKind.GitLab, DisplayName = "gitlab", BaseUrl = _provider.BaseUrl, OauthClientId = "client", OauthClientSecretEnc = encryptor.Encrypt("secret") }; + var payload = new OAuthPayload { AccessToken = "at-0", RefreshToken = "rt-0", ExpiresAt = DateTimeOffset.UtcNow.AddHours(2) }; + var credential = new Credential { Id = Guid.NewGuid(), TeamId = _teamId, ProviderInstanceId = instance.Id, AuthType = AuthType.OAuth, DisplayName = "gitlab oauth", EncryptedPayload = encryptor.Encrypt(serializer.Serialize(payload)), ExpiresDate = payload.ExpiresAt, Status = CredentialStatus.Active }; + var repository = new Repository { Id = Guid.NewGuid(), TeamId = _teamId, ProviderInstanceId = instance.Id, CredentialId = credential.Id, ExternalId = suffix, NamespacePath = "acme", Name = "repo", FullPath = "acme/repo", DefaultBranch = "main", Visibility = RepositoryVisibility.Private, WebUrl = $"{_provider.BaseUrl}/acme/repo", CloneUrlHttps = Remote.Url, Status = RepositoryStatus.Active }; + + db.ProviderInstance.Add(instance); + db.Credential.Add(credential); + db.Repository.Add(repository); + await db.SaveChangesAsync().ConfigureAwait(false); + + _credentialId = credential.Id; + _repositoryId = repository.Id; + } + + public void Dispose() + { + Remote.DisposeAsync().AsTask().GetAwaiter().GetResult(); + _provider.Dispose(); + try { Directory.Delete(_root, recursive: true); } catch { /* best-effort */ } + } + } +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/GitPublishRemoteFixture.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/GitPublishRemoteFixture.cs index 59d20f9c5..1d2c5526b 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/GitPublishRemoteFixture.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/GitPublishRemoteFixture.cs @@ -83,7 +83,24 @@ internal sealed class GitPublishRemoteFixture : IAsyncDisposable public IReadOnlyList HarvestedAuthorizations { get { lock (_gate) return _harvestedAuthorizations.ToList(); } } /// The Authorization header that carries , as git sends it. - public static string TokenAuthorization => "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{FakeToken}")); + public static string TokenAuthorization => BasicAuthorization("x-access-token", FakeToken); + + /// The Authorization header git sends for and . + public static string BasicAuthorization(string username, string password) => "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{username}:{password}")); + + private volatile string _acceptedAuthorization = TokenAuthorization; + + /// + /// Demand this credential from now on instead of — a remote whose accepted token moves, as an + /// OAuth provider's does when it rotates a pair: the access token it issued before is refused the moment the new one + /// exists. Credentials presented to a write endpoint and refused are listed in . + /// + public void AcceptOnly(string username, string password) => _acceptedAuthorization = BasicAuthorization(username, password); + + /// The Authorization headers a git or LFS write endpoint refused, in the order they arrived — what the remote saw a stale credential do. + public IReadOnlyList RefusedAuthorizations { get { lock (_gate) return _refusedAuthorizations.ToList(); } } + + private readonly List _refusedAuthorizations = new(); public async Task StartAsync() { @@ -282,7 +299,7 @@ private async Task ServeHarvesterAsync(HttpListenerContext context) if (IsRefAdvertisement(context, path)) { await ServeGitAsync(context, path, anonymous: true); return; } await context.Request.InputStream.CopyToAsync(Stream.Null, _stopping.Token); - Unauthorized(context); + Unauthorized(context, recordRefusal: false); } finally { context.Response.Close(); } } @@ -296,10 +313,12 @@ private void RedirectToHarvester(HttpListenerContext context, string path) context.Response.RedirectLocation = _harvesterAuthority + path + context.Request.Url!.Query; } - private static bool AuthOk(HttpListenerContext context) => string.Equals(context.Request.Headers["Authorization"], TokenAuthorization, StringComparison.Ordinal); + private bool AuthOk(HttpListenerContext context) => string.Equals(context.Request.Headers["Authorization"], _acceptedAuthorization, StringComparison.Ordinal); - private void Unauthorized(HttpListenerContext context) + private void Unauthorized(HttpListenerContext context, bool recordRefusal = true) { + if (recordRefusal && context.Request.Headers["Authorization"] is { } presented) lock (_gate) _refusedAuthorizations.Add(presented); + context.Response.StatusCode = 401; context.Response.Headers["WWW-Authenticate"] = "Basic realm=\"fixture\""; } diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/RepositoryWorkspaceResolverTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/RepositoryWorkspaceResolverTests.cs index ba4ee66d9..ffdbbc69d 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/RepositoryWorkspaceResolverTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/RepositoryWorkspaceResolverTests.cs @@ -251,6 +251,78 @@ public async Task A_multi_repo_workspace_resolves_every_repo_into_a_provision() provision.Primary!.Alias.ShouldBe("web"); } + // ─── The repository a token was resolved for, and the source that resolves it again ────────────────── + + [Fact] + public async Task Every_resolved_request_names_the_repository_and_team_its_token_was_resolved_for() + { + // The workspace provider re-asks for the credential by these two ids right before it pushes, so a request that + // lost them would silently push with the token it was provisioned with — hours stale on a long run. + var teamId = await SeedTeamAsync(); + var githubId = await SeedRepoAsync(teamId, ProviderKind.GitHub, "https://github.com/org/web.git", "main", token: "ghp_web"); + var anonymousId = await SeedRepoAsync(teamId, ProviderKind.GitLab, "https://gitlab.com/org/public.git", "main", token: null); + + var legacy = await ResolveAsync(TaskFor(githubId), teamId); + var authored = await ResolveAsync(new AgentTask { Goal = "g", Harness = "h", Model = "m", Workspace = WorkspaceSpec.FromRepository(githubId) }, teamId); + var anonymous = await ResolveAsync(TaskFor(anonymousId), teamId); + + foreach (var (request, repositoryId) in new[] { (legacy, githubId), (authored, githubId), (anonymous, anonymousId) }) + { + request.ShouldNotBeNull(); + request.RepositoryId.ShouldBe(repositoryId); + request.TeamId.ShouldBe(teamId); + } + + var multi = await ResolveProvisionAsync(new AgentTask + { + Goal = "g", Harness = "h", Model = "m", + Workspace = new WorkspaceSpec { PrimaryAlias = "web", Repositories = new[] { new WorkspaceRepositorySpec { Alias = "web", RepositoryId = githubId, Access = WorkspaceAccess.Write, IsPrimary = true }, new WorkspaceRepositorySpec { Alias = "docs", RepositoryId = anonymousId, Access = WorkspaceAccess.Read } } }, + }, teamId); + + multi!.Repositories.Single(r => r.Alias == "web").CloneRequest.RepositoryId.ShouldBe(githubId); + multi.Repositories.Single(r => r.Alias == "docs").CloneRequest.RepositoryId.ShouldBe(anonymousId, "each repo of a multi-repo workspace names its own repository"); + } + + [Fact] + public async Task The_token_source_answers_the_credential_the_repository_holds_now_not_the_one_it_held_when_first_asked() + { + // Nothing is cached: the source opens a fresh scope per ask and the resolver loads the row untracked, so a credential + // the repository was re-linked to between two asks is the one the second ask gets. + var teamId = await SeedTeamAsync(); + var repoId = await SeedRepoAsync(teamId, ProviderKind.GitLab, "https://gitlab.com/org/repo.git", "main", token: "glpat_first"); + var source = _fixture.BeginScope().Resolve(); + + (await source.ResolveAsync(repoId, teamId, CancellationToken.None)).ShouldBe("glpat_first"); + + await RelinkToANewTokenAsync(repoId, teamId, "glpat_second"); + + (await source.ResolveAsync(repoId, teamId, CancellationToken.None)).ShouldBe("glpat_second"); + } + + [Fact] + public async Task The_token_source_answers_null_for_a_repository_with_no_credential() + { + var teamId = await SeedTeamAsync(); + var repoId = await SeedRepoAsync(teamId, ProviderKind.GitHub, "https://github.com/org/public.git", "main", token: null); + + (await _fixture.BeginScope().Resolve().ResolveAsync(repoId, teamId, CancellationToken.None)).ShouldBeNull(); + } + + [Fact] + public async Task The_token_source_fails_closed_for_an_unknown_foreign_or_deleted_repository() + { + var teamId = await SeedTeamAsync(); + var otherTeam = await SeedTeamAsync(); + var foreignId = await SeedRepoAsync(otherTeam, ProviderKind.GitHub, "https://github.com/org/r.git", "main", token: "ghp_foreign"); + var deletedId = await SeedRepoAsync(teamId, ProviderKind.GitHub, "https://github.com/org/d.git", "main", token: "ghp_deleted"); + await SoftDeleteRepoAsync(deletedId); + var source = _fixture.BeginScope().Resolve(); + + await Should.ThrowAsync(() => source.ResolveAsync(Guid.NewGuid(), teamId, CancellationToken.None)); + await Should.ThrowAsync(() => source.ResolveAsync(foreignId, teamId, CancellationToken.None)); + await Should.ThrowAsync(() => source.ResolveAsync(deletedId, teamId, CancellationToken.None)); + } + // ─── Helpers ───────────────────────────────────────────────────────────── private static AgentTask TaskFor(Guid repositoryId) => new() { Goal = "g", Harness = "h", Model = "m", RepositoryId = repositoryId }; @@ -321,6 +393,22 @@ private async Task SeedRepoAsync(Guid teamId, ProviderKind provider, strin return repoId; } + /// Re-link a seeded repo to a new credential holding — what the relink command does to the binding. + private async Task RelinkToANewTokenAsync(Guid repoId, Guid teamId, string token) + { + using var scope = _fixture.BeginScope(); + var db = scope.Resolve(); + var encryptor = scope.Resolve(); + var serializer = scope.Resolve(); + var repo = await db.Repository.SingleAsync(r => r.Id == repoId); + + var credentialId = Guid.NewGuid(); + db.Credential.Add(new Credential { Id = credentialId, TeamId = teamId, ProviderInstanceId = repo.ProviderInstanceId, AuthType = AuthType.Pat, DisplayName = "relinked", Status = CredentialStatus.Active, EncryptedPayload = encryptor.Encrypt(serializer.Serialize(new PatPayload { Token = token })) }); + repo.CredentialId = credentialId; + + await db.SaveChangesAsync(); + } + /// Soft-delete a seeded repo (set DeletedDate) so the clone gate's DeletedDate == null filter must refuse it. private async Task SoftDeleteRepoAsync(Guid repoId) { diff --git a/backend/tests/CodeSpace.UnitTests/Architecture/FailureTaxonomyTests.cs b/backend/tests/CodeSpace.UnitTests/Architecture/FailureTaxonomyTests.cs index 1b4f06526..e4ab727be 100644 --- a/backend/tests/CodeSpace.UnitTests/Architecture/FailureTaxonomyTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Architecture/FailureTaxonomyTests.cs @@ -75,6 +75,7 @@ public void The_wire_codes_are_pinned() FailureCodes.ProviderError.ShouldBe("provider_error"); FailureCodes.RateLimited.ShouldBe("rate_limited"); FailureCodes.CredentialDisconnected.ShouldBe("credential_disconnected"); + FailureCodes.GitCredentialRejected.ShouldBe("git_credential_rejected"); FailureCodes.ModelEndpointKeyRequired.ShouldBe("model_endpoint_key_required"); FailureCodes.ProviderEndpointCredentialsBound.ShouldBe("provider_endpoint_credentials_bound"); FailureCodes.ProviderEndpointSecretRequired.ShouldBe("provider_endpoint_secret_required"); diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorAcceptanceTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorAcceptanceTests.cs index ebc42d036..c6149196f 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorAcceptanceTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorAcceptanceTests.cs @@ -438,6 +438,64 @@ public async Task A_multi_repo_result_with_no_produced_branch_anywhere_fails_clo grader.Calls.ShouldBe(0); } + // ── A push the remote refused for its credential: the grade names it instead of "no-branch-or-repo" ────────── + + [Fact] + public async Task A_contract_whose_push_was_refused_for_its_credential_fails_closed_naming_the_credential_and_is_infra() + { + var (executor, grader) = NewExecutor(new BenchmarkGrade { Passed = true, Detail = "unused" }); + + var rejected = Succeeded() with { ProducedBranch = null, PublishError = "The remote refused the git credential for https://example.test/r.git", PublishFailureCode = "git_credential_rejected" }; + var result = await executor.GradeAcceptanceIfPresentAsync(Run(), TaskWith(Spec("sh", "check.sh")), rejected, workspace: null, CancellationToken.None); + + result.Status.ShouldBe(AgentRunStatus.Failed, "the contract still was not met — only the reason it states changes"); + result.AcceptanceDetail.ShouldBe("credential-rejected"); + result.AcceptancePassed.ShouldBe(false); + grader.Calls.ShouldBe(0, "there is nothing to clone — fail closed, never a phantom pass"); + AgentAcceptanceContract.IsInfraFailure(result.AcceptanceDetail, workPresent: false).ShouldBeTrue("the check never ran, and no revise round can change a credential the remote refuses"); + AgentRunExecutor.ReviseReasonFor(TaskWith(Spec("sh", "check.sh")), result).ShouldBeNull("the revise loop is never spent on a credential"); + } + + [Fact] + public async Task A_multi_repo_contract_whose_pushes_were_refused_for_the_credential_names_the_credential_too() + { + var (executor, grader) = NewExecutor(new BenchmarkGrade { Passed = true, Detail = "unused" }); + + var multi = Succeeded() with + { + RepositoryResults = new[] { new RepositoryRunResult { RepositoryId = Guid.NewGuid(), Alias = "api", PublishError = "refused", PublishFailureCode = "git_credential_rejected" } }, + }; + + var result = await executor.GradeAcceptanceIfPresentAsync(Run(), TaskWith(Spec("sh", "check.sh")), multi, workspace: null, CancellationToken.None); + + result.Status.ShouldBe(AgentRunStatus.Failed); + result.AcceptanceDetail.ShouldBe("credential-rejected"); + grader.Calls.ShouldBe(0); + } + + [Fact] + public async Task A_contract_whose_push_failed_for_any_other_reason_keeps_the_long_standing_detail() + { + var (executor, _) = NewExecutor(new BenchmarkGrade { Passed = true, Detail = "unused" }); + + var failed = Succeeded() with { ProducedBranch = null, PublishError = "git push failed: connection reset" }; + var result = await executor.GradeAcceptanceIfPresentAsync(Run(), TaskWith(Spec("sh", "check.sh")), failed, workspace: null, CancellationToken.None); + + result.AcceptanceDetail.ShouldBe("no-branch-or-repo", "byte-identical for every publish failure that is not a rejected credential"); + } + + [Fact] + public void The_credential_rejected_detail_is_pinned_and_infra_with_or_without_visible_work() + { + // The executor writes this string and the shared infra classification reads it: renaming one side silently turns a + // credential problem back into "the agent did nothing" for the decider, the receipts and the revise loop. + AgentAcceptanceContract.CredentialRejectedDetail.ShouldBe("credential-rejected"); + AgentAcceptanceContract.IsInfraFailure("credential-rejected", workPresent: false).ShouldBeTrue(); + AgentAcceptanceContract.IsInfraFailure("credential-rejected", workPresent: true).ShouldBeTrue(); + AgentAcceptanceContract.IsInfraFailure("repo 'api': credential-rejected", workPresent: false).ShouldBeTrue("sees through the multi-repo display tag like every other infra detail"); + AgentAcceptanceContract.IsInfraFailure("no-branch-or-repo", workPresent: false).ShouldBeFalse("unchanged: with no work the fix is to do the work"); + } + [Fact] public async Task A_contract_with_no_branch_and_no_patch_fails_closed_without_grading() { diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorPushTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorPushTests.cs index db0d9f9e3..74032b81a 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorPushTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/AgentRunExecutorPushTests.cs @@ -1,3 +1,4 @@ +using System.Text.Json; using CodeSpace.Core.Persistence.Entities; using CodeSpace.Core.Services.Agents; using CodeSpace.Core.Services.Agents.Publish; @@ -508,6 +509,88 @@ await Should.ThrowAsync(async () => await executor.PushProducedBranchIfEnabledAsync(new(runId, runId, ClaimedEpoch), DefaultTask, SucceededWithChanges(), handle, CancellationToken.None)); } + // ─── A credential the remote rejected: not retried, recorded with its code, the run stays Succeeded ── + + [Fact] + public async Task A_credential_the_remote_rejected_is_not_retried_and_is_recorded_with_its_code_on_a_run_that_stays_succeeded() + { + // The workspace provider has already resolved the credential again and presented the fresh one; presenting it a + // third time could only repeat the refusal (and each refused login counts against the account). The failure is + // recorded — message AND stable code — and the run keeps the Succeeded it earned: its work was done, the publish was + // the side effect that failed, and every caller that reads Succeeded must keep reading it. + var (runId, executor, runs) = NewExecutor(epoch: ClaimedEpoch); + var rejection = new GitCredentialRejectedException("The remote refused the git credential for https://example.test/r.git, so the branch was not published. Reconnect the provider."); + var handle = new RecordingPushHandle { ThrowOnPush = rejection }; + + var result = await executor.PushProducedBranchIfEnabledAsync(new(runId, runId, ClaimedEpoch), DefaultTask, SucceededWithChanges(), handle, CancellationToken.None); + + result.Status.ShouldBe(AgentRunStatus.Succeeded, "a rejected push never flips a Succeeded run"); + result.ExitReason.ShouldBe("completed"); + result.ProducedBranch.ShouldBeNull(); + result.PublishError.ShouldBe(rejection.Message); + result.PublishFailureCode.ShouldBe("git_credential_rejected", "the typed failure is recorded on the run, where a reader can branch on it without reading prose"); + handle.PushCallCount.ShouldBe(1, "a rejected credential is not retried — the retry budget is for failures a retry can fix"); + runs.AppendedEvents.Count.ShouldBe(1); + runs.AppendedEvents[0].Kind.ShouldBe(AgentEventKind.Warning); + runs.AppendedEvents[0].Text.ShouldContain("Reconnect the provider"); + } + + [Fact] + public async Task A_generic_push_failure_records_no_failure_code_and_still_burns_the_retry_budget() + { + var (runId, executor, _) = NewExecutor(epoch: ClaimedEpoch); + var handle = new RecordingPushHandle { ThrowOnPush = new WorkspaceException("git push failed: connection reset") }; + + var result = await executor.PushProducedBranchIfEnabledAsync(new(runId, runId, ClaimedEpoch), DefaultTask, SucceededWithChanges(), handle, CancellationToken.None); + + result.PublishError.ShouldBe("git push failed: connection reset"); + result.PublishFailureCode.ShouldBeNull("only a failure a retry cannot fix has a code; the message is all there is to say about the rest"); + handle.PushCallCount.ShouldBe(AgentRunExecutor.PushMaxAttempts); + } + + [Fact] + public async Task A_multi_repo_credential_rejection_is_isolated_to_its_repo_not_retried_and_recorded_on_that_repo_alone() + { + var (runId, executor, runs) = NewExecutor(epoch: ClaimedEpoch); + var handle = new MultiRepoRecordingPushHandle { RejectForAliases = { "api" } }; + + var result = await executor.PushProducedBranchIfEnabledAsync(new(runId, runId, ClaimedEpoch), DefaultTask, MultiRepoSucceeded(runId), handle, CancellationToken.None); + + result.Status.ShouldBe(AgentRunStatus.Succeeded); + var web = result.RepositoryResults.Single(r => r.Alias == "web"); + var api = result.RepositoryResults.Single(r => r.Alias == "api"); + web.ProducedBranch.ShouldBe(AgentRunExecutor.BuildBranchName(runId, ClaimedEpoch), "the sibling's branch is kept"); + web.PublishFailureCode.ShouldBeNull(); + api.ProducedBranch.ShouldBeNull(); + api.PublishError.ShouldNotBeNullOrEmpty(); + api.PublishFailureCode.ShouldBe("git_credential_rejected"); + handle.CallCountByAlias["api"].ShouldBe(1, "a rejected credential is not retried"); + runs.AppendedEvents.Single().Text.ShouldStartWith("Could not push the agent's changes to a branch: [api]"); + } + + [Fact] + public void The_publish_failure_code_is_the_failures_own_code_for_a_rejected_credential_and_null_for_everything_else() + { + AgentRunExecutor.PublishFailureCodeOf(new GitCredentialRejectedException("rejected")).ShouldBe("git_credential_rejected"); + AgentRunExecutor.PublishFailureCodeOf(new WorkspaceException("a plain failure")).ShouldBeNull(); + } + + [Fact] + public void The_publish_failure_code_is_omitted_from_a_stored_result_that_has_none_and_survives_a_round_trip_when_it_has_one() + { + // Rolling deploy / rollback: a result written without the field must read as it always did, and the field must not + // appear in the JSON of any result that has none — an older reader never meets a property it does not know. + var withoutCode = JsonSerializer.Serialize(new AgentRunResult { Status = AgentRunStatus.Succeeded, ExitReason = "completed", PublishError = "x", RepositoryResults = new[] { new RepositoryRunResult { Alias = "web" } } }, AgentJson.Options); + withoutCode.ShouldNotContain("ublishFailureCode", Case.Insensitive, "null is omitted, so every earlier result serializes byte-identically"); + + var withCode = JsonSerializer.Serialize(new AgentRunResult { Status = AgentRunStatus.Succeeded, ExitReason = "completed", PublishError = "x", PublishFailureCode = "git_credential_rejected", RepositoryResults = new[] { new RepositoryRunResult { Alias = "web", PublishFailureCode = "git_credential_rejected" } } }, AgentJson.Options); + var read = JsonSerializer.Deserialize(withCode, AgentJson.Options)!; + read.PublishFailureCode.ShouldBe("git_credential_rejected"); + read.RepositoryResults.Single().PublishFailureCode.ShouldBe("git_credential_rejected"); + + JsonSerializer.Deserialize(withoutCode, AgentJson.Options)!.PublishFailureCode.ShouldBeNull("a result stored before the field existed reads as having none"); + } + // ─── Helpers ───────────────────────────────────────────────────────────── private static AgentRunResult SucceededWithChanges() => @@ -592,6 +675,9 @@ private sealed class MultiRepoRecordingPushHandle : IWorkspaceHandle, IWorkspace public HashSet NullForAliases { get; } = new(); public HashSet ThrowForAliases { get; } = new(); + /// Aliases whose push throws a — the workspace provider's verdict that the remote refused the credential even after it was resolved again. + public HashSet RejectForAliases { get; } = new(); + /// Per-alias call counter — so a test can prove ONE repo's transient failure retries in isolation from its siblings. public Dictionary CallCountByAlias { get; } = new(); @@ -614,6 +700,7 @@ private sealed class MultiRepoRecordingPushHandle : IWorkspaceHandle, IWorkspace if (CallCountByAlias[alias] <= failFirstN) throw new WorkspaceException($"git push failed for '{alias}': *** rejected (transient)"); if (failFirstN == 0 && ThrowForAliases.Contains(alias)) throw new WorkspaceException($"git push failed for '{alias}': *** rejected"); + if (RejectForAliases.Contains(alias)) throw new GitCredentialRejectedException($"The remote refused the git credential for https://example.test/{alias}.git, so the branch was not published."); var pushed = NullForAliases.Contains(alias) ? null : branchName; PushedByAlias[alias] = pushed; diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/LocalGitWorkspaceProviderCredentialTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/LocalGitWorkspaceProviderCredentialTests.cs new file mode 100644 index 000000000..9de0de902 --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Workflows/LocalGitWorkspaceProviderCredentialTests.cs @@ -0,0 +1,462 @@ +using CodeSpace.Core.Services.Agents.Sandbox; +using CodeSpace.Core.Services.Agents.Workspace; +using CodeSpace.Core.Services.Agents.Workspace.Providers; +using CodeSpace.Messages.Agents; +using CodeSpace.Messages.Exceptions; +using CodeSpace.Messages.Failures; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; + +namespace CodeSpace.UnitTests.Workflows; + +/// +/// The credential a pushes with is resolved when the push runs, not when the clone was +/// made: a run outlives the access token it was provisioned with, and a GitLab access token is revoked the moment any caller +/// refreshes it. These tests pin the provider's side of that against a SCRIPTED runner (no git, no network, no database): which +/// commands ask the , which token each carries, what a refusal does — one retry with a +/// DIFFERENT credential, never the same one twice — and that every credential it ever held stays out of what it throws. +/// +/// The refusal text the script answers with is git's own, captured from git 2.33 against a remote that answered 401 by +/// AgentPublishCredentialRefusalFlowTests.What_real_git_prints_for_a_refused_credential_is_what_the_classifier_reads +/// — the integration test that fails if git's wording ever stops matching, so this fixture cannot drift from production +/// unseen. The real chain (resolver, refresher, database, git, a remote that revokes) is composed in the integration tests. +/// +[Trait("Category", "Unit")] +public sealed class LocalGitWorkspaceProviderCredentialTests +{ + private const string Url = "https://example.test/repo.git"; + private const string Branch = "codespace/run"; + + /// What git 2.33 wrote to stderr when the remote answered a push with 401 after it was offered a credential. + private const string GitRefusal = "fatal: Authentication failed for 'https://example.test/repo.git/'"; + + // ─── Which commands ask, and what each carries ─────────────────────────────────── + + [Fact] + public async Task Every_command_that_reaches_the_remote_asks_the_token_source_first_and_carries_what_it_answers_and_no_local_command_asks() + { + var source = new ScriptedSource("push-token", "readback-token"); + await using var publish = await PrepareAsync(new ScriptedRunner(), source); + + await publish.PushAsync(); + + var remoteSpecs = publish.Runner.RemoteSpecs; + remoteSpecs.Select(Subcommand).ShouldBe(new[] { "push", "ls-remote" }, "fixture check: the push and its readback are the commands that reach the remote"); + TokenedGitSpecs.CarriesTheCredential(remoteSpecs[0], "oauth2", "push-token").ShouldBeTrue("the push carries the credential resolved for it"); + TokenedGitSpecs.CarriesTheCredential(remoteSpecs[1], "oauth2", "readback-token").ShouldBeTrue("the readback resolves its own — the push may have run for minutes"); + source.Asked.ShouldBe(2, "one ask per remote command; the bundles, the checkout, the commit and the imports reach no remote and never ask"); + publish.Runner.PostCloneSpecs.Where(s => TokenedGitSpecs.CarriesTheCredential(s, "oauth2", "clone-token")).ShouldBeEmpty("the token the clone was provisioned with is never presented once the source answers"); + } + + [Fact] + public async Task The_lfs_upload_the_push_and_the_readback_each_ask_in_that_order() + { + var source = new ScriptedSource("lfs-token", "push-token", "readback-token"); + await using var publish = await PrepareAsync(new ScriptedRunner(), source); + publish.PlantALfsObject(); + + await publish.PushAsync(); + + publish.Runner.RemoteSpecs.Select(Subcommand).ShouldBe(new[] { "lfs", "push", "ls-remote" }); + publish.Runner.RemoteSpecs.Select(s => s.Environment["CODESPACE_GIT_PASSWORD"]).ShouldBe(new[] { "lfs-token", "push-token", "readback-token" }, "each command carries the credential resolved immediately before it"); + source.Asked.ShouldBe(3); + } + + [Fact] + public async Task A_second_push_of_the_same_workspace_resolves_its_credential_again() + { + // An S6 revise round re-pushes the same branch from the same workspace, possibly an hour later. + var source = new ScriptedSource("first-push", "first-readback", "second-push", "second-readback"); + await using var publish = await PrepareAsync(new ScriptedRunner(), source); + + await publish.PushAsync(); + await publish.PushAsync(); + + publish.Runner.RemoteSpecs.Select(s => s.Environment["CODESPACE_GIT_PASSWORD"]).ShouldBe(new[] { "first-push", "first-readback", "second-push", "second-readback" }); + } + + [Fact] + public async Task A_request_that_names_no_repository_keeps_the_token_it_was_given_and_never_asks() + { + var source = new ScriptedSource("never-used"); + await using var publish = await PrepareAsync(new ScriptedRunner(), source, repositoryNamed: false); + + await publish.PushAsync(); + + publish.Runner.RemoteSpecs.ShouldAllBe(s => TokenedGitSpecs.CarriesTheCredential(s, "oauth2", "clone-token")); + source.Asked.ShouldBe(0, "byte-identical for every request built without the resolver"); + } + + [Fact] + public async Task A_provider_with_no_token_source_keeps_the_token_it_was_given() + { + await using var publish = await PrepareAsync(new ScriptedRunner(), source: null, repositoryNamed: true); + + await publish.PushAsync(); + + publish.Runner.RemoteSpecs.ShouldAllBe(s => TokenedGitSpecs.CarriesTheCredential(s, "oauth2", "clone-token")); + } + + [Fact] + public async Task An_anonymous_clone_still_publishes_nothing_and_never_asks() + { + var source = new ScriptedSource("never-used"); + await using var publish = await PrepareAsync(new ScriptedRunner(), source, cloneToken: null); + + (await publish.PushAsync()).ShouldBeNull("a clone with no credential has nothing to push with — no failure, no branch"); + + publish.Runner.RemoteSpecs.ShouldBeEmpty(); + source.Asked.ShouldBe(0); + } + + // ─── A refusal ─────────────────────────────────────────────────────────────────── + + [Fact] + public async Task A_refused_push_is_tried_once_more_with_the_credential_resolved_again_when_it_differs() + { + var source = new ScriptedSource("stale-token", "fresh-token", "readback-token"); + var runner = new ScriptedRunner { Remote = (spec, _) => Subcommand(spec) == "push" && spec.Environment["CODESPACE_GIT_PASSWORD"] == "stale-token" ? Refusal() : null }; + await using var publish = await PrepareAsync(runner, source); + + (await publish.PushAsync()).ShouldBe(Branch); + + runner.RemoteSpecs.Select(Subcommand).ShouldBe(new[] { "push", "push", "ls-remote" }); + runner.RemoteSpecs.Select(s => s.Environment["CODESPACE_GIT_PASSWORD"]).ShouldBe(new[] { "stale-token", "fresh-token", "readback-token" }); + source.Asked.ShouldBe(3, "the push, the resolve for its one retry, and the readback"); + } + + [Fact] + public async Task A_refused_push_whose_credential_resolves_to_the_same_one_is_not_presented_again_and_is_typed() + { + var source = new ScriptedSource("dead-token"); + var runner = new ScriptedRunner { Remote = (spec, _) => Subcommand(spec) == "push" ? Refusal() : null }; + await using var publish = await PrepareAsync(runner, source); + + var failure = await Should.ThrowAsync(() => publish.PushAsync()); + + runner.RemoteSpecs.Count(s => Subcommand(s) == "push").ShouldBe(1, "the same refused token is never put to the remote a second time"); + failure.Code.ShouldBe(FailureCodes.GitCredentialRejected); + failure.Kind.ShouldBe(FailureKind.PreconditionRequired); + failure.Message.ShouldContain(Url, customMessage: "the message names the remote, without userinfo"); + failure.Message.ShouldContain("returned the same one"); + failure.Message.ShouldContain("Reconnect the provider"); + failure.Message.ShouldNotContain("dead-token", Case.Insensitive); + runner.RemoteSpecs.ShouldNotContain(s => Subcommand(s) == "ls-remote", "a push that never landed has nothing to read back"); + } + + [Fact] + public async Task A_push_refused_with_two_different_credentials_is_typed_after_exactly_two_tries() + { + var source = new ScriptedSource("first-bad", "second-bad"); + var runner = new ScriptedRunner { Remote = (spec, _) => Subcommand(spec) == "push" ? Refusal() : null }; + await using var publish = await PrepareAsync(runner, source); + + var failure = await Should.ThrowAsync(() => publish.PushAsync()); + + runner.RemoteSpecs.Count(s => Subcommand(s) == "push").ShouldBe(2); + failure.Message.ShouldContain("again after resolving a fresh one"); + failure.Message.ShouldNotContain("first-bad", Case.Insensitive); + failure.Message.ShouldNotContain("second-bad", Case.Insensitive); + } + + [Fact] + public async Task A_refused_lfs_upload_gets_the_same_one_fresh_try_before_the_push_is_attempted() + { + var source = new ScriptedSource("stale-token", "fresh-token", "push-token", "readback-token"); + var runner = new ScriptedRunner { Remote = (spec, _) => Subcommand(spec) == "lfs" && spec.Environment["CODESPACE_GIT_PASSWORD"] == "stale-token" ? Refusal("refused this credential") : null }; + await using var publish = await PrepareAsync(runner, source); + publish.PlantALfsObject(); + + (await publish.PushAsync()).ShouldBe(Branch); + + runner.RemoteSpecs.Select(Subcommand).ShouldBe(new[] { "lfs", "lfs", "push", "ls-remote" }); + } + + [Fact] + public async Task A_remote_that_refuses_the_action_rather_than_the_credential_stays_the_plain_failure_and_is_not_resolved_again() + { + // A 403: the credential was accepted and the action refused (a read-only token, a protected branch). No other token + // for the same account changes that, and the push retry above this decides what to do with a plain failure. + var source = new ScriptedSource("a-token"); + var runner = new ScriptedRunner { Remote = (spec, _) => Subcommand(spec) == "push" ? new SandboxResult { Status = SandboxStatus.Failed, ExitCode = 128, Stdout = "", Stderr = "remote: You are not allowed to push code to this project.\nfatal: unable to access 'https://example.test/repo.git/': The requested URL returned error: 403" } : null }; + await using var publish = await PrepareAsync(runner, source); + + var failure = await Should.ThrowAsync(() => publish.PushAsync()); + + failure.Message.ShouldContain("403"); + runner.RemoteSpecs.Count(s => Subcommand(s) == "push").ShouldBe(1); + source.Asked.ShouldBe(1, "no second resolve for a failure that is not a refusal of the credential"); + } + + [Fact] + public async Task A_refusal_on_a_request_that_names_no_repository_stays_the_plain_failure() + { + // Nothing to resolve again, so nothing to retry and nothing new to say: the caller sees what it always saw. + var runner = new ScriptedRunner { Remote = (spec, _) => Subcommand(spec) == "push" ? Refusal() : null }; + await using var publish = await PrepareAsync(runner, new ScriptedSource("never-used"), repositoryNamed: false); + + var failure = await Should.ThrowAsync(() => publish.PushAsync()); + + failure.Message.ShouldContain("Authentication failed"); + runner.RemoteSpecs.Count(s => Subcommand(s) == "push").ShouldBe(1); + } + + [Fact] + public async Task Every_credential_the_repository_carried_is_redacted_from_what_the_provider_throws() + { + // The remote can echo a credential back; both the one that was refused and the one resolved for the retry are secrets. + var source = new ScriptedSource("stale-secret-111", "fresh-secret-222"); + var runner = new ScriptedRunner { Remote = (spec, _) => Subcommand(spec) == "push" ? new SandboxResult { Status = SandboxStatus.Failed, ExitCode = 128, Stdout = "", Stderr = $"remote: HTTP Basic: Access denied. The token was {spec.Environment["CODESPACE_GIT_PASSWORD"]}\n{GitRefusal}" } : null }; + await using var publish = await PrepareAsync(runner, source); + + var failure = await Should.ThrowAsync(() => publish.PushAsync()); + + failure.Message.ShouldNotContain("stale-secret-111", Case.Insensitive); + failure.Message.ShouldNotContain("fresh-secret-222", Case.Insensitive); + failure.Message.ShouldContain("***", customMessage: "the echo is redacted, not dropped"); + } + + // ─── When the credential cannot be resolved ────────────────────────────────────── + + [Fact] + public async Task A_credential_disconnected_while_the_run_worked_is_typed_and_nothing_reaches_the_remote() + { + var source = new ScriptedSource { Answer = _ => throw new CredentialDisconnectedException(Guid.NewGuid(), "GitLab · someone") }; + await using var publish = await PrepareAsync(new ScriptedRunner(), source); + + var failure = await Should.ThrowAsync(() => publish.PushAsync()); + + failure.Code.ShouldBe(FailureCodes.GitCredentialRejected); + failure.Message.ShouldContain("was disconnected"); + failure.InnerException.ShouldBeOfType(); + publish.Runner.RemoteSpecs.ShouldBeEmpty("nothing was put to the remote"); + } + + [Fact] + public async Task A_failure_to_resolve_the_credential_is_a_plain_workspace_failure_the_push_retry_can_retry() + { + // A store hiccup, a provider that did not answer: transient until proven otherwise, and the retry above the provider + // only retries a WorkspaceException — an escaping raw exception would flip the run that produced the work. + var source = new ScriptedSource { Answer = _ => throw new TimeoutException("the credential store did not answer") }; + await using var publish = await PrepareAsync(new ScriptedRunner(), source); + + var failure = await Should.ThrowAsync(() => publish.PushAsync()); + + failure.Message.ShouldContain("Could not resolve the git credential to push with"); + failure.Message.ShouldContain("the credential store did not answer"); + failure.InnerException.ShouldBeOfType(); + publish.Runner.RemoteSpecs.ShouldBeEmpty(); + } + + [Fact] + public async Task A_repository_with_no_credential_any_more_has_nothing_to_push_with() + { + var source = new ScriptedSource { Answer = _ => null }; + await using var publish = await PrepareAsync(new ScriptedRunner(), source); + + var failure = await Should.ThrowAsync(() => publish.PushAsync()); + + failure.Message.ShouldContain("no credential bound any more"); + publish.Runner.RemoteSpecs.ShouldBeEmpty("an anonymous push is never attempted for a repository that was provisioned with a credential"); + } + + [Fact] + public async Task The_runs_own_cancellation_while_resolving_the_credential_propagates_unwrapped() + { + using var cancelled = new CancellationTokenSource(); + var source = new ScriptedSource("never-answered"); + await using var publish = await PrepareAsync(new ScriptedRunner(), source); + + await cancelled.CancelAsync(); + + await Should.ThrowAsync(() => publish.PushAsync(cancelled.Token)); + publish.Runner.RemoteSpecs.ShouldBeEmpty(); + } + + [Fact] + public async Task A_provider_call_that_timed_out_while_resolving_the_credential_is_a_retryable_workspace_failure_not_a_cancelled_run() + { + // An HTTP timeout inside the OAuth refresh surfaces as a TaskCanceledException while the run's own token is untouched: + // letting it through would end the run as cancelled instead of recording a failed publish. + var source = new ScriptedSource { Answer = _ => throw new TaskCanceledException("the token endpoint timed out", new TimeoutException()) }; + await using var publish = await PrepareAsync(new ScriptedRunner(), source); + + var failure = await Should.ThrowAsync(() => publish.PushAsync()); + + failure.Message.ShouldContain("Could not resolve the git credential to push with"); + failure.InnerException.ShouldBeOfType(); + } + + [Fact] + public async Task Each_repository_of_a_multi_repo_workspace_asks_for_its_own_credential() + { + var (webId, apiId, teamId) = (Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid()); + var source = new ScriptedSource { Answer = _ => "any-token" }; + var asked = new List<(Guid RepositoryId, Guid TeamId)>(); + source.OnAsk = (repositoryId, team) => asked.Add((repositoryId, team)); + var runner = new ScriptedRunner(); + var provider = new LocalGitWorkspaceProvider(new SandboxRunnerRegistry(new[] { runner }), source, NullLogger.Instance, Path.Combine(Path.GetTempPath(), "cs-credunit-" + Guid.NewGuid().ToString("N"))); + + await using var handle = await provider.PrepareAsync(new WorkspaceProvisionRequest + { + PrimaryAlias = "web", + Repositories = new[] + { + new WorkspaceRepositoryProvision { Alias = "web", CloneRequest = new WorkspaceRequest { RepositoryUrl = "https://example.test/web.git", Token = "web-clone", TokenUsername = "oauth2", RepositoryId = webId, TeamId = teamId }, Access = WorkspaceAccess.Write, IsPrimary = true }, + new WorkspaceRepositoryProvision { Alias = "api", CloneRequest = new WorkspaceRequest { RepositoryUrl = "https://example.test/api.git", Token = "api-clone", TokenUsername = "oauth2", RepositoryId = apiId, TeamId = teamId }, Access = WorkspaceAccess.Write }, + }, + }, CancellationToken.None); + + await ((IWorkspacePushHandle)handle).PushChangesAsync("web", Branch, CancellationToken.None); + await ((IWorkspacePushHandle)handle).PushChangesAsync("api", Branch, CancellationToken.None); + + asked.Select(a => $"{a.RepositoryId}/{a.TeamId}").ShouldBe(new[] { $"{webId}/{teamId}", $"{webId}/{teamId}", $"{apiId}/{teamId}", $"{apiId}/{teamId}" }, customMessage: "each repo's push and readback ask for the repository they were resolved for, never a sibling's"); + } + + [Fact] + public async Task A_readback_whose_credential_cannot_be_resolved_withholds_the_confirmation_but_keeps_the_push() + { + // The push already landed. A credential that cannot be resolved a moment later costs only the confirmation. + var source = new ScriptedSource { Answer = ask => ask == 1 ? "push-token" : throw new TimeoutException("the credential store did not answer") }; + await using var publish = await PrepareAsync(new ScriptedRunner(), source); + + (await publish.PushAsync()).ShouldBe(Branch, "the branch was pushed"); + publish.LastPushedCommitSha().ShouldBeNull("arrival stays unconfirmed — never fabricated"); + publish.Runner.RemoteSpecs.Select(Subcommand).ShouldBe(new[] { "push" }, "the readback never ran without a credential"); + } + + // ─── Classifying a refusal ─────────────────────────────────────────────────────── + + [Theory] + [InlineData("fatal: Authentication failed for 'https://example.test/repo.git/'", true)] // git, after a 401 on a credential it offered (captured from git 2.33) + [InlineData("remote: HTTP Basic: Access denied", true)] // the body GitLab sends with the 401 + [InlineData("fatal: unable to access 'https://example.test/repo.git/': The requested URL returned error: 401", true)] // curl's own words + [InlineData("the remote refused this credential; not offering it again", true)] // our credential helper, what git-lfs reads after the 401 + [InlineData("FATAL: AUTHENTICATION FAILED FOR 'https://example.test/'", true)] // case-insensitive + [InlineData("fatal: unable to access 'https://example.test/repo.git/': The requested URL returned error: 403", false)] // accepted, not authorised: no other token fixes it + [InlineData("remote: You are not allowed to push code to this project.", false)] + [InlineData("fatal: unable to access 'https://example.test/repo.git/': Could not resolve host: example.test", false)] + [InlineData("fatal: could not read Username for 'https://example.test': terminal prompts disabled", false)] + [InlineData("", false)] + [InlineData(null, false)] + public void IsCredentialRefusal_reads_what_git_gitlab_and_the_helper_say(string? stderr, bool expected) => + LocalGitWorkspaceProvider.IsCredentialRefusal(stderr).ShouldBe(expected); + + [Fact] + public void The_helpers_own_refusal_message_is_one_of_the_markers() + { + // Drift detector: the helper's text is written in TokenedGitCommand and read here. Reword one and not the other and + // an LFS upload the remote refused would be retried as if nothing had been learned. + LocalGitWorkspaceProvider.CredentialRefusalMarkers.ShouldContain(marker => TokenedGitCommand.CredentialHelper.Contains(marker, StringComparison.Ordinal)); + LocalGitWorkspaceProvider.CredentialRefusalMarkers.ShouldBe(new[] { "Authentication failed for", "HTTP Basic: Access denied", "returned error: 401", "refused this credential" }, "adding or removing a marker changes which refusals get a retry — decide it here"); + } + + // ─── Harness ───────────────────────────────────────────────────────────────────── + + private static SandboxResult Refusal(string stderr = GitRefusal) => new() { Status = SandboxStatus.Failed, ExitCode = 128, Stdout = "", Stderr = stderr }; + + /// The git subcommand, past any leading -c key=value and -C dir. + private static string Subcommand(SandboxSpec spec) + { + var i = 0; + while (spec.Args[i] is "-c" or "-C") i += 2; + return spec.Args[i]; + } + + private static async Task PrepareAsync(ScriptedRunner runner, ScriptedSource? source, bool repositoryNamed = true, string? cloneToken = "clone-token") + { + var workspacesRoot = Path.Combine(Path.GetTempPath(), "cs-credunit-" + Guid.NewGuid().ToString("N")); + var provider = new LocalGitWorkspaceProvider(new SandboxRunnerRegistry(new[] { runner }), source, NullLogger.Instance, workspacesRoot); + var handle = await provider.PrepareAsync(WorkspaceProvisionRequest.FromSingle(new WorkspaceRequest + { + RepositoryUrl = Url, Token = cloneToken, TokenUsername = "oauth2", + RepositoryId = repositoryNamed ? Guid.NewGuid() : null, TeamId = repositoryNamed ? Guid.NewGuid() : null, + }), CancellationToken.None); + + runner.MarkCloneDone(); + return new PublishUnderTest(handle, runner, workspacesRoot); + } + + private sealed class PublishUnderTest(IWorkspaceHandle handle, ScriptedRunner runner, string workspacesRoot) : IAsyncDisposable + { + public ScriptedRunner Runner { get; } = runner; + + public async Task PushAsync(CancellationToken cancellationToken = default) => await ((IWorkspacePushHandle)handle).PushChangesAsync(Branch, cancellationToken); + + public string? LastPushedCommitSha() => ((IWorkspacePushHandle)handle).LastPushedCommitSha(); + + /// An LFS object in the clone's store, so the publish runs the LFS upload before the push. + public void PlantALfsObject() + { + var oid = "abcd" + new string('0', 60); + var path = Path.Combine(handle.Directory, ".git", "lfs", "objects", "ab", "cd", oid); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, "an lfs object the branch points at\n"); + } + + public async ValueTask DisposeAsync() + { + await handle.DisposeAsync(); + + try { Directory.Delete(workspacesRoot, recursive: true); } catch { /* best-effort: the root holds nothing once the workspace is removed */ } + } + } + + /// The credential store, from a script: answers each ask with the next token (repeating the last) unless decides, and counts the asks. + private sealed class ScriptedSource : IWorkspaceTokenSource + { + private readonly string[] _tokens; + private int _asked; + + public ScriptedSource(params string[] tokens) { _tokens = tokens; } + + public int Asked => Volatile.Read(ref _asked); + + /// When set, decides each ask (numbered from 1) instead of the script of tokens. + public Func? Answer { get; set; } + + /// Called with the ids each ask carried, before it is answered. + public Action? OnAsk { get; set; } + + public Task ResolveAsync(Guid repositoryId, Guid teamId, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var ask = Interlocked.Increment(ref _asked); + OnAsk?.Invoke(repositoryId, teamId); + + return Task.FromResult(Answer is not null ? Answer(ask) : _tokens[Math.Min(ask, _tokens.Length) - 1]); + } + } + + /// Answers success with a 40-char sha for every command — the local ones, and the remote ones unless decides otherwise — except that the bundle is never "within the base" (the branch adds objects). Records every spec; the ones after the clone are the publish. + private sealed class ScriptedRunner : ISandboxRunner + { + private int _cloneSpecs = -1; + + public string Kind => "local"; + + public List Specs { get; } = new(); + + /// For a command that reaches the remote (a tokened one), the answer to give it — numbered by order among them — or null for success. + public Func? Remote { get; set; } + + public IReadOnlyList PostCloneSpecs => Specs.Skip(_cloneSpecs).ToList(); + + /// The commands after the clone that reached the remote: every tokened one (they carry the credential in their environment). + public IReadOnlyList RemoteSpecs => PostCloneSpecs.Where(s => s.Environment.ContainsKey("CODESPACE_GIT_PASSWORD")).ToList(); + + public void MarkCloneDone() => _cloneSpecs = Specs.Count; + + public Task RunAsync(SandboxSpec spec, CancellationToken cancellationToken) + { + Specs.Add(spec); + + if (_cloneSpecs >= 0 && spec.Environment.ContainsKey("CODESPACE_GIT_PASSWORD") && Remote?.Invoke(spec, RemoteSpecs.Count) is { } scripted) return Task.FromResult(scripted); + + if (spec.Args.Contains("--is-ancestor")) return Task.FromResult(new SandboxResult { Status = SandboxStatus.Failed, ExitCode = 1, Stdout = "", Stderr = "" }); + + return Task.FromResult(new SandboxResult { Status = SandboxStatus.Success, ExitCode = 0, Stdout = new string('a', 40), Stderr = "" }); + } + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/RepositoryWorkspaceTokenSourceTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/RepositoryWorkspaceTokenSourceTests.cs new file mode 100644 index 000000000..fcc4af4af --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Workflows/RepositoryWorkspaceTokenSourceTests.cs @@ -0,0 +1,100 @@ +using CodeSpace.Core.Services.Agents.Workspace; +using CodeSpace.Messages.Agents; +using Microsoft.Extensions.DependencyInjection; +using Shouldly; + +namespace CodeSpace.UnitTests.Workflows; + +/// +/// is the one place a workspace's credential is asked for again, so it must ask +/// the way the provisioning resolve asked — for the repository and team it was given, through the scoped resolver — and keep +/// nothing between asks: one DI scope per ask (a long-lived scope's DbContext would hand back the very credential row this +/// exists to re-read), disposed when the ask ends. The real resolver, refresher and database behind it are composed in the +/// integration tests. +/// +[Trait("Category", "Unit")] +public sealed class RepositoryWorkspaceTokenSourceTests +{ + [Fact] + public async Task It_asks_the_resolver_for_the_repository_and_team_it_was_given_and_returns_its_token() + { + var repositoryId = Guid.NewGuid(); + var teamId = Guid.NewGuid(); + var resolver = new FakeResolver(Request(token: "the-token")); + var source = new RepositoryWorkspaceTokenSource(new CountingScopes(resolver)); + + (await source.ResolveAsync(repositoryId, teamId, CancellationToken.None)).ShouldBe("the-token"); + + resolver.Asked.ShouldBe(new[] { (repositoryId, teamId) }); + } + + [Fact] + public async Task It_answers_null_for_a_request_with_no_token_an_anonymous_repository() + { + var source = new RepositoryWorkspaceTokenSource(new CountingScopes(new FakeResolver(Request(token: null)))); + + (await source.ResolveAsync(Guid.NewGuid(), Guid.NewGuid(), CancellationToken.None)).ShouldBeNull(); + } + + [Fact] + public async Task It_opens_one_scope_per_ask_and_disposes_it_whether_the_ask_succeeds_or_fails() + { + var scopes = new CountingScopes(new FakeResolver(Request(token: "t"))); + var source = new RepositoryWorkspaceTokenSource(scopes); + + await source.ResolveAsync(Guid.NewGuid(), Guid.NewGuid(), CancellationToken.None); + await source.ResolveAsync(Guid.NewGuid(), Guid.NewGuid(), CancellationToken.None); + + scopes.Created.ShouldBe(2, "nothing is kept between asks"); + scopes.Disposed.ShouldBe(2); + + var failing = new CountingScopes(new FakeResolver(new WorkspaceException("Repository not found for this team."))); + + await Should.ThrowAsync(() => new RepositoryWorkspaceTokenSource(failing).ResolveAsync(Guid.NewGuid(), Guid.NewGuid(), CancellationToken.None)); + + failing.Disposed.ShouldBe(1, "a failing ask releases its scope too"); + } + + private static WorkspaceRequest Request(string? token) => new() { RepositoryUrl = "https://example.test/repo.git", Token = token }; + + private sealed class FakeResolver : IAgentWorkspaceResolver + { + private readonly WorkspaceRequest? _request; + private readonly Exception? _failure; + + public FakeResolver(WorkspaceRequest request) { _request = request; } + public FakeResolver(Exception failure) { _failure = failure; } + + public List<(Guid RepositoryId, Guid TeamId)> Asked { get; } = new(); + + public Task ResolveByRepositoryIdAsync(Guid repositoryId, Guid teamId, CancellationToken cancellationToken, string? @ref = null, bool softFallback = false, string? pinnedSha = null) + { + Asked.Add((repositoryId, teamId)); + + return _failure is not null ? throw _failure : Task.FromResult(_request); + } + + public Task ResolveAsync(AgentTask task, Guid teamId, CancellationToken cancellationToken) => throw new NotSupportedException("the token source asks by repository id"); + } + + /// Hands out a scope whose provider serves the resolver, counting scopes created and disposed. + private sealed class CountingScopes(IAgentWorkspaceResolver resolver) : IServiceScopeFactory + { + public int Created { get; private set; } + public int Disposed { get; private set; } + + public IServiceScope CreateScope() + { + Created++; + + return new Scope(new ServiceCollection().AddSingleton(resolver).BuildServiceProvider(), () => Disposed++); + } + + private sealed class Scope(IServiceProvider provider, Action onDispose) : IServiceScope + { + public IServiceProvider ServiceProvider { get; } = provider; + + public void Dispose() => onDispose(); + } + } +}