diff --git a/backend/src/CodeSpace.Core/CodeSpaceModule.cs b/backend/src/CodeSpace.Core/CodeSpaceModule.cs index 820d461fb..bb6f2a5bc 100644 --- a/backend/src/CodeSpace.Core/CodeSpaceModule.cs +++ b/backend/src/CodeSpace.Core/CodeSpaceModule.cs @@ -106,6 +106,9 @@ private static void RegisterAmbient(ContainerBuilder builder) // TimeProvider — the OAuth state store + token refresh use this to compute // expiry; test fixtures swap it for a deterministic clock to make TTL tests reliable. builder.RegisterInstance(TimeProvider.System).As().SingleInstance(); + + // What the credential health sweep may spend on one provider and on one pass; a test registers shorter ones. + builder.RegisterInstance(Messages.Dtos.Credentials.CredentialHealthDeadlines.Default).SingleInstance(); } private void RegisterSettings(ContainerBuilder builder) diff --git a/backend/src/CodeSpace.Core/Handlers/CommandHandlers/Credentials/ProbeCredentialsCommandHandler.cs b/backend/src/CodeSpace.Core/Handlers/CommandHandlers/Credentials/ProbeCredentialsCommandHandler.cs new file mode 100644 index 000000000..7a97d06cf --- /dev/null +++ b/backend/src/CodeSpace.Core/Handlers/CommandHandlers/Credentials/ProbeCredentialsCommandHandler.cs @@ -0,0 +1,21 @@ +using CodeSpace.Core.Services.Credentials; +using CodeSpace.Messages.Commands.Credentials; +using MediatR; + +namespace CodeSpace.Core.Handlers.CommandHandlers.Credentials; + +/// Thin dispatcher (Rule 16) — the production caller of . +public sealed class ProbeCredentialsCommandHandler : IRequestHandler +{ + private readonly ICredentialHealthService _health; + + public ProbeCredentialsCommandHandler(ICredentialHealthService health) + { + _health = health; + } + + public async Task Handle(ProbeCredentialsCommand request, CancellationToken cancellationToken) + { + return await _health.ProbeBatchAsync(request.BatchSize, cancellationToken).ConfigureAwait(false); + } +} diff --git a/backend/src/CodeSpace.Core/Jobs/RecurringJobs/CredentialHealthProbeRecurringJob.cs b/backend/src/CodeSpace.Core/Jobs/RecurringJobs/CredentialHealthProbeRecurringJob.cs new file mode 100644 index 000000000..5244af27a --- /dev/null +++ b/backend/src/CodeSpace.Core/Jobs/RecurringJobs/CredentialHealthProbeRecurringJob.cs @@ -0,0 +1,17 @@ +using CodeSpace.Messages.Commands.Credentials; +using MediatR; + +namespace CodeSpace.Core.Jobs.RecurringJobs; + +/// Every ten minutes: ask each provider whether a bounded batch of credentials still works — a personal, project or group token that expired or was revoked at GitLab is marked as needing a reconnect, and a token's real expiry is recorded, instead of every run finding out at its first call (thin Rule-14 dispatcher). A validated credential is not asked again for hours, so an idle tick is one small read. +public sealed class CredentialHealthProbeRecurringJob : IRecurringJob +{ + private readonly IMediator _mediator; + + public CredentialHealthProbeRecurringJob(IMediator mediator) { _mediator = mediator; } + + public string JobId => nameof(CredentialHealthProbeRecurringJob); + public string CronExpression => "*/10 * * * *"; + + public async Task Execute() => await _mediator.Send(new ProbeCredentialsCommand()).ConfigureAwait(false); +} diff --git a/backend/src/CodeSpace.Core/Jobs/RecurringJobs/DisconnectedCredentialBindingRepairRecurringJob.cs b/backend/src/CodeSpace.Core/Jobs/RecurringJobs/DisconnectedCredentialBindingRepairRecurringJob.cs index 865393805..50fbf80b4 100644 --- a/backend/src/CodeSpace.Core/Jobs/RecurringJobs/DisconnectedCredentialBindingRepairRecurringJob.cs +++ b/backend/src/CodeSpace.Core/Jobs/RecurringJobs/DisconnectedCredentialBindingRepairRecurringJob.cs @@ -3,7 +3,7 @@ namespace CodeSpace.Core.Jobs.RecurringJobs; -/// Every five minutes: move the repositories and group hooks still bound to a disconnected credential onto its owner's later reconnect — the rows stranded before the connect paths carried them forward, and any an older pod strands during a rolling deploy (thin Rule-14 dispatcher). A moved binding is no longer a candidate, so an idle tick is one small read. +/// Every five minutes: move the repositories and group hooks still bound to a disconnected (or provider-refused) credential onto its owner's later reconnect — the rows stranded before the connect paths carried them forward, and any an older pod strands during a rolling deploy (thin Rule-14 dispatcher). A moved binding is no longer a candidate, so an idle tick is one small read. public sealed class DisconnectedCredentialBindingRepairRecurringJob : IRecurringJob { private readonly IMediator _mediator; diff --git a/backend/src/CodeSpace.Core/Services/Agents/Workspace/Providers/LocalGitWorkspaceProvider.cs b/backend/src/CodeSpace.Core/Services/Agents/Workspace/Providers/LocalGitWorkspaceProvider.cs index 35a71987e..5ea8bc783 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/Workspace/Providers/LocalGitWorkspaceProvider.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/Workspace/Providers/LocalGitWorkspaceProvider.cs @@ -975,6 +975,10 @@ private async Task UseCurrentCredentialAsync(MaterializedRepo repo, Cancellation { throw new GitCredentialRejectedException($"The credential this repository pushes with was disconnected while the run worked, so its branch could not be published. {ReconnectRemedy} ({Redact(ex.Message, repo.Token)})", ex); } + catch (CredentialNeedsReconnectException ex) + { + throw new GitCredentialRejectedException($"The credential this repository pushes with was refused by its provider while the run worked, so its branch could not be published. {ReconnectRemedy} ({Redact(ex.Message, repo.Token)})", ex); + } catch (Exception ex) when (!cancellationToken.IsCancellationRequested) { // The run's own cancellation is the only thing allowed through unwrapped: a provider call that timed out diff --git a/backend/src/CodeSpace.Core/Services/Credentials/CredentialHealthDerivation.cs b/backend/src/CodeSpace.Core/Services/Credentials/CredentialHealthDerivation.cs new file mode 100644 index 000000000..620dea2ac --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Credentials/CredentialHealthDerivation.cs @@ -0,0 +1,40 @@ +using CodeSpace.Messages.Enums; + +namespace CodeSpace.Core.Services.Credentials; + +/// +/// What to read off a credential, from what is recorded about it — computed when it is listed, never stored, so it cannot +/// disagree with the columns it is derived from. Only a static token (a personal, project or group access token) has an +/// expiry worth showing: an OAuth credential's expires_date is its access token's, which renews itself every couple +/// of hours, and reading it as "expiring" would flag every healthy connection. +/// +public static class CredentialHealthDerivation +{ + /// How long before a static token expires it reads . GitLab caps a token's life at a year, so a month is a warning a person can act on. + public static readonly TimeSpan ExpiringWithin = TimeSpan.FromDays(30); + + /// + /// Disconnected beats everything (there is nothing left to be healthy); a provider's refusal — recorded, or an expiry that + /// has passed — beats a departed owner (it needs reconnecting whoever holds it); then a token about to expire; then fine. + /// + public static CredentialHealth Derive(CredentialStatus status, AuthType authType, DateTimeOffset? expiresDate, bool ownerLeft, DateTimeOffset now) + { + if (status == CredentialStatus.Revoked) return CredentialHealth.Revoked; + if (status is CredentialStatus.Error or CredentialStatus.Expired) return CredentialHealth.Rejected; + if (HasExpired(authType, expiresDate, now)) return CredentialHealth.Rejected; + if (ownerLeft) return CredentialHealth.OwnerLeft; + + return IsExpiringSoon(authType, expiresDate, now) ? CredentialHealth.Expiring : CredentialHealth.Active; + } + + /// Whole days left before a static token expires (0 once it has); null for any other credential, and for a token with no recorded expiry. + public static int? DaysToExpiry(AuthType authType, DateTimeOffset? expiresDate, DateTimeOffset now) => + IsStaticToken(authType) && expiresDate is { } at ? (int)Math.Max(0, Math.Floor((at - now).TotalDays)) : null; + + /// The credentials whose provider-side expiry CodeSpace records: access tokens a person created, not tokens the provider issued for a grant. + public static bool IsStaticToken(AuthType authType) => authType is AuthType.Pat or AuthType.ProjectAccessToken or AuthType.GroupAccessToken; + + private static bool HasExpired(AuthType authType, DateTimeOffset? expiresDate, DateTimeOffset now) => IsStaticToken(authType) && expiresDate is { } at && at <= now; + + private static bool IsExpiringSoon(AuthType authType, DateTimeOffset? expiresDate, DateTimeOffset now) => IsStaticToken(authType) && expiresDate is { } at && at - now <= ExpiringWithin; +} diff --git a/backend/src/CodeSpace.Core/Services/Credentials/CredentialHealthService.cs b/backend/src/CodeSpace.Core/Services/Credentials/CredentialHealthService.cs new file mode 100644 index 000000000..a41e94670 --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Credentials/CredentialHealthService.cs @@ -0,0 +1,226 @@ +using CodeSpace.Core.DependencyInjection; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.OAuth; +using CodeSpace.Core.Services.Providers; +using CodeSpace.Core.Services.Providers.Capabilities; +using CodeSpace.Messages.Dtos.Credentials; +using CodeSpace.Messages.Dtos.Providers; +using CodeSpace.Messages.Enums; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace CodeSpace.Core.Services.Credentials; + +public sealed class CredentialHealthService : ICredentialHealthService, IScopedDependency +{ + private const string SweepName = "credential-health-probe"; + + /// How long a validation holds a credential out of the sweep. A token that stops working is found within this, and the number of provider calls a deployment spends on healthy credentials is a few a day each. + public static readonly TimeSpan ValidatedStaleAfter = TimeSpan.FromHours(6); + + /// How long a credential marked Error waits before the provider is asked again: long enough that a dead one costs a call an hour, short enough that one the provider accepts again (an account unblocked, an OAuth client put back) heals within it. + public static readonly TimeSpan ErrorRecheckAfter = TimeSpan.FromHours(1); + + /// The advisory lock a pass holds ("CREDHELT" in ASCII), so a tick that finds the previous pass still running skips. In the key space shares, where a credential's key is 64 random bits. + public const long PassLockKey = 0x4352_4544_4845_4C54; + + private readonly CodeSpaceDbContext _db; + private readonly IProviderRegistry _registry; + private readonly ICredentialHealthWriter _writer; + private readonly ISweepFailureBackoff _backoff; + private readonly ICrossProcessTryLock _passLock; + private readonly CredentialHealthDeadlines _deadlines; + private readonly TimeProvider _clock; + private readonly ILogger _logger; + + public CredentialHealthService(CodeSpaceDbContext db, IProviderRegistry registry, ICredentialHealthWriter writer, ISweepFailureBackoff backoff, ICrossProcessTryLock passLock, CredentialHealthDeadlines deadlines, TimeProvider clock, ILogger logger) + { + _db = db; + _registry = registry; + _writer = writer; + _backoff = backoff; + _passLock = passLock; + _deadlines = deadlines; + _clock = clock; + _logger = logger; + } + + public async Task ProbeBatchAsync(int batchSize, CancellationToken cancellationToken) + { + await using var pass = await _passLock.TryAcquireAsync(PassLockKey, cancellationToken).ConfigureAwait(false); + + if (pass is null) return SkipOverlappingPass(); + + return await RunPassWithinBudgetAsync(batchSize, cancellationToken).ConfigureAwait(false); + } + + /// A pass that is already running — here or on another worker — is not queued behind: a tick that arrives while it works would only wait for the provider the first one is waiting for. + private int SkipOverlappingPass() + { + _logger.LogInformation("{Sweep}: the previous pass is still running; this tick is skipped", SweepName); + + return 0; + } + + /// + /// The pass, ended at wherever it is — a probe in flight is cut, and no + /// further one starts. What the pass had recorded stays recorded and the rest is due again next tick, so running out of time + /// costs nothing but the credentials not reached. + /// + private async Task RunPassWithinBudgetAsync(int batchSize, CancellationToken cancellationToken) + { + var probed = 0; + + using var budget = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + budget.CancelAfter(_deadlines.Pass); + + async Task ProbeAndCountAsync(DueCredential candidate, CancellationToken token) + { + var answered = await ProbeOneAsync(candidate, token).ConfigureAwait(false); + + if (answered) probed++; + + return answered; + } + + try + { + return await new SweepPass(SweepName, _backoff, _logger).RunAsync(batchSize, LoadDueAsync, ProbeAndCountAsync, budget.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (budget.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + _logger.LogWarning("{Sweep}: the pass ran out of its {Budget}s budget after {Probed} credentials; the rest are probed by the next tick", SweepName, _deadlines.Pass.TotalSeconds, probed); + + return probed; + } + } + + /// + /// The next page of due credentials in id order, behind a cursor (see ). Due means in + /// service with a token to present, of a type the providers can probe, and either Active and never validated or not + /// since , or Error and last confirmed over ago — a + /// verdict's date is when the row was last written, which a confirmation refreshes. + /// + private async Task> LoadDueAsync(Guid? after, int take, CancellationToken cancellationToken) + { + var now = _clock.GetUtcNow(); + var validatedBefore = now - ValidatedStaleAfter; + var confirmedBefore = now - ErrorRecheckAfter; + + var due = _db.Credential.AsNoTracking() + .Where(c => c.DeletedDate == null && c.EncryptedPayload != string.Empty) + .Where(c => c.AuthType == AuthType.Pat || c.AuthType == AuthType.ProjectAccessToken || c.AuthType == AuthType.GroupAccessToken || c.AuthType == AuthType.OAuth) + .Where(c => (c.Status == CredentialStatus.Active && (c.LastValidatedDate == null || c.LastValidatedDate < validatedBefore)) || (c.Status == CredentialStatus.Error && c.LastModifiedDate < confirmedBefore)); + + if (after is Guid cursor) due = due.Where(c => c.Id.CompareTo(cursor) > 0); + + var ids = await due.OrderBy(c => c.Id).Select(c => c.Id).Take(take).ToListAsync(cancellationToken).ConfigureAwait(false); + + return ids.Select(id => new DueCredential(id)).ToList(); + } + + /// Probe one credential and record what the provider said, each on connections of their own. False when it was no longer due: disconnected, retired or lifted meanwhile. + private async Task ProbeOneAsync(DueCredential candidate, CancellationToken cancellationToken) + { + var credential = await _db.Credential.AsNoTracking().Include(c => c.ProviderInstance) + .SingleOrDefaultAsync(c => c.Id == candidate.Id && c.DeletedDate == null && c.EncryptedPayload != string.Empty && (c.Status == CredentialStatus.Active || c.Status == CredentialStatus.Error), cancellationToken).ConfigureAwait(false); + + if (credential == null) return false; + + var result = await ProbeAsync(credential.ProviderInstance, credential, cancellationToken).ConfigureAwait(false); + + cancellationToken.ThrowIfCancellationRequested(); + + return await RecordAsync(credential, result, cancellationToken).ConfigureAwait(false); + } + + /// + /// What the provider says about the credential now. A credential recorded as Error is probed as if it were not: the + /// verdict in its row stops anything resolving its token, and the probe is how the verdict is lifted. The copy is read + /// without tracking and never saved, so the pretence reaches no row. + /// + private async Task ProbeAsync(ProviderInstance instance, Credential credential, CancellationToken cancellationToken) + { + var provider = instance.Provider; + var probe = _registry.Require(provider); + var context = new ProviderContext(instance, credential); + + credential.Status = CredentialStatus.Active; + + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(_deadlines.Probe); + + try + { + // The token reaches a provider that honours it, which then stops; one that does not — an SDK with no per-call + // token, a call that blocks its thread — is walked away from at the deadline all the same, on a pool thread of its + // own so that a blocked one does not hold this pass's. + return await Task.Run(() => probe.ProbeCredentialAsync(context, deadline.Token), CancellationToken.None).WaitAsync(deadline.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + return new CredentialProbeResult { IsValid = false, Error = $"{provider} did not answer within {_deadlines.Probe.TotalSeconds:0.#}s" }; + } + } + + /// + /// Accepted: validated, and a static token's real expiry kept when the provider described the token. Refused (401): marked + /// Error. Anything else is no answer — nothing is written, and the failure sets the credential aside after a few in a + /// row. Written against the payload the probe ended with (a refresh it triggered may have replaced the one read), so a + /// reconnect that replaced the row meanwhile is never overwritten by a verdict on the old one. + /// + private async Task RecordAsync(Credential credential, CredentialProbeResult result, CancellationToken cancellationToken) + { + var provider = credential.ProviderInstance.Provider; + + if (result.IsValid) + { + await _writer.MarkValidatedAsync(credential.Id, credential.EncryptedPayload, _clock.GetUtcNow(), ExpiryOf(credential, result), cancellationToken).ConfigureAwait(false); + + return true; + } + + if (!result.IsRejected) throw new InvalidOperationException($"The {provider} probe of credential {credential.Id} got no answer ({result.Error}); nothing was recorded"); + + var marked = await _writer.MarkRejectedAsync(credential.Id, credential.EncryptedPayload, $"token rejected by {provider}", cancellationToken).ConfigureAwait(false); + + _logger.LogWarning("Credential {CredentialId} ({CredentialName}) was refused by {Provider}: marked Error ({Marked}). It needs to be reconnected", credential.Id, credential.DisplayName, provider, marked); + + return true; + } + + public async Task ValidateNewAsync(Credential credential, ProviderInstance instance, CancellationToken cancellationToken) + { + if (!CredentialHealthDerivation.IsStaticToken(credential.AuthType) || !_registry.TryGet(instance.Provider, out _)) return; + + var result = await TryProbeAsync(instance, credential, cancellationToken).ConfigureAwait(false); + + if (result is not { IsValid: true }) return; + + credential.LastValidatedDate = _clock.GetUtcNow(); + + if (ExpiryOf(credential, result) is { } expiry) credential.ExpiresDate = expiry.ExpiresAt; + } + + /// The probe of a credential that is not saved yet, or null when it could not be asked: a failure here is the provider's, and adding the credential does not wait on it or fail with it. The caller's own cancellation still ends the add. + private async Task TryProbeAsync(ProviderInstance instance, Credential credential, CancellationToken cancellationToken) + { + try + { + return await ProbeAsync(instance, credential, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (!cancellationToken.IsCancellationRequested) + { + _logger.LogWarning(ex, "{Sweep}: the new credential {CredentialId} could not be validated as it was added; it is left for the sweep", SweepName, credential.Id); + + return null; + } + } + + /// A static token's expiry as its provider reports it — recorded even when it is "none" — but only when the provider described the token; an OAuth credential's expiry is its access token's and is not this sweep's to write. + private static ExpiryRecord? ExpiryOf(Credential credential, CredentialProbeResult result) => + CredentialHealthDerivation.IsStaticToken(credential.AuthType) && result.Active.HasValue ? new ExpiryRecord(result.ExpiresAt) : null; + + private sealed record DueCredential(Guid Id) : ISweepCandidate; +} diff --git a/backend/src/CodeSpace.Core/Services/Credentials/CredentialHealthWriter.cs b/backend/src/CodeSpace.Core/Services/Credentials/CredentialHealthWriter.cs new file mode 100644 index 000000000..49c399100 --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Credentials/CredentialHealthWriter.cs @@ -0,0 +1,60 @@ +using CodeSpace.Core.DependencyInjection; +using CodeSpace.Core.Settings.Database; +using CodeSpace.Messages.Constants; +using Npgsql; + +namespace CodeSpace.Core.Services.Credentials; + +public sealed class CredentialHealthWriter : ICredentialHealthWriter, IScopedDependency +{ + private readonly CodeSpaceConnectionString _connectionString; + + public CredentialHealthWriter(CodeSpaceConnectionString connectionString) { _connectionString = connectionString; } + + public async Task MarkRejectedAsync(Guid credentialId, string expectedEncryptedPayload, string lastError, CancellationToken cancellationToken) + { + // The first reason stays: it is the most specific one (what the refresh was refused with), and a later confirmation of + // the same verdict only has the date to add. + const string sql = "UPDATE credential SET status = 'Error', last_error = CASE WHEN status = 'Error' AND last_error IS NOT NULL THEN last_error ELSE @error END, last_modified_date = @now, last_modified_by = @user " + Guard; + + return await ExecuteAsync(sql, credentialId, expectedEncryptedPayload, cmd => + { + cmd.Parameters.AddWithValue("@error", lastError); + cmd.Parameters.AddWithValue("@now", DateTimeOffset.UtcNow); + cmd.Parameters.AddWithValue("@user", SystemUsers.SeederId); + }, cancellationToken).ConfigureAwait(false); + } + + public async Task MarkValidatedAsync(Guid credentialId, string expectedEncryptedPayload, DateTimeOffset validatedAt, ExpiryRecord? expiry, CancellationToken cancellationToken) + { + // Setting an Error credential back to Active is a change of the row, so it bumps last_modified_date (the clock a + // recheck of an Error credential is measured from); confirming one that was fine already is not. + const string sql = "UPDATE credential SET last_modified_date = CASE WHEN status = 'Error' THEN @now ELSE last_modified_date END, status = 'Active', last_error = NULL, last_validated_date = @validated, expires_date = CASE WHEN @recordsExpiry THEN @expires ELSE expires_date END " + Guard; + + return await ExecuteAsync(sql, credentialId, expectedEncryptedPayload, cmd => + { + cmd.Parameters.AddWithValue("@now", DateTimeOffset.UtcNow); + cmd.Parameters.AddWithValue("@validated", validatedAt.ToUniversalTime()); + cmd.Parameters.AddWithValue("@recordsExpiry", expiry is not null); + cmd.Parameters.AddWithValue("@expires", (object?)expiry?.ExpiresAt?.ToUniversalTime() ?? DBNull.Value); + }, cancellationToken).ConfigureAwait(false); + } + + /// Only a live row that is still in service (Active, or Error awaiting a verdict) and still holds the payload the caller judged: a disconnected or retired row, and one a reconnect replaced, is not this verdict's to change. + private const string Guard = "WHERE id = @id AND encrypted_payload = @expected AND status IN ('Active', 'Error') AND deleted_date IS NULL"; + + private async Task ExecuteAsync(string sql, Guid credentialId, string expectedEncryptedPayload, Action parameters, CancellationToken cancellationToken) + { + // Detached connection: a verdict must not join an outer EF transaction that may roll back (see ICredentialHealthWriter). + await using var conn = new NpgsqlConnection(_connectionString.Value); + await conn.OpenAsync(cancellationToken).ConfigureAwait(false); + + await using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + cmd.Parameters.AddWithValue("@id", credentialId); + cmd.Parameters.AddWithValue("@expected", expectedEncryptedPayload); + parameters(cmd); + + return await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false) > 0; + } +} diff --git a/backend/src/CodeSpace.Core/Services/Credentials/CredentialRelinkService.Probe.cs b/backend/src/CodeSpace.Core/Services/Credentials/CredentialRelinkService.Probe.cs index a477b0c9e..c5e4fdeb2 100644 --- a/backend/src/CodeSpace.Core/Services/Credentials/CredentialRelinkService.Probe.cs +++ b/backend/src/CodeSpace.Core/Services/Credentials/CredentialRelinkService.Probe.cs @@ -89,9 +89,9 @@ private async Task CanReadAsync(IRepositoryCatalogCapability catalog, Prov /// /// The provider answered, and the answer is about the target: this token is not let in (401, 403 or an insufficient /// scope), cannot see that project (404), or — for an OAuth connection — the grant itself was refused when its token was - /// refreshed. Anything else (a 5xx, a 429, a dropped connection, a local rate limit) says nothing about the target. + /// refreshed (the provider's answer, or the credential already marked as needing a reconnect for it). Anything else (a 5xx, a 429, a dropped connection, a local rate limit) says nothing about the target. /// - private static bool TargetIsRefused(Exception ex) => ex is ProviderInsufficientScopeException or OAuthExchangeException or ProviderApiException { StatusCode: 401 or 403 or 404 }; + private static bool TargetIsRefused(Exception ex) => ex is ProviderInsufficientScopeException or OAuthExchangeException or CredentialNeedsReconnectException or ProviderApiException { StatusCode: 401 or 403 or 404 }; private string UnreadableMessage(List unreadable) { diff --git a/backend/src/CodeSpace.Core/Services/Credentials/CredentialResolver.cs b/backend/src/CodeSpace.Core/Services/Credentials/CredentialResolver.cs index 9f22de95c..3bd58bdeb 100644 --- a/backend/src/CodeSpace.Core/Services/Credentials/CredentialResolver.cs +++ b/backend/src/CodeSpace.Core/Services/Credentials/CredentialResolver.cs @@ -25,10 +25,28 @@ public CredentialPayload Resolve(Credential credential) return _serializer.Deserialize(credential.AuthType, json); } - /// A disconnect revokes the row and blanks its payload, so there is nothing to decrypt — say so, instead of letting Data Protection call the empty payload one it never protected. + /// + /// A disconnect revokes the row and blanks its payload, so there is nothing to decrypt — say so, instead of letting Data + /// Protection call the empty payload one it never protected. A credential its provider refused for good (marked + /// Error, with the reason) is not asked again: there is a payload, and presenting it only earns the same refusal — + /// a refused refresh token, a 401 on every call — so it fails here, naming the credential and what the provider said. + /// + /// For an hour, and for an OAuth credential only (): past that a use is let through to + /// the refresher, which asks the provider once more — a still-dead grant is refused again and re-marked, which dates the + /// verdict afresh, so it costs one refused call an hour; one the provider accepts again (an OAuth client put back, an + /// account unblocked) is refreshed and the pair it issues lifts the verdict, instead of failing runs until the sweep + /// next looks. A static token has nothing to refresh, and the sweep re-asks it on the same schedule. + /// private static void EnsureConnected(Credential credential) { if (credential.Status == CredentialStatus.Revoked) throw new CredentialDisconnectedException(credential.Id, credential.DisplayName); + + if (credential.Status == CredentialStatus.Error && !IsDueToBeAskedAgain(credential)) + throw new CredentialNeedsReconnectException(credential.Id, credential.DisplayName, credential.LastError ?? "its provider refused it"); } + + /// An OAuth credential whose verdict was last written, or last confirmed, longer ago than — the window the sweep re-asks an Error credential after. + private static bool IsDueToBeAskedAgain(Credential credential) => + credential.AuthType == AuthType.OAuth && DateTimeOffset.UtcNow - credential.LastModifiedDate >= CredentialHealthService.ErrorRecheckAfter; } diff --git a/backend/src/CodeSpace.Core/Services/Credentials/CredentialRevocationService.cs b/backend/src/CodeSpace.Core/Services/Credentials/CredentialRevocationService.cs index 23698c829..640ccb24d 100644 --- a/backend/src/CodeSpace.Core/Services/Credentials/CredentialRevocationService.cs +++ b/backend/src/CodeSpace.Core/Services/Credentials/CredentialRevocationService.cs @@ -1,8 +1,6 @@ -using CodeSpace.Core.Authorization; using CodeSpace.Core.DependencyInjection; using CodeSpace.Core.Persistence.Db; using CodeSpace.Core.Persistence.Entities; -using CodeSpace.Core.Services.Workflows; using CodeSpace.Messages.Enums; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; @@ -107,21 +105,8 @@ private async Task RetireDepartedOneAsync(DepartedCredential candidate, Ca } } - /// - /// Live Personal credentials whose person has no standing in the credential's team (): no - /// membership row there, and not an instance Admin. The person is the owner, or — for a token stored without one, as - /// adding a token without naming an owner did before the owner defaulted to the caller — whoever added it. - /// - /// Only a person with an account. An owner always has one (the column is a foreign key); a creator stamp is not - /// one — it can be empty, or name nobody when a script wrote the row — and then nothing says who lent the token or - /// whether they left. That token is kept: retiring it would break every repository bound through it on a guess. A - /// soft-deleted account is still somebody, and goes like anyone else without standing. - /// - private IQueryable Departed() => - _db.Credential - .Where(c => c.Ownership == CredentialOwnership.Personal && c.DeletedDate == null) - .Where(c => _db.User.IgnoreQueryFilters().Any(u => u.Id == (c.OwnerUserId ?? c.CreatedBy))) - .Where(ExpressionPredicate.Not(TeamStanding.Of(_db, c => c.TeamId, c => c.OwnerUserId ?? c.CreatedBy))); + /// The credentials the sweep retires — . + private IQueryable Departed() => DepartedCredentials.Of(_db); /// /// Clear the token material — nothing in CodeSpace can present it once it is gone — and take the row out of the diff --git a/backend/src/CodeSpace.Core/Services/Credentials/CredentialService.cs b/backend/src/CodeSpace.Core/Services/Credentials/CredentialService.cs index e173afde0..fbc7537cb 100644 --- a/backend/src/CodeSpace.Core/Services/Credentials/CredentialService.cs +++ b/backend/src/CodeSpace.Core/Services/Credentials/CredentialService.cs @@ -32,10 +32,11 @@ public sealed partial class CredentialService : ICredentialService, IScopedDepen private readonly IScopeChecker _scopeChecker; private readonly ICredentialSuccessionService _succession; private readonly ICredentialRevocationService _revocation; + private readonly ICredentialHealthService _health; private readonly ITeamMemberService _members; private readonly ILogger _logger; - public CredentialService(CodeSpaceDbContext db, ICurrentTeam currentTeam, ICurrentUser currentUser, IPayloadEncryptor encryptor, ICredentialPayloadSerializer serializer, IOAuthGrantRevoker grants, IProviderRegistry registry, IProviderModuleCatalog modules, IScopeChecker scopeChecker, ICredentialSuccessionService succession, ICredentialRevocationService revocation, ITeamMemberService members, ILogger logger) + public CredentialService(CodeSpaceDbContext db, ICurrentTeam currentTeam, ICurrentUser currentUser, IPayloadEncryptor encryptor, ICredentialPayloadSerializer serializer, IOAuthGrantRevoker grants, IProviderRegistry registry, IProviderModuleCatalog modules, IScopeChecker scopeChecker, ICredentialSuccessionService succession, ICredentialRevocationService revocation, ICredentialHealthService health, ITeamMemberService members, ILogger logger) { _db = db; _currentTeam = currentTeam; @@ -48,6 +49,7 @@ public CredentialService(CodeSpaceDbContext db, ICurrentTeam currentTeam, ICurre _scopeChecker = scopeChecker; _succession = succession; _revocation = revocation; + _health = health; _members = members; _logger = logger; } @@ -63,7 +65,7 @@ public async Task> ListAsync(Guid? providerInst // Repository picker. Two credentials with the same DisplayName (e.g. after a rename) // would otherwise be ambiguous; OwnerUserName disambiguates without forcing the user // to think about display-name uniqueness. - return await query + var summaries = await query .OrderByDescending(c => c.CreatedDate) .Select(c => new CredentialSummary { @@ -82,6 +84,26 @@ public async Task> ListAsync(Guid? providerInst CreatedDate = c.CreatedDate }) .ToListAsync(cancellationToken).ConfigureAwait(false); + + return await WithHealthAsync(summaries, teamId, cancellationToken).ConfigureAwait(false); + } + + /// + /// What to read off each listed credential (): the columns it is derived from are + /// already on the summary, and whether its person has left is one more read for the team's list — the same rule the + /// retirement sweep uses (), so the list never promises a retirement the sweep will not make. + /// + private async Task> WithHealthAsync(List summaries, Guid teamId, CancellationToken cancellationToken) + { + var ids = summaries.Select(s => s.Id).ToList(); + var left = await DepartedCredentials.Of(_db).AsNoTracking().Where(c => c.TeamId == teamId && ids.Contains(c.Id)).Select(c => c.Id).ToListAsync(cancellationToken).ConfigureAwait(false); + var now = DateTimeOffset.UtcNow; + + return summaries.Select(s => s with + { + Health = CredentialHealthDerivation.Derive(s.Status, s.AuthType, s.ExpiresDate, left.Contains(s.Id), now), + DaysToExpiry = CredentialHealthDerivation.DaysToExpiry(s.AuthType, s.ExpiresDate, now) + }).ToList(); } public async Task GetUsageAsync(Guid credentialId, CancellationToken cancellationToken) @@ -162,7 +184,7 @@ public async Task AddAsync(AddCredentialInput input, CancellationToken can // boundary (a confused-deputy seam: the platform would pair that instance's host + OAuth client // with this team's token). Conflate not-found with not-in-team — a 404 leaks nothing about // another team's instances. - await EnsureProviderInstanceInTeamAsync(input.ProviderInstanceId, teamId, cancellationToken).ConfigureAwait(false); + var instance = await LoadProviderInstanceInTeamAsync(input.ProviderInstanceId, teamId, cancellationToken).ConfigureAwait(false); var ownerUserId = OwnerOf(input); @@ -184,6 +206,9 @@ public async Task AddAsync(AddCredentialInput input, CancellationToken can Status = CredentialStatus.Active }; + // Before it is tracked: the validation stamps the new row, and a provider that answers slowly holds nothing the unit of work has written. + await _health.ValidateNewAsync(credential, instance, cancellationToken).ConfigureAwait(false); + await _db.Credential.AddAsync(credential, cancellationToken).ConfigureAwait(false); await _succession.CarryForwardAsync(credential, cancellationToken).ConfigureAwait(false); @@ -250,13 +275,9 @@ public async Task ListAccessibleRepositoriesAsync(Guid cre return await catalog.PagedListAccessibleAsync(context, search, clampedPage, clampedPerPage, cancellationToken).ConfigureAwait(false); } - private async Task EnsureProviderInstanceInTeamAsync(Guid providerInstanceId, Guid teamId, CancellationToken cancellationToken) - { - var inTeam = await _db.ProviderInstance.AsNoTracking() - .AnyAsync(p => p.Id == providerInstanceId && p.TeamId == teamId && p.DeletedDate == null, cancellationToken).ConfigureAwait(false); - - if (!inTeam) throw new KeyNotFoundException($"Provider instance {providerInstanceId} not found."); - } + private async Task LoadProviderInstanceInTeamAsync(Guid providerInstanceId, Guid teamId, CancellationToken cancellationToken) => + await _db.ProviderInstance.AsNoTracking().FirstOrDefaultAsync(p => p.Id == providerInstanceId && p.TeamId == teamId && p.DeletedDate == null, cancellationToken).ConfigureAwait(false) + ?? throw new KeyNotFoundException($"Provider instance {providerInstanceId} not found."); /// /// A team-service credential belongs to the team, not a person — it never carries an owner, whatever it was handed. A diff --git a/backend/src/CodeSpace.Core/Services/Credentials/CredentialSuccessionService.cs b/backend/src/CodeSpace.Core/Services/Credentials/CredentialSuccessionService.cs index a355d2bd1..298910a36 100644 --- a/backend/src/CodeSpace.Core/Services/Credentials/CredentialSuccessionService.cs +++ b/backend/src/CodeSpace.Core/Services/Credentials/CredentialSuccessionService.cs @@ -10,9 +10,12 @@ namespace CodeSpace.Core.Services.Credentials; /// -/// One rule, applied at connect time and by the recurring repair: what is bound through a REVOKED credential — its +/// One rule, applied at connect time and by the recurring repair: what is bound through a DEAD credential — its /// repositories and the group hooks it registered — follows that credential's owner to the credential they connected -/// after it on the same provider instance in the same team. Ownership is the whole key — a teammate connecting the same +/// after it on the same provider instance in the same team. Dead is a revoked credential, and one its provider refused for +/// good (Error, with its payload kept): the failure for either says to reconnect, and a person who does has no +/// Disconnect to click first — the Providers list shows only live credentials, so a refused one is already gone from it. +/// Ownership is the whole key — a teammate connecting the same /// provider is a different identity, and moving the owner's bindings onto it would change who they act as. A live /// credential has no successor, whatever its owner connects beside it — its bindings move only when it is disconnected /// (, with the destination a person named, or — when the owner disconnects their own — the @@ -46,7 +49,7 @@ public async Task CarryForwardAsync(Credential successor, CancellationToken canc { if (successor.OwnerUserId is not Guid ownerId) return; - var predecessorIds = await LoadDisconnectedPredecessorIdsAsync(successor, ownerId, cancellationToken).ConfigureAwait(false); + var predecessorIds = await LoadDeadPredecessorIdsAsync(successor, ownerId, cancellationToken).ConfigureAwait(false); if (predecessorIds.Count == 0) return; @@ -61,25 +64,28 @@ public async Task RepairStrandedAsync(int batchSize, CancellationToken canc public async Task CountAsync(CancellationToken cancellationToken) => await StrandedWithSuccessor().CountAsync(cancellationToken).ConfigureAwait(false); /// - /// The owner's revoked credentials on the successor's instance and team. Filtered on the tracked entities, not in - /// SQL: a token link revokes the credential it replaces in the same unit of work, so that row is Revoked in memory - /// before it is Revoked in the database. + /// The owner's dead credentials on the successor's instance and team — revoked, or refused by their provider. Filtered on + /// the tracked entities, not in SQL: a token link revokes the credential it replaces in the same unit of work, so that row + /// is Revoked in memory before it is Revoked in the database. No date is asked of a refused one here: the successor is + /// being created now, after whatever the owner's credentials last did. /// - private async Task> LoadDisconnectedPredecessorIdsAsync(Credential successor, Guid ownerId, CancellationToken cancellationToken) + private async Task> LoadDeadPredecessorIdsAsync(Credential successor, Guid ownerId, CancellationToken cancellationToken) { var owned = await _db.Credential .Where(c => c.TeamId == successor.TeamId && c.ProviderInstanceId == successor.ProviderInstanceId && c.OwnerUserId == ownerId && c.Id != successor.Id) .ToListAsync(cancellationToken).ConfigureAwait(false); - return owned.Where(c => c.Status == CredentialStatus.Revoked).Select(c => c.Id).ToList(); + return owned.Where(c => c.Status is CredentialStatus.Revoked or CredentialStatus.Error).Select(c => c.Id).ToList(); } /// - /// Revoked owned credentials that something live is still bound to, each paired with its successor: the owner's - /// newest Active credential on the same instance and team connected at or after the revocation (a revoked row is not - /// modified again, so its last-modified date is when it was revoked; a token link revokes the credential it replaces - /// in the same save that creates the new one, hence at-or-after). A credential with no successor — a disconnect - /// nobody reconnected — is not a candidate, so it never crowds a repairable one out of the batch. Ordered by id and + /// Dead owned credentials that something live is still bound to, each paired with its successor: the owner's + /// newest Active credential on the same instance and team connected at or after the moment the dead one stopped working. A revoked row is not modified again, so its last-modified date is when it was revoked (a token link revokes + /// the credential it replaces in the same save that creates the new one, hence at-or-after). A refused row IS written again, + /// every hour its verdict is confirmed, so its last-modified date says nothing about when it died: it was last known good + /// at its last validation, or at its creation if it never had one, and a credential connected since is the owner's + /// reconnect — while one connected before it last worked was beside it, not after it. A credential with no successor — a + /// disconnect nobody reconnected — is not a candidate, so it never crowds a repairable one out of the batch. Ordered by id and /// read in pages: is the last id of the previous page, so the pass walks the whole backlog /// instead of re-reading its first rows every time. /// @@ -99,18 +105,18 @@ private async Task> LoadStrandedAsync(Guid? after, int /// private IQueryable StrandedWithSuccessor(Guid? after = null) { - var revoked = _db.Credential.AsNoTracking().Where(dead => dead.Status == CredentialStatus.Revoked && dead.OwnerUserId != null); + var deadCredentials = _db.Credential.AsNoTracking().Where(c => (c.Status == CredentialStatus.Revoked || c.Status == CredentialStatus.Error) && c.OwnerUserId != null); - if (after is Guid cursor) revoked = revoked.Where(dead => dead.Id.CompareTo(cursor) > 0); + if (after is Guid cursor) deadCredentials = deadCredentials.Where(c => c.Id.CompareTo(cursor) > 0); - return revoked + return deadCredentials .Where(dead => _db.Repository.Any(r => r.CredentialId == dead.Id && r.TeamId == dead.TeamId && r.ProviderInstanceId == dead.ProviderInstanceId && r.DeletedDate == null) || _db.ConnectionWebhook.Any(w => w.CredentialId == dead.Id && w.ProviderInstanceId == dead.ProviderInstanceId && WebhookRegistrationLifecycle.InService.Contains(w.RegistrationStatus))) .Select(dead => new StrandedRow { DeadId = dead.Id, SuccessorId = _db.Credential - .Where(c => c.TeamId == dead.TeamId && c.ProviderInstanceId == dead.ProviderInstanceId && c.OwnerUserId == dead.OwnerUserId && c.Status == CredentialStatus.Active && c.DeletedDate == null && c.CreatedDate >= dead.LastModifiedDate) + .Where(c => c.TeamId == dead.TeamId && c.ProviderInstanceId == dead.ProviderInstanceId && c.OwnerUserId == dead.OwnerUserId && c.Status == CredentialStatus.Active && c.DeletedDate == null && c.CreatedDate >= (dead.Status == CredentialStatus.Revoked ? dead.LastModifiedDate : (dead.LastValidatedDate ?? dead.CreatedDate))) .OrderByDescending(c => c.CreatedDate).ThenByDescending(c => c.Id) .Select(c => (Guid?)c.Id) .FirstOrDefault() @@ -118,7 +124,7 @@ private IQueryable StrandedWithSuccessor(Guid? after = null) .Where(candidate => candidate.SuccessorId != null); } - /// Move one disconnected credential's bindings and save them on their own, then hand its revived hooks to the registrar. + /// Move one dead credential's bindings and save them on their own, then hand its revived hooks to the registrar. private async Task RepairAsync(StrandedCredential candidate, CancellationToken cancellationToken) { try diff --git a/backend/src/CodeSpace.Core/Services/Credentials/DepartedCredentials.cs b/backend/src/CodeSpace.Core/Services/Credentials/DepartedCredentials.cs new file mode 100644 index 000000000..cd98ae450 --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Credentials/DepartedCredentials.cs @@ -0,0 +1,32 @@ +using CodeSpace.Core.Authorization; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Workflows; +using CodeSpace.Messages.Enums; +using Microsoft.EntityFrameworkCore; + +namespace CodeSpace.Core.Services.Credentials; + +/// +/// The one definition of a credential whose person has left, asked by the sweep that retires them and by the credential list +/// that says one is about to be: written once because the two must agree, and a list that called a credential "owner left" +/// by a looser rule would promise a retirement that never comes. +/// +internal static class DepartedCredentials +{ + /// + /// Live Personal credentials whose person has no standing in the credential's team (): no + /// membership row there, and not an instance Admin. The person is the owner, or — for a token stored without one, as + /// adding a token without naming an owner did before the owner defaulted to the caller — whoever added it. + /// + /// Only a person with an account. An owner always has one (the column is a foreign key); a creator stamp is not + /// one — it can be empty, or name nobody when a script wrote the row — and then nothing says who lent the token or + /// whether they left. That token is kept: retiring it would break every repository bound through it on a guess. A + /// soft-deleted account is still somebody, and goes like anyone else without standing. + /// + public static IQueryable Of(CodeSpaceDbContext db) => + db.Credential + .Where(c => c.Ownership == CredentialOwnership.Personal && c.DeletedDate == null) + .Where(c => db.User.IgnoreQueryFilters().Any(u => u.Id == (c.OwnerUserId ?? c.CreatedBy))) + .Where(ExpressionPredicate.Not(TeamStanding.Of(db, c => c.TeamId, c => c.OwnerUserId ?? c.CreatedBy))); +} diff --git a/backend/src/CodeSpace.Core/Services/Credentials/ICredentialHealthService.cs b/backend/src/CodeSpace.Core/Services/Credentials/ICredentialHealthService.cs new file mode 100644 index 000000000..7a1dca510 --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Credentials/ICredentialHealthService.cs @@ -0,0 +1,32 @@ +using CodeSpace.Core.Persistence.Entities; + +namespace CodeSpace.Core.Services.Credentials; + +/// +/// Keeps what CodeSpace records about a credential's health in step with what its provider says, without waiting for a run to +/// find out: asks each provider whether a bounded batch of credentials still works and records the answer. +/// +public interface ICredentialHealthService +{ + /// + /// Probes up to credentials that are due (never validated, or validated long enough ago; + /// one marked Error only after an hour) and returns how many the provider answered about. A credential it accepts is + /// validated — its expiry recorded if it is a static token, an Error lifted; one it refuses with a 401 is marked Error. A + /// probe that got no answer changes nothing, counts for nothing, and is not asked again by the same pass. + /// + /// A pass is bounded in time as well as in credentials (): + /// a provider that stops answering costs one credential its probe deadline, not the worker, and the pass ends at its budget — + /// ahead of the next tick. A tick that finds the previous pass still running returns 0 at once and asks no provider. + /// + Task ProbeBatchAsync(int batchSize, CancellationToken cancellationToken); + + /// + /// For a credential about to be added: asks its provider about the token — within the probe deadline — and, when the + /// provider accepts it, records on that it was just validated and, for a static token the + /// provider describes, when it expires; so the credential reads right from the first list instead of after the sweep's + /// first pass. Never fails and never refuses: a provider that cannot be reached, is silent, or refuses the token leaves the + /// credential as it was, due for the sweep — which is what marks one the provider refuses. Does nothing for a credential that + /// is not a static token, or whose provider has no probe. Tracked changes to only. + /// + Task ValidateNewAsync(Credential credential, ProviderInstance instance, CancellationToken cancellationToken); +} diff --git a/backend/src/CodeSpace.Core/Services/Credentials/ICredentialHealthWriter.cs b/backend/src/CodeSpace.Core/Services/Credentials/ICredentialHealthWriter.cs new file mode 100644 index 000000000..44de83178 --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Credentials/ICredentialHealthWriter.cs @@ -0,0 +1,21 @@ +namespace CodeSpace.Core.Services.Credentials; + +/// +/// Records what a provider said about a credential: it refused it (Status=Error and why), or accepted it (validated, +/// and any Error cleared). Like every write goes straight to the row on a +/// connection of its own, never inside the caller's transaction: a verdict is a fact about the provider, and a caller that +/// rolls back must not unsay it. Every write is conditional on the row still holding the payload the caller judged — a +/// disconnect, a reconnect or a re-authorization that replaced it meanwhile is never overwritten by a verdict about the old +/// one — and returns whether it landed. +/// +public interface ICredentialHealthWriter +{ + /// Marks the credential Error with , if it is still Active or already Error and still holds . A repeat keeps the first reason and refreshes the date, which is when the verdict was last confirmed. + Task MarkRejectedAsync(Guid credentialId, string expectedEncryptedPayload, string lastError, CancellationToken cancellationToken); + + /// Records and, when asked, the token's real expiry; clears Error and its text. Under the same conditions as . + Task MarkValidatedAsync(Guid credentialId, string expectedEncryptedPayload, DateTimeOffset validatedAt, ExpiryRecord? expiry, CancellationToken cancellationToken); +} + +/// The expiry a validation records for a static token: null means the provider says it never expires. Absent (a null ) leaves the column alone, as for an OAuth credential, whose expiry is its access token's and is the refresher's to write. +public sealed record ExpiryRecord(DateTimeOffset? ExpiresAt); diff --git a/backend/src/CodeSpace.Core/Services/Credentials/ICredentialSuccessionService.cs b/backend/src/CodeSpace.Core/Services/Credentials/ICredentialSuccessionService.cs index 6bfb4bc2a..c206337d8 100644 --- a/backend/src/CodeSpace.Core/Services/Credentials/ICredentialSuccessionService.cs +++ b/backend/src/CodeSpace.Core/Services/Credentials/ICredentialSuccessionService.cs @@ -4,16 +4,18 @@ namespace CodeSpace.Core.Services.Credentials; /// -/// Moves what a person bound through a credential they disconnected onto the credential they connect next on the same -/// provider instance. Reconnecting mints a new credential row; without this, every repository and group hook bound -/// through the old one kept pointing at a revoked row whose token material is gone. +/// Moves what a person bound through a credential that died — one they disconnected, or one its provider refused for good — +/// onto the credential they connect next on the same provider instance. Reconnecting mints a new credential row; without +/// this, every repository and group hook bound through the old one kept pointing at a row whose token material is gone, +/// or that its provider no longer accepts. /// public interface ICredentialSuccessionService { /// /// Re-point every live repository and in-service group hook bound through one of 's - /// owner's revoked credentials on the same provider instance and team at , lifting the - /// "needs a new credential" error the disconnect set and reviving the hooks its failures buried. A credential with no + /// owner's dead credentials (revoked, or refused by their provider) on the same provider instance and team at + /// , lifting the "needs a new credential" error a disconnect set and reviving the hooks its + /// failures buried. A credential with no /// owner (team-service) succeeds nobody. Tracked changes only — the caller's unit of work saves them with the /// successor, and the revived hooks are dispatched after it commits. /// @@ -21,7 +23,7 @@ public interface ICredentialSuccessionService /// /// Apply the same rule to bindings a connect path left behind — rows stranded before connects carried them forward, - /// or by an older pod during a rolling deploy: up to revoked credentials that still have + /// or by an older pod during a rolling deploy: up to dead credentials that still have /// live bindings AND an owner's reconnect to follow, each saved on its own. Returns how many were repaired. A /// credential that fails is skipped and logged, and the pass goes on to the ones after it — however many fail, they /// cannot crowd a repairable credential out of the pass — and one that has failed several passes in a row is set aside diff --git a/backend/src/CodeSpace.Core/Services/Credentials/IStrandedCredentialCensus.cs b/backend/src/CodeSpace.Core/Services/Credentials/IStrandedCredentialCensus.cs index 9b2ac269b..929e8e4be 100644 --- a/backend/src/CodeSpace.Core/Services/Credentials/IStrandedCredentialCensus.cs +++ b/backend/src/CodeSpace.Core/Services/Credentials/IStrandedCredentialCensus.cs @@ -8,7 +8,7 @@ namespace CodeSpace.Core.Services.Credentials; public interface IStrandedCredentialCensus { /// - /// The revoked credentials that something live is still bound to AND whose owner has since connected another — exactly + /// The dead (revoked, or refused by their provider) credentials that something live is still bound to AND whose owner has since connected another — exactly /// the rows takes, counted without its batch bound. A /// backlog that stays above zero across passes is a sweep that is not running, or one candidate that fails every time. /// diff --git a/backend/src/CodeSpace.Core/Services/OAuth/CredentialPayloadWriter.cs b/backend/src/CodeSpace.Core/Services/OAuth/CredentialPayloadWriter.cs index fd6eb5755..24060367f 100644 --- a/backend/src/CodeSpace.Core/Services/OAuth/CredentialPayloadWriter.cs +++ b/backend/src/CodeSpace.Core/Services/OAuth/CredentialPayloadWriter.cs @@ -4,6 +4,7 @@ using CodeSpace.Core.Settings.Database; using CodeSpace.Messages.Constants; using CodeSpace.Messages.Credentials; +using CodeSpace.Messages.Enums; using Npgsql; namespace CodeSpace.Core.Services.OAuth; @@ -37,7 +38,9 @@ public async Task UpdatePayloadAsync(Credential credential, string expecte await conn.OpenAsync(cancellationToken).ConfigureAwait(false); await using var cmd = conn.CreateCommand(); - cmd.CommandText = "UPDATE credential SET encrypted_payload = @enc, expires_date = @exp, last_modified_date = @now, last_modified_by = @user WHERE id = @id AND encrypted_payload = @expected"; + // A pair the provider just issued for this row is proof the provider accepts the grant, so an Error that recorded its refusal + // is cleared with it (the status and the text it carried are read from the row, not assumed, hence the CASE). + cmd.CommandText = "UPDATE credential SET encrypted_payload = @enc, expires_date = @exp, last_modified_date = @now, last_modified_by = @user, status = CASE WHEN status = 'Error' THEN 'Active' ELSE status END, last_error = CASE WHEN status = 'Error' THEN NULL ELSE last_error END WHERE id = @id AND encrypted_payload = @expected"; cmd.Parameters.AddWithValue("@enc", encrypted); cmd.Parameters.AddWithValue("@exp", (object?)expiresDate ?? DBNull.Value); cmd.Parameters.AddWithValue("@now", now); @@ -54,6 +57,13 @@ public async Task UpdatePayloadAsync(Credential credential, string expecte credential.ExpiresDate = expiresDate; credential.LastModifiedDate = now; credential.LastModifiedBy = SystemUsers.SeederId; + + if (credential.Status == CredentialStatus.Error) + { + credential.Status = CredentialStatus.Active; + credential.LastError = null; + } + _trackedEntry.MarkPayloadStored(credential); return true; diff --git a/backend/src/CodeSpace.Core/Services/OAuth/ICrossProcessTryLock.cs b/backend/src/CodeSpace.Core/Services/OAuth/ICrossProcessTryLock.cs new file mode 100644 index 000000000..5a28ca92b --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/OAuth/ICrossProcessTryLock.cs @@ -0,0 +1,15 @@ +namespace CodeSpace.Core.Services.OAuth; + +/// +/// A sibling of for work that must not run twice at once and should simply be skipped, not +/// queued, when something else is already doing it: a recurring sweep whose previous pass is still going. Same key space as +/// . +/// +public interface ICrossProcessTryLock +{ + /// + /// Takes the lock if no one holds it and returns a handle whose disposal releases it. Returns null at once, without + /// waiting, when another holder has it. + /// + Task TryAcquireAsync(long key, CancellationToken cancellationToken); +} diff --git a/backend/src/CodeSpace.Core/Services/OAuth/IOAuthTokenRecovery.cs b/backend/src/CodeSpace.Core/Services/OAuth/IOAuthTokenRecovery.cs new file mode 100644 index 000000000..840de1da7 --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/OAuth/IOAuthTokenRecovery.cs @@ -0,0 +1,32 @@ +using CodeSpace.Core.Services.Providers; +using CodeSpace.Messages.Credentials; + +namespace CodeSpace.Core.Services.OAuth; + +/// +/// What an OAuth credential does when the token it holds cannot be used as the row stands: its provider answered 401 to an +/// access token it was just handed, or this process holds a pair the provider issued that the row does not. A sibling of +/// , which decides from the token's expiry and so never sees an access token the provider +/// retired early — another consumer rotated the pair and stored it, or the provider's token lifetime is shorter than the +/// expires_in it reported. The refresher's callers resolve a token once; this is the way back when it was refused. +/// +public interface IOAuthTokenRecovery +{ + /// + /// Under the credential's refresh locks, from the STORED row: a token other than + /// is the pair another caller already replaced it with, and is adopted; the very token that was refused is rotated once, + /// with the stored refresh token. Null when there is nothing to try: no refresh token, or the credential was rotated + /// moments ago and the new token is refused too — the 401 is not the token's age, and rotating again would only spend + /// refresh tokens. A rotation the provider refuses for good is CredentialNeedsReconnectException, as in a refresh. + /// + Task RecoverAfterRejectionAsync(ProviderContext context, string rejectedAccessToken, CancellationToken cancellationToken); + + /// + /// Stores the pair this process holds for the credential, if it holds one: the provider rotated it and every attempt to + /// store it failed, so the row still holds the refresh token the provider consumed. Asked before the credential is read, so + /// a verdict another process wrote over that row — its refusal of the consumed token marked the credential Error — does not stand in front of a + /// live pair; storing it lifts the verdict, as any refresh the provider accepts does. Nothing happens, and no lock is + /// taken, when this process holds no pair for the credential or the credential is disconnected. + /// + Task StoreKeptPairAsync(ProviderContext context, CancellationToken cancellationToken); +} diff --git a/backend/src/CodeSpace.Core/Services/OAuth/OAuthExchangeException.cs b/backend/src/CodeSpace.Core/Services/OAuth/OAuthExchangeException.cs index ce1160c9a..f7fa9cfeb 100644 --- a/backend/src/CodeSpace.Core/Services/OAuth/OAuthExchangeException.cs +++ b/backend/src/CodeSpace.Core/Services/OAuth/OAuthExchangeException.cs @@ -6,9 +6,26 @@ namespace CodeSpace.Core.Services.OAuth; /// Thrown when an OAuth provider rejects a code-exchange or refresh request. Wraps the /// provider's RFC 6749 error code + description for diagnostics. NEVER includes raw secrets /// (client_secret, refresh_token, etc.) in the message. +/// +/// A refusal of a REFRESH that names the grant or the client () is a verdict, not a +/// hiccup: the refresher turns it into CredentialNeedsReconnectException and marks the credential. Anything else — an +/// HTTP error with no OAuth body, a provider that is down, a malformed answer — says nothing about the grant and is left as +/// it is. /// public sealed class OAuthExchangeException : Exception, IFailure { + /// + /// The RFC 6749 §5.2 error codes (and GitHub's own spellings of them) that refuse THIS grant or THIS client — the ones a + /// retry can never turn into a success: the refresh token is unknown, used, revoked or expired, or the app's id or secret + /// is not the one the grant was issued to. Everything else (http_503, temporarily_unavailable, + /// server_error, invalid_request, malformed_response, an unknown code) is not a judgement of the + /// grant: a malformed or unavailable exchange is not a dead credential. Pinned by test. + /// + internal static readonly IReadOnlySet FinalRefusalErrors = new HashSet(StringComparer.Ordinal) + { + "invalid_grant", "invalid_client", "unauthorized_client", "bad_refresh_token", "incorrect_client_credentials" + }; + public FailureKind Kind => FailureKind.Invalid; public string Code => FailureCodes.OAuthExchangeFailed; @@ -22,7 +39,21 @@ public OAuthExchangeException(string error, string? description, string? provide ProviderBody = providerBody; } + private OAuthExchangeException(OAuthExchangeException original, Guid credentialId) + : base($"{original.Message} (credential {credentialId})", original) + { + Error = original.Error; + Description = original.Description; + ProviderBody = original.ProviderBody; + } + public string Error { get; } public string? Description { get; } public string? ProviderBody { get; } + + /// True when the provider's answer refuses the grant or the client for good (). + public bool IsFinalRefusal => FinalRefusalErrors.Contains(Error); + + /// The same refusal, its text naming the credential whose refresh it answered — the provider's answer carries no credential, and an operator reading a run's error cannot tell which row to look at without one. + public OAuthExchangeException ForCredential(Guid credentialId) => new(this, credentialId); } diff --git a/backend/src/CodeSpace.Core/Services/OAuth/OAuthTokenRefresher.cs b/backend/src/CodeSpace.Core/Services/OAuth/OAuthTokenRefresher.cs index 9692708f8..54ce77d09 100644 --- a/backend/src/CodeSpace.Core/Services/OAuth/OAuthTokenRefresher.cs +++ b/backend/src/CodeSpace.Core/Services/OAuth/OAuthTokenRefresher.cs @@ -5,6 +5,8 @@ using CodeSpace.Core.Services.Providers; using CodeSpace.Core.Services.Providers.Auth; using CodeSpace.Messages.Credentials; +using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Exceptions; using Microsoft.Extensions.Logging; namespace CodeSpace.Core.Services.OAuth; @@ -34,8 +36,19 @@ public interface IOAuthTokenRefresher /// proven able to seal it BEFORE the provider is asked; the store is retried on fresh connections; a pair that still /// cannot be stored is kept in this process, logged, and stored by the credential's next use; and a compare-and-swap lost /// to a row that is not empty never drops it — the row's pair wins only if it is at least as new. +/// +/// A refusal that names the grant or the client () and finds the row +/// unchanged is final: the credential is marked Error with the reason, on a connection of its own, and the caller gets +/// — until then the row stayed Active with nothing to say why, and every run +/// re-asked the provider for the same refusal. It is also what answers when the +/// pair it rotates after a 401 is refused for good. +/// +/// A pair this process kept is also stored when the credential is next READ, not only when it is next refreshed +/// (, asked by the OAuth strategies first): another process that was refused the consumed +/// refresh token has marked the credential Error by then, and the fail-fast on that verdict would otherwise stand in +/// front of the one live pair. /// -public sealed class OAuthTokenRefresher : IOAuthTokenRefresher, IScopedDependency +public sealed class OAuthTokenRefresher : IOAuthTokenRefresher, IOAuthTokenRecovery, IScopedDependency { private static readonly ConcurrentDictionary RefreshLocks = new(); private static readonly TimeSpan RefreshBuffer = TimeSpan.FromMinutes(2); @@ -49,6 +62,14 @@ public sealed class OAuthTokenRefresher : IOAuthTokenRefresher, IScopedDependenc /// Plaintext for proving the key ring can seal before the provider is asked to rotate. Never stored, never logged. private const string SealProbe = "oauth-rotation-seal-probe"; + /// How long after this process rotated a credential's pair a refusal of the NEW access token is left alone: a token issued moments ago that is refused is not aged out, and rotating again would burn a refresh token on a 401 that has another cause. + private static readonly TimeSpan RejectionRotationCooldown = TimeSpan.FromMinutes(1); + + private const int PruneRotationsAbove = 1024; + + /// When this process last had the provider rotate each credential's pair — what is measured from. + private static readonly ConcurrentDictionary RecentRotations = new(); + /// /// Pairs the provider issued that no attempt could store, by credential: what the row held when the refresh began /// (the compare-and-swap's expected value) and the pair. Only this process knows them; the next use of the credential in @@ -64,9 +85,10 @@ public sealed class OAuthTokenRefresher : IOAuthTokenRefresher, IScopedDependenc private readonly TimeProvider _clock; private readonly ICrossProcessLock _crossProcessLock; private readonly ITrackedCredentialEntry _trackedEntry; + private readonly ICredentialHealthWriter _healthWriter; private readonly ILogger _logger; - public OAuthTokenRefresher(IOAuthClientRegistry oauthClients, ICredentialPayloadWriter payloadWriter, ICredentialPayloadReader payloadReader, ICredentialResolver credentialResolver, IPayloadEncryptor encryptor, TimeProvider clock, ICrossProcessLock crossProcessLock, ITrackedCredentialEntry trackedEntry, ILogger logger) + public OAuthTokenRefresher(IOAuthClientRegistry oauthClients, ICredentialPayloadWriter payloadWriter, ICredentialPayloadReader payloadReader, ICredentialResolver credentialResolver, IPayloadEncryptor encryptor, TimeProvider clock, ICrossProcessLock crossProcessLock, ITrackedCredentialEntry trackedEntry, ICredentialHealthWriter healthWriter, ILogger logger) { _oauthClients = oauthClients; _payloadWriter = payloadWriter; @@ -76,6 +98,7 @@ public OAuthTokenRefresher(IOAuthClientRegistry oauthClients, ICredentialPayload _clock = clock; _crossProcessLock = crossProcessLock; _trackedEntry = trackedEntry; + _healthWriter = healthWriter; _logger = logger; } @@ -100,8 +123,7 @@ public async Task RefreshIfNeededAsync(ProviderContext context, OA /// /// Runs under the lock. Refreshes only if the stored token still needs it. A refusal is final unless the row changed - /// meanwhile: emptied by a revocation, which does not take the lock (reported as revoked), or — only when this - /// refresher's lock session was lost mid-refresh — refreshed by the pod that took the lock over, whose token we use. + /// meanwhile (see ). /// private async Task RefreshStoredPayloadAsync(ProviderContext context, CancellationToken cancellationToken) { @@ -110,17 +132,103 @@ private async Task RefreshStoredPayloadAsync(ProviderContext conte if (!ShouldRefresh(latest)) return latest; + return await RotateOrClassifyAsync(context, stored, latest).ConfigureAwait(false); + } + + public async Task RecoverAfterRejectionAsync(ProviderContext context, string rejectedAccessToken, CancellationToken cancellationToken) => + await UnderRefreshLockAsync(context, ct => RecoverStoredPayloadAsync(context, rejectedAccessToken, ct), cancellationToken).ConfigureAwait(false); + + public async Task StoreKeptPairAsync(ProviderContext context, CancellationToken cancellationToken) + { + if (context.Credential.Status == CredentialStatus.Revoked || !KeptRotations.ContainsKey(context.Credential.Id)) return; + + await UnderRefreshLockAsync(context, async ct => await ReadStoredAfterKeptPairAsync(context, ct).ConfigureAwait(false), cancellationToken).ConfigureAwait(false); + } + + /// The credential's two refresh locks — this process's, then every process's — held for . + private async Task UnderRefreshLockAsync(ProviderContext context, Func> work, CancellationToken cancellationToken) + { + var sem = RefreshLocks.GetOrAdd(context.Credential.Id, _ => new SemaphoreSlim(1, 1)); + await sem.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + await using var crossLock = await _crossProcessLock.AcquireAsync(CredentialIdToLockKey(context.Credential.Id), cancellationToken).ConfigureAwait(false); + + return await work(cancellationToken).ConfigureAwait(false); + } + finally + { + sem.Release(); + } + } + + /// + /// Runs under the lock, from the stored row. Whatever the row holds that is not the refused token and is still good is + /// what another caller replaced it with, and is the answer. Otherwise the refused token is the credential's token, and + /// the provider is asked once for a new pair — unless there is no refresh token to ask with, or the pair was rotated + /// moments ago and its token is refused too (). + /// + private async Task RecoverStoredPayloadAsync(ProviderContext context, string rejectedAccessToken, CancellationToken cancellationToken) + { + var stored = await ReadStoredAfterKeptPairAsync(context, cancellationToken).ConfigureAwait(false); + var latest = Adopt(context.Credential, stored); + + if (latest.AccessToken != rejectedAccessToken && !ShouldRefresh(latest)) return latest; + if (string.IsNullOrEmpty(latest.RefreshToken) || RotatedRecently(context.Credential.Id)) return null; + + return await RotateOrClassifyAsync(context, stored, latest).ConfigureAwait(false); + } + + /// + /// Rotates the pair and stores it. A refusal is final unless the row changed meanwhile: emptied by a revocation, which + /// does not take the lock (reported as revoked), or — only when this refresher's lock session was lost mid-refresh — + /// refreshed by the pod that took the lock over, whose token we use. A refusal that stands is the provider's verdict on + /// the grant (). + /// + private async Task RotateOrClassifyAsync(ProviderContext context, string stored, OAuthPayload latest) + { try { return await RotateAndStoreAsync(context, stored, latest).ConfigureAwait(false); } - catch (OAuthExchangeException) + catch (OAuthExchangeException refusal) { var storedSinceRefusal = await _payloadReader.ReadEncryptedPayloadAsync(context.Credential.Id, CancellationToken.None).ConfigureAwait(false); - if (storedSinceRefusal == stored) throw; + if (storedSinceRefusal != stored) return Adopt(context.Credential, storedSinceRefusal); + + throw await ClassifyRefusalAsync(context, stored, refusal).ConfigureAwait(false); + } + } + + /// + /// What the caller is told about a refusal that stands. One that names the grant or the client is final: the credential + /// is marked Error () and the failure says to reconnect, with the credential named. + /// Any other (a provider that is down, an answer that is not OAuth) is not a verdict on the credential, so it stays the + /// provider's failure — now naming the credential it was for. + /// + private async Task ClassifyRefusalAsync(ProviderContext context, string stored, OAuthExchangeException refusal) + { + if (!refusal.IsFinalRefusal) return refusal.ForCredential(context.Credential.Id); + + await RecordRefusalAsync(context, stored, refusal).ConfigureAwait(false); + + return new CredentialNeedsReconnectException(context.Credential.Id, context.Credential.DisplayName, refusal.Error, refusal); + } + + /// Marks the credential Error over the row the refusal was judged against, and says so in the log. A store that cannot be reached must not hide the refusal it was recording, so it is logged and the refusal still surfaces. + private async Task RecordRefusalAsync(ProviderContext context, string stored, OAuthExchangeException refusal) + { + try + { + var marked = await _healthWriter.MarkRejectedAsync(context.Credential.Id, stored, $"needs reconnect: {refusal.Error}", CancellationToken.None).ConfigureAwait(false); - return Adopt(context.Credential, storedSinceRefusal); + _logger.LogWarning("Credential {CredentialId} ({CredentialName}) needs to be reconnected: its provider refused the OAuth grant ({OAuthError}). Marked Error: {Marked}", context.Credential.Id, context.Credential.DisplayName, refusal.Error, marked); + } + catch (Exception ex) + { + _logger.LogError(ex, "Credential {CredentialId} ({CredentialName}) needs to be reconnected (its provider refused the OAuth grant: {OAuthError}), and recording that failed", context.Credential.Id, context.Credential.DisplayName, refusal.Error); } } @@ -151,9 +259,25 @@ private async Task RotateAndStoreAsync(ProviderContext context, st var refreshed = await ExchangeRefreshAsync(context, latest, CancellationToken.None).ConfigureAwait(false); + NoteRotation(context.Credential.Id); + return await PersistOrAdoptAsync(context, stored, refreshed).ConfigureAwait(false); } + private void NoteRotation(Guid credentialId) + { + var now = _clock.GetUtcNow(); + + RecentRotations[credentialId] = now; + + if (RecentRotations.Count <= PruneRotationsAbove) return; + + foreach (var (id, at) in RecentRotations) + if (now - at >= RejectionRotationCooldown) RecentRotations.TryRemove(id, out _); + } + + private bool RotatedRecently(Guid credentialId) => RecentRotations.TryGetValue(credentialId, out var at) && _clock.GetUtcNow() - at < RejectionRotationCooldown; + /// /// The writer seals the new pair only after the provider has issued it, so a Data Protection fault there — a key-ring /// store that is down, a key that cannot be created — would lose a pair already issued. Hitting the same fault here diff --git a/backend/src/CodeSpace.Core/Services/OAuth/PostgresAdvisoryLock.cs b/backend/src/CodeSpace.Core/Services/OAuth/PostgresAdvisoryLock.cs index 2251dc25e..49292bbab 100644 --- a/backend/src/CodeSpace.Core/Services/OAuth/PostgresAdvisoryLock.cs +++ b/backend/src/CodeSpace.Core/Services/OAuth/PostgresAdvisoryLock.cs @@ -5,7 +5,8 @@ namespace CodeSpace.Core.Services.OAuth; /// -/// Postgres advisory lock. Holds a dedicated connection for the duration of the lock — the +/// Postgres advisory lock — waited for () or tried and skipped +/// (). Holds a dedicated connection for the duration of the lock — the /// lock is session-scoped, so it auto-releases when the session ends (covers process-crash /// recovery). DisposeAsync issues an explicit pg_advisory_unlock and then closes the connection. /// @@ -16,7 +17,7 @@ namespace CodeSpace.Core.Services.OAuth; /// the lock however the acquire or the unlock went. Multiplexing, which requires pooling, is turned off with it: the /// configured string is otherwise used as it is. /// -public sealed class PostgresAdvisoryLock : ICrossProcessLock, IScopedDependency +public sealed class PostgresAdvisoryLock : ICrossProcessLock, ICrossProcessTryLock, IScopedDependency { private readonly CodeSpaceConnectionString _connectionString; @@ -24,8 +25,7 @@ public sealed class PostgresAdvisoryLock : ICrossProcessLock, IScopedDependency public async Task AcquireAsync(long key, CancellationToken cancellationToken) { - var conn = new NpgsqlConnection(SessionConnectionString()); - await conn.OpenAsync(cancellationToken).ConfigureAwait(false); + var conn = await OpenSessionAsync(cancellationToken).ConfigureAwait(false); try { @@ -43,6 +43,38 @@ public async Task AcquireAsync(long key, CancellationToken can return new Handle(conn, key); } + public async Task TryAcquireAsync(long key, CancellationToken cancellationToken) + { + var conn = await OpenSessionAsync(cancellationToken).ConfigureAwait(false); + + try + { + await using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT pg_try_advisory_lock(@key)"; + cmd.Parameters.AddWithValue("@key", key); + + if (await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false) is true) return new Handle(conn, key); + } + catch + { + await conn.DisposeAsync().ConfigureAwait(false); + throw; + } + + await conn.DisposeAsync().ConfigureAwait(false); + + return null; + } + + private async Task OpenSessionAsync(CancellationToken cancellationToken) + { + var conn = new NpgsqlConnection(SessionConnectionString()); + + await conn.OpenAsync(cancellationToken).ConfigureAwait(false); + + return conn; + } + /// Multiplexing needs the pool this session must not use, so a deployed string that sets it would otherwise fail every acquire. private string SessionConnectionString() => new NpgsqlConnectionStringBuilder(_connectionString.Value) { Pooling = false, Multiplexing = false }.ConnectionString; diff --git a/backend/src/CodeSpace.Core/Services/Providers/Auth/IProviderAuthResolver.cs b/backend/src/CodeSpace.Core/Services/Providers/Auth/IProviderAuthResolver.cs index c0aa40a91..3eff19092 100644 --- a/backend/src/CodeSpace.Core/Services/Providers/Auth/IProviderAuthResolver.cs +++ b/backend/src/CodeSpace.Core/Services/Providers/Auth/IProviderAuthResolver.cs @@ -8,4 +8,13 @@ namespace CodeSpace.Core.Services.Providers.Auth; public interface IProviderAuthResolver { Task ResolveAsync(ProviderContext context, CancellationToken cancellationToken); + + /// + /// The caller presented , which handed out, and the provider + /// answered 401. Returns the auth to retry with once — another caller's newer token adopted from the stored credential, or + /// a rotation this one made — or null when the rejection stands: a static token, a credential that has nothing to rotate + /// with, or one rotated moments ago. A resolver with no way to recover answers null, which is every caller's cue to surface + /// the 401 as it always did. + /// + Task InvalidateAsync(ProviderContext context, string rejectedToken, CancellationToken cancellationToken) => Task.FromResult(null); } diff --git a/backend/src/CodeSpace.Core/Services/Providers/Auth/IRecoverableProviderAuthStrategy.cs b/backend/src/CodeSpace.Core/Services/Providers/Auth/IRecoverableProviderAuthStrategy.cs new file mode 100644 index 000000000..23398f5ba --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Providers/Auth/IRecoverableProviderAuthStrategy.cs @@ -0,0 +1,12 @@ +namespace CodeSpace.Core.Services.Providers.Auth; + +/// +/// An auth strategy whose token can be replaced after the provider refused it. A sibling of +/// rather than a member of it: a static token (a personal, project or group access token) has nothing to replace it with, so +/// the question has no answer for most strategies, and the resolver asks only the ones that implement this. +/// +public interface IRecoverableProviderAuthStrategy : IProviderAuthStrategy +{ + /// The auth to present instead of , which the provider answered 401; null when there is none to try. + Task InvalidateAsync(ProviderContext context, string rejectedToken, CancellationToken cancellationToken); +} diff --git a/backend/src/CodeSpace.Core/Services/Providers/Auth/ProviderAuthResolver.cs b/backend/src/CodeSpace.Core/Services/Providers/Auth/ProviderAuthResolver.cs index fe036f026..f28bdd27a 100644 --- a/backend/src/CodeSpace.Core/Services/Providers/Auth/ProviderAuthResolver.cs +++ b/backend/src/CodeSpace.Core/Services/Providers/Auth/ProviderAuthResolver.cs @@ -23,4 +23,17 @@ public async Task ResolveAsync(ProviderContext context, Cancellati return auth; } + + public async Task InvalidateAsync(ProviderContext context, string rejectedToken, CancellationToken cancellationToken) + { + var key = (context.Instance.Provider, context.Credential.AuthType); + + if (!_strategies.TryGetValue(key, out var strategy) || strategy is not IRecoverableProviderAuthStrategy recoverable) return null; + + var auth = await recoverable.InvalidateAsync(context, rejectedToken, cancellationToken).ConfigureAwait(false); + + if (auth is not null) PresentedProviderCredentials.Remember(context.Instance, auth.Token); + + return auth; + } } diff --git a/backend/src/CodeSpace.Core/Services/Providers/Capabilities/IRepositoryMemberStandingCapability.cs b/backend/src/CodeSpace.Core/Services/Providers/Capabilities/IRepositoryMemberStandingCapability.cs index 4328d30ff..ac97eb6b1 100644 --- a/backend/src/CodeSpace.Core/Services/Providers/Capabilities/IRepositoryMemberStandingCapability.cs +++ b/backend/src/CodeSpace.Core/Services/Providers/Capabilities/IRepositoryMemberStandingCapability.cs @@ -13,6 +13,6 @@ namespace CodeSpace.Core.Services.Providers.Capabilities; /// public interface IRepositoryMemberStandingCapability : IProviderCapability { - /// Member or None from the provider's answer, Unknown when it answered with an error. A provider that cannot be reached throws; either way a caller treats the author as not a member. + /// Member or None from the provider's answer. An answer that says nothing about the user — the credential refused, the provider erroring or not reached — throws, carrying the cause, so a caller can tell "not a member" from "could not find out" and name the difference to whoever must act on it; it is never returned as . Task GetMemberStandingAsync(ProviderContext context, RemoteRepository repository, string userExternalId, CancellationToken cancellationToken); } diff --git a/backend/src/CodeSpace.Core/Services/Providers/GitHub/Auth/GitHubOAuthAuthStrategy.cs b/backend/src/CodeSpace.Core/Services/Providers/GitHub/Auth/GitHubOAuthAuthStrategy.cs index 19caf9295..b1b9d0bfb 100644 --- a/backend/src/CodeSpace.Core/Services/Providers/GitHub/Auth/GitHubOAuthAuthStrategy.cs +++ b/backend/src/CodeSpace.Core/Services/Providers/GitHub/Auth/GitHubOAuthAuthStrategy.cs @@ -10,11 +10,13 @@ public sealed class GitHubOAuthAuthStrategy : IProviderAuthStrategy { private readonly ICredentialResolver _credentialResolver; private readonly IOAuthTokenRefresher _refresher; + private readonly IOAuthTokenRecovery _recovery; - public GitHubOAuthAuthStrategy(ICredentialResolver credentialResolver, IOAuthTokenRefresher refresher) + public GitHubOAuthAuthStrategy(ICredentialResolver credentialResolver, IOAuthTokenRefresher refresher, IOAuthTokenRecovery recovery) { _credentialResolver = credentialResolver; _refresher = refresher; + _recovery = recovery; } public ProviderKind Kind => ProviderKind.GitHub; @@ -22,6 +24,8 @@ public GitHubOAuthAuthStrategy(ICredentialResolver credentialResolver, IOAuthTok public async Task ResolveAsync(ProviderContext context, CancellationToken cancellationToken) { + await _recovery.StoreKeptPairAsync(context, cancellationToken).ConfigureAwait(false); + var payload = _credentialResolver.Resolve(context.Credential) as OAuthPayload ?? throw new InvalidOperationException($"Credential {context.Credential.Id} has AuthType=OAuth but payload is not OAuthPayload — encrypted blob may be corrupt"); diff --git a/backend/src/CodeSpace.Core/Services/Providers/GitHub/GitHubRepositoryProvider.cs b/backend/src/CodeSpace.Core/Services/Providers/GitHub/GitHubRepositoryProvider.cs index 40b2cad41..8a0fce8a8 100644 --- a/backend/src/CodeSpace.Core/Services/Providers/GitHub/GitHubRepositoryProvider.cs +++ b/backend/src/CodeSpace.Core/Services/Providers/GitHub/GitHubRepositoryProvider.cs @@ -784,7 +784,7 @@ public async Task ProbeCredentialAsync(ProviderContext co } catch (Exception ex) { - return new CredentialProbeResult { IsValid = false, Error = ex.Message }; + return CredentialProbeResult.FromFailure(ex); } } diff --git a/backend/src/CodeSpace.Core/Services/Providers/GitLab/Auth/GitLabOAuthAuthStrategy.cs b/backend/src/CodeSpace.Core/Services/Providers/GitLab/Auth/GitLabOAuthAuthStrategy.cs index c68226756..e722198a2 100644 --- a/backend/src/CodeSpace.Core/Services/Providers/GitLab/Auth/GitLabOAuthAuthStrategy.cs +++ b/backend/src/CodeSpace.Core/Services/Providers/GitLab/Auth/GitLabOAuthAuthStrategy.cs @@ -6,15 +6,17 @@ namespace CodeSpace.Core.Services.Providers.GitLab.Auth; -public sealed class GitLabOAuthAuthStrategy : IProviderAuthStrategy +public sealed class GitLabOAuthAuthStrategy : IRecoverableProviderAuthStrategy { private readonly ICredentialResolver _credentialResolver; private readonly IOAuthTokenRefresher _refresher; + private readonly IOAuthTokenRecovery _recovery; - public GitLabOAuthAuthStrategy(ICredentialResolver credentialResolver, IOAuthTokenRefresher refresher) + public GitLabOAuthAuthStrategy(ICredentialResolver credentialResolver, IOAuthTokenRefresher refresher, IOAuthTokenRecovery recovery) { _credentialResolver = credentialResolver; _refresher = refresher; + _recovery = recovery; } public ProviderKind Kind => ProviderKind.GitLab; @@ -22,6 +24,8 @@ public GitLabOAuthAuthStrategy(ICredentialResolver credentialResolver, IOAuthTok public async Task ResolveAsync(ProviderContext context, CancellationToken cancellationToken) { + await _recovery.StoreKeptPairAsync(context, cancellationToken).ConfigureAwait(false); + var payload = _credentialResolver.Resolve(context.Credential) as OAuthPayload ?? throw new InvalidOperationException($"Credential {context.Credential.Id} has AuthType=OAuth but payload is not OAuthPayload — encrypted blob may be corrupt"); @@ -29,4 +33,11 @@ public async Task ResolveAsync(ProviderContext context, Cancellati return new ResolvedAuth { Token = effective.AccessToken, ExpiresAt = effective.ExpiresAt }; } + + public async Task InvalidateAsync(ProviderContext context, string rejectedToken, CancellationToken cancellationToken) + { + var recovered = await _recovery.RecoverAfterRejectionAsync(context, rejectedToken, cancellationToken).ConfigureAwait(false); + + return recovered is null ? null : new ResolvedAuth { Token = recovered.AccessToken, ExpiresAt = recovered.ExpiresAt }; + } } diff --git a/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabCredential.cs b/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabCredential.cs index 9a59efed3..1385cbbde 100644 --- a/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabCredential.cs +++ b/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabCredential.cs @@ -12,29 +12,42 @@ namespace CodeSpace.Core.Services.Providers.GitLab; /// PAT and fail with an OAuth token as soon as a redirect dropped the Bearer, and the typed client (Bearer only) fail on /// every long access token. /// -/// A struct that overrides : a record's generated one would print the token into whatever +/// An OAuth credential also carries the way to ask for a replacement when GitLab answers 401 to its token +/// (). The replacement STAYS: this one object is what the typed client and every raw call of a +/// provider call present, so once a request has been refused and the token replaced, the requests after it — the write that +/// follows a read, the next page — carry the replacement without being refused first. A static token carries no way to ask. +/// +/// A class that overrides : a record's generated one would print the token into whatever /// message or log line a value of this type is formatted into. /// -internal readonly struct GitLabCredential +internal sealed class GitLabCredential { internal const string PrivateTokenHeader = "PRIVATE-TOKEN"; internal const string AuthorizationHeader = "Authorization"; - public GitLabCredential(string token, AuthType type) + private readonly Func>? _recover; + private volatile string _token; + + /// Given the token GitLab refused, the token to try instead — null when there is none. Absent for a credential that cannot be replaced. + public GitLabCredential(string token, AuthType type, Func>? recover = null) { - Token = token; + _token = token; Type = type; + _recover = recover; } - public string Token { get; } + /// The token presented now: the one it was made with until a refusal replaced it. + public string Token => _token; public AuthType Type { get; } /// The header this credential's type names: PRIVATE-TOKEN for an access token, Authorization for anything else (OAuth). public string HeaderName => IsAccessToken ? PrivateTokenHeader : AuthorizationHeader; - public string HeaderValue => IsAccessToken ? Token : $"Bearer {Token}"; + public string HeaderValue => ValueOf(_token); + + private string ValueOf(string token) => IsAccessToken ? token : $"Bearer {token}"; private bool IsAccessToken => Type is AuthType.Pat or AuthType.ProjectAccessToken or AuthType.GroupAccessToken; @@ -43,7 +56,7 @@ public void Attach(HttpRequestMessage request) { request.Headers.Remove(PrivateTokenHeader); request.Headers.Remove(AuthorizationHeader); - request.Headers.TryAddWithoutValidation(HeaderName, HeaderValue); + request.Headers.TryAddWithoutValidation(HeaderName, ValueOf(_token)); } /// The same, for NGitLab's request, whose headers it has already written an Authorization: Bearer into. @@ -51,7 +64,29 @@ public void Attach(WebHeaderCollection headers) { headers.Remove(PrivateTokenHeader); headers.Remove(AuthorizationHeader); - headers[HeaderName] = HeaderValue; + headers[HeaderName] = ValueOf(_token); + } + + /// Whether a 401 is worth asking about. + public bool CanRecover => _recover is not null; + + /// + /// GitLab answered 401 to . True when the credential now carries another token to send the + /// request again with: a request that was refused alongside this one may already have replaced it, which is taken as the + /// answer without asking again; otherwise the way to ask is used once. False when the 401 stands. + /// + public async Task RecoverAsync(string rejectedToken, CancellationToken cancellationToken) + { + if (_recover is null) return false; + if (_token != rejectedToken) return true; + + var replacement = await _recover(rejectedToken, cancellationToken).ConfigureAwait(false); + + if (replacement is null) return false; + + _token = replacement; + + return true; } public override string ToString() => $"{HeaderName} credential ({Type})"; diff --git a/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabHttp.cs b/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabHttp.cs index 66fdfef22..2c07fd163 100644 --- a/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabHttp.cs +++ b/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabHttp.cs @@ -1,3 +1,4 @@ +using System.Net; using CodeSpace.Core.Services.OutboundHttp; using CodeSpace.Messages.Enums; using CodeSpace.Messages.Exceptions; @@ -12,6 +13,12 @@ namespace CodeSpace.Core.Services.Providers.GitLab; /// with that credential attached again. A redirect to another origin — http to https, a host alias, another port — is not /// followed, so nothing is sent there, and fails with naming where GitLab /// pointed. The raw calls' client had followed every redirect itself, keeping a PRIVATE-TOKEN across it. +/// +/// A 401 to an OAuth credential is sent once more with the credential +/// returns (a newer token another caller stored, or a rotation), by both request paths: GitLab retires an access token the +/// moment its pair is refreshed, however much of its lifetime is left, so a token read from the row an instant ago can be +/// refused. A request that carries a body is sent again too, by the raw path, which holds its body; a 401 happens before +/// GitLab acts on a request, so repeating it repeats nothing. /// internal static class GitLabHttp { @@ -33,9 +40,25 @@ internal static class GitLabHttp /// /// sent with . A redirect that was not followed throws /// instead of coming back as a 3xx the caller would read as some other - /// failure, so no call site has to know about redirects. + /// failure, so no call site has to know about redirects. A 401 to a credential that can be replaced + /// () is answered by sending the request once more with the replacement; the + /// second answer, whatever it is, is the answer. /// public static async Task SendAsync(HttpRequestMessage request, GitLabCredential credential, HttpCompletionOption completion, CancellationToken cancellationToken) + { + var replay = await CopyForReplayAsync(request, credential, cancellationToken).ConfigureAwait(false); + var presented = credential.Token; + var response = await SendOnceAsync(request, credential, completion, cancellationToken).ConfigureAwait(false); + + if (replay is null || response.StatusCode != HttpStatusCode.Unauthorized) return response; + if (!await credential.RecoverAsync(presented, cancellationToken).ConfigureAwait(false)) return response; + + response.Dispose(); + + return await SendOnceAsync(replay, credential, completion, cancellationToken).ConfigureAwait(false); + } + + private static async Task SendOnceAsync(HttpRequestMessage request, GitLabCredential credential, HttpCompletionOption completion, CancellationToken cancellationToken) { var requested = request.RequestUri!; @@ -52,6 +75,30 @@ public static async Task SendAsync(HttpRequestMessage reque throw new ProviderBaseUrlRedirectsException(ProviderKind.GitLab, requested, target); } + /// + /// The request as it was handed over, for sending again after a 401 — made before the first send because a redirect + /// rewrites the request it follows. Null for a credential that cannot be replaced, which never sends twice. The body is + /// read into memory (the raw calls' bodies are small JSON strings), which also lets the first send read it again. + /// + private static async Task CopyForReplayAsync(HttpRequestMessage request, GitLabCredential credential, CancellationToken cancellationToken) + { + if (!credential.CanRecover) return null; + + var copy = new HttpRequestMessage(request.Method, request.RequestUri) { Version = request.Version, VersionPolicy = request.VersionPolicy }; + + foreach (var header in request.Headers) copy.Headers.TryAddWithoutValidation(header.Key, header.Value); + + if (request.Content is not { } content) return copy; + + var body = new ByteArrayContent(await content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false)); + + foreach (var header in content.Headers) body.Headers.TryAddWithoutValidation(header.Key, header.Value); + + copy.Content = body; + + return copy; + } + /// reading the whole answer. public static Task SendAsync(HttpRequestMessage request, GitLabCredential credential, CancellationToken cancellationToken) => SendAsync(request, credential, HttpCompletionOption.ResponseContentRead, cancellationToken); } diff --git a/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabRepositoryProvider.MemberStanding.cs b/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabRepositoryProvider.MemberStanding.cs index cdebfb519..d2f570a65 100644 --- a/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabRepositoryProvider.MemberStanding.cs +++ b/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabRepositoryProvider.MemberStanding.cs @@ -2,6 +2,7 @@ using CodeSpace.Core.Services.Providers.Capabilities; using CodeSpace.Messages.Dtos.Providers; using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Exceptions; namespace CodeSpace.Core.Services.Providers.GitLab; @@ -9,29 +10,69 @@ namespace CodeSpace.Core.Services.Providers.GitLab; /// A merge request author's standing on the project, for the trigger's authors filter. GitLab's hook names the author /// and not their access level, so this asks GET /projects/:id/members/all/:user_id — "all" so a role inherited /// from a parent group counts like a direct one. +/// +/// Only two answers say anything about the author: 200 (their role) and 404 (they hold none). Every other answer is +/// not an answer about the author — a 401 or 403 for the credential, a 429 or a 5xx for GitLab — so it is thrown as +/// , carrying the status, rather than returned as Unknown, which a members-only +/// trigger reads as "not a member" and refuses the author's own pull request for. A 429 or 5xx is asked again once +/// first (), inside the caller's deadline. /// public sealed partial class GitLabRepositoryProvider : IRepositoryMemberStandingCapability { /// Developer: the first role that can push to the project — GitLab's counterpart of a GitHub collaborator. internal const int MemberAccessLevelFloor = 30; + /// The wait before a 429 or 5xx is asked again. Short on purpose: the whole lookup has the trigger's few seconds (PullRequestTriggerAdmission.AuthorLookupTimeout), and a longer Retry-After is not waited out. + internal static readonly TimeSpan StandingRetryDelay = TimeSpan.FromMilliseconds(250); + public async Task GetMemberStandingAsync(ProviderContext context, RemoteRepository repository, string userExternalId, CancellationToken cancellationToken) { var (host, credential) = await ResolveCredentialAsync(context, cancellationToken).ConfigureAwait(false); var url = $"{host.TrimEnd('/')}/api/v4/projects/{Uri.EscapeDataString(repository.ExternalId)}/members/all/{Uri.EscapeDataString(userExternalId)}"; - using var request = new HttpRequestMessage(HttpMethod.Get, url); - - using var response = await GitLabHttp.SendAsync(request, credential, cancellationToken).ConfigureAwait(false); + using var response = await SendStandingRequestAsync(url, credential, cancellationToken).ConfigureAwait(false); if ((int)response.StatusCode == 404) return PullRequestAuthorAssociation.None; - if (!response.IsSuccessStatusCode) return PullRequestAuthorAssociation.Unknown; + if (!response.IsSuccessStatusCode) throw StandingRefused(response); var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); return MapMemberStanding(ParseMemberAccessLevel(json)); } + /// The lookup, asked again once after if GitLab answered 429 or 5xx. + private static async Task SendStandingRequestAsync(string url, GitLabCredential credential, CancellationToken cancellationToken) + { + using var first = new HttpRequestMessage(HttpMethod.Get, url); + + var response = await GitLabHttp.SendAsync(first, credential, cancellationToken).ConfigureAwait(false); + + if (!IsWorthAskingAgain(response.StatusCode)) return response; + + response.Dispose(); + + await Task.Delay(StandingRetryDelay, cancellationToken).ConfigureAwait(false); + + using var second = new HttpRequestMessage(HttpMethod.Get, url); + + return await GitLabHttp.SendAsync(second, credential, cancellationToken).ConfigureAwait(false); + } + + private static bool IsWorthAskingAgain(System.Net.HttpStatusCode status) => (int)status == 429 || (int)status >= 500; + + /// + /// The answer that is not an answer about the author, as the failure it is: GitLab's status and reason phrase, never its + /// body (an error page can echo what it was sent). Raised here, where the status is known, rather than by the resilience + /// layer: this call does not pass through it, and its retry is the one above. + /// + private static ProviderApiException StandingRefused(HttpResponseMessage response) + { + var status = (int)response.StatusCode; + var reason = $"{status} {response.ReasonPhrase}".TrimEnd(); + + return new ProviderApiException(ProviderKind.GitLab, status, nameof(GetMemberStandingAsync), reason, new HttpRequestException(reason, null, response.StatusCode)); + } + /// Developer and above is a member; Guest and Reporter can see the project but hold no say over its code. internal static PullRequestAuthorAssociation MapMemberStanding(int? accessLevel) => accessLevel >= MemberAccessLevelFloor ? PullRequestAuthorAssociation.Member : PullRequestAuthorAssociation.None; diff --git a/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabRepositoryProvider.Writes.cs b/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabRepositoryProvider.Writes.cs new file mode 100644 index 000000000..6889ccca7 --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabRepositoryProvider.Writes.cs @@ -0,0 +1,50 @@ +using System.Net; +using NGitLab; + +namespace CodeSpace.Core.Services.Providers.GitLab; + +/// +/// A write through NGitLab's typed client, sent again when GitLab refused its token. +/// answers a 401 to an OAuth credential by asking for a replacement, and for a read sends the request again with it; a write it +/// cannot (NGitLab wrote the body to a stream it does not hand back), so it settles the credential and lets the 401 stand. That left +/// the first merge, comment or issue after another consumer rotated the pair failing with the very 401 recovery exists +/// for, whichever call came first — a read before it settles the token, and a write that opens a call is the first request. +/// +/// The provider can send it: a 401 is GitLab's answer before it acts on a request, so repeating the call repeats +/// nothing, and the credential the client presents already carries the replacement. Once — a replacement refused too is the +/// answer. +/// +public sealed partial class GitLabRepositoryProvider +{ + /// + /// , made again once if GitLab answered 401 and the refusal replaced the credential's token — the + /// token this attempt began with is no longer the one the credential carries. A 401 that left the token as it was (a static + /// token, or no replacement to offer) stands, and so does any other failure. + /// + private static Func> ResentOnceIfRefused(GitLabCredential credential, Func> write) => async cancellationToken => + { + var presented = credential.Token; + + try + { + return await write(cancellationToken).ConfigureAwait(false); + } + catch (GitLabException refusal) when (refusal.StatusCode == HttpStatusCode.Unauthorized && credential.Token != presented) + { + return await write(cancellationToken).ConfigureAwait(false); + } + }; + + /// for a write that returns nothing. + private static Func ResentOnceIfRefused(GitLabCredential credential, Func write) + { + var resending = ResentOnceIfRefused(credential, async cancellationToken => + { + await write(cancellationToken).ConfigureAwait(false); + + return null; + }); + + return resending; + } +} diff --git a/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabRepositoryProvider.cs b/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabRepositoryProvider.cs index ccdc2b1b6..54cdca8d6 100644 --- a/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabRepositoryProvider.cs +++ b/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabRepositoryProvider.cs @@ -311,7 +311,7 @@ public async Task GetPullRequestAsync(ProviderContext context public async Task OpenPullRequestAsync(ProviderContext context, RemoteRepository repository, OpenPullRequestInput input, CancellationToken cancellationToken) { - var client = await BuildClientAsync(context, cancellationToken).ConfigureAwait(false); + var (client, _, credential) = await BuildAuthedAsync(context, cancellationToken).ConfigureAwait(false); var projectId = int.Parse(repository.ExternalId); // GitLab marks a draft via a "Draft:" title prefix (the modern replacement for work_in_progress); @@ -328,7 +328,7 @@ public async Task OpenPullRequestAsync(ProviderContext contex { // A freshly-opened MR carries no labels, so no project-label-colour lookup is needed. return await _resilience.ExecuteNonIdempotentAsync(context.Instance, nameof(OpenPullRequestAsync), - _ => Task.FromResult(ToRemotePullRequestDetail(client.GetMergeRequest(projectId).Create(create), EmptyLabelColors)), + ResentOnceIfRefused(credential, _ => Task.FromResult(ToRemotePullRequestDetail(client.GetMergeRequest(projectId).Create(create), EmptyLabelColors))), _ => Task.FromResult(FindOpenedMergeRequest(client, projectId, input)), cancellationToken).ConfigureAwait(false); } @@ -365,7 +365,7 @@ public async Task OpenPullRequestAsync(ProviderContext contex public async Task MergePullRequestAsync(ProviderContext context, RemoteRepository repository, int number, MergePullRequestInput input, CancellationToken cancellationToken) { - var client = await BuildClientAsync(context, cancellationToken).ConfigureAwait(false); + var (client, _, credential) = await BuildAuthedAsync(context, cancellationToken).ConfigureAwait(false); var mergeRequests = client.GetMergeRequest(int.Parse(repository.ExternalId)); // GitLab's Accept covers merge + squash + remove-source-branch in one call. Rebase-merge isn't a @@ -382,7 +382,7 @@ public async Task MergePullRequestAsync(ProviderCo // A merge is one-way: re-sent after it landed, GitLab answers 405 and fails a merge that succeeded. So a // retry first re-reads the merge request and takes an existing merge as the answer. var accepted = await _resilience.ExecuteNonIdempotentAsync(context.Instance, nameof(MergePullRequestAsync), - _ => Task.FromResult(mergeRequests.Accept(number, merge)), + ResentOnceIfRefused(credential, _ => Task.FromResult(mergeRequests.Accept(number, merge))), ct => FindMergedAsync(mergeRequests, number, ct), cancellationToken).ConfigureAwait(false); @@ -492,7 +492,7 @@ private async Task CountIssuesViaXTotalAsync(ProviderContext public async Task CreateIssueAsync(ProviderContext context, RemoteRepository repository, CreateIssueInput input, CancellationToken cancellationToken) { - var client = await BuildClientAsync(context, cancellationToken).ConfigureAwait(false); + var (client, _, credential) = await BuildAuthedAsync(context, cancellationToken).ConfigureAwait(false); var projectId = int.Parse(repository.ExternalId); var marker = IdempotencyMarker.New(); @@ -507,7 +507,7 @@ public async Task CreateIssueAsync(ProviderContext context, RemoteR }; var issue = await _resilience.ExecuteNonIdempotentAsync(context.Instance, nameof(CreateIssueAsync), - _ => Task.FromResult(client.Issues.Create(create)), + ResentOnceIfRefused(credential, _ => Task.FromResult(client.Issues.Create(create))), _ => Task.FromResult(FindMarkedIssue(client, projectId, marker)), cancellationToken).ConfigureAwait(false); @@ -622,14 +622,14 @@ private static string ClassifyGitLabSystemNote(string? body) public async Task CommentIssueAsync(ProviderContext context, RemoteRepository repository, int number, string body, CancellationToken cancellationToken) { - var client = await BuildClientAsync(context, cancellationToken).ConfigureAwait(false); + var (client, _, credential) = await BuildAuthedAsync(context, cancellationToken).ConfigureAwait(false); var marker = IdempotencyMarker.New(); // GitLab issue comments are "notes" on a separate client; IssueId here is the iid (the `number`). var notes = client.GetProjectIssueNoteClient(int.Parse(repository.ExternalId)); var note = await _resilience.ExecuteNonIdempotentAsync(context.Instance, nameof(CommentIssueAsync), - _ => Task.FromResult(notes.Create(new ProjectIssueNoteCreate { IssueId = number, Body = IdempotencyMarker.Append(body, marker) })), + ResentOnceIfRefused(credential, _ => Task.FromResult(notes.Create(new ProjectIssueNoteCreate { IssueId = number, Body = IdempotencyMarker.Append(body, marker) }))), _ => Task.FromResult(notes.ForIssue(number).FirstOrDefault(n => IdempotencyMarker.IsIn(n.Body, marker))), cancellationToken).ConfigureAwait(false); @@ -645,9 +645,9 @@ public async Task CommentIssueAsync(ProviderContext context, public async Task CloseIssueAsync(ProviderContext context, RemoteRepository repository, int number, CancellationToken cancellationToken) { - var client = await BuildClientAsync(context, cancellationToken).ConfigureAwait(false); + var (client, _, credential) = await BuildAuthedAsync(context, cancellationToken).ConfigureAwait(false); - return await _resilience.ExecuteAsync(context.Instance, nameof(CloseIssueAsync), _ => + return await _resilience.ExecuteAsync(context.Instance, nameof(CloseIssueAsync), ResentOnceIfRefused(credential, _ => { var projectId = int.Parse(repository.ExternalId); @@ -661,7 +661,7 @@ public async Task CloseIssueAsync(ProviderContext context, RemoteRe }); return Task.FromResult(ToRemoteIssue(closed)); - }, cancellationToken).ConfigureAwait(false); + }), cancellationToken).ConfigureAwait(false); } private static readonly IReadOnlyDictionary EmptyLabelColors = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -920,7 +920,7 @@ private static IReadOnlyList FloorByPipelineStatus(List< public async Task PostCommentAsync(ProviderContext context, RemoteRepository repository, int number, string body, CancellationToken cancellationToken) { - var client = await BuildClientAsync(context, cancellationToken).ConfigureAwait(false); + var (client, _, credential) = await BuildAuthedAsync(context, cancellationToken).ConfigureAwait(false); var marker = IdempotencyMarker.New(); // NGitLab v11: IMergeRequestClient.Comments(iid) returns IMergeRequestCommentClient. @@ -929,7 +929,7 @@ public async Task PostCommentAsync(ProviderContext con var commentsClient = client.GetMergeRequest(int.Parse(repository.ExternalId)).Comments(number); var comment = await _resilience.ExecuteNonIdempotentAsync(context.Instance, nameof(PostCommentAsync), - _ => Task.FromResult(commentsClient.Add(new NGitLab.Models.MergeRequestCommentCreate { Body = IdempotencyMarker.Append(body, marker) })), + ResentOnceIfRefused(credential, _ => Task.FromResult(commentsClient.Add(new NGitLab.Models.MergeRequestCommentCreate { Body = IdempotencyMarker.Append(body, marker) }))), _ => Task.FromResult(commentsClient.All.FirstOrDefault(c => IdempotencyMarker.IsIn(c.Body, marker))), cancellationToken).ConfigureAwait(false); @@ -1135,42 +1135,117 @@ public async Task ProbeCredentialAsync(ProviderContext co { try { - var client = await BuildClientAsync(context, cancellationToken).ConfigureAwait(false); + var identity = await ReadIdentityAsync(context, cancellationToken).ConfigureAwait(false); - var identity = await _resilience.ExecuteAsync(context.Instance, nameof(ProbeCredentialAsync), _ => - { - var current = client.Users.Current; - return Task.FromResult((Id: current.Id.ToString(), Name: current.Username)); - }, cancellationToken).ConfigureAwait(false); - - var scopes = await TryFetchTokenScopesAsync(context, cancellationToken).ConfigureAwait(false); + var self = await TryReadTokenSelfAsync(context, cancellationToken).ConfigureAwait(false); return new CredentialProbeResult { - IsValid = true, + IsValid = self?.Active != false, + IsRejected = self?.Active == false, + Error = self?.Active == false ? "GitLab reports the token is no longer active (revoked or expired)" : null, AuthenticatedUserExternalId = identity.Id, AuthenticatedUserName = identity.Name, - GrantedScopes = scopes + GrantedScopes = self?.Scopes, + ExpiresAt = self?.ExpiresAt, + Active = self?.Active }; } catch (Exception ex) { - return new CredentialProbeResult { IsValid = false, Error = ex.Message }; + return CredentialProbeResult.FromFailure(ex); } } /// - /// Reads the token's OWN granted scopes via GET /api/v4/personal_access_tokens/self so - /// capability warnings reflect the real PAT (GitLab returns the exact scope list the user ticked). - /// Best-effort: the endpoint is PAT-only, so OAuth / group / impersonation tokens get 401/403 — any - /// non-success or failure returns null, which the capability check reads as "scopes unknown" (no - /// false warnings) rather than "no scopes". Mirrors the raw-HTTP pattern used by CountViaXTotalAsync. + /// Who the token is, from GET /api/v4/user — asked over the raw client rather than NGitLab's Users.Current, + /// which is synchronous: it ignores the caller's token, blocks its thread for the HTTP client's five minutes and, being + /// a connection that never answered, is retried twice more, so a probe's deadline could not stop it. A 401 to an OAuth + /// credential is answered as every raw call answers it (). + /// A refusal is raised here, outside the resilience layer, already classified — as raises + /// its own — so the layer does not wrap it again or ask a 4xx twice; only a 5xx is thrown inside it, to be asked again. /// - private async Task?> TryFetchTokenScopesAsync(ProviderContext context, CancellationToken cancellationToken) + private async Task<(string Id, string Name)> ReadIdentityAsync(ProviderContext context, CancellationToken cancellationToken) { var (host, credential) = await ResolveCredentialAsync(context, cancellationToken).ConfigureAwait(false); + var url = $"{host.TrimEnd('/')}/api/v4/user"; + + var answer = await _resilience.ExecuteAsync(context.Instance, nameof(ProbeCredentialAsync), async ct => await AskIdentityAsync(url, credential, ct).ConfigureAwait(false), cancellationToken).ConfigureAwait(false); + + return answer.Refusal is { } refusal ? throw refusal : ParseIdentity(answer.Body!); + } + + private static async Task AskIdentityAsync(string url, GitLabCredential credential, CancellationToken cancellationToken) + { + using var request = new HttpRequestMessage(HttpMethod.Get, url); + using var response = await GitLabHttp.SendAsync(request, credential, cancellationToken).ConfigureAwait(false); + + if ((int)response.StatusCode >= 500) throw new HttpRequestException(StatusLine(response), null, response.StatusCode); + + if (response.IsSuccessStatusCode) return new IdentityAnswer(await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false), null); + + return new IdentityAnswer(null, await IdentityRefusalAsync(response, credential, cancellationToken).ConfigureAwait(false)); + } + + /// + /// GitLab's refusal of the identity call: a 403 tagged insufficient_scope is the scope gap it names (the body is + /// read for that tag, with the token struck from it), any other answer is its status and reason phrase — never its body, + /// which an error page can fill with what it was sent. + /// + private static async Task IdentityRefusalAsync(HttpResponseMessage response, GitLabCredential credential, CancellationToken cancellationToken) + { + var status = (int)response.StatusCode; + var refused = new ProviderApiException(ProviderKind.GitLab, status, nameof(ProbeCredentialAsync), StatusLine(response), new HttpRequestException(StatusLine(response), null, response.StatusCode)); + + if (status != 403) return refused; + + var body = CredentialNeedles.Strike(await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false), [credential.Token]); + + return (Exception?)_unapproveScopeClassifier.ClassifyScope(status, body, body, nameof(ProbeCredentialAsync)) ?? refused; + } + + private static string StatusLine(HttpResponseMessage response) => $"{(int)response.StatusCode} {response.ReasonPhrase}".TrimEnd(); + + /// What the identity call came back with: the user's body, or the refusal to raise. + private sealed record IdentityAnswer(string? Body, Exception? Refusal); + + /// The user's id and username from the /user body; anything else is not a GitLab user and fails the probe without saying anything about the token. + internal static (string Id, string Name) ParseIdentity(string json) + { + try + { + using var doc = JsonDocument.Parse(json); + var root = doc.RootElement; + + if (root.ValueKind == JsonValueKind.Object && root.TryGetProperty("id", out var id) && id.TryGetInt64(out var userId) && root.TryGetProperty("username", out var name) && name.ValueKind == JsonValueKind.String) + return (userId.ToString(System.Globalization.CultureInfo.InvariantCulture), name.GetString()!); + } + catch (JsonException) + { + // Not JSON: the same answer as JSON that is not a user. + } + + throw new InvalidOperationException("GitLab answered GET /user with something that is not a user"); + } + + /// + /// Reads the token's OWN record via GET /api/v4/personal_access_tokens/self: the scopes it was granted, so + /// capability warnings reflect the real PAT (GitLab returns the exact scope list the user ticked), and when it expires + /// and whether it is still live, which nothing else tells CodeSpace — GitLab gives every new personal, project and + /// group token an expiry, and a token that reaches it answers 401 to everything. Best-effort: the endpoint is + /// token-only, so OAuth / impersonation tokens get 401/403 — any non-success or failure returns null, which reads as + /// "unknown" (no false warnings, no false expiry) rather than "no scopes". Mirrors the raw-HTTP pattern used by + /// CountViaXTotalAsync. + /// + private async Task TryReadTokenSelfAsync(ProviderContext context, CancellationToken cancellationToken) + { + var (host, resolved) = await ResolveCredentialAsync(context, cancellationToken).ConfigureAwait(false); var url = $"{host.TrimEnd('/')}/api/v4/personal_access_tokens/self"; + // Without the way to ask for a replacement: an OAuth token is answered 401 here by design, and that 401 says nothing + // about the token — asking would rotate a working pair every time the credential is probed. + var credential = new GitLabCredential(resolved.Token, resolved.Type); + using var request = new HttpRequestMessage(HttpMethod.Get, url); try @@ -1179,7 +1254,7 @@ public async Task ProbeCredentialAsync(ProviderContext co if (!response.IsSuccessStatusCode) return null; var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - return ParseTokenScopes(json); + return ParseTokenSelf(json); } catch { @@ -1187,6 +1262,47 @@ public async Task ProbeCredentialAsync(ProviderContext co } } + /// What personal_access_tokens/self says about a token. Each part is null when the body does not say it. + internal sealed record GitLabTokenSelf(IReadOnlyList? Scopes, DateTimeOffset? ExpiresAt, bool? Active); + + /// + /// Pure parse of the personal_access_tokens/self body: the scopes (), expires_at + /// — a date, which GitLab takes as midnight UTC at its start, or a timestamp — and whether the token is live: active, + /// and never when revoked is true. Null (not a default) for whatever is absent or malformed, so a caller never + /// reads "does not say" as "never expires" or "not active". + /// + internal static GitLabTokenSelf ParseTokenSelf(string json) + { + try + { + using var doc = JsonDocument.Parse(json); + var root = doc.RootElement; + + if (root.ValueKind != JsonValueKind.Object) return new GitLabTokenSelf(null, null, null); + + return new GitLabTokenSelf(ParseTokenScopes(json), ReadExpiry(root), ReadActive(root)); + } + catch (JsonException) + { + return new GitLabTokenSelf(null, null, null); + } + } + + private static DateTimeOffset? ReadExpiry(JsonElement root) + { + if (!root.TryGetProperty("expires_at", out var value) || value.ValueKind != JsonValueKind.String) return null; + + return DateTimeOffset.TryParse(value.GetString(), System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AssumeUniversal | System.Globalization.DateTimeStyles.AdjustToUniversal, out var at) ? at : null; + } + + private static bool? ReadActive(JsonElement root) + { + if (root.TryGetProperty("revoked", out var revoked) && revoked.ValueKind == JsonValueKind.True) return false; + if (root.TryGetProperty("active", out var active) && active.ValueKind is JsonValueKind.True or JsonValueKind.False) return active.GetBoolean(); + + return null; + } + /// /// Pure parse of GitLab's personal_access_tokens/self body → the scopes array. /// Returns null (not empty) when the field is absent, empty, or the JSON is malformed, so callers @@ -1476,14 +1592,14 @@ private static ProviderCallDiagnostic DescribeHookFailure(Exception exception, C public async Task DeleteWebhookAsync(ProviderContext context, RemoteRepository repository, string externalWebhookId, CancellationToken cancellationToken) { - var client = await BuildClientAsync(context, cancellationToken).ConfigureAwait(false); + var (client, _, credential) = await BuildAuthedAsync(context, cancellationToken).ConfigureAwait(false); - await _resilience.ExecuteAsync(context.Instance, nameof(DeleteWebhookAsync), _ => + await _resilience.ExecuteAsync(context.Instance, nameof(DeleteWebhookAsync), ResentOnceIfRefused(credential, _ => { var projectId = int.Parse(repository.ExternalId); client.GetRepository(projectId).ProjectHooks.Delete(int.Parse(externalWebhookId)); return Task.CompletedTask; - }, cancellationToken).ConfigureAwait(false); + }), cancellationToken).ConfigureAwait(false); } public bool VerifySignature(string body, IReadOnlyDictionary headers, string secret) => _signatureVerifier.Verify(body, headers, secret); @@ -1850,9 +1966,17 @@ private async Task BuildClientAsync(ProviderContext context, Cance var auth = await _authResolver.ResolveAsync(context, cancellationToken).ConfigureAwait(false); var host = string.IsNullOrWhiteSpace(context.Instance.ApiUrl) ? context.Instance.BaseUrl : context.Instance.ApiUrl; - return (host, new GitLabCredential(auth.Token, context.Credential.AuthType)); + return (host, new GitLabCredential(auth.Token, context.Credential.AuthType, RecoveryFor(context))); } + /// + /// How a call answered 401 asks for another token: through the resolver, which adopts the one a newer refresh stored or + /// rotates. Only an OAuth credential has one — a personal, project or group access token is the same token whenever it is + /// asked for, so its 401 is its answer. + /// + private Func>? RecoveryFor(ProviderContext context) => + context.Credential.AuthType != AuthType.OAuth ? null : async (rejected, cancellationToken) => (await _authResolver.InvalidateAsync(context, rejected, cancellationToken).ConfigureAwait(false))?.Token; + private static RemoteRepository ToRemoteRepository(Project project) => new() { ExternalId = project.Id.ToString(), diff --git a/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabRequestOptions.cs b/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabRequestOptions.cs index e135d1973..87186b1e7 100644 --- a/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabRequestOptions.cs +++ b/backend/src/CodeSpace.Core/Services/Providers/GitLab/GitLabRequestOptions.cs @@ -27,6 +27,13 @@ namespace CodeSpace.Core.Services.Providers.GitLab; /// to another origin than the connection's is refused before the credential is attached, with the same /// : nothing is sent there. /// +/// The rejection. A 401 to an OAuth credential asks for a replacement — a +/// newer token another caller stored, or a rotation — and a GET or HEAD is sent again with it, once. A request that changes +/// something is not: NGitLab has written its body to a stream it does not hand back, so it cannot be sent again. Asking still +/// settles the credential — it carries the replacement from then on, for the requests that follow in the same call, and the +/// stored token is the new one for every call after — and the 401 that this one got is the answer here. The provider, which can +/// repeat the whole call, does so once for a write that opens it (GitLabRepositoryProvider.ResentOnceIfRefused). +/// /// Construct with (0, TimeSpan.Zero), what is: no retries of its own — the /// provider's resilience wrapper retries — and its 5-minute timeout. /// @@ -46,10 +53,27 @@ public GitLabRequestOptions(GitLabCredential credential, string host) : base(0, public override WebResponse GetResponse(HttpWebRequest request) { var current = Prepare(request); + var presented = _credential.Token; + var rejected = false; for (var hop = 0; ; hop++) { - var response = (HttpWebResponse)current.GetResponse(); + HttpWebResponse response; + + try + { + response = (HttpWebResponse)current.GetResponse(); + } + catch (WebException failure) when (!rejected && IsRejection(failure) && _credential.CanRecover) + { + rejected = true; + + // NGitLab's synchronous surface has no async to await in; nothing here holds a synchronization context. + if (ReplayAfterRejectionAsync(current, failure, presented, CancellationToken.None).GetAwaiter().GetResult() is not { } replay) throw; + + current = replay; + continue; + } if (FollowTarget(current, response, hop) is not { } next) return response; @@ -61,10 +85,26 @@ public override WebResponse GetResponse(HttpWebRequest request) public override async Task GetResponseAsync(HttpWebRequest request, CancellationToken cancellationToken) { var current = Prepare(request); + var presented = _credential.Token; + var rejected = false; for (var hop = 0; ; hop++) { - var response = await SendAsync(current, cancellationToken).ConfigureAwait(false); + HttpWebResponse response; + + try + { + response = await SendAsync(current, cancellationToken).ConfigureAwait(false); + } + catch (WebException failure) when (!rejected && IsRejection(failure) && _credential.CanRecover) + { + rejected = true; + + if (await ReplayAfterRejectionAsync(current, failure, presented, cancellationToken).ConfigureAwait(false) is not { } replay) throw; + + current = replay; + continue; + } if (FollowTarget(current, response, hop) is not { } next) return response; @@ -73,6 +113,26 @@ public override async Task GetResponseAsync(HttpWebRequest request, } } + /// GitLab answered 401: reports every 4xx as an exception that carries the response. + private static bool IsRejection(WebException failure) => failure.Response is HttpWebResponse { StatusCode: HttpStatusCode.Unauthorized }; + + /// + /// Asks for the credential to try instead of the one that was refused, and returns the request to send again with it — or + /// null when the 401 stands: nothing to try, or a request that cannot be sent again (anything but a GET or HEAD). + /// + private async Task ReplayAfterRejectionAsync(HttpWebRequest sent, WebException failure, string presented, CancellationToken cancellationToken) + { + if (!await _credential.RecoverAsync(presented, cancellationToken).ConfigureAwait(false) || !IsReadOnly(sent.Method)) return null; + + failure.Response?.Dispose(); + + var next = Follow(sent, sent.RequestUri); + + _credential.Attach(next.Headers); + + return next; + } + /// One request, aborted with its caller's token — as the base class sends it. private static async Task SendAsync(HttpWebRequest request, CancellationToken cancellationToken) { diff --git a/backend/src/CodeSpace.Core/Services/Providers/Identity/UserProviderIdentityService.cs b/backend/src/CodeSpace.Core/Services/Providers/Identity/UserProviderIdentityService.cs index 68d381b7c..85257b3f3 100644 --- a/backend/src/CodeSpace.Core/Services/Providers/Identity/UserProviderIdentityService.cs +++ b/backend/src/CodeSpace.Core/Services/Providers/Identity/UserProviderIdentityService.cs @@ -58,6 +58,11 @@ public async Task LinkByPatAsync(Guid providerInsta // reflect this PAT — same as the OAuth flow stores token.GrantedScopes. Null when the provider // couldn't expose them; the capability check treats that as "unknown", never a false warning. credential.Scopes = profile.GrantedScopes?.ToList(); + + // The probe just validated the token, and GitLab tells it when the token expires (every new one has an expiry, and + // one that reaches it answers 401 to everything) — recorded so the list can say so ahead of time. + credential.LastValidatedDate = DateTimeOffset.UtcNow; + credential.ExpiresDate = profile.ExpiresAt; await ReplaceExistingLinkAsync(userId, providerInstanceId, cancellationToken).ConfigureAwait(false); await _db.Credential.AddAsync(credential, cancellationToken).ConfigureAwait(false); diff --git a/backend/src/CodeSpace.Core/Services/Workflows/RunSources/Admission/Exceptions/PullRequestStandingUnavailableException.cs b/backend/src/CodeSpace.Core/Services/Workflows/RunSources/Admission/Exceptions/PullRequestStandingUnavailableException.cs new file mode 100644 index 000000000..e4fb9e093 --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Workflows/RunSources/Admission/Exceptions/PullRequestStandingUnavailableException.cs @@ -0,0 +1,32 @@ +using CodeSpace.Messages.Constants; +using CodeSpace.Messages.Failures; + +namespace CodeSpace.Core.Services.Workflows.RunSources.Admission.Exceptions; + +/// +/// An activation that admits only members could not learn whether a pull request's author — or whoever pushed its new +/// commits — is one: the provider's lookup failed, and a failure says nothing about the person. Distinct from +/// , which is a refusal of someone whose standing is known. Costs that +/// activation's run only, and is recorded under the reason the failure names so the audit row says what to fix: the +/// credential () or the lookup +/// (). +/// +public sealed class PullRequestStandingUnavailableException : Exception, IFailure +{ + public PullRequestStandingUnavailableException(string reason, string detail, string auditKey) : base(detail) + { + Reason = reason; + AuditKey = auditKey; + } + + /// The value the refusal is recorded under. + public string Reason { get; } = WorkflowRunRequestRejectionReasons.AuthorStandingUnknown; + + /// What the refusal row is deduplicated on — one per (activation, PR, kind) per day, however many deliveries arrive. + public string AuditKey { get; } + + /// A rejected credential is put right by a person; a lookup that did not work is the provider's and may work next time. + public FailureKind Kind => Reason == WorkflowRunRequestRejectionReasons.CredentialRejected ? FailureKind.PreconditionRequired : FailureKind.Unavailable; + + public string Code => Reason == WorkflowRunRequestRejectionReasons.CredentialRejected ? FailureCodes.CredentialNeedsReconnect : FailureCodes.ProviderError; +} diff --git a/backend/src/CodeSpace.Core/Services/Workflows/RunSources/Admission/PullRequestTriggerAdmission.cs b/backend/src/CodeSpace.Core/Services/Workflows/RunSources/Admission/PullRequestTriggerAdmission.cs index 8dc4c2609..38b83bce1 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/RunSources/Admission/PullRequestTriggerAdmission.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/RunSources/Admission/PullRequestTriggerAdmission.cs @@ -6,9 +6,11 @@ using CodeSpace.Core.Services.Providers.Capabilities; using CodeSpace.Core.Services.Webhooks; using CodeSpace.Core.Services.Workflows.RunSources.Admission.Exceptions; +using CodeSpace.Messages.Constants; using CodeSpace.Messages.Enums; using CodeSpace.Messages.Events; using CodeSpace.Messages.Events.PullRequest; +using CodeSpace.Messages.Exceptions; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; @@ -41,6 +43,9 @@ public sealed class PullRequestTriggerAdmission : IPullRequestTriggerAdmission, private IPullRequestOriginEvent? _standingsFor; private readonly Dictionary _standings = new(); + /// Why a user's standing is Unknown, for the users whose lookup failed — what separates "not a member" from "could not find out". + private readonly Dictionary _standingFailures = new(); + public PullRequestTriggerAdmission(CodeSpaceDbContext db, IProviderRegistry providers, IWebhookClaimStore claims, ILogger logger) { _db = db; @@ -76,9 +81,24 @@ private void EnsureAuthorAdmitted(WorkflowActivation activation, IPullRequestOri { if (PullRequestTriggerAuthors.Admits(required, pullRequest.Origin.AuthorAssociation)) return; + ThrowIfStandingUnavailable(activation, pullRequest, pullRequest.Origin.AuthorExternalId, "its author"); + throw new PullRequestAuthorRefusedException(AuthorRefusal(pullRequest), BuildRefusalAuditKey("author", activation.Id, pullRequest, DateTimeOffset.UtcNow)); } + /// + /// A standing that is Unknown because the lookup FAILED is not a verdict on the person, so it is not refused as one: + /// the refusal names what failed — the credential, or the lookup, with its status — and what to do about it. + /// + private void ThrowIfStandingUnavailable(WorkflowActivation activation, IPullRequestOriginEvent pullRequest, string? userExternalId, string whose) + { + if (userExternalId is null || !_standingFailures.TryGetValue(userExternalId, out var failure)) return; + + var audit = BuildRefusalAuditKey(failure.Reason == WorkflowRunRequestRejectionReasons.CredentialRejected ? "credential-rejected" : "standing-unknown", activation.Id, pullRequest, DateTimeOffset.UtcNow); + + throw new PullRequestStandingUnavailableException(failure.Reason, StandingUnavailable(pullRequest, whose, failure), audit); + } + /// The pusher's standing is asked only when the rule needs it: someone other than the author pushed to a fork head. private async Task EnsurePusherAdmittedAsync(WorkflowActivation activation, IPullRequestOriginEvent pullRequest, string required, CancellationToken cancellationToken) { @@ -88,6 +108,8 @@ private async Task EnsurePusherAdmittedAsync(WorkflowActivation activation, IPul if (PullRequestTriggerAuthors.AdmitsPusher(required, pullRequest.Origin)) return; + ThrowIfStandingUnavailable(activation, pullRequest, pullRequest.Origin.PusherExternalId, "whoever pushed its new commits"); + throw new PullRequestAuthorRefusedException(PusherRefusal(pullRequest), BuildRefusalAuditKey("pusher", activation.Id, pullRequest, DateTimeOffset.UtcNow)); } @@ -113,6 +135,11 @@ private static string PusherRefusal(IPullRequestOriginEvent pullRequest) => private static string Describe(PullRequestAuthorAssociation association) => association.ToString().ToLowerInvariant(); + private static string StandingUnavailable(IPullRequestOriginEvent pullRequest, string whose, StandingFailure failure) => + failure.Reason == WorkflowRunRequestRejectionReasons.CredentialRejected + ? $"pull request #{pullRequest.Number}: could not check whether {whose} is a member — {failure.Cause}. Reconnect the credential, or re-link the repository to a live one; nobody's standing is in question, so loosening 'Pull requests from' is not the remedy" + : $"pull request #{pullRequest.Number}: could not find out whether {whose} is a member — {failure.Cause}. Nothing is known about them, so this is not a refusal of them; the same pull request is admitted once the lookup works"; + /// /// GitLab's payload names the author but not their standing; ask the provider, once per delivery, when it can be /// asked. A provider that cannot answer leaves the standing Unknown, which no members-only trigger admits. @@ -139,7 +166,11 @@ private async Task CompletePusherAssociationAsync(IPullRequestOriginEvent pullRe /// A user's standing on this event's repository, asked at most once per delivery — an unanswered lookup costs up to its timeout, and every matching activation repeating it would outlast the provider's own. private async Task StandingOfAsync(IPullRequestOriginEvent pullRequest, string userExternalId, CancellationToken cancellationToken) { - if (!ReferenceEquals(_standingsFor, pullRequest)) _standings.Clear(); + if (!ReferenceEquals(_standingsFor, pullRequest)) + { + _standings.Clear(); + _standingFailures.Clear(); + } _standingsFor = pullRequest; @@ -156,7 +187,15 @@ private async Task LookUpStandingAsync(Guid reposi var repository = await _db.Repository.AsNoTracking().Include(r => r.ProviderInstance).Include(r => r.Credential) .SingleOrDefaultAsync(r => r.Id == repositoryId, cancellationToken).ConfigureAwait(false); - if (repository?.Credential == null) return PullRequestAuthorAssociation.Unknown; + if (repository == null) return PullRequestAuthorAssociation.Unknown; + + if (repository.Credential == null) + { + _standingFailures[userExternalId] = new StandingFailure(WorkflowRunRequestRejectionReasons.AuthorStandingUnknown, "the repository has no credential bound to ask the provider with"); + + return PullRequestAuthorAssociation.Unknown; + } + if (!_providers.TryGet(repository.ProviderInstance.Provider, out var standing) || standing == null) return PullRequestAuthorAssociation.Unknown; return await AskProviderAsync(standing, repository, userExternalId, cancellationToken).ConfigureAwait(false); @@ -174,7 +213,36 @@ private async Task AskProviderAsync(IRepositoryMem catch (Exception ex) when (!cancellationToken.IsCancellationRequested) { _logger.LogWarning(ex, "Could not read the standing of user {UserExternalId} on repository {RepositoryId}; a members-only trigger will not admit them", userExternalId, repository.Id); + + _standingFailures[userExternalId] = ClassifyFailure(ex, repository); + return PullRequestAuthorAssociation.Unknown; } } + + /// + /// What a failed lookup says about whom to blame. The credential is refused — a 401, a grant that cannot be refreshed, a + /// disconnection — or the lookup did not work: a 403 (the credential lacks the scope or role to read members), a 429 or a + /// 5xx, no answer in time, anything else. + /// + private StandingFailure ClassifyFailure(Exception failure, Repository repository) + { + var provider = repository.ProviderInstance.Provider; + var credential = repository.Credential!.DisplayName; + + return failure switch + { + CredentialNeedsReconnectException => new StandingFailure(WorkflowRunRequestRejectionReasons.CredentialRejected, $"its provider refused the repository's credential '{credential}' and it needs to be reconnected"), + CredentialDisconnectedException => new StandingFailure(WorkflowRunRequestRejectionReasons.CredentialRejected, $"the repository's credential '{credential}' was disconnected"), + ProviderApiException { StatusCode: 401 } => new StandingFailure(WorkflowRunRequestRejectionReasons.CredentialRejected, $"{provider} answered HTTP 401 to the repository's credential '{credential}'"), + ProviderApiException { StatusCode: 403 } => new StandingFailure(WorkflowRunRequestRejectionReasons.AuthorStandingUnknown, $"{provider} refused the lookup (HTTP 403): the repository's credential '{credential}' needs the read_api scope and a role on the project to read its members"), + ProviderInsufficientScopeException scope => new StandingFailure(WorkflowRunRequestRejectionReasons.AuthorStandingUnknown, $"{provider} refused the lookup: the repository's credential '{credential}' lacks the {string.Join(", ", scope.MissingScopes)} scope"), + ProviderApiException api => new StandingFailure(WorkflowRunRequestRejectionReasons.AuthorStandingUnknown, $"{provider} answered HTTP {api.StatusCode} to the lookup"), + OperationCanceledException => new StandingFailure(WorkflowRunRequestRejectionReasons.AuthorStandingUnknown, $"{provider} did not answer within {AuthorLookupTimeout.TotalSeconds:0} seconds"), + _ => new StandingFailure(WorkflowRunRequestRejectionReasons.AuthorStandingUnknown, $"the lookup failed ({failure.GetType().Name})") + }; + } + + /// Why a lookup failed: the value it is refused under, and the cause in words. + private sealed record StandingFailure(string Reason, string Cause); } diff --git a/backend/src/CodeSpace.Core/Services/Workflows/RunSources/RunSourceDispatcher.cs b/backend/src/CodeSpace.Core/Services/Workflows/RunSources/RunSourceDispatcher.cs index 17e0a47ba..52837fa38 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/RunSources/RunSourceDispatcher.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/RunSources/RunSourceDispatcher.cs @@ -177,6 +177,12 @@ await _db.Repository.AsNoTracking() _logger.LogInformation("Webhook activation refused a pull request author. ActivationId={ActivationId} WorkflowId={WorkflowId} TeamId={TeamId} Reason={Reason}", activation.Id, activation.WorkflowId, activation.Workflow.TeamId, ex.Message); await AuditRefusalAsync(activation, normalizedEvent, WorkflowRunRequestRejectionReasons.AuthorNotMember, ex.Message, ex.AuditKey, cancellationToken).ConfigureAwait(false); } + catch (PullRequestStandingUnavailableException ex) + { + anyRefused = true; + _logger.LogWarning("Webhook activation could not read a pull request author's standing. ActivationId={ActivationId} WorkflowId={WorkflowId} TeamId={TeamId} Reason={Reason} Detail={Detail}", activation.Id, activation.WorkflowId, activation.Workflow.TeamId, ex.Reason, ex.Message); + await AuditRefusalAsync(activation, normalizedEvent, ex.Reason, ex.Message, ex.AuditKey, cancellationToken).ConfigureAwait(false); + } catch (PullRequestTriggerDebouncedException ex) { anyRefused = true; diff --git a/backend/src/CodeSpace.Messages/Commands/Credentials/ProbeCredentialsCommand.cs b/backend/src/CodeSpace.Messages/Commands/Credentials/ProbeCredentialsCommand.cs new file mode 100644 index 000000000..0356bee1a --- /dev/null +++ b/backend/src/CodeSpace.Messages/Commands/Credentials/ProbeCredentialsCommand.cs @@ -0,0 +1,27 @@ +using CodeSpace.Messages.Mediation; + +namespace CodeSpace.Messages.Commands.Credentials; + +/// +/// Ask each provider whether a bounded batch of credentials still works, and record what it answered: a credential it +/// accepts gets its validation date (and a static token its real expiry); one it refuses with a definite 401 is marked +/// Error with the reason, so the credential list says so and a run fails fast with credential_needs_reconnect +/// instead of at its first provider call. An answer that is not an answer — a dropped connection, a timeout, a 5xx — says +/// nothing about the credential and changes nothing. Credentials last validated recently are not candidates, and a +/// credential already marked Error is asked again only after an hour, so it heals by itself if the provider accepts it +/// again. NOT tenant-scoped: a system-wide sweep that runs without an actor context (mirrors +/// RetireDepartedMembersCredentialsCommand). +/// +/// A pass ends at its time budget and a provider that stops answering costs one credential its probe deadline +/// (CredentialHealthDeadlines); a tick that finds the previous pass still running skips. Both leave the credentials not +/// reached due for the next tick. +/// +/// NOT transactional (): each credential is probed and recorded on its own +/// connection, and one that fails stays a candidate while the pass carries on. One transaction around the pass would hold +/// every row it touched across every provider call it makes. +/// +public sealed record ProbeCredentialsCommand : ICommand, INonTransactionalCommand +{ + /// Credentials probed per tick — bounds each pass, which also attempts a bounded multiple of it: credentials that fail to answer do not count, are not retried by the same pass, and after several passes in a row are set aside for a while. + public int BatchSize { get; init; } = 25; +} diff --git a/backend/src/CodeSpace.Messages/Constants/WorkflowRunRequestRejectionReasons.cs b/backend/src/CodeSpace.Messages/Constants/WorkflowRunRequestRejectionReasons.cs index 91a2d8c58..f9f054607 100644 --- a/backend/src/CodeSpace.Messages/Constants/WorkflowRunRequestRejectionReasons.cs +++ b/backend/src/CodeSpace.Messages/Constants/WorkflowRunRequestRejectionReasons.cs @@ -80,6 +80,24 @@ public static class WorkflowRunRequestRejectionReasons /// public const string AuthorNotMember = "author_not_member"; + /// + /// A pull-request trigger that admits only members could not learn whether the author is one: the provider did not + /// answer the standing lookup within its deadline, answered with an error (a 403 for a credential without the scope or + /// role to read members, a 429, a 5xx), or the repository has no credential to ask with. Unlike + /// nothing was learned about the author, so setting 'Pull requests from' to Anyone is not + /// the remedy — the detail names the status and what the credential needs, and the same pull request is admitted once + /// the lookup works. + /// + public const string AuthorStandingUnknown = "author_standing_unknown"; + + /// + /// A pull-request trigger that admits only members could not ask the provider for the author's standing because the + /// repository's credential is refused: the provider answered 401, its OAuth grant can no longer be refreshed, or it was + /// disconnected. Remedy: reconnect the credential (the detail names it) or re-link the repository; the author's standing + /// is not in question. + /// + public const string CredentialRejected = "credential_rejected"; + /// /// The same activation already started a run for this pull request at the same head commit moments ago. Events for /// one head within the debounce window start one run, not one per event, so closing and reopening a PR cannot be used diff --git a/backend/src/CodeSpace.Messages/Dtos/Credentials/CredentialHealthDeadlines.cs b/backend/src/CodeSpace.Messages/Dtos/Credentials/CredentialHealthDeadlines.cs new file mode 100644 index 000000000..0fa424a50 --- /dev/null +++ b/backend/src/CodeSpace.Messages/Dtos/Credentials/CredentialHealthDeadlines.cs @@ -0,0 +1,12 @@ +namespace CodeSpace.Messages.Dtos.Credentials; + +/// +/// How long the credential health sweep may spend. bounds what one credential costs: a provider that +/// takes longer is given up on, and the credential counts as having had no answer. bounds the whole +/// pass, in-flight probe included, and is kept under the recurring job's interval so a pass is over before the next tick +/// starts. One instance is registered for the process (); a test registers a shorter one. +/// +public sealed record CredentialHealthDeadlines(TimeSpan Probe, TimeSpan Pass) +{ + public static readonly CredentialHealthDeadlines Default = new(TimeSpan.FromSeconds(30), TimeSpan.FromMinutes(5)); +} diff --git a/backend/src/CodeSpace.Messages/Dtos/Credentials/CredentialSummary.cs b/backend/src/CodeSpace.Messages/Dtos/Credentials/CredentialSummary.cs index 445e882ad..b3b7b0cd4 100644 --- a/backend/src/CodeSpace.Messages/Dtos/Credentials/CredentialSummary.cs +++ b/backend/src/CodeSpace.Messages/Dtos/Credentials/CredentialSummary.cs @@ -28,4 +28,10 @@ public sealed record CredentialSummary public DateTimeOffset? LastValidatedDate { get; init; } public string? LastError { get; init; } public required DateTimeOffset CreatedDate { get; init; } + + /// What to read off the credential — derived from its status, a static token's expiry and its owner's standing; see . + public CredentialHealth Health { get; init; } = CredentialHealth.Active; + + /// Whole days until a static token expires at its provider; null for an OAuth credential (its access token renews itself) and for a token with no recorded expiry. + public int? DaysToExpiry { get; init; } } diff --git a/backend/src/CodeSpace.Messages/Dtos/Providers/CredentialProbeResult.cs b/backend/src/CodeSpace.Messages/Dtos/Providers/CredentialProbeResult.cs index 3ce5b824f..d168481f1 100644 --- a/backend/src/CodeSpace.Messages/Dtos/Providers/CredentialProbeResult.cs +++ b/backend/src/CodeSpace.Messages/Dtos/Providers/CredentialProbeResult.cs @@ -1,3 +1,5 @@ +using CodeSpace.Messages.Exceptions; + namespace CodeSpace.Messages.Dtos.Providers; public sealed record CredentialProbeResult @@ -7,4 +9,25 @@ public sealed record CredentialProbeResult public IReadOnlyList? GrantedScopes { get; init; } public string? AuthenticatedUserExternalId { get; init; } public string? AuthenticatedUserName { get; init; } + + /// When a static token expires at its provider (GitLab's expires_at). Null when the token never expires — or when the provider did not describe the token at all, which being null tells apart. + public DateTimeOffset? ExpiresAt { get; init; } + + /// What the provider said about the token being live (GitLab's active and revoked); null when it did not describe the token, in which case is unknown rather than "never". + public bool? Active { get; init; } + + /// + /// The provider ANSWERED and refused the credential — a 401, or a grant that can no longer be refreshed. Distinct from + /// being false for lack of an answer (a dropped connection, a timeout, a 5xx), which says nothing + /// about the credential: only a rejection is evidence to act on. + /// + public bool IsRejected { get; init; } + + /// The answer for a probe that threw: rejected when the provider refused the credential, otherwise merely not valid. + public static CredentialProbeResult FromFailure(Exception failure) => new() + { + IsValid = false, + Error = failure.Message, + IsRejected = failure is ProviderApiException { StatusCode: 401 } or CredentialNeedsReconnectException + }; } diff --git a/backend/src/CodeSpace.Messages/Enums/CredentialHealth.cs b/backend/src/CodeSpace.Messages/Enums/CredentialHealth.cs new file mode 100644 index 000000000..e011ba1d4 --- /dev/null +++ b/backend/src/CodeSpace.Messages/Enums/CredentialHealth.cs @@ -0,0 +1,27 @@ +using System.Text.Json.Serialization; + +namespace CodeSpace.Messages.Enums; + +/// +/// What a person should read off a credential, derived from what is recorded about it (never stored): whether it works, +/// is about to stop, was refused by its provider, was disconnected, or lent by someone no longer in the team. Serialized +/// as the lower-case names clients branch on, so a rename is a breaking change. +/// +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum CredentialHealth +{ + /// Nothing recorded says it will not work. + [JsonStringEnumMemberName("active")] Active, + + /// A static token whose provider-side expiry is inside the warning window (DaysToExpiry says how long is left). + [JsonStringEnumMemberName("expiring")] Expiring, + + /// The provider refused it — a dead OAuth grant, a rejected or expired static token — and it needs reconnecting. + [JsonStringEnumMemberName("rejected")] Rejected, + + /// Disconnected: its token material is gone. + [JsonStringEnumMemberName("revoked")] Revoked, + + /// A person's credential whose person no longer has standing in the team; the retirement sweep is about to retire it. + [JsonStringEnumMemberName("owner_left")] OwnerLeft +} diff --git a/backend/src/CodeSpace.Messages/Exceptions/CredentialNeedsReconnectException.cs b/backend/src/CodeSpace.Messages/Exceptions/CredentialNeedsReconnectException.cs new file mode 100644 index 000000000..da45cb00d --- /dev/null +++ b/backend/src/CodeSpace.Messages/Exceptions/CredentialNeedsReconnectException.cs @@ -0,0 +1,35 @@ +using CodeSpace.Messages.Failures; + +namespace CodeSpace.Messages.Exceptions; + +/// +/// Thrown when something tries to authenticate through a credential whose provider has refused it for good: the OAuth grant +/// behind it answered a refresh with invalid_grant (the stored pair is dead, or the provider's OAuth client changed +/// under it), or a static token was rejected by the provider or has expired. The credential row is marked +/// Error with the reason, so the next use fails here without asking the provider again. The remedy is a person's: the +/// owner reconnects the provider (or adds the token again), or a team Admin re-links what is bound through it to a live +/// credential. A retry as-is reads the same dead credential, which is why this is not a transient provider failure. +/// +public sealed class CredentialNeedsReconnectException : Exception, IFailure +{ + public FailureKind Kind => FailureKind.PreconditionRequired; + + public string Code => FailureCodes.CredentialNeedsReconnect; + + public IReadOnlyDictionary? Details => new Dictionary { ["credentialId"] = CredentialId, ["credentialName"] = DisplayName }; + + public Guid CredentialId { get; } + + public string DisplayName { get; } + + /// What the provider said or what was recorded when the credential was marked, e.g. invalid_grant — never a secret. + public string Reason { get; } + + public CredentialNeedsReconnectException(Guid credentialId, string displayName, string reason, Exception? inner = null) + : base($"The credential '{displayName}' ({credentialId}) cannot authenticate: its provider refused it ({reason}). It needs to be reconnected — its owner reconnects the provider (or adds the token again), or a team Admin re-links the repository to a live credential of the same provider. Until then nothing bound through it can authenticate.", inner) + { + CredentialId = credentialId; + DisplayName = displayName; + Reason = reason; + } +} diff --git a/backend/src/CodeSpace.Messages/Failures/FailureCodes.cs b/backend/src/CodeSpace.Messages/Failures/FailureCodes.cs index 3d4a47ce2..570f71e86 100644 --- a/backend/src/CodeSpace.Messages/Failures/FailureCodes.cs +++ b/backend/src/CodeSpace.Messages/Failures/FailureCodes.cs @@ -55,6 +55,9 @@ public static class FailureCodes /// Something tried to authenticate through a credential that was disconnected — revoked, its token material destroyed. Remedy: reconnect the provider (bound repositories follow their owner's reconnect) or re-link to an active credential; a retry as-is reads the same dead row. public const string CredentialDisconnected = "credential_disconnected"; + /// A credential's grant or token was refused by its provider and cannot be used again as it is — an OAuth refresh answered invalid_grant, a static token the provider rejects or has expired. Remedy: reconnect the provider (or add the token again); a retry as-is is refused the same way, so nothing bound through the credential authenticates until then. + public const string CredentialNeedsReconnect = "credential_needs_reconnect"; + /// The remote refused the git credential a push carried even after it was resolved again immediately before the push (or the credential was disconnected while the run worked), so the agent's branch was not published. Remedy: reconnect the provider, or re-link the repository to an active credential, then re-run — a retry as-is is refused the same way. public const string GitCredentialRejected = "git_credential_rejected"; diff --git a/backend/tests/CodeSpace.E2ETests/Credentials/AccessibleRepositoriesGateE2ETests.cs b/backend/tests/CodeSpace.E2ETests/Credentials/AccessibleRepositoriesGateE2ETests.cs index dc6b1962e..27587e90d 100644 --- a/backend/tests/CodeSpace.E2ETests/Credentials/AccessibleRepositoriesGateE2ETests.cs +++ b/backend/tests/CodeSpace.E2ETests/Credentials/AccessibleRepositoriesGateE2ETests.cs @@ -45,17 +45,20 @@ public async Task Only_the_owner_of_a_personal_credential_or_a_repos_manage_hold var world = await SeedAsync(gitLab.BaseUrl); var (credentialId, token) = world.Credential(holder); + // Adding a token asks its provider about it (GET /user), so each token was presented once when it was added; the listing is what is counted from there. + var presentedWhenAdded = gitLab.Carrying(token).Count; + var listing = await _tenants.SendAsync(HttpMethod.Get, $"/api/credentials/{credentialId}/accessible-repositories", world.User(caller), world.Team); listing.StatusCode.ShouldBe(expected, $"{caller} listing through {holder} — {await TenantHarness.DescribeAsync(listing)}"); if (expected == HttpStatusCode.OK) { - gitLab.Carrying(token).ShouldNotBeEmpty("fixture check: an allowed listing presents the token to GitLab"); + gitLab.Carrying(token).Count.ShouldBeGreaterThan(presentedWhenAdded, "fixture check: an allowed listing presents the token to GitLab"); return; } - gitLab.Carrying(token).ShouldBeEmpty($"{caller} was refused before the {holder} token was ever presented to the provider"); + gitLab.Carrying(token).Count.ShouldBe(presentedWhenAdded, $"{caller} was refused before the {holder} token was presented to the provider again"); } private sealed record World(Guid Team, Guid Owner, Guid Admin, Guid Member, Guid Viewer, (Guid, string) OwnersPat, (Guid, string) MembersPat, (Guid, string) TeamToken) diff --git a/backend/tests/CodeSpace.E2ETests/Credentials/CredentialHealthE2ETests.cs b/backend/tests/CodeSpace.E2ETests/Credentials/CredentialHealthE2ETests.cs new file mode 100644 index 000000000..36308ee9a --- /dev/null +++ b/backend/tests/CodeSpace.E2ETests/Credentials/CredentialHealthE2ETests.cs @@ -0,0 +1,163 @@ +using System.Net; +using System.Text.Json; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Credentials; +using CodeSpace.E2ETests.Infrastructure; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.Messages.Commands.Credentials; +using CodeSpace.Messages.Constants; +using CodeSpace.Messages.Enums; +using MediatR; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Shouldly; + +namespace CodeSpace.E2ETests.Credentials; + +/// +/// 🟢 High fidelity: a person links a GitLab personal access token through the real HTTP surface +/// (POST /api/me/identities/pat: the real mediator pipeline, the real GitLab provider over NGitLab and raw HTTP, the +/// probe that reads the token's own record) against real Postgres and a on a loopback +/// port, and reads what the team's credential list says about it (GET /api/credentials). The one production class +/// driven directly is the recurring probe's command, sent through the host's own mediator — the Hangfire tick that would +/// send it is not what is under test. +/// +/// The story: GitLab gives the token an expiry three days out, so the list says it is expiring with the days +/// left, before anything has failed. Then GitLab starts refusing the token; within one sweep the credential is +/// rejected and says why, and the next use of it — here, listing the repositories it can reach — fails with a typed +/// credential_needs_reconnect that names it, instead of GitLab's 401 surfacing as a 422 at the first call a run +/// makes. +/// +[Trait("Category", "E2E")] +[Trait("Category", "ProviderE2E")] +[Trait("Surface", "Http")] +public sealed class CredentialHealthE2ETests : IClassFixture +{ + private readonly TaskLaunchApiFactory _factory; + private readonly TenantHarness _tenants; + + public CredentialHealthE2ETests(TaskLaunchApiFactory factory) + { + _factory = factory; + _tenants = new TenantHarness(factory); + } + + [Fact] + public async Task A_token_that_expires_soon_reads_expiring_and_once_gitlab_refuses_it_one_sweep_marks_it_and_its_next_use_names_what_to_do() + { + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.AddAccessToken("glpat-e2e-expiring", "nacho", DateTimeOffset.UtcNow.AddDays(3)); + var world = await SeedWorldAsync(gitlab).ConfigureAwait(false); + + var linked = await _tenants.SendAsync(HttpMethod.Post, "/api/me/identities/pat", world.UserId, world.TeamId, new { providerInstanceId = world.InstanceId, accessToken = "glpat-e2e-expiring" }).ConfigureAwait(false); + linked.StatusCode.ShouldBe(HttpStatusCode.OK, await TenantHarness.DescribeAsync(linked).ConfigureAwait(false)); + + var expiring = await ReadSummaryAsync(world, "GitLab · nacho").ConfigureAwait(false); + + expiring.GetProperty("health").GetString().ShouldBe("expiring", "the list says so before anything has failed"); + expiring.GetProperty("daysToExpiry").GetInt32().ShouldBeInRange(2, 3, "GitLab's expires_at is a date: it begins at midnight UTC, between two and three days from now"); + expiring.GetProperty("status").GetString().ShouldBe("Active"); + expiring.GetProperty("lastValidatedDate").ValueKind.ShouldNotBe(JsonValueKind.Null, "the link just validated it"); + + // GitLab starts refusing the token. The credential is not due again for hours, so it is made due. + gitlab.RevokeAccessToken("glpat-e2e-expiring"); + var credentialId = expiring.GetProperty("id").GetGuid(); + await _tenants.DbAsync(async db => await db.Database.ExecuteSqlInterpolatedAsync($"UPDATE credential SET last_validated_date = now() - interval '7 hours' WHERE id = {credentialId}")).ConfigureAwait(false); + + await SweepAsync().ConfigureAwait(false); + + var rejected = await ReadSummaryAsync(world, "GitLab · nacho").ConfigureAwait(false); + + rejected.GetProperty("health").GetString().ShouldBe("rejected"); + rejected.GetProperty("status").GetString().ShouldBe("Error"); + rejected.GetProperty("lastError").GetString().ShouldBe("token rejected by GitLab"); + + var use = await _tenants.SendAsync(HttpMethod.Get, $"/api/credentials/{credentialId}/accessible-repositories", world.UserId, world.TeamId).ConfigureAwait(false); + var body = JsonDocument.Parse(await use.Content.ReadAsStringAsync().ConfigureAwait(false)).RootElement; + + ((int)use.StatusCode).ShouldBe(428, await TenantHarness.DescribeAsync(use).ConfigureAwait(false)); + body.GetProperty("code").GetString().ShouldBe("credential_needs_reconnect"); + body.GetProperty("credentialId").GetGuid().ShouldBe(credentialId, "the failure names the credential, so a client can link to it"); + body.GetProperty("credentialName").GetString().ShouldBe("GitLab · nacho"); + } + + [Fact] + public async Task A_token_added_through_the_credentials_api_reads_expiring_at_once_without_waiting_for_a_sweep() + { + // The group and project tokens a team shares, and a personal token added for someone, do not come through the identity + // link: POST /api/credentials/pat. GitLab describes the token as it is added, so the very first list says how long it has. + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.AddAccessToken("glpat-e2e-added", "nacho", DateTimeOffset.UtcNow.AddDays(3)); + var world = await SeedWorldAsync(gitlab).ConfigureAwait(false); + + var added = await _tenants.SendAsync(HttpMethod.Post, "/api/credentials/pat", world.UserId, world.TeamId, new { providerInstanceId = world.InstanceId, displayName = "added token", token = "glpat-e2e-added" }).ConfigureAwait(false); + added.StatusCode.ShouldBe(HttpStatusCode.OK, await TenantHarness.DescribeAsync(added).ConfigureAwait(false)); + + var expiring = await ReadSummaryAsync(world, "added token").ConfigureAwait(false); + + expiring.GetProperty("health").GetString().ShouldBe("expiring", "the list says so from the first read, before any sweep has run"); + expiring.GetProperty("daysToExpiry").GetInt32().ShouldBeInRange(2, 3); + expiring.GetProperty("status").GetString().ShouldBe("Active"); + expiring.GetProperty("lastValidatedDate").ValueKind.ShouldNotBe(JsonValueKind.Null, "adding it validated it"); + } + + [Fact] + public async Task A_credential_list_keeps_every_field_it_had_and_reads_active_for_a_token_with_nothing_to_say() + { + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.AddAccessToken("glpat-e2e-forever", "nacho"); + var world = await SeedWorldAsync(gitlab).ConfigureAwait(false); + + (await _tenants.SendAsync(HttpMethod.Post, "/api/me/identities/pat", world.UserId, world.TeamId, new { providerInstanceId = world.InstanceId, accessToken = "glpat-e2e-forever" }).ConfigureAwait(false)).StatusCode.ShouldBe(HttpStatusCode.OK); + + var summary = await ReadSummaryAsync(world, "GitLab · nacho").ConfigureAwait(false); + + foreach (var property in new[] { "id", "teamId", "providerInstanceId", "ownerUserId", "ownerUserName", "ownership", "authType", "displayName", "status", "expiresDate", "lastValidatedDate", "lastError", "createdDate" }) + summary.TryGetProperty(property, out _).ShouldBeTrue($"'{property}' was in the list before and must stay"); + + summary.GetProperty("health").GetString().ShouldBe("active"); + summary.GetProperty("expiresDate").ValueKind.ShouldBe(JsonValueKind.Null, "a token that never expires has no expiry to show"); + summary.GetProperty("daysToExpiry").ValueKind.ShouldBe(JsonValueKind.Null); + } + + // ─── Plumbing ────────────────────────────────────────────────────────────── + + private sealed record World(Guid UserId, Guid TeamId, Guid InstanceId); + + /// One Owner in one team and a token-only GitLab instance on the stub's loopback address. + private async Task SeedWorldAsync(GitLabDoorkeeperStub gitlab) + { + var user = await _tenants.AccountAsync().ConfigureAwait(false); + var team = await _tenants.TeamAsync((user, TeamRole.Owner)).ConfigureAwait(false); + var instance = new ProviderInstance { Id = Guid.NewGuid(), TeamId = team, Provider = ProviderKind.GitLab, DisplayName = $"gitlab-{Guid.NewGuid():N}"[..14], BaseUrl = gitlab.BaseUrl, CreatedBy = SystemUsers.SeederId, LastModifiedBy = SystemUsers.SeederId }; + + await _tenants.DbAsync(async db => + { + db.ProviderInstance.Add(instance); + await db.SaveChangesAsync(); + + return 0; + }).ConfigureAwait(false); + + return new World(user, team, instance.Id); + } + + private async Task ReadSummaryAsync(World world, string displayName) + { + var response = await _tenants.SendAsync(HttpMethod.Get, "/api/credentials", world.UserId, world.TeamId).ConfigureAwait(false); + + response.StatusCode.ShouldBe(HttpStatusCode.OK, await TenantHarness.DescribeAsync(response).ConfigureAwait(false)); + + using var json = JsonDocument.Parse(await response.Content.ReadAsStringAsync().ConfigureAwait(false)); + + return json.RootElement.EnumerateArray().Single(c => c.GetProperty("displayName").GetString() == displayName).Clone(); + } + + private async Task SweepAsync() + { + using var scope = _factory.Services.CreateScope(); + + await scope.ServiceProvider.GetRequiredService().Send(new ProbeCredentialsCommand()).ConfigureAwait(false); + } +} diff --git a/backend/tests/CodeSpace.E2ETests/Repositories/ListPagingPastTheCeilingE2ETests.cs b/backend/tests/CodeSpace.E2ETests/Repositories/ListPagingPastTheCeilingE2ETests.cs index 11b2f5861..74530d154 100644 --- a/backend/tests/CodeSpace.E2ETests/Repositories/ListPagingPastTheCeilingE2ETests.cs +++ b/backend/tests/CodeSpace.E2ETests/Repositories/ListPagingPastTheCeilingE2ETests.cs @@ -60,13 +60,13 @@ public async Task A_page_too_large_for_an_int_is_refused_and_not_answered_with_a await using var gitLab = LoopbackResponder.Start(OneRowPerList); var (viewer, team, repositoryId) = await SeedRepositoryAsync(gitLab.Port); - // Connecting the provider may already have asked GitLab (the endpoint probe on add); only what the refused list request asks counts. - var asked = gitLab.Requests.Count; + // Connecting the provider and adding the token already ask GitLab (the endpoint probe, GET /user and the token's own record), so what the refused request must add to is counted from there. + var askedWhenSeeded = gitLab.Requests.Count; var response = await _tenants.SendAsync(HttpMethod.Get, $"/api/repositories/{repositoryId}/{list}?page=3000000000&perPage=100", viewer, team); response.StatusCode.ShouldBe(HttpStatusCode.BadRequest, await TenantHarness.DescribeAsync(response)); - gitLab.Requests.Count.ShouldBe(asked, "a refused request reaches no provider"); + gitLab.Requests.Count.ShouldBe(askedWhenSeeded, "a refused request reaches no provider"); } private async Task ListAsync(Guid viewer, Guid team, Guid repositoryId, string list, int page) diff --git a/backend/tests/CodeSpace.E2ETests/Repositories/ProviderInstanceDeleteRevokeE2ETests.cs b/backend/tests/CodeSpace.E2ETests/Repositories/ProviderInstanceDeleteRevokeE2ETests.cs index 1869a0481..a0411ecdb 100644 --- a/backend/tests/CodeSpace.E2ETests/Repositories/ProviderInstanceDeleteRevokeE2ETests.cs +++ b/backend/tests/CodeSpace.E2ETests/Repositories/ProviderInstanceDeleteRevokeE2ETests.cs @@ -46,6 +46,9 @@ public async Task Deleting_a_connection_revokes_its_oauth_grants_at_the_provider var canary = Guid.NewGuid().ToString("N"); var world = await SeedConnectionAsync(gitLab.BaseUrl, canary); + // Adding the personal access token asked GitLab about it (GET /user), which is not what a delete sends; the delete is counted from there. + var presentedWhenAdded = gitLab.Carrying($"glpat-{canary}").Count; + var delete = await _tenants.SendAsync(HttpMethod.Delete, $"/api/provider-instances/{world.InstanceId}", world.Admin, world.Team); delete.StatusCode.ShouldBe(HttpStatusCode.OK, await TenantHarness.DescribeAsync(delete)); @@ -54,7 +57,7 @@ public async Task Deleting_a_connection_revokes_its_oauth_grants_at_the_provider var revokes = gitLab.Requests.Where(r => r.Method == "POST" && r.PathAndQuery == "/oauth/revoke").ToList(); revokes.Select(r => (r.Form["token"], r.Form["token_type_hint"])).ShouldBe(new[] { ($"rt-{canary}", "refresh_token"), ($"at-{canary}", "access_token") }, "GitLab must be asked to revoke the refresh token, then the access token — check the post-commit revoke in ProviderInstanceService and the stub's request log if this fails"); revokes.ShouldAllBe(r => r.Form["client_id"] == world.ClientId && r.Form["client_secret"] == $"client-secret-{canary}"); - gitLab.Carrying($"glpat-{canary}").ShouldBeEmpty("a personal access token has no revocation endpoint and is never sent anywhere by a delete"); + gitLab.Carrying($"glpat-{canary}").Count.ShouldBe(presentedWhenAdded, "a personal access token has no revocation endpoint and is never sent anywhere by a delete"); var stored = await StoredAsync(world); stored.Payloads.ShouldAllBe(p => p == string.Empty, "a deleted connection's tokens must not stay decryptable in the database"); diff --git a/backend/tests/CodeSpace.IntegrationTests/Credentials/CredentialAddHealthFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Credentials/CredentialAddHealthFlowTests.cs new file mode 100644 index 000000000..99ae7e23f --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Credentials/CredentialAddHealthFlowTests.cs @@ -0,0 +1,144 @@ +using System.Diagnostics; +using Autofac; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.IntegrationTests.Webhooks; +using CodeSpace.IntegrationTests.Workflows.Infrastructure; +using CodeSpace.Messages.Commands.Credentials; +using CodeSpace.Messages.Constants; +using CodeSpace.Messages.Credentials; +using CodeSpace.Messages.Dtos.Credentials; +using CodeSpace.Messages.Enums; +using MediatR; +using Microsoft.EntityFrameworkCore; +using Shouldly; + +namespace CodeSpace.IntegrationTests.Credentials; + +/// +/// 🟢 High fidelity — a token added through the credentials API (POST /api/credentials/pat and +/// /group-access-token: personal, and the group token a team shares) is described by its provider as it is added, as a +/// token linked for a person already is: GitLab says when it expires (every new token has an expiry, and one that reaches it +/// answers 401 to everything), and the credential records that and the validation, so the list says expiring from the +/// first read instead of waiting for the sweep's first pass. The real AddCredentialCommand pipeline, the real GitLab +/// provider over loopback, real Postgres. +/// +/// Adding never depends on the answer: a provider that is unreachable, silent or refuses the token leaves the credential +/// added as it always was, unstamped and due for the sweep — which is what marks a token GitLab refuses. +/// +[Collection(PostgresCollection.Name)] +[Trait("Category", "Integration")] +public class CredentialAddHealthFlowTests +{ + private readonly PostgresFixture _fixture; + + public CredentialAddHealthFlowTests(PostgresFixture fixture) { _fixture = fixture; } + + [Theory] + [InlineData(AuthType.Pat, CredentialOwnership.Personal)] + [InlineData(AuthType.GroupAccessToken, CredentialOwnership.TeamService)] + public async Task Adding_a_token_records_when_gitlab_says_it_expires_and_that_it_was_just_validated(AuthType type, CredentialOwnership ownership) + { + using var gitlab = new GitLabDoorkeeperStub(); + var expires = DateTimeOffset.UtcNow.AddDays(3); + gitlab.AddAccessToken("glpat-added", "nacho", expires); + var (team, seed) = await SeedAsync(gitlab.BaseUrl).ConfigureAwait(false); + + var id = await AddAsync(team, seed, "glpat-added", type, ownership).ConfigureAwait(false); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, id).ConfigureAwait(false); + + row.Status.ShouldBe(CredentialStatus.Active); + row.ExpiresDate.ShouldBe(new DateTimeOffset(expires.UtcDateTime.Date, TimeSpan.Zero), "GitLab's expires_at is a date; the row records the instant it begins"); + row.LastValidatedDate.ShouldNotBeNull().ShouldBeGreaterThan(DateTimeOffset.UtcNow.AddMinutes(-2)); + row.LastError.ShouldBeNull(); + } + + [Fact] + public async Task Adding_a_token_gitlab_says_never_expires_records_no_expiry() + { + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.AddAccessToken("glpat-forever", "nacho"); + var (team, seed) = await SeedAsync(gitlab.BaseUrl).ConfigureAwait(false); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, await AddAsync(team, seed, "glpat-forever").ConfigureAwait(false)).ConfigureAwait(false); + + row.ExpiresDate.ShouldBeNull(); + row.LastValidatedDate.ShouldNotBeNull("GitLab accepted it, which is worth recording even with no expiry to say"); + } + + [Fact] + public async Task A_token_added_while_the_provider_is_unreachable_is_added_as_it_always_was_and_left_due_for_the_sweep() + { + var gitlab = new GitLabDoorkeeperStub(); + var (team, seed) = await SeedAsync(gitlab.BaseUrl).ConfigureAwait(false); + gitlab.Dispose(); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, await AddAsync(team, seed, "glpat-offline").ConfigureAwait(false)).ConfigureAwait(false); + + row.Status.ShouldBe(CredentialStatus.Active); + row.LastValidatedDate.ShouldBeNull("nothing was learned, so it is still due"); + row.ExpiresDate.ShouldBeNull(); + } + + [Fact] + public async Task A_token_added_while_the_provider_is_silent_is_added_within_the_probe_deadline_and_left_due_for_the_sweep() + { + using var silent = new SilentProviderHost(); + var (team, seed) = await SeedAsync(silent.BaseUrl).ConfigureAwait(false); + var clock = Stopwatch.StartNew(); + + var id = await AddAsync(team, seed, "glpat-silent", deadlines: new CredentialHealthDeadlines(Probe: TimeSpan.FromSeconds(1), Pass: TimeSpan.FromMinutes(1))).ConfigureAwait(false); + + clock.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(15), "adding a token does not wait on a provider that has stopped answering"); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, id).ConfigureAwait(false); + + row.Status.ShouldBe(CredentialStatus.Active); + row.LastValidatedDate.ShouldBeNull(); + } + + [Fact] + public async Task A_token_gitlab_refuses_is_still_added_and_the_sweep_is_what_marks_it() + { + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.AddAccessToken("glpat-revoked", "nacho"); + gitlab.RevokeAccessToken("glpat-revoked"); + var (team, seed) = await SeedAsync(gitlab.BaseUrl).ConfigureAwait(false); + + var id = await AddAsync(team, seed, "glpat-revoked").ConfigureAwait(false); + + var added = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, id).ConfigureAwait(false); + + added.Status.ShouldBe(CredentialStatus.Active, "adding has never validated the token against its provider and still does not refuse one"); + added.LastValidatedDate.ShouldBeNull("a refusal is not a validation"); + + await GitLabCredentialSeed.QuietEveryOtherCredentialAsync(_fixture, id).ConfigureAwait(false); + + using (var scope = _fixture.BeginScope()) + await scope.Resolve().Send(new ProbeCredentialsCommand()).ConfigureAwait(false); + + (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, id).ConfigureAwait(false)).Status.ShouldBe(CredentialStatus.Error, "the sweep found out within its first pass"); + } + + // ─── Plumbing ────────────────────────────────────────────────────────────── + + private async Task<((Guid TeamId, Guid UserId) Team, SeededGitLabCredential Seed)> SeedAsync(string gitLabBaseUrl) + { + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture).ConfigureAwait(false); + var seed = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitLabBaseUrl, "glpat-other", existingTeam: team).ConfigureAwait(false); + + return (team, seed); + } + + /// POST /api/credentials/pat or /group-access-token, as the team's Admin. + private async Task AddAsync((Guid TeamId, Guid UserId) team, SeededGitLabCredential seed, string token, AuthType type = AuthType.Pat, CredentialOwnership ownership = CredentialOwnership.Personal, CredentialHealthDeadlines? deadlines = null) + { + CredentialPayload payload = type == AuthType.GroupAccessToken ? new GroupAccessTokenPayload { Token = token } : new PatPayload { Token = token }; + + using var scope = _fixture.BeginScopeAs(team.UserId, team.TeamId, Roles.Admin); + using var run = deadlines is null ? null : scope.BeginLifetimeScope(b => b.RegisterInstance(deadlines).SingleInstance()); + + return await (run ?? scope).Resolve().Send(new AddCredentialCommand { ProviderInstanceId = seed.InstanceId, DisplayName = $"added {token}", Payload = payload, Ownership = ownership }).ConfigureAwait(false); + } +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Credentials/CredentialHealthDeadlineFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Credentials/CredentialHealthDeadlineFlowTests.cs new file mode 100644 index 000000000..55c074912 --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Credentials/CredentialHealthDeadlineFlowTests.cs @@ -0,0 +1,214 @@ +using System.Diagnostics; +using Autofac; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Credentials; +using CodeSpace.Core.Services.OAuth; +using CodeSpace.Core.Services.Providers; +using CodeSpace.Core.Services.Providers.Capabilities; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.IntegrationTests.Webhooks; +using CodeSpace.Messages.Commands.Credentials; +using CodeSpace.Messages.Dtos.Credentials; +using CodeSpace.Messages.Dtos.Providers; +using CodeSpace.Messages.Enums; +using MediatR; +using Shouldly; +using Xunit.Abstractions; + +namespace CodeSpace.IntegrationTests.Credentials; + +/// +/// 🟢 High fidelity — what the recurring credential health probe costs a worker when a provider stops answering, from the real +/// ProbeCredentialsCommand through the real GitLab provider to a loopback server that accepts the connection and never +/// replies (), over real Postgres. The sweep runs on the control pool beside webhooks and +/// resume; a probe that waits on a silent provider for the HTTP client's whole timeout (five minutes, three attempts) holds a +/// worker for a quarter of an hour, and a tick every ten minutes then piles up behind it until the pool has none left. +/// +/// What it pins: one credential costs the sweep at most the probe deadline, and nothing is recorded about it (no answer +/// says nothing about a token); the whole pass ends at its budget, in-flight probe included, ahead of the next tick; a probe +/// that ignores the token it was given — a provider SDK with no cancellation, a call that blocks its thread — is walked +/// away from at the deadline all the same; and a tick that finds the previous pass still running skips instead of piling +/// up. The deadlines are shortened by registering a , so no test waits a minute. +/// +[Collection(PostgresCollection.Name)] +[Trait("Category", "Integration")] +public class CredentialHealthDeadlineFlowTests +{ + private static readonly TimeSpan Generous = TimeSpan.FromSeconds(20); + + private readonly PostgresFixture _fixture; + private readonly ITestOutputHelper _output; + + public CredentialHealthDeadlineFlowTests(PostgresFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + } + + [Fact] + public async Task A_provider_that_never_answers_costs_the_sweep_its_probe_deadline_and_nothing_is_recorded_about_the_credential() + { + using var silent = new SilentProviderHost(); + var seed = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, silent.BaseUrl, "glpat-silent").ConfigureAwait(false); + await GitLabCredentialSeed.QuietEveryOtherCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + var (probed, took) = await TimedSweepAsync(new CredentialHealthDeadlines(Probe: TimeSpan.FromSeconds(1), Pass: TimeSpan.FromMinutes(1))).ConfigureAwait(false); + + _output.WriteLine($"sweep took {took.TotalSeconds:0.0}s; connections={silent.Connections}"); + took.ShouldBeLessThan(Generous, "the sweep gives up on a silent provider at the probe deadline, not at the HTTP client's five minutes"); + probed.ShouldBe(0, "a credential the provider did not answer about is not one it answered about"); + silent.Connections.ShouldBe(1, "a call given up on at its deadline is not sent again"); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + row.Status.ShouldBe(CredentialStatus.Active, "no answer is not a verdict: marking the token dead on a deadline would stop every run through it"); + row.LastValidatedDate.ShouldBeNull("and it stays due, so the next tick asks again"); + row.LastError.ShouldBeNull(); + } + + [Fact] + public async Task A_pass_ends_at_its_budget_with_a_probe_still_in_flight_and_does_not_go_on_to_the_next_credential() + { + using var silent = new SilentProviderHost(); + var seeds = new List(); + + for (var i = 0; i < 3; i++) seeds.Add(await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, silent.BaseUrl, $"glpat-silent-{i}").ConfigureAwait(false)); + + await GitLabCredentialSeed.QuietEveryOtherCredentialAsync(_fixture, seeds.Select(s => s.CredentialId).ToArray()).ConfigureAwait(false); + + var (probed, took) = await TimedSweepAsync(new CredentialHealthDeadlines(Probe: TimeSpan.FromSeconds(30), Pass: TimeSpan.FromSeconds(2))).ConfigureAwait(false); + + _output.WriteLine($"sweep took {took.TotalSeconds:0.0}s; connections={silent.Connections}"); + took.ShouldBeLessThan(TimeSpan.FromSeconds(10), "the budget cuts the probe that is in flight; waiting for its 30-second deadline would overrun it"); + probed.ShouldBe(0); + silent.Connections.ShouldBe(1, "a pass that is out of budget starts no further probe"); + + foreach (var seed in seeds) + (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).Status.ShouldBe(CredentialStatus.Active, "a pass that ran out of time condemns nobody"); + } + + [Theory] + [InlineData(ProbeIgnores.TheTokenWhileAwaiting)] + [InlineData(ProbeIgnores.TheTokenWhileBlockingItsThread)] + public async Task A_probe_that_ignores_its_token_is_walked_away_from_at_the_deadline(ProbeIgnores how) + { + var seed = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, "http://127.0.0.1:1", "glpat-ignoring").ConfigureAwait(false); + await GitLabCredentialSeed.QuietEveryOtherCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + var calls = new ProbeCalls(); + var registry = new ProbeOnlyRegistry(new UncancellableProbe(how, calls)); + + var (probed, took) = await TimedSweepAsync(new CredentialHealthDeadlines(Probe: TimeSpan.FromSeconds(1), Pass: TimeSpan.FromMinutes(1)), registry).ConfigureAwait(false); + + _output.WriteLine($"{how}: sweep took {took.TotalSeconds:0.0}s"); + took.ShouldBeLessThan(TimeSpan.FromSeconds(10), "a provider SDK that takes no token cannot be cancelled, but the sweep need not wait for it"); + probed.ShouldBe(0); + calls.Started.ShouldBe(1); + (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).Status.ShouldBe(CredentialStatus.Active); + } + + [Fact] + public async Task A_tick_that_finds_the_previous_pass_still_running_skips_instead_of_piling_up_behind_it() + { + using var silent = new SilentProviderHost(); + var seed = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, silent.BaseUrl, "glpat-overlap").ConfigureAwait(false); + await GitLabCredentialSeed.QuietEveryOtherCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + var deadlines = new CredentialHealthDeadlines(Probe: TimeSpan.FromSeconds(1), Pass: TimeSpan.FromMinutes(1)); + + IAsyncDisposable runningPass; + + using (var scope = _fixture.BeginScope()) + runningPass = (await scope.Resolve().AcquireAsync(CredentialHealthService.PassLockKey, CancellationToken.None).ConfigureAwait(false)); + + var (skipped, took) = await TimedSweepAsync(deadlines).ConfigureAwait(false); + + took.ShouldBeLessThan(TimeSpan.FromSeconds(5), "a skipped tick returns at once; it does not wait for the pass it overlaps"); + skipped.ShouldBe(0); + silent.Connections.ShouldBe(0, "while a pass is running, a second one asks no provider anything"); + + await runningPass.DisposeAsync().ConfigureAwait(false); + + await TimedSweepAsync(deadlines).ConfigureAwait(false); + + silent.Connections.ShouldBe(1, "once the pass is over the next tick runs"); + } + + public enum ProbeIgnores { TheTokenWhileAwaiting, TheTokenWhileBlockingItsThread } + + // ─── Plumbing ────────────────────────────────────────────────────────────── + + private async Task<(int Probed, TimeSpan Took)> TimedSweepAsync(CredentialHealthDeadlines deadlines, IProviderRegistry? registry = null) + { + using var scope = _fixture.BeginScope(b => + { + b.RegisterInstance(deadlines).SingleInstance(); + + if (registry is not null) b.RegisterInstance(registry).As().SingleInstance(); + }); + + var clock = Stopwatch.StartNew(); + + // On a thread of its own, bounded: a sweep that ignores its deadlines must fail the test rather than hang the suite. + var sweep = Task.Run(() => scope.Resolve().Send(new ProbeCredentialsCommand { BatchSize = 25 })); + + if (await Task.WhenAny(sweep, Task.Delay(TimeSpan.FromSeconds(45))) != sweep) throw new ShouldAssertException("the sweep was still running after 45 seconds — nothing bounded it"); + + return (await sweep.ConfigureAwait(false), clock.Elapsed); + } + + /// A registry that answers the credential probe with for every provider. + private sealed class ProbeOnlyRegistry : IProviderRegistry + { + private readonly UncancellableProbe _probe; + + public ProbeOnlyRegistry(UncancellableProbe probe) { _probe = probe; } + + public TCapability Require(ProviderKind kind) where TCapability : IProviderCapability => (TCapability)(object)_probe; + + public bool TryGet(ProviderKind kind, out TCapability? capability) where TCapability : class, IProviderCapability + { + capability = _probe as TCapability; + + return capability is not null; + } + + public IReadOnlyList GetCapabilities(ProviderKind kind) => new[] { typeof(ICredentialProbeCapability) }; + } + + private sealed class ProbeCalls + { + private int _started; + + public int Started => Volatile.Read(ref _started); + + public void Start() => Interlocked.Increment(ref _started); + } + + /// + /// A provider call that takes no cancellation token, as an SDK built on an HTTP client with no per-call token does — and, in + /// the second shape, one that blocks the thread that calls it. A struct, not a class: the fixture registers every class + /// of this assembly that is a provider capability, for every test, and this one must exist only in the registry the test + /// hands the sweep. + /// + private readonly struct UncancellableProbe : ICredentialProbeCapability + { + private readonly ProbeIgnores _how; + private readonly ProbeCalls _calls; + + public UncancellableProbe(ProbeIgnores how, ProbeCalls calls) + { + _how = how; + _calls = calls; + } + + public ProviderKind Kind => ProviderKind.GitLab; + + public Task ProbeCredentialAsync(ProviderContext context, CancellationToken cancellationToken) + { + _calls.Start(); + + if (_how == ProbeIgnores.TheTokenWhileBlockingItsThread) Thread.Sleep(TimeSpan.FromSeconds(30)); + + return Task.Delay(TimeSpan.FromSeconds(30), CancellationToken.None).ContinueWith(_ => new CredentialProbeResult { IsValid = true }, TaskScheduler.Default); + } + } +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Credentials/CredentialHealthSweepFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Credentials/CredentialHealthSweepFlowTests.cs new file mode 100644 index 000000000..9a5939fbd --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Credentials/CredentialHealthSweepFlowTests.cs @@ -0,0 +1,352 @@ +using Autofac; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Credentials; +using CodeSpace.Core.Services.Providers; +using CodeSpace.Core.Services.Providers.Auth; +using CodeSpace.Core.Services.Providers.Identity; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.IntegrationTests.Workflows.Infrastructure; +using CodeSpace.Messages.Commands.Credentials; +using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Exceptions; +using MediatR; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit.Abstractions; + +namespace CodeSpace.IntegrationTests.Credentials; + +/// +/// 🟢 High fidelity — the recurring credential health probe, from the real ProbeCredentialsCommand through the real +/// GitLab provider (NGitLab and raw HTTP, with the credential in the header its type names) to the credential row, over real +/// Postgres. The one fake is GitLab, a that answers 401 to a token that was revoked or +/// reached its expiry, describes a token at personal_access_tokens/self, and honours OAuth grants as Doorkeeper does. +/// +/// What it pins: a personal, project or group token that expired or was revoked at GitLab is found without a run +/// having to fail on it — marked Error with a reason, so its next use fails fast and typed; one GitLab accepts is +/// validated and its real expiry recorded (a token linked before anything recorded one gets it now); an answer that is not +/// an answer — GitLab unreachable — changes nothing; and a credential marked Error is asked again only after an hour, healing +/// by itself if GitLab accepts it again. +/// +/// The sweep is deployment-wide, so each test says which credentials are its own and records every other as validated +/// (): a test asserts the rows it owns, never the tally. +/// +[Collection(PostgresCollection.Name)] +[Trait("Category", "Integration")] +public class CredentialHealthSweepFlowTests +{ + private readonly PostgresFixture _fixture; + private readonly ITestOutputHelper _output; + + public CredentialHealthSweepFlowTests(PostgresFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + } + + [Fact] + public async Task A_token_gitlab_accepts_is_validated_with_its_real_expiry_and_one_it_refuses_is_marked_and_fails_fast_afterwards() + { + using var gitlab = new GitLabDoorkeeperStub(); + var expires = DateTimeOffset.UtcNow.AddDays(10); + gitlab.AddAccessToken("glpat-live", "nacho", expires); + gitlab.AddAccessToken("glpat-revoked", "nacho"); + gitlab.RevokeAccessToken("glpat-revoked"); + var live = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, "glpat-live").ConfigureAwait(false); + var recordedExpiry = AsStored(DateTimeOffset.UtcNow.AddDays(20)); + var revoked = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, "glpat-revoked", expiresDate: recordedExpiry).ConfigureAwait(false); + await GitLabCredentialSeed.QuietEveryOtherCredentialAsync(_fixture, live.CredentialId, revoked.CredentialId).ConfigureAwait(false); + + await SweepAsync().ConfigureAwait(false); + + Dump(gitlab); + var liveRow = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, live.CredentialId).ConfigureAwait(false); + + liveRow.Status.ShouldBe(CredentialStatus.Active); + liveRow.LastValidatedDate.ShouldNotBeNull().ShouldBeGreaterThan(DateTimeOffset.UtcNow.AddMinutes(-2)); + liveRow.ExpiresDate.ShouldBe(new DateTimeOffset(expires.UtcDateTime.Date, TimeSpan.Zero), "GitLab's expires_at is a date; the row records the instant it begins"); + liveRow.LastError.ShouldBeNull(); + + var revokedRow = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, revoked.CredentialId).ConfigureAwait(false); + + revokedRow.Status.ShouldBe(CredentialStatus.Error); + revokedRow.LastError.ShouldBe("token rejected by GitLab"); + revokedRow.ExpiresDate.ShouldBe(recordedExpiry, "a verdict does not rewrite what was recorded about the token's expiry"); + revokedRow.LastValidatedDate.ShouldBeNull(); + + var asked = gitlab.Log.Count; + var failure = await Should.ThrowAsync(() => ResolveAsync(revoked.RepositoryId)).ConfigureAwait(false); + + failure.CredentialId.ShouldBe(revoked.CredentialId); + failure.Message.ShouldContain("token rejected by GitLab"); + gitlab.Log.Count.ShouldBe(asked, "the failure comes from what the row says: GitLab is not asked again"); + (await ResolveAsync(live.RepositoryId).ConfigureAwait(false)).Token.ShouldBe("glpat-live", "an accepted token resolves as it always did"); + } + + [Fact] + public async Task A_probe_that_gets_no_answer_changes_nothing() + { + // GitLab is unreachable: nothing is known about the token, so nothing is recorded — least of all that it is dead. + var gitlab = new GitLabDoorkeeperStub(); + var seed = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, "glpat-any").ConfigureAwait(false); + gitlab.Dispose(); + await GitLabCredentialSeed.QuietEveryOtherCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + await SweepAsync().ConfigureAwait(false); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + row.Status.ShouldBe(CredentialStatus.Active); + row.LastValidatedDate.ShouldBeNull("it is still due: the next tick asks again"); + row.LastError.ShouldBeNull(); + } + + [Fact] + public async Task A_pass_is_bounded_by_its_batch_and_the_next_one_takes_up_where_the_validated_left_off() + { + using var gitlab = new GitLabDoorkeeperStub(); + var seeds = new List(); + + for (var i = 0; i < 3; i++) + { + gitlab.AddAccessToken($"glpat-batch-{i}", "nacho"); + seeds.Add(await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, $"glpat-batch-{i}").ConfigureAwait(false)); + } + + await GitLabCredentialSeed.QuietEveryOtherCredentialAsync(_fixture, seeds.Select(s => s.CredentialId).ToArray()).ConfigureAwait(false); + + await SweepAsync(batchSize: 2).ConfigureAwait(false); + (await CountValidatedAsync(seeds).ConfigureAwait(false)).ShouldBe(2, "a pass probes at most its batch"); + + await SweepAsync(batchSize: 2).ConfigureAwait(false); + (await CountValidatedAsync(seeds).ConfigureAwait(false)).ShouldBe(3, "a validated credential is not a candidate, so the next pass reaches the one left"); + } + + [Fact] + public async Task A_credential_validated_recently_is_not_asked_about_and_one_validated_long_ago_is() + { + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.AddAccessToken("glpat-recent", "nacho"); + gitlab.AddAccessToken("glpat-stale", "nacho"); + var recent = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, "glpat-recent").ConfigureAwait(false); + var stale = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, "glpat-stale").ConfigureAwait(false); + var validatedHourAgo = AsStored(DateTimeOffset.UtcNow.AddHours(-1)); + await GitLabCredentialSeed.SetLastValidatedAsync(_fixture, recent.CredentialId, validatedHourAgo).ConfigureAwait(false); + await GitLabCredentialSeed.SetLastValidatedAsync(_fixture, stale.CredentialId, DateTimeOffset.UtcNow.AddHours(-7)).ConfigureAwait(false); + await GitLabCredentialSeed.QuietEveryOtherCredentialAsync(_fixture, recent.CredentialId, stale.CredentialId).ConfigureAwait(false); + await GitLabCredentialSeed.SetLastValidatedAsync(_fixture, recent.CredentialId, validatedHourAgo).ConfigureAwait(false); + await GitLabCredentialSeed.SetLastValidatedAsync(_fixture, stale.CredentialId, DateTimeOffset.UtcNow.AddHours(-7)).ConfigureAwait(false); + + await SweepAsync().ConfigureAwait(false); + + (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, recent.CredentialId).ConfigureAwait(false)).LastValidatedDate.ShouldBe(validatedHourAgo, "validated within the last six hours: GitLab is not asked"); + (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, stale.CredentialId).ConfigureAwait(false)).LastValidatedDate.ShouldNotBeNull().ShouldBeGreaterThan(DateTimeOffset.UtcNow.AddMinutes(-2)); + gitlab.Count(StubEvent.TokenApi200).ShouldBe(2, "one user call and one self call, for the stale credential only"); + } + + [Fact] + public async Task A_credential_marked_Error_is_asked_again_only_after_an_hour_and_heals_when_gitlab_accepts_it() + { + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.AddAccessToken("glpat-back", "nacho"); + var seed = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, "glpat-back").ConfigureAwait(false); + await MarkErrorAsync(seed.CredentialId, "token rejected by GitLab").ConfigureAwait(false); + await GitLabCredentialSeed.QuietEveryOtherCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + await SweepAsync().ConfigureAwait(false); + + (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).Status.ShouldBe(CredentialStatus.Error, "marked a moment ago: not due"); + gitlab.Count(StubEvent.TokenApi200).ShouldBe(0, "GitLab was not asked"); + + await GitLabCredentialSeed.SetLastModifiedAsync(_fixture, seed.CredentialId, DateTimeOffset.UtcNow.AddHours(-2)).ConfigureAwait(false); + + await SweepAsync().ConfigureAwait(false); + + var healed = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + healed.Status.ShouldBe(CredentialStatus.Active, "GitLab accepts the token again — an account unblocked, an expiry extended — so the Error is lifted"); + healed.LastError.ShouldBeNull(); + healed.LastValidatedDate.ShouldNotBeNull(); + (await ResolveAsync(seed.RepositoryId).ConfigureAwait(false)).Token.ShouldBe("glpat-back", "and the token is usable again"); + } + + [Fact] + public async Task A_credential_still_refused_after_an_hour_keeps_its_first_reason_and_waits_another_hour() + { + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.AddAccessToken("glpat-gone", "nacho"); + gitlab.RevokeAccessToken("glpat-gone"); + var seed = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, "glpat-gone").ConfigureAwait(false); + await MarkErrorAsync(seed.CredentialId, "needs reconnect: invalid_grant").ConfigureAwait(false); + await GitLabCredentialSeed.SetLastModifiedAsync(_fixture, seed.CredentialId, DateTimeOffset.UtcNow.AddHours(-2)).ConfigureAwait(false); + await GitLabCredentialSeed.QuietEveryOtherCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + await SweepAsync().ConfigureAwait(false); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + row.Status.ShouldBe(CredentialStatus.Error); + row.LastError.ShouldBe("needs reconnect: invalid_grant", "the first reason is the most specific one"); + row.LastModifiedDate.ShouldBeGreaterThan(DateTimeOffset.UtcNow.AddMinutes(-2), "the confirmation is dated, which is what the next recheck is measured from"); + } + + [Fact] + public async Task A_tokens_expiry_is_recorded_when_it_was_unknown_and_cleared_when_gitlab_says_it_never_expires() + { + using var gitlab = new GitLabDoorkeeperStub(); + var expires = DateTimeOffset.UtcNow.AddDays(10); + gitlab.AddAccessToken("glpat-dated", "nacho", expires); + gitlab.AddAccessToken("glpat-forever", "nacho"); + var dated = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, "glpat-dated", AuthType.ProjectAccessToken).ConfigureAwait(false); + var forever = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, "glpat-forever", AuthType.GroupAccessToken, expiresDate: DateTimeOffset.UtcNow.AddDays(5)).ConfigureAwait(false); + await GitLabCredentialSeed.QuietEveryOtherCredentialAsync(_fixture, dated.CredentialId, forever.CredentialId).ConfigureAwait(false); + + await SweepAsync().ConfigureAwait(false); + + (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, dated.CredentialId).ConfigureAwait(false)).ExpiresDate.ShouldBe(new DateTimeOffset(expires.UtcDateTime.Date, TimeSpan.Zero), "a token linked before anything recorded its expiry gets it from the sweep"); + (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, forever.CredentialId).ConfigureAwait(false)).ExpiresDate.ShouldBeNull("GitLab described the token and gave it no expiry: what was recorded before is not true"); + } + + [Fact] + public async Task An_oauth_credential_is_validated_without_touching_its_access_tokens_expiry_and_a_dead_grant_is_marked_by_the_refusal_itself() + { + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.Seed(GitLabDoorkeeperStub.DefaultClientId, "nacho", "at-0", "rt-0"); + var live = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: false).ConfigureAwait(false); + var dead = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-dead", "rt-dead").ConfigureAwait(false); + var accessTokenExpiry = (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, live.CredentialId).ConfigureAwait(false)).ExpiresDate; + await GitLabCredentialSeed.QuietEveryOtherCredentialAsync(_fixture, live.CredentialId, dead.CredentialId).ConfigureAwait(false); + + await SweepAsync().ConfigureAwait(false); + + Dump(gitlab); + var liveRow = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, live.CredentialId).ConfigureAwait(false); + + liveRow.Status.ShouldBe(CredentialStatus.Active); + liveRow.LastValidatedDate.ShouldNotBeNull(); + liveRow.ExpiresDate.ShouldBe(accessTokenExpiry, "an OAuth credential's expiry is its access token's, which the refresher writes"); + + var deadRow = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, dead.CredentialId).ConfigureAwait(false); + + deadRow.Status.ShouldBe(CredentialStatus.Error); + deadRow.LastError.ShouldBe("needs reconnect: invalid_grant", "the refusal that found it is what the row says, not the probe's generic reason"); + } + + [Fact] + public async Task A_disconnected_or_retired_credential_is_never_probed() + { + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.AddAccessToken("glpat-disconnected", "nacho"); + gitlab.AddAccessToken("glpat-retired", "nacho"); + var disconnected = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, "glpat-disconnected").ConfigureAwait(false); + var retired = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, "glpat-retired").ConfigureAwait(false); + + using (var scope = _fixture.BeginScope()) + { + var db = scope.Resolve(); + + await db.Database.ExecuteSqlInterpolatedAsync($"UPDATE credential SET status = 'Revoked', encrypted_payload = '' WHERE id = {disconnected.CredentialId}").ConfigureAwait(false); + await db.Database.ExecuteSqlInterpolatedAsync($"UPDATE credential SET deleted_date = now() WHERE id = {retired.CredentialId}").ConfigureAwait(false); + } + + await GitLabCredentialSeed.QuietEveryOtherCredentialAsync(_fixture, disconnected.CredentialId, retired.CredentialId).ConfigureAwait(false); + + await SweepAsync().ConfigureAwait(false); + + gitlab.Log.ShouldBeEmpty("neither has a token to present, so GitLab was never asked"); + (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, disconnected.CredentialId).ConfigureAwait(false)).Status.ShouldBe(CredentialStatus.Revoked); + } + + [Fact] + public async Task A_verdict_is_written_only_over_the_payload_it_judged_and_never_into_a_disconnected_row() + { + using var gitlab = new GitLabDoorkeeperStub(); + var seed = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, "glpat-judged").ConfigureAwait(false); + var judged = (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).EncryptedPayload; + + using var scope = _fixture.BeginScope(); + var writer = scope.Resolve(); + + (await writer.MarkRejectedAsync(seed.CredentialId, "a payload the row no longer holds", "token rejected by GitLab", CancellationToken.None).ConfigureAwait(false)).ShouldBeFalse("a reconnect replaced the row since the verdict was judged"); + (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).Status.ShouldBe(CredentialStatus.Active); + + (await writer.MarkRejectedAsync(seed.CredentialId, judged, "token rejected by GitLab", CancellationToken.None).ConfigureAwait(false)).ShouldBeTrue(); + (await writer.MarkRejectedAsync(seed.CredentialId, judged, "a later, vaguer reason", CancellationToken.None).ConfigureAwait(false)).ShouldBeTrue(); + (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).LastError.ShouldBe("token rejected by GitLab", "the first reason stays"); + + (await writer.MarkValidatedAsync(seed.CredentialId, judged, DateTimeOffset.UtcNow, expiry: null, CancellationToken.None).ConfigureAwait(false)).ShouldBeTrue(); + + var healed = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + healed.Status.ShouldBe(CredentialStatus.Active); + healed.LastError.ShouldBeNull(); + + await _fixture.BeginScope().Resolve().Database.ExecuteSqlInterpolatedAsync($"UPDATE credential SET status = 'Revoked', encrypted_payload = '' WHERE id = {seed.CredentialId}").ConfigureAwait(false); + + (await writer.MarkRejectedAsync(seed.CredentialId, string.Empty, "token rejected by GitLab", CancellationToken.None).ConfigureAwait(false)).ShouldBeFalse("a disconnected row is not this verdict's to change"); + (await writer.MarkValidatedAsync(seed.CredentialId, string.Empty, DateTimeOffset.UtcNow, expiry: null, CancellationToken.None).ConfigureAwait(false)).ShouldBeFalse(); + (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).Status.ShouldBe(CredentialStatus.Revoked); + } + + [Fact] + public async Task Linking_a_token_records_when_gitlab_says_it_expires_and_that_it_was_just_validated() + { + using var gitlab = new GitLabDoorkeeperStub(); + var expires = DateTimeOffset.UtcNow.AddDays(3); + gitlab.AddAccessToken("glpat-linked", "nacho", expires); + var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture).ConfigureAwait(false); + var seed = await GitLabCredentialSeed.SeedAccessTokenAsync(_fixture, gitlab.BaseUrl, "glpat-other", existingTeam: team).ConfigureAwait(false); + + using var scope = _fixture.BeginScopeAs(team.UserId, team.TeamId, CodeSpace.Messages.Constants.Roles.Admin); + var identity = await scope.Resolve().LinkByPatAsync(seed.InstanceId, "glpat-linked", CancellationToken.None).ConfigureAwait(false); + + var credential = await scope.Resolve().UserProviderIdentity.AsNoTracking().Where(i => i.Id == identity.Id).Select(i => i.Credential).SingleAsync().ConfigureAwait(false); + + credential.ExpiresDate.ShouldBe(new DateTimeOffset(expires.UtcDateTime.Date, TimeSpan.Zero)); + credential.LastValidatedDate.ShouldNotBeNull().ShouldBeGreaterThan(DateTimeOffset.UtcNow.AddMinutes(-2)); + credential.Status.ShouldBe(CredentialStatus.Active); + } + + // ─── Plumbing ────────────────────────────────────────────────────────────── + + private async Task SweepAsync(int batchSize = 25) + { + using var scope = _fixture.BeginScope(); + + return await scope.Resolve().Send(new ProbeCredentialsCommand { BatchSize = batchSize }).ConfigureAwait(false); + } + + private async Task ResolveAsync(Guid repositoryId) + { + using var scope = _fixture.BeginScope(); + var repo = await CredentialRefreshHarness.LoadUntrackedAsync(scope, repositoryId).ConfigureAwait(false); + + return await scope.Resolve().ResolveAsync(new ProviderContext(repo.ProviderInstance, repo.Credential!), CancellationToken.None).ConfigureAwait(false); + } + + private async Task CountValidatedAsync(IEnumerable seeds) + { + var validated = 0; + + foreach (var seed in seeds) + if ((await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).LastValidatedDate is { } at && at > DateTimeOffset.UtcNow.AddMinutes(-2)) validated++; + + return validated; + } + + private async Task MarkErrorAsync(Guid credentialId, string lastError) + { + using var scope = _fixture.BeginScope(); + + await scope.Resolve().Database.ExecuteSqlInterpolatedAsync($"UPDATE credential SET status = 'Error', last_error = {lastError}, last_modified_date = now() WHERE id = {credentialId}").ConfigureAwait(false); + } + + private void Dump(GitLabDoorkeeperStub gitlab) + { + foreach (var entry in gitlab.Log) _output.WriteLine($" {entry.At:HH:mm:ss.fff} {entry.Kind}: {entry.Detail}"); + } + + /// A timestamp as Postgres stores it: timestamptz keeps microseconds, a .NET clock on Linux reads 100ns ticks, so an expected value must be cut to what the row can hold. + private static DateTimeOffset AsStored(DateTimeOffset value) => new(value.Ticks - value.Ticks % TimeSpan.TicksPerMicrosecond, value.Offset); +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Credentials/DeadGrantFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Credentials/DeadGrantFlowTests.cs new file mode 100644 index 000000000..5ad2fde57 --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Credentials/DeadGrantFlowTests.cs @@ -0,0 +1,201 @@ +using Autofac; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Services.Credentials; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Failures; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit.Abstractions; + +namespace CodeSpace.IntegrationTests.Credentials; + +/// +/// 🟢 High fidelity — an OAuth grant GitLab refuses for good, from the token request to the credential row. The real auth +/// resolver → OAuth strategy → refresher → Postgres advisory lock → the real GitLabOAuthClient → health writer, over +/// real Postgres; the one fake is GitLab, a whose refresh tokens are single use and which +/// refuses a refresh token it never issued, or one presented by another OAuth application, as Doorkeeper does. +/// +/// The shape this pins is the invisible one: the stored pair is dead, every run presents it and is refused with the +/// provider's invalid_grant, the credential stays Active and nothing says why — and a run that fails with a string +/// nobody can link to anything. The refusal is now the provider's verdict: the credential is marked with it, the failure is +/// typed (credential_needs_reconnect, naming the credential), and the next use does not ask GitLab again. +/// +[Collection(PostgresCollection.Name)] +[Trait("Category", "Integration")] +public class DeadGrantFlowTests +{ + private const string NeedsReconnect = "credential_needs_reconnect"; + + private readonly PostgresFixture _fixture; + private readonly ITestOutputHelper _output; + + public DeadGrantFlowTests(PostgresFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + } + + [Fact] + public async Task B2_a_dead_refresh_token_is_a_typed_failure_the_credential_is_marked_and_the_next_use_does_not_ask_gitlab_again() + { + using var gitlab = new GitLabDoorkeeperStub(); + // GitLab never issued this pair — the shape of a pair that was rotated away and never stored. + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-dead", "rt-dead").ConfigureAwait(false); + + var first = await Should.ThrowAsync(() => CredentialRefreshHarness.ResolveTokenAsync(_fixture, seed.RepositoryId)).ConfigureAwait(false); + + Dump(gitlab); + ((IFailure)first).Code.ShouldBe(NeedsReconnect, $"a refused grant is told apart from a provider fault — {first.GetType().Name}: {first.Message}"); + first.Message.ShouldContain(seed.CredentialId.ToString(), customMessage: "an operator reading a run's error must be able to tell which credential it is"); + first.Message.ShouldContain("GitLab · NACHO.L"); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + row.Status.ShouldBe(CredentialStatus.Error, "the row says what the provider said, instead of staying Active with nothing to show"); + row.LastError.ShouldBe("needs reconnect: invalid_grant"); + + var second = await Should.ThrowAsync(() => CredentialRefreshHarness.ResolveTokenAsync(_fixture, seed.RepositoryId)).ConfigureAwait(false); + + ((IFailure)second).Code.ShouldBe(NeedsReconnect); + gitlab.Count(StubEvent.Refused).ShouldBe(1, "the second use failed from what the row says; presenting the dead refresh token again would only earn the same refusal"); + } + + [Theory] + [InlineData("app2", "secret2", "invalid_grant")] // the instance's OAuth client id was changed: the grant belongs to the old application + [InlineData("app1", "not-the-secret", "invalid_client")] // the secret was changed: the client no longer authenticates + public async Task S8_an_edited_oauth_client_is_classified_as_what_the_provider_called_it(string clientId, string clientSecret, string oauthError) + { + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.AddApplication("app2", "secret2"); + gitlab.Seed(GitLabDoorkeeperStub.DefaultClientId, "nacho", "at-0", "rt-0"); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0").ConfigureAwait(false); + + await EditInstanceClientAsync(seed.InstanceId, clientId, clientSecret).ConfigureAwait(false); + + var failure = await Should.ThrowAsync(() => CredentialRefreshHarness.ResolveTokenAsync(_fixture, seed.RepositoryId)).ConfigureAwait(false); + + Dump(gitlab); + ((IFailure)failure).Code.ShouldBe(NeedsReconnect, $"{failure.GetType().Name}: {failure.Message}"); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + row.Status.ShouldBe(CredentialStatus.Error); + row.LastError.ShouldBe($"needs reconnect: {oauthError}"); + gitlab.IsRefreshTokenLive("rt-0").ShouldBeTrue("the refused attempts consumed nothing: the grant itself is intact, which is what lets a reverted edit heal it"); + } + + [Fact] + public async Task S8_a_reverted_client_edit_heals_the_credential_on_its_next_use_once_the_recheck_window_has_passed() + { + // The admin put the client id back. Nothing about the grant was ever wrong — GitLab consumed nothing — so the credential + // must not stay "needs reconnect" until the sweep's next pass: a run that finds the window passed asks again, and the + // refresh GitLab accepts lifts the verdict. Inside the window it still fails from what the row says. + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.AddApplication("app2", "secret2"); + gitlab.Seed(GitLabDoorkeeperStub.DefaultClientId, "nacho", "at-0", "rt-0"); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0").ConfigureAwait(false); + + await EditInstanceClientAsync(seed.InstanceId, "app2", "secret2").ConfigureAwait(false); + await Should.ThrowAsync(() => CredentialRefreshHarness.ResolveTokenAsync(_fixture, seed.RepositoryId)).ConfigureAwait(false); + + await EditInstanceClientAsync(seed.InstanceId, GitLabDoorkeeperStub.DefaultClientId, GitLabDoorkeeperStub.DefaultClientSecret).ConfigureAwait(false); + + var inside = await Should.ThrowAsync(() => CredentialRefreshHarness.ResolveTokenAsync(_fixture, seed.RepositoryId)).ConfigureAwait(false); + + ((IFailure)inside).Code.ShouldBe(NeedsReconnect, "marked a moment ago: not asked again yet"); + gitlab.Count(StubEvent.Refused).ShouldBe(1, "fixture check: GitLab was asked once, by the first use"); + + await GitLabCredentialSeed.SetLastModifiedAsync(_fixture, seed.CredentialId, DateTimeOffset.UtcNow.AddHours(-2)).ConfigureAwait(false); + + var token = await CredentialRefreshHarness.ResolveTokenAsync(_fixture, seed.RepositoryId).ConfigureAwait(false); + + Dump(gitlab); + token.ShouldBe("at-2", "the use after the window asked GitLab, which accepted the grant"); + + var healed = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + healed.Status.ShouldBe(CredentialStatus.Active, "the pair GitLab issued lifts the verdict"); + healed.LastError.ShouldBeNull(); + } + + [Fact] + public async Task A_credential_still_refused_after_the_window_costs_one_refused_call_and_waits_another_window() + { + using var gitlab = new GitLabDoorkeeperStub(); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-dead", "rt-dead").ConfigureAwait(false); + + await Should.ThrowAsync(() => CredentialRefreshHarness.ResolveTokenAsync(_fixture, seed.RepositoryId)).ConfigureAwait(false); + await GitLabCredentialSeed.SetLastModifiedAsync(_fixture, seed.CredentialId, DateTimeOffset.UtcNow.AddHours(-2)).ConfigureAwait(false); + + var afterTheWindow = await Should.ThrowAsync(() => CredentialRefreshHarness.ResolveTokenAsync(_fixture, seed.RepositoryId)).ConfigureAwait(false); + + ((IFailure)afterTheWindow).Code.ShouldBe(NeedsReconnect, "still refused: the same typed failure, not a provider fault"); + gitlab.Count(StubEvent.Refused).ShouldBe(2, "the use after the window asked GitLab once more"); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + row.Status.ShouldBe(CredentialStatus.Error); + row.LastError.ShouldBe("needs reconnect: invalid_grant"); + row.LastModifiedDate.ShouldBeGreaterThan(DateTimeOffset.UtcNow.AddMinutes(-2), "the confirmation dates the verdict afresh, which is what the next window is measured from"); + + await Should.ThrowAsync(() => CredentialRefreshHarness.ResolveTokenAsync(_fixture, seed.RepositoryId)).ConfigureAwait(false); + + gitlab.Count(StubEvent.Refused).ShouldBe(2, "inside the new window it fails from what the row says"); + } + + [Fact] + public async Task A_refresh_that_fails_without_a_verdict_leaves_the_credential_as_it_was() + { + // GitLab is down: no OAuth answer at all, so nothing is known about the grant and nothing is recorded about it. + var gitlab = new GitLabDoorkeeperStub(); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0").ConfigureAwait(false); + gitlab.Dispose(); + + await Should.ThrowAsync(() => CredentialRefreshHarness.ResolveTokenAsync(_fixture, seed.RepositoryId)).ConfigureAwait(false); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + row.Status.ShouldBe(CredentialStatus.Active); + row.LastError.ShouldBeNull(); + } + + [Fact] + public async Task A_credential_disconnected_while_it_was_being_refused_is_not_marked() + { + // The verdict is written against the row it was judged on: a Disconnect that blanked the row meanwhile is not a + // credential to put an Error on. + using var gitlab = new GitLabDoorkeeperStub(); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-dead", "rt-dead").ConfigureAwait(false); + + using (var scope = _fixture.BeginScope()) + await scope.Resolve().Database.ExecuteSqlInterpolatedAsync($"UPDATE credential SET status = 'Revoked', encrypted_payload = '' WHERE id = {seed.CredentialId}").ConfigureAwait(false); + + var failure = await Should.ThrowAsync(() => CredentialRefreshHarness.ResolveTokenAsync(_fixture, seed.RepositoryId)).ConfigureAwait(false); + + ((IFailure)failure).Code.ShouldBe(FailureCodes.CredentialDisconnected); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + row.Status.ShouldBe(CredentialStatus.Revoked); + row.LastError.ShouldBeNull(); + } + + private async Task EditInstanceClientAsync(Guid instanceId, string clientId, string clientSecret) + { + using var scope = _fixture.BeginScope(); + var db = scope.Resolve(); + var instance = await db.ProviderInstance.SingleAsync(i => i.Id == instanceId).ConfigureAwait(false); + + instance.OauthClientId = clientId; + instance.OauthClientSecretEnc = scope.Resolve().Encrypt(clientSecret); + await db.SaveChangesAsync().ConfigureAwait(false); + } + + private void Dump(GitLabDoorkeeperStub gitlab) + { + _output.WriteLine($"GitLab saw: rotated={gitlab.Count(StubEvent.Rotated)} refused={gitlab.Count(StubEvent.Refused)}"); + + foreach (var entry in gitlab.Log) _output.WriteLine($" {entry.At:HH:mm:ss.fff} {entry.Kind}: {entry.Detail}"); + } +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Credentials/MrBurstFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Credentials/MrBurstFlowTests.cs index 6f5cb2847..070c9fc7b 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Credentials/MrBurstFlowTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Credentials/MrBurstFlowTests.cs @@ -113,6 +113,112 @@ public async Task S11_a_delivery_whose_standing_lookup_gives_up_mid_refresh_stil (await CountRunsAsync(world.TeamId, world.WorkflowId).ConfigureAwait(false)).ShouldBeGreaterThan(0, "the next delivery's standing lookup got an answer, so a members-only activation admitted its author"); } + // ─── What a failed standing lookup says ─────────────────────────────────────── + + [Fact] + public async Task B2_a_dead_refresh_token_refuses_a_members_only_trigger_as_a_rejected_credential_not_as_a_stranger_while_an_open_trigger_still_starts() + { + // GitLab never issued the pair the row holds. Before, the dead grant read as "author whose standing is unknown": a + // members-only activation refused the author as not a member, with no log line, while the credential stayed Active. + using var gitlab = new GitLabDoorkeeperStub(); + var world = await SeedWorldAsync(gitlab, gitLabKnowsThePair: false, siblingAuthors: "any").ConfigureAwait(false); + + await DeliverAsync(world, MergeRequestHook("reopen", iid: 8, headSha: "head-1")).ConfigureAwait(false); + + Dump(gitlab); + (await RefusalsAsync(world, "credential_rejected").ConfigureAwait(false)).ShouldHaveSingleItem("the members-only activation says its credential was refused").ShouldContain("GitLab · NACHO.L"); + (await RefusalsAsync(world, "author_not_member").ConfigureAwait(false)).ShouldBeEmpty("nothing is known about the author, so no one was refused as a stranger"); + (await CountRunsAsync(world.TeamId, world.WorkflowId).ConfigureAwait(false)).ShouldBe(0); + (await CountRunsAsync(world.TeamId, world.SiblingWorkflowId).ConfigureAwait(false)).ShouldBe(1, "an activation that admits anyone does not need the author's standing, so a dead credential does not stop it"); + gitlab.Count(StubEvent.Refused).ShouldBe(1, "one refusal for the whole delivery: the second activation reuses the answer"); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, world.Credential.CredentialId).ConfigureAwait(false); + + row.Status.ShouldBe(CredentialStatus.Error); + row.LastError.ShouldBe("needs reconnect: invalid_grant"); + + await DeliverAsync(world, MergeRequestHook("reopen", iid: 8, headSha: "head-2")).ConfigureAwait(false); + + gitlab.Count(StubEvent.Refused).ShouldBe(1, "the next delivery fails from what the row says and does not present the dead refresh token again"); + (await RefusalsAsync(world, "credential_rejected").ConfigureAwait(false)).ShouldHaveSingleItem("refusals are recorded once per activation and pull request a day"); + } + + [Fact] + public async Task A_standing_lookup_refused_with_403_names_the_status_and_the_scope_it_needs_and_leaves_the_credential_alone() + { + using var gitlab = new GitLabDoorkeeperStub(); + var world = await SeedWorldAsync(gitlab, siblingAuthors: "any").ConfigureAwait(false); + gitlab.FailMembers(403); + + await DeliverAsync(world, MergeRequestHook("reopen", iid: 9, headSha: "head-1")).ConfigureAwait(false); + + Dump(gitlab); + var refusal = (await RefusalsAsync(world, "author_standing_unknown").ConfigureAwait(false)).ShouldHaveSingleItem("the lookup failed, so the refusal says that"); + + refusal.ShouldContain("403"); + refusal.ShouldContain("read_api"); + (await RefusalsAsync(world, "author_not_member").ConfigureAwait(false)).ShouldBeEmpty("the author was not found to be anyone's stranger"); + (await RefusalsAsync(world, "credential_rejected").ConfigureAwait(false)).ShouldBeEmpty("a 403 is a missing scope or role, not a dead token"); + (await CountRunsAsync(world.TeamId, world.SiblingWorkflowId).ConfigureAwait(false)).ShouldBe(1); + gitlab.Count(StubEvent.MembersFailed).ShouldBe(1, "a 403 is not asked again"); + (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, world.Credential.CredentialId).ConfigureAwait(false)).Status.ShouldBe(CredentialStatus.Active); + } + + [Fact] + public async Task A_standing_lookup_that_fails_once_with_a_server_error_is_asked_again_and_the_author_is_admitted() + { + using var gitlab = new GitLabDoorkeeperStub(); + var world = await SeedWorldAsync(gitlab).ConfigureAwait(false); + gitlab.FailMembers(500, times: 1); + + await DeliverAsync(world, MergeRequestHook("reopen", iid: 10, headSha: "head-1")).ConfigureAwait(false); + + Dump(gitlab); + gitlab.Count(StubEvent.MembersFailed).ShouldBe(1); + gitlab.Count(StubEvent.Api200).ShouldBe(1, "the lookup that failed is not an answer; the second ask was one"); + (await RefusalsAsync(world, "author_standing_unknown").ConfigureAwait(false)).ShouldBeEmpty(); + (await RefusalsAsync(world, "author_not_member").ConfigureAwait(false)).ShouldBeEmpty(); + (await CountRunsAsync(world.TeamId, world.WorkflowId).ConfigureAwait(false)).ShouldBe(1, "one 500 no longer costs the author's own merge request its run"); + (await CountRunsAsync(world.TeamId, world.SiblingWorkflowId).ConfigureAwait(false)).ShouldBe(1); + } + + [Fact] + public async Task A_standing_lookup_that_keeps_failing_is_asked_twice_and_refused_as_unknown_with_the_status() + { + using var gitlab = new GitLabDoorkeeperStub(); + var world = await SeedWorldAsync(gitlab).ConfigureAwait(false); + gitlab.FailMembers(503); + + await DeliverAsync(world, MergeRequestHook("reopen", iid: 11, headSha: "head-1")).ConfigureAwait(false); + + gitlab.Count(StubEvent.MembersFailed).ShouldBe(2, "asked, then asked once more, then given up on"); + var refusals = await RefusalsAsync(world, "author_standing_unknown").ConfigureAwait(false); + + refusals.Count.ShouldBe(2, "one refusal row per members-only activation"); + refusals.ShouldAllBe(refusal => refusal.Contains("503"), "each names the status"); + (await RefusalsAsync(world, "author_not_member").ConfigureAwait(false)).ShouldBeEmpty(); + (await CountRunsAsync(world.TeamId, world.WorkflowId).ConfigureAwait(false)).ShouldBe(0, "Members still means members: a standing nobody could read is not admitted"); + } + + [Fact] + public async Task A_repository_with_no_credential_to_ask_with_is_refused_as_unknown_not_as_a_stranger() + { + using var gitlab = new GitLabDoorkeeperStub(); + var world = await SeedWorldAsync(gitlab).ConfigureAwait(false); + + using (var scope = _fixture.BeginScope()) + await scope.Resolve().Database.ExecuteSqlInterpolatedAsync($"UPDATE repository SET credential_id = NULL WHERE id = {world.Credential.RepositoryId}").ConfigureAwait(false); + + await DeliverAsync(world, MergeRequestHook("reopen", iid: 12, headSha: "head-1")).ConfigureAwait(false); + + var refusals = await RefusalsAsync(world, "author_standing_unknown").ConfigureAwait(false); + + refusals.Count.ShouldBe(2); + refusals.ShouldAllBe(refusal => refusal.Contains("no credential")); + (await RefusalsAsync(world, "author_not_member").ConfigureAwait(false)).ShouldBeEmpty(); + gitlab.Log.ShouldBeEmpty("there was nothing to ask GitLab with"); + } + // ─── Deliveries ──────────────────────────────────────────────────────────── /// GitLab's Merge Request Hook for MR , authored by user 51 on an internal project. No oldrev: the head did not move. @@ -143,7 +249,7 @@ await scope.Resolve().Send(new ReceiveWebhookCommand private void Dump(GitLabDoorkeeperStub gitlab) { - _output.WriteLine($"GitLab saw: rotated={gitlab.Count(StubEvent.Rotated)} refused={gitlab.Count(StubEvent.Refused)} api-200={gitlab.Count(StubEvent.Api200)} api-401={gitlab.Count(StubEvent.Api401)}"); + _output.WriteLine($"GitLab saw: rotated={gitlab.Count(StubEvent.Rotated)} refused={gitlab.Count(StubEvent.Refused)} api-200={gitlab.Count(StubEvent.Api200)} api-401={gitlab.Count(StubEvent.Api401)} members-failed={gitlab.Count(StubEvent.MembersFailed)}"); foreach (var entry in gitlab.Log) _output.WriteLine($" {entry.At:HH:mm:ss.fff} {entry.Kind}: {entry.Detail}"); } @@ -163,6 +269,15 @@ private async Task CountRunsAsync(Guid teamId, Guid workflowId) return await scope.Resolve().WorkflowRun.AsNoTracking().CountAsync(r => r.TeamId == teamId && r.WorkflowId == workflowId).ConfigureAwait(false); } + /// The detail of each Rejected request of the team whose reason is , as the audit view's filter reads it: the reason, a colon, the free text. + private async Task> RefusalsAsync(World world, string reason) + { + using var scope = _fixture.BeginScope(); + + return await scope.Resolve().WorkflowRunRequest.AsNoTracking() + .Where(r => r.TeamId == world.TeamId && r.Status == WorkflowRunRequestStatus.Rejected && r.Error!.StartsWith(reason + ":")).Select(r => r.Error!).ToListAsync().ConfigureAwait(false); + } + private async Task DebouncedRefusalsAsync(World world) { using var scope = _fixture.BeginScope(); @@ -183,9 +298,9 @@ private sealed record World(SeededGitLabCredential Credential, Guid WorkflowId, /// workflows whose pull-request triggers admit only members, and the repository's webhook. GitLab knows the pair the /// row holds. /// - private async Task SeedWorldAsync(GitLabDoorkeeperStub gitlab) + private async Task SeedWorldAsync(GitLabDoorkeeperStub gitlab, bool gitLabKnowsThePair = true, string siblingAuthors = "members") { - gitlab.Seed(GitLabDoorkeeperStub.DefaultClientId, "nacho", "at-0", "rt-0"); + if (gitLabKnowsThePair) gitlab.Seed(GitLabDoorkeeperStub.DefaultClientId, "nacho", "at-0", "rt-0"); var team = await WorkflowsTestSeed.SeedTeamAsync(_fixture).ConfigureAwait(false); var credential = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: true, RepositoryVisibility.Internal, team).ConfigureAwait(false); @@ -193,8 +308,8 @@ private async Task SeedWorldAsync(GitLabDoorkeeperStub gitlab) var sibling = await CreateWorkflowAsync(team.TeamId, await AddAdminAsync(team.TeamId).ConfigureAwait(false)).ConfigureAwait(false); var secret = $"gl-sec-{Guid.NewGuid():N}"; - await SeedMembersOnlyActivationAsync(workflow, team.UserId).ConfigureAwait(false); - await SeedMembersOnlyActivationAsync(sibling, team.UserId).ConfigureAwait(false); + await SeedActivationAsync(workflow, team.UserId, "members").ConfigureAwait(false); + await SeedActivationAsync(sibling, team.UserId, siblingAuthors).ConfigureAwait(false); return new World(credential, workflow, sibling, await SeedWebhookAsync(credential.RepositoryId, secret).ConfigureAwait(false), secret); } @@ -219,12 +334,12 @@ private async Task CreateWorkflowAsync(Guid teamId, Guid userId) return await scope.Resolve().Send(new CreateWorkflowCommand { Name = "burst-" + Guid.NewGuid().ToString("N")[..8], Description = null, Definition = WorkflowsTestSeed.MinimalDefinition(), Activations = new List(), Enabled = true }).ConfigureAwait(false); } - private async Task SeedMembersOnlyActivationAsync(Guid workflowId, Guid userId) + private async Task SeedActivationAsync(Guid workflowId, Guid userId, string authors) { using var scope = _fixture.BeginScope(); var db = scope.Resolve(); - db.WorkflowActivation.Add(new WorkflowActivation { Id = Guid.NewGuid(), WorkflowId = workflowId, TypeKey = Opened, ConfigJson = """{"authors":"members"}""", Enabled = true, CreatedBy = userId, LastModifiedBy = userId }); + db.WorkflowActivation.Add(new WorkflowActivation { Id = Guid.NewGuid(), WorkflowId = workflowId, TypeKey = Opened, ConfigJson = $$"""{"authors":"{{authors}}"}""", Enabled = true, CreatedBy = userId, LastModifiedBy = userId }); await db.SaveChangesAsync().ConfigureAwait(false); } diff --git a/backend/tests/CodeSpace.IntegrationTests/Credentials/RefreshLiftsAnErrorFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Credentials/RefreshLiftsAnErrorFlowTests.cs new file mode 100644 index 000000000..cf087461b --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Credentials/RefreshLiftsAnErrorFlowTests.cs @@ -0,0 +1,96 @@ +using Autofac; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Services.OAuth; +using CodeSpace.Core.Services.Providers; +using CodeSpace.Core.Services.Providers.Auth; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.Messages.Credentials; +using CodeSpace.Messages.Enums; +using Microsoft.EntityFrameworkCore; +using Shouldly; + +namespace CodeSpace.IntegrationTests.Credentials; + +/// +/// 🟢 High fidelity — a pair the provider just issued is proof it accepts the grant, so the payload writer that stores it +/// lifts an Error the credential carried, and the text that explained it. That matters when a verdict and a refresh +/// cross: a pod refused the consumed refresh token and marked the row, or the lock session was lost mid-rotation and another +/// pod judged it — and a refresh then lands. Nothing else lifts the verdict at that moment (the health sweep only looks at +/// an Error credential an hour later), so a credential holding a live pair would read "needs reconnect" until it did. +/// +/// Real Postgres and the real payload writer; the race is made exactly as it happens — the entity is loaded, the row is +/// marked, then the refresh runs from the entity — against a that honours the pair. +/// +[Collection(PostgresCollection.Name)] +[Trait("Category", "Integration")] +public class RefreshLiftsAnErrorFlowTests +{ + private const string Reason = "needs reconnect: invalid_grant"; + + private readonly PostgresFixture _fixture; + + public RefreshLiftsAnErrorFlowTests(PostgresFixture fixture) { _fixture = fixture; } + + [Fact] + public async Task A_refresh_the_provider_accepts_lifts_the_Error_a_racing_pod_marked_after_this_one_loaded_the_credential() + { + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.Seed(GitLabDoorkeeperStub.DefaultClientId, "nacho", "at-0", "rt-0"); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0").ConfigureAwait(false); + + using var scope = _fixture.BeginScope(); + var repo = await CredentialRefreshHarness.LoadUntrackedAsync(scope, seed.RepositoryId).ConfigureAwait(false); + + await MarkErrorAsync(seed.CredentialId, Reason).ConfigureAwait(false); + + var auth = await scope.Resolve().ResolveAsync(new ProviderContext(repo.ProviderInstance, repo.Credential!), CancellationToken.None).ConfigureAwait(false); + + auth.Token.ShouldBe("at-2", "the entity was loaded before the verdict, so it presents the pair the provider still honours"); + gitlab.Count(StubEvent.Rotated).ShouldBe(1); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + row.Status.ShouldBe(CredentialStatus.Active, "the provider accepted the grant, so the verdict is lifted with the pair it issued — not left for the sweep an hour later"); + row.LastError.ShouldBeNull(); + (await GitLabCredentialSeed.ReadStoredPayloadAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).RefreshToken.ShouldBe("rt-2"); + } + + [Theory] + [InlineData(CredentialStatus.Error, CredentialStatus.Active, null)] + [InlineData(CredentialStatus.Active, CredentialStatus.Active, null)] + [InlineData(CredentialStatus.Expired, CredentialStatus.Expired, "kept as it was")] + public async Task The_payload_writer_lifts_an_Error_with_its_text_and_leaves_any_other_status_as_it_was(CredentialStatus before, CredentialStatus after, string? lastErrorAfter) + { + using var gitlab = new GitLabDoorkeeperStub(); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0").ConfigureAwait(false); + + await SetStatusAsync(seed.CredentialId, before, before == CredentialStatus.Active ? null : before == CredentialStatus.Error ? Reason : "kept as it was").ConfigureAwait(false); + + var loaded = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + var issued = new OAuthPayload { AccessToken = "at-new", RefreshToken = "rt-new", ExpiresAt = DateTimeOffset.UtcNow.AddHours(2) }; + + using var scope = _fixture.BeginScope(); + + (await scope.Resolve().UpdatePayloadAsync(loaded, loaded.EncryptedPayload, issued, CancellationToken.None).ConfigureAwait(false)).ShouldBeTrue(); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + row.Status.ShouldBe(after); + row.LastError.ShouldBe(lastErrorAfter); + loaded.Status.ShouldBe(after, "the caller's entity says what the row now holds, so a later read in its scope does not fail fast on a verdict that is gone"); + loaded.LastError.ShouldBe(lastErrorAfter); + (await GitLabCredentialSeed.ReadStoredPayloadAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).RefreshToken.ShouldBe("rt-new"); + } + + // ─── Plumbing ────────────────────────────────────────────────────────────── + + private Task MarkErrorAsync(Guid credentialId, string lastError) => SetStatusAsync(credentialId, CredentialStatus.Error, lastError); + + private async Task SetStatusAsync(Guid credentialId, CredentialStatus status, string? lastError) + { + using var scope = _fixture.BeginScope(); + var name = status.ToString(); + + await scope.Resolve().Database.ExecuteSqlInterpolatedAsync($"UPDATE credential SET status = {name}, last_error = {lastError} WHERE id = {credentialId}").ConfigureAwait(false); + } +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Credentials/UnauthorizedRecoveryFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Credentials/UnauthorizedRecoveryFlowTests.cs new file mode 100644 index 000000000..728ff50ab --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Credentials/UnauthorizedRecoveryFlowTests.cs @@ -0,0 +1,214 @@ +using System.Text.Json; +using Autofac; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Credentials; +using CodeSpace.Core.Services.Providers; +using CodeSpace.Core.Services.Providers.Capabilities; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.Messages.Credentials; +using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Exceptions; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit.Abstractions; + +namespace CodeSpace.IntegrationTests.Credentials; + +/// +/// 🟢 High fidelity — GitLab refuses an access token CodeSpace has just read from the credential row, and the call is sent +/// again with the token that replaces it. The real GitLab provider (NGitLab on HttpWebRequest) → auth resolver → OAuth +/// strategy → refresher → Postgres advisory lock → the real OAuth client → payload writer, over real Postgres; the one fake is +/// GitLab, a that retires an access token with its row the moment the pair is refreshed, +/// however much of its life is left, and refuses an access token past the lifetime GitLab gave it however much of it the +/// row says is left. +/// +/// The two shapes that reach a 401: another consumer refreshed the pair and stored it, so the entity a caller loaded +/// before still carries a token GitLab retired (what is stored is adopted, nothing is rotated); and the access token's +/// lifetime at GitLab is shorter than the row says (the stored token is the one refused, and the refresh token is live, so +/// the pair is rotated once). Before, either was a 422 for up to two hours. +/// +[Collection(PostgresCollection.Name)] +[Trait("Category", "Integration")] +public class UnauthorizedRecoveryFlowTests +{ + private readonly PostgresFixture _fixture; + private readonly ITestOutputHelper _output; + + public UnauthorizedRecoveryFlowTests(PostgresFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + } + + [Fact] + public async Task A_token_another_consumer_replaced_is_adopted_from_the_row_and_nothing_is_rotated() + { + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.Seed(GitLabDoorkeeperStub.DefaultClientId, "nacho", "at-0", "rt-0"); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: false).ConfigureAwait(false); + + using var scope = _fixture.BeginScope(); + var repo = await CredentialRefreshHarness.LoadUntrackedAsync(scope, seed.RepositoryId).ConfigureAwait(false); + + // Another pod refreshes the pair and stores it. This caller's entity, loaded before, still holds at-0, which GitLab has retired. + var rotated = await RefreshBehindTheEntityAsync(gitlab, "rt-0").ConfigureAwait(false); + await StoreAsync(seed.CredentialId, rotated).ConfigureAwait(false); + + var branches = await ListBranchesAsync(scope, repo).ConfigureAwait(false); + + Dump(gitlab); + branches.ShouldBeEmpty("the call was answered"); + gitlab.Count(StubEvent.Api401).ShouldBe(1, "the stale token was refused once"); + gitlab.Count(StubEvent.Api200).ShouldBe(1, "and the call was sent again with the token the other consumer stored"); + gitlab.Count(StubEvent.Rotated).ShouldBe(1, "only the other consumer's refresh: adopting needs none"); + gitlab.Count(StubEvent.Refused).ShouldBe(0); + (await GitLabCredentialSeed.ReadStoredPayloadAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).RefreshToken.ShouldBe("rt-2", "the row is as the other consumer left it"); + } + + [Fact] + public async Task A_token_gitlab_retired_before_its_reported_lifetime_is_rotated_once_and_the_call_is_sent_again() + { + using var gitlab = new GitLabDoorkeeperStub(); + var seed = await SeedTokenGitLabRetiresAfterOneSecondAsync(gitlab).ConfigureAwait(false); + + using var scope = _fixture.BeginScope(); + var repo = await CredentialRefreshHarness.LoadUntrackedAsync(scope, seed.RepositoryId).ConfigureAwait(false); + + await ListBranchesAsync(scope, repo).ConfigureAwait(false); + + Dump(gitlab); + gitlab.Count(StubEvent.Api401).ShouldBe(1); + gitlab.Count(StubEvent.Rotated).ShouldBe(1, "the refresh token was live, so the pair was rotated once"); + gitlab.Count(StubEvent.Api200).ShouldBe(1); + gitlab.Count(StubEvent.Refused).ShouldBe(0); + (await GitLabCredentialSeed.ReadStoredPayloadAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).RefreshToken.ShouldBe("rt-2", "the pair GitLab issued is the credential's now"); + (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).Status.ShouldBe(CredentialStatus.Active); + } + + [Fact] + public async Task A_burst_of_calls_refused_together_rotates_the_pair_once_and_every_call_is_answered() + { + using var gitlab = new GitLabDoorkeeperStub(); + var seed = await SeedTokenGitLabRetiresAfterOneSecondAsync(gitlab).ConfigureAwait(false); + + var calls = await Task.WhenAll(Enumerable.Range(0, 5).Select(_ => CallAsync(seed.RepositoryId))).ConfigureAwait(false); + + Dump(gitlab); + calls.ShouldAllBe(failure => failure == null, $"every call is answered — {string.Join("; ", calls.Where(c => c != null).Select(c => c!.Message))}"); + gitlab.Count(StubEvent.Rotated).ShouldBe(1, "one rotation for the whole burst: whoever got the lock rotated, and the rest adopted what it stored"); + gitlab.Count(StubEvent.Refused).ShouldBe(0, $"a call presented a refresh token GitLab had already consumed: {string.Join(", ", gitlab.RefusedRefreshTokens())}"); + gitlab.Count(StubEvent.Api401).ShouldBeGreaterThan(0, "fixture check: the burst did meet the retired token"); + } + + [Fact] + public async Task A_token_rotated_moments_ago_that_is_refused_too_is_not_rotated_again_and_the_401_is_the_answer() + { + using var gitlab = new GitLabDoorkeeperStub(); + var seed = await SeedTokenGitLabRetiresAfterOneSecondAsync(gitlab).ConfigureAwait(false); + (await CallAsync(seed.RepositoryId).ConfigureAwait(false)).ShouldBeNull("fixture check: the first call recovered by rotating"); + + await RevokeAtGitLabAsync(gitlab, "at-2").ConfigureAwait(false); + + var failure = await CallAsync(seed.RepositoryId).ConfigureAwait(false); + + Dump(gitlab); + failure.ShouldBeOfType("what GitLab said stands").StatusCode.ShouldBe(401); + gitlab.Count(StubEvent.Rotated).ShouldBe(1, "a token issued moments ago that is refused is not aged out: rotating again would only spend a refresh token on another cause"); + gitlab.Count(StubEvent.Refused).ShouldBe(0); + (await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).Status.ShouldBe(CredentialStatus.Active, "a 401 alone is not a verdict on the grant"); + } + + [Fact] + public async Task A_pair_gitlab_has_revoked_behind_a_token_that_looks_live_is_a_typed_failure_and_the_credential_is_marked() + { + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.Seed(GitLabDoorkeeperStub.DefaultClientId, "nacho", "at-0", "rt-0"); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: false).ConfigureAwait(false); + await RevokeAtGitLabAsync(gitlab, "at-0").ConfigureAwait(false); + + var failure = await CallAsync(seed.RepositoryId).ConfigureAwait(false); + + Dump(gitlab); + failure.ShouldBeOfType("the 401 led to a rotation GitLab refused for good, which is what the caller is told instead of the 401").CredentialId.ShouldBe(seed.CredentialId); + + var row = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + row.Status.ShouldBe(CredentialStatus.Error); + row.LastError.ShouldBe("needs reconnect: invalid_grant"); + } + + // ─── Plumbing ────────────────────────────────────────────────────────────── + + /// A credential whose row says its token lives two hours, holding a pair GitLab gave a one-second lifetime — a second later GitLab refuses the token, and its refresh token is live. + private async Task SeedTokenGitLabRetiresAfterOneSecondAsync(GitLabDoorkeeperStub gitlab) + { + gitlab.ExpiresInSeconds = 1; + gitlab.Seed(GitLabDoorkeeperStub.DefaultClientId, "nacho", "at-0", "rt-0"); + gitlab.ExpiresInSeconds = 7200; + + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0", expired: false).ConfigureAwait(false); + + await Task.Delay(TimeSpan.FromMilliseconds(1300)).ConfigureAwait(false); + + return seed; + } + + /// One call a run makes of the provider, in a scope of its own: the repository loaded untracked, then ListBranchesAsync. Null when it was answered, the failure when it was not. + private async Task CallAsync(Guid repositoryId) + { + try + { + using var scope = _fixture.BeginScope(); + + await ListBranchesAsync(scope, await CredentialRefreshHarness.LoadUntrackedAsync(scope, repositoryId).ConfigureAwait(false)).ConfigureAwait(false); + + return null; + } + catch (Exception ex) + { + return ex; + } + } + + private static async Task> ListBranchesAsync(ILifetimeScope scope, Repository repo) + { + var source = scope.Resolve().Require(ProviderKind.GitLab); + + return await source.ListBranchesAsync(new ProviderContext(repo.ProviderInstance, repo.Credential!), repo.ToRemoteRepository(), CancellationToken.None).ConfigureAwait(false); + } + + /// What another consumer does: presents the refresh token to GitLab directly and gets the new pair. + private static async Task RefreshBehindTheEntityAsync(GitLabDoorkeeperStub gitlab, string refreshToken) + { + using var http = new HttpClient(); + var form = new Dictionary { ["grant_type"] = "refresh_token", ["client_id"] = GitLabDoorkeeperStub.DefaultClientId, ["client_secret"] = GitLabDoorkeeperStub.DefaultClientSecret, ["refresh_token"] = refreshToken }; + using var response = await http.PostAsync($"{gitlab.BaseUrl}/oauth/token", new FormUrlEncodedContent(form)).ConfigureAwait(false); + using var json = JsonDocument.Parse(await response.Content.ReadAsStringAsync().ConfigureAwait(false)); + + return new OAuthPayload { AccessToken = json.RootElement.GetProperty("access_token").GetString()!, RefreshToken = json.RootElement.GetProperty("refresh_token").GetString(), ExpiresAt = DateTimeOffset.UtcNow.AddHours(2) }; + } + + private static async Task RevokeAtGitLabAsync(GitLabDoorkeeperStub gitlab, string token) + { + using var http = new HttpClient(); + + await http.PostAsync($"{gitlab.BaseUrl}/oauth/revoke", new FormUrlEncodedContent(new Dictionary { ["token"] = token })).ConfigureAwait(false); + } + + /// Stores a pair into the row as another pod's refresher would. + private async Task StoreAsync(Guid credentialId, OAuthPayload payload) + { + using var scope = _fixture.BeginScope(); + var sealedPair = scope.Resolve().Encrypt(scope.Resolve().Serialize(payload)); + + await scope.Resolve().Database.ExecuteSqlInterpolatedAsync($"UPDATE credential SET encrypted_payload = {sealedPair}, expires_date = {payload.ExpiresAt} WHERE id = {credentialId}").ConfigureAwait(false); + } + + private void Dump(GitLabDoorkeeperStub gitlab) + { + _output.WriteLine($"GitLab saw: rotated={gitlab.Count(StubEvent.Rotated)} refused={gitlab.Count(StubEvent.Refused)} api-200={gitlab.Count(StubEvent.Api200)} api-401={gitlab.Count(StubEvent.Api401)}"); + + foreach (var entry in gitlab.Log) _output.WriteLine($" {entry.At:HH:mm:ss.fff} {entry.Kind}: {entry.Detail}"); + } +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Hangfire/RecurringJobScheduleKeeperFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Hangfire/RecurringJobScheduleKeeperFlowTests.cs index 7644cba38..424e56233 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Hangfire/RecurringJobScheduleKeeperFlowTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Hangfire/RecurringJobScheduleKeeperFlowTests.cs @@ -30,7 +30,7 @@ public sealed class RecurringJobScheduleKeeperFlowTests { private const string LoadFailure = "Hangfire.Common.JobLoadException: Could not load the job. See inner exception for the details. ---> Newtonsoft.Json.JsonSerializationException: Error converting value \"CodeSpace.Core.Jobs.RecurringJobs.DepartedMemberCredentialRetirementRecurringJob, CodeSpace.Core\" to type 'System.Type'."; - private static readonly string[] AddedSinceThePreviousBuild = [nameof(DepartedMemberCredentialRetirementRecurringJob), nameof(DisconnectedCredentialBindingRepairRecurringJob), nameof(WebhookClaimPurgeRecurringJob), nameof(PackCloneUrlBackfillRecurringJob)]; + private static readonly string[] AddedSinceThePreviousBuild = [nameof(DepartedMemberCredentialRetirementRecurringJob), nameof(DisconnectedCredentialBindingRepairRecurringJob), nameof(WebhookClaimPurgeRecurringJob), nameof(PackCloneUrlBackfillRecurringJob), nameof(CredentialHealthProbeRecurringJob)]; private readonly PostgresFixture _fixture; diff --git a/backend/tests/CodeSpace.IntegrationTests/Infrastructure/GitLabCredentialSeed.cs b/backend/tests/CodeSpace.IntegrationTests/Infrastructure/GitLabCredentialSeed.cs index b6414e89b..5cd4f9d78 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Infrastructure/GitLabCredentialSeed.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Infrastructure/GitLabCredentialSeed.cs @@ -46,6 +46,68 @@ public static async Task SeedAsync(PostgresFixture fixtu return new SeededGitLabCredential(userId, team, instance.Id, credential.Id, repository.Id, repository.ExternalId); } + /// + /// A team's connection by a personal, project or group access token instead of an OAuth grant: an instance whose OAuth + /// client is the stub's (so a test may also hold OAuth credentials on it), a static-token credential holding + /// , and a repository bound through it. is what CodeSpace has + /// recorded about the token's expiry — nothing, for a token linked before it recorded any. + /// + public static async Task SeedAccessTokenAsync(PostgresFixture fixture, string gitLabBaseUrl, string token, AuthType type = AuthType.Pat, DateTimeOffset? expiresDate = null, (Guid TeamId, Guid UserId)? existingTeam = null) + { + using var scope = fixture.BeginScope(); + var db = scope.Resolve(); + var encryptor = scope.Resolve(); + var serializer = scope.Resolve(); + + var suffix = Guid.NewGuid().ToString("N")[..8]; + var (team, userId) = existingTeam ?? await AddTeamAsync(db, suffix).ConfigureAwait(false); + var instance = new ProviderInstance { Id = Guid.NewGuid(), TeamId = team, Provider = ProviderKind.GitLab, DisplayName = "gitlab", BaseUrl = gitLabBaseUrl, ApiUrl = gitLabBaseUrl, OauthClientId = GitLabDoorkeeperStub.DefaultClientId, OauthClientSecretEnc = encryptor.Encrypt(GitLabDoorkeeperStub.DefaultClientSecret) }; + CredentialPayload payload = type switch + { + AuthType.ProjectAccessToken => new ProjectAccessTokenPayload { Token = token }, + AuthType.GroupAccessToken => new GroupAccessTokenPayload { Token = token }, + _ => new PatPayload { Token = token } + }; + var credential = new Credential { Id = Guid.NewGuid(), TeamId = team, ProviderInstanceId = instance.Id, OwnerUserId = userId, AuthType = type, DisplayName = $"GitLab · {suffix}", EncryptedPayload = encryptor.Encrypt(serializer.Serialize(payload)), ExpiresDate = expiresDate, Status = CredentialStatus.Active }; + var repository = new Repository { Id = Guid.NewGuid(), TeamId = team, ProviderInstanceId = instance.Id, CredentialId = credential.Id, ExternalId = Random.Shared.Next(100_000, 999_999).ToString(), NamespacePath = "acme", Name = "api", FullPath = "acme/api", DefaultBranch = "main", Visibility = RepositoryVisibility.Private, WebUrl = $"{gitLabBaseUrl}/acme/api", CloneUrlHttps = $"{gitLabBaseUrl}/acme/api.git", Status = RepositoryStatus.Active }; + + db.ProviderInstance.Add(instance); + db.Credential.Add(credential); + db.Repository.Add(repository); + await db.SaveChangesAsync().ConfigureAwait(false); + + return new SeededGitLabCredential(userId, team, instance.Id, credential.Id, repository.Id, repository.ExternalId); + } + + /// + /// The credential health sweep is deployment-wide: every credential an earlier test of this database left Active and + /// never validated is due ahead of the ones a test owns, and a bounded pass could spend its batch on them. A test that + /// runs the sweep says which credentials are its own, and every other Active one is recorded as validated just now — a + /// test never asserts the sweep's tally, only the rows it owns. + /// + public static async Task QuietEveryOtherCredentialAsync(PostgresFixture fixture, params Guid[] mine) + { + using var scope = fixture.BeginScope(); + + await scope.Resolve().Database.ExecuteSqlInterpolatedAsync($"UPDATE credential SET last_validated_date = now() WHERE status = 'Active' AND deleted_date IS NULL AND NOT (id = ANY({mine}))").ConfigureAwait(false); + } + + /// Writes the credential's last-modified date, which is what an Error credential's recheck is measured from. + public static async Task SetLastModifiedAsync(PostgresFixture fixture, Guid credentialId, DateTimeOffset at) + { + using var scope = fixture.BeginScope(); + + await scope.Resolve().Database.ExecuteSqlInterpolatedAsync($"UPDATE credential SET last_modified_date = {at} WHERE id = {credentialId}").ConfigureAwait(false); + } + + /// Writes the credential's validation date, which is what an Active credential's next probe is measured from. + public static async Task SetLastValidatedAsync(PostgresFixture fixture, Guid credentialId, DateTimeOffset? at) + { + using var scope = fixture.BeginScope(); + + await scope.Resolve().Database.ExecuteSqlInterpolatedAsync($"UPDATE credential SET last_validated_date = {at} WHERE id = {credentialId}").ConfigureAwait(false); + } + /// What the credential row holds now, decrypted — what the next refresh would present. public static async Task ReadStoredPayloadAsync(PostgresFixture fixture, Guid credentialId) { diff --git a/backend/tests/CodeSpace.IntegrationTests/Infrastructure/GitLabDoorkeeperStub.cs b/backend/tests/CodeSpace.IntegrationTests/Infrastructure/GitLabDoorkeeperStub.cs index 16fb86e2a..75ccccff0 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Infrastructure/GitLabDoorkeeperStub.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Infrastructure/GitLabDoorkeeperStub.cs @@ -26,7 +26,13 @@ public enum StubEvent /// A git smart-HTTP request was challenged: no credentials, a dead token, or an OAuth token under a login other than oauth2. Git401, /// A request the stub has no route for. - Unrouted + Unrouted, + /// A REST call carried a personal, project or group access token (PRIVATE-TOKEN) that is live. + TokenApi200, + /// A REST call carried an access token that is unknown, revoked or past its expiry — GitLab's 401 for a token that stopped working. + TokenApi401, + /// The members lookup was answered with a scripted failure () instead of the author's role. + MembersFailed } /// @@ -40,6 +46,12 @@ public enum StubEvent /// pair a winner was issued survives a loser's refusal. A client whose id or secret does not match the issuing /// application is refused with invalid_client or invalid_grant. /// +/// Access tokens. A personal, project or group access token is not part of an OAuth grant: it is added with +/// , travels as PRIVATE-TOKEN on REST (GitLab resolves that header as exactly that, whatever +/// the token's length), answers 401 once it is revoked () or past the expiry it was given, and +/// describes itself — scopes, expires_at as a date, active — at GET /api/v4/personal_access_tokens/self, +/// which an OAuth token cannot call (404). +/// /// Endpoints. POST /oauth/token (refresh_token and authorization_code grants; the code /// authz:<user> issues <user> a first pair), POST /oauth/revoke (RFC 7009: 200 whatever /// it was given), GET /api/v4/user and GET /api/v4/projects/:id/members/all/:user_id (Bearer only — GitLab @@ -71,7 +83,10 @@ public sealed class GitLabDoorkeeperStub : IDisposable private readonly List _rows = new(); private readonly List _log = new(); private readonly Dictionary _applications = new() { [DefaultClientId] = DefaultClientSecret }; + private readonly Dictionary _accessTokens = new(); private int _sequence; + private int _membersFailureStatus; + private int _membersFailuresLeft; public GitLabDoorkeeperStub() { @@ -112,6 +127,31 @@ public void AddApplication(string clientId, string clientSecret) lock (_gate) { _applications[clientId] = clientSecret; } } + /// + /// A personal, project or group access token, live from now until (never, when null) or + /// until . is who it speaks for. + /// + public void AddAccessToken(string token, string user, DateTimeOffset? expiresAt = null) + { + lock (_gate) { _accessTokens[token] = new AccessToken(user, expiresAt); } + } + + /// GitLab refuses the token from now on, as it does one an owner revoked. + public void RevokeAccessToken(string token) + { + lock (_gate) { _accessTokens[token].Revoked = true; } + } + + /// The next members lookups (all of them when omitted) are answered with instead of the author's role: a 403 for a token without the scope to read members, a 429, a 5xx. + public void FailMembers(int status, int times = int.MaxValue) + { + lock (_gate) + { + _membersFailureStatus = status; + _membersFailuresLeft = times; + } + } + /// A pair issued before the test began: what the credential row held when the story starts. public TokenRow Seed(string app, string user, string accessToken, string refreshToken) { @@ -190,7 +230,7 @@ private Reply Route(HttpListenerRequest request, string body) if (request.HttpMethod == "POST" && path == "/oauth/token") return Token(HttpUtility.ParseQueryString(body)); if (request.HttpMethod == "POST" && path == "/oauth/revoke") return Revoke(HttpUtility.ParseQueryString(body)); - if (request.HttpMethod == "GET" && path.StartsWith("/api/v4/", StringComparison.Ordinal)) return Api(request.Headers["Authorization"], path); + if (request.HttpMethod == "GET" && path.StartsWith("/api/v4/", StringComparison.Ordinal)) return IsKnownAccessToken(request.Headers["Authorization"], request.Headers["PRIVATE-TOKEN"]) ? TokenApi(request.Headers["PRIVATE-TOKEN"]!, path) : Api(request.Headers["Authorization"], path); if (request.HttpMethod == "GET" && path.EndsWith("/info/refs", StringComparison.Ordinal)) return GitAdvertisement(request.Headers["Authorization"], HttpUtility.ParseQueryString(request.Url.Query)["service"]); Note(StubEvent.Unrouted, $"{request.HttpMethod} {request.Url.PathAndQuery}", null); @@ -282,18 +322,60 @@ private Reply Api(string? authorization, string path) lock (_gate) { var row = FindLiveByAccessToken(BearerOf(authorization)); + var scripted = row == null ? null : ScriptedMembersFailure(path); - Note(row == null ? StubEvent.Api401 : StubEvent.Api200, $"{path} Authorization={Describe(authorization)}", BearerOf(authorization)); + if (scripted == null) Note(row == null ? StubEvent.Api401 : StubEvent.Api200, $"{path} Authorization={Describe(authorization)}", BearerOf(authorization)); if (row == null) return Reply.Json(401, UnauthorizedBody) with { Headers = new Dictionary { ["WWW-Authenticate"] = "Bearer realm=\"Protected by OAuth 2.0\", error=\"invalid_token\"" } }; - return Reply.Json(200, ApiBody(row, path)); + if (path == "/api/v4/personal_access_tokens/self") return Reply.Json(404, """{"message":"404 Not Found"}"""); + + return scripted ?? Reply.Json(200, ApiBody(row.User, path)); } } - private string ApiBody(TokenRow row, string path) + /// A request by an access token this stub was given: its PRIVATE-TOKEN, with no Bearer (a Bearer is read as OAuth, as before). An unknown PRIVATE-TOKEN is an OAuth lookup that fails. + private bool IsKnownAccessToken(string? authorization, string? privateToken) { - if (path == "/api/v4/user") return JsonSerializer.Serialize(new { id = 1, username = row.User }); + lock (_gate) { return authorization == null && privateToken != null && _accessTokens.ContainsKey(privateToken); } + } + + /// A REST call by a personal, project or group access token — PRIVATE-TOKEN. Runs under the stub's lock. + private Reply TokenApi(string privateToken, string path) + { + lock (_gate) + { + var token = _accessTokens.GetValueOrDefault(privateToken); + var live = token != null && !token.Revoked && (token.ExpiresAt == null || token.ExpiresAt > DateTimeOffset.UtcNow); + + var scripted = live ? ScriptedMembersFailure(path) : null; + + if (scripted == null) Note(live ? StubEvent.TokenApi200 : StubEvent.TokenApi401, $"{path} PRIVATE-TOKEN", privateToken); + + if (!live) return Reply.Json(401, UnauthorizedBody); + if (path == "/api/v4/personal_access_tokens/self") return Reply.Json(200, SelfBody(token!)); + + return scripted ?? Reply.Json(200, ApiBody(token!.User, path)); + } + } + + private static string SelfBody(AccessToken token) => + JsonSerializer.Serialize(new { id = 1, name = "stub", scopes = new[] { "api" }, active = true, revoked = false, expires_at = token.ExpiresAt?.UtcDateTime.ToString("yyyy-MM-dd") }); + + /// The members lookup's scripted failure, while any are left (). + private Reply? ScriptedMembersFailure(string path) + { + if (!path.Contains("/members/all/", StringComparison.Ordinal) || _membersFailuresLeft <= 0) return null; + + _membersFailuresLeft--; + Note(StubEvent.MembersFailed, $"{path} answered {_membersFailureStatus}", null); + + return Reply.Json(_membersFailureStatus, """{"message":"scripted failure"}"""); + } + + private string ApiBody(string user, string path) + { + if (path == "/api/v4/user") return JsonSerializer.Serialize(new { id = 1, username = user }); var member = path.IndexOf("/members/all/", StringComparison.Ordinal); @@ -419,6 +501,19 @@ private static (HttpListener Listener, string BaseUrl) ListenOnFreeLoopbackPort( } } + private sealed class AccessToken + { + public AccessToken(string user, DateTimeOffset? expiresAt) + { + User = user; + ExpiresAt = expiresAt; + } + + public string User { get; } + public DateTimeOffset? ExpiresAt { get; } + public bool Revoked { get; set; } + } + /// One token row, as Doorkeeper keeps it: the access token and its refresh token together. public sealed class TokenRow { diff --git a/backend/tests/CodeSpace.IntegrationTests/Infrastructure/GitLabDoorkeeperStubTests.cs b/backend/tests/CodeSpace.IntegrationTests/Infrastructure/GitLabDoorkeeperStubTests.cs index 2b475ea5c..8a24d5522 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Infrastructure/GitLabDoorkeeperStubTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Infrastructure/GitLabDoorkeeperStubTests.cs @@ -170,6 +170,52 @@ public async Task Git_takes_a_bearer_token_and_challenges_a_request_with_no_cred _gitlab.Count(StubEvent.Git401).ShouldBe(1); } + [Fact] + public async Task An_access_token_travels_as_private_token_describes_itself_and_stops_when_it_is_revoked_or_expires() + { + _gitlab.AddAccessToken("glpat-live", "nacho", DateTimeOffset.UtcNow.AddDays(3)); + _gitlab.AddAccessToken("glpat-forever", "nacho"); + _gitlab.AddAccessToken("glpat-lapsed", "nacho", DateTimeOffset.UtcNow.AddMinutes(-1)); + _gitlab.AddAccessToken("glpat-revoked", "nacho"); + _gitlab.RevokeAccessToken("glpat-revoked"); + + (await GetAsync("/api/v4/user", ("PRIVATE-TOKEN", "glpat-live")).ConfigureAwait(false)).Status.ShouldBe(HttpStatusCode.OK); + (await GetAsync("/api/v4/user", ("PRIVATE-TOKEN", "glpat-lapsed")).ConfigureAwait(false)).Status.ShouldBe(HttpStatusCode.Unauthorized, "GitLab refuses a token that reached its expiry"); + (await GetAsync("/api/v4/user", ("PRIVATE-TOKEN", "glpat-revoked")).ConfigureAwait(false)).Status.ShouldBe(HttpStatusCode.Unauthorized); + (await GetAsync("/api/v4/user", ("PRIVATE-TOKEN", "glpat-unknown")).ConfigureAwait(false)).Status.ShouldBe(HttpStatusCode.Unauthorized, "a PRIVATE-TOKEN the stub was not given is looked up as an OAuth token, which fails as before"); + (await GetAsync("/api/v4/user", ("Authorization", "Bearer glpat-live")).ConfigureAwait(false)).Status.ShouldBe(HttpStatusCode.Unauthorized, "an access token is not an OAuth token: GitLab looks a Bearer up as OAuth alone"); + + var self = await GetAsync("/api/v4/personal_access_tokens/self", ("PRIVATE-TOKEN", "glpat-live")).ConfigureAwait(false); + + self.Json.GetProperty("expires_at").GetString().ShouldBe(DateTimeOffset.UtcNow.AddDays(3).UtcDateTime.ToString("yyyy-MM-dd"), "a date, as GitLab spells it"); + self.Json.GetProperty("active").GetBoolean().ShouldBeTrue(); + (await GetAsync("/api/v4/personal_access_tokens/self", ("PRIVATE-TOKEN", "glpat-forever")).ConfigureAwait(false)).Json.GetProperty("expires_at").ValueKind.ShouldBe(JsonValueKind.Null); + _gitlab.Count(StubEvent.TokenApi200).ShouldBe(3); + _gitlab.Count(StubEvent.TokenApi401).ShouldBe(2, "the lapsed and the revoked token"); + } + + [Fact] + public async Task An_oauth_token_cannot_describe_itself_as_an_access_token_does() + { + _gitlab.Seed("app1", "nacho", "at-0", "rt-0"); + + (await GetAsync("/api/v4/personal_access_tokens/self", ("Authorization", "Bearer at-0")).ConfigureAwait(false)).Status.ShouldBe(HttpStatusCode.NotFound); + } + + [Fact] + public async Task A_scripted_members_failure_answers_the_next_lookups_for_either_kind_of_token_and_then_the_role_again() + { + _gitlab.Seed("app1", "nacho", "at-0", "rt-0"); + _gitlab.AddAccessToken("glpat-live", "nacho"); + _gitlab.FailMembers(403, times: 2); + + (await GetAsync("/api/v4/projects/42/members/all/7", ("Authorization", "Bearer at-0")).ConfigureAwait(false)).Status.ShouldBe(HttpStatusCode.Forbidden); + (await GetAsync("/api/v4/projects/42/members/all/7", ("PRIVATE-TOKEN", "glpat-live")).ConfigureAwait(false)).Status.ShouldBe(HttpStatusCode.Forbidden); + (await GetAsync("/api/v4/projects/42/members/all/7", ("Authorization", "Bearer at-0")).ConfigureAwait(false)).Json.GetProperty("access_level").GetInt32().ShouldBe(30); + (await GetAsync("/api/v4/user", ("Authorization", "Bearer at-0")).ConfigureAwait(false)).Status.ShouldBe(HttpStatusCode.OK, "only the members lookup is scripted"); + _gitlab.Count(StubEvent.MembersFailed).ShouldBe(2); + } + [Fact] public async Task A_route_the_stub_does_not_know_is_counted_rather_than_silently_answered() { diff --git a/backend/tests/CodeSpace.IntegrationTests/OAuth/OAuthTokenRefreshRaceFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/OAuth/OAuthTokenRefreshRaceFlowTests.cs index 08a85899f..184135346 100644 --- a/backend/tests/CodeSpace.IntegrationTests/OAuth/OAuthTokenRefreshRaceFlowTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/OAuth/OAuthTokenRefreshRaceFlowTests.cs @@ -14,6 +14,7 @@ using CodeSpace.Messages.Constants; using CodeSpace.Messages.Credentials; using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Exceptions; using MediatR; using Microsoft.EntityFrameworkCore; using Npgsql; @@ -188,15 +189,16 @@ public async Task A_run_cancelled_after_the_provider_rotated_its_token_leaves_th } [Fact] - public async Task A_refused_refresh_with_nothing_newer_stored_surfaces_the_providers_refusal() + public async Task A_refused_refresh_with_nothing_newer_stored_is_the_providers_verdict_and_says_the_credential_needs_reconnecting() { using var provider = new StubProviderHost(); var endpoint = RotatingTokenEndpoint.Serve(provider, liveRefreshToken: "rt-revoked-elsewhere"); var seed = await SeedExpiredOAuthRepositoryAsync(provider.BaseUrl, refreshToken: "rt-0").ConfigureAwait(false); - var refusal = await Should.ThrowAsync(() => ResolveTokenAsync(seed.RepositoryId)).ConfigureAwait(false); + var verdict = await Should.ThrowAsync(() => ResolveTokenAsync(seed.RepositoryId)).ConfigureAwait(false); - refusal.Error.ShouldBe("invalid_grant"); + verdict.CredentialId.ShouldBe(seed.CredentialId); + verdict.InnerException.ShouldBeOfType().Error.ShouldBe("invalid_grant", "the provider's own answer stays reachable for whoever diagnoses it"); endpoint.Requests.ShouldBe(1, customMessage: "a refusal with nothing newer stored is final — no retry loop against the provider"); var stored = await ReadStoredPayloadAsync(seed.CredentialId).ConfigureAwait(false); @@ -258,7 +260,8 @@ private async Task ReadStoredPayloadAsync(Guid credentialId) using var scope = _fixture.BeginScope(); var credential = await ReadCredentialAsync(credentialId).ConfigureAwait(false); - return (OAuthPayload)scope.Resolve().Resolve(credential); + // Decrypted directly, not resolved: the resolver refuses a credential its provider refused (Error), and what the row holds is the question. + return (OAuthPayload)scope.Resolve().Deserialize(credential.AuthType, scope.Resolve().Decrypt(credential.EncryptedPayload)); } private async Task ReadCredentialAsync(Guid credentialId) diff --git a/backend/tests/CodeSpace.IntegrationTests/OAuth/ProviderReconnectFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/OAuth/ProviderReconnectFlowTests.cs index 1a95c5197..73fd1bc87 100644 --- a/backend/tests/CodeSpace.IntegrationTests/OAuth/ProviderReconnectFlowTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/OAuth/ProviderReconnectFlowTests.cs @@ -71,7 +71,7 @@ public async Task A_run_after_reconnecting_an_expired_connection_clones_with_the var repoId = await SeedRepositoryAsync(world, stale); world.OAuth.RefreshFailure = new OAuthExchangeException("invalid_grant", "The provided authorization grant is invalid, expired, revoked", null); - await Should.ThrowAsync(() => ResolveCloneTokenAsync(world, repoId)); + await Should.ThrowAsync(() => ResolveCloneTokenAsync(world, repoId)); await DisconnectAsync(world, stale); var reconnected = await ReconnectAsync(world, reconnectWith, reconnectedToken); @@ -86,6 +86,52 @@ public async Task A_run_after_reconnecting_an_expired_connection_clones_with_the repo.LastError.ShouldBeNull(); } + [Theory] + [InlineData(ReconnectBy.OAuth, "at-reconnected")] + [InlineData(ReconnectBy.TokenLink, "glpat-reconnected")] + [InlineData(ReconnectBy.CredentialsApi, "glpat-reconnected")] + public async Task A_run_after_reconnecting_beside_a_connection_its_provider_refused_clones_with_the_reconnected_token(ReconnectBy reconnectWith, string reconnectedToken) + { + // The refusal marks the credential Error and leaves it listed as it was; the Providers dialog lists only Active ones, so + // the person sees nothing connected and clicks Connect — there is no Disconnect first. The failure told them to + // reconnect, and what was bound through the refused credential has to follow that, as it follows a disconnect's. + var world = await SeedWorldAsync(); + var refused = await ConnectOAuthAsync(world, world.OwnerId, Token("at-stale", "rt-stale", TimeSpan.FromMinutes(-5))); + var repoId = await SeedRepositoryAsync(world, refused); + var hookId = await SeedConnectionWebhookAsync(world, refused, RepositoryWebhookRegistrationStatus.DeadLettered); + + world.OAuth.RefreshFailure = new OAuthExchangeException("invalid_grant", "The provided authorization grant is invalid, expired, revoked", null); + await Should.ThrowAsync(() => ResolveCloneTokenAsync(world, repoId)); + + using var dispatches = Jobs().ManualExecution(); + var reconnected = await ReconnectAsync(world, reconnectWith, reconnectedToken); + + (await ResolveCloneTokenAsync(world, repoId)).ShouldBe(reconnectedToken, "a run after the reconnect must clone with the reconnected token — not fail naming a credential the person has just replaced"); + + var repo = await LoadRepositoryAsync(repoId); + var hook = await LoadConnectionWebhookAsync(hookId); + + repo.CredentialId.ShouldBe(reconnected, "the binding follows the owner's reconnect, so the repository row tells the truth about what it authenticates with"); + repo.Status.ShouldBe(RepositoryStatus.Active); + hook.CredentialId.ShouldBe(reconnected, "a group hook registered through the refused credential follows with the repository"); + hook.RegistrationStatus.ShouldBe(RepositoryWebhookRegistrationStatus.Enqueued, "and a dead-lettered one is revived, as a disconnect's reconnect revives it"); + } + + [Fact] + public async Task Another_members_connect_does_not_adopt_a_refused_credentials_repositories() + { + var world = await SeedWorldAsync(); + var refused = await ConnectOAuthAsync(world, world.OwnerId, Token("at-owner", "rt-owner", TimeSpan.FromMinutes(-5))); + var repoId = await SeedRepositoryAsync(world, refused); + + world.OAuth.RefreshFailure = new OAuthExchangeException("invalid_grant", "dead", null); + await Should.ThrowAsync(() => ResolveCloneTokenAsync(world, repoId)); + + await ConnectOAuthAsync(world, world.MemberId, Token("at-member", "rt-member", TimeSpan.FromHours(2))); + + (await LoadRepositoryAsync(repoId)).CredentialId.ShouldBe(refused, "a teammate is a different identity: the owner's repositories still need the owner (or a deliberate re-link)"); + } + [Fact] public async Task Linking_a_token_over_a_live_oauth_link_carries_its_repositories_to_the_token() { @@ -186,7 +232,7 @@ public async Task A_run_on_a_disconnect_nobody_reconnected_names_the_disconnect( failure.Message.ShouldContain("'owner's GitLab' was disconnected", Case.Sensitive, "the run must say which credential is gone and what to do — not report the blanked payload as a Data Protection fault"); } - public enum StrandedShape { ReconnectedAfterTheDisconnect, TokenLinkedInTheSameSave, OnlyAnOlderLiveCredential, ReconnectedByAnotherMember, ReconnectedThenDisconnectedAgain, BoundToALiveCredentialWithANewerSibling } + public enum StrandedShape { ReconnectedAfterTheDisconnect, TokenLinkedInTheSameSave, OnlyAnOlderLiveCredential, ReconnectedByAnotherMember, ReconnectedThenDisconnectedAgain, BoundToALiveCredentialWithANewerSibling, RefusedThenReconnected, RefusedWithOnlyAnOlderLiveCredential, RefusedWithASiblingConnectedBeforeItLastWorked, RefusedThenReconnectedByAnotherMember } [Theory] [InlineData(StrandedShape.ReconnectedAfterTheDisconnect, true)] @@ -195,6 +241,10 @@ public enum StrandedShape { ReconnectedAfterTheDisconnect, TokenLinkedInTheSameS [InlineData(StrandedShape.ReconnectedByAnotherMember, false)] [InlineData(StrandedShape.ReconnectedThenDisconnectedAgain, false)] [InlineData(StrandedShape.BoundToALiveCredentialWithANewerSibling, false)] + [InlineData(StrandedShape.RefusedThenReconnected, true)] + [InlineData(StrandedShape.RefusedWithOnlyAnOlderLiveCredential, false)] + [InlineData(StrandedShape.RefusedWithASiblingConnectedBeforeItLastWorked, false)] + [InlineData(StrandedShape.RefusedThenReconnectedByAnotherMember, false)] public async Task The_repair_sweep_moves_a_stranded_binding_only_to_its_owners_reconnect(StrandedShape shape, bool moves) { // The rows a connect path without the carry-forward leaves behind — the code before it, or an older pod still @@ -203,7 +253,7 @@ public async Task The_repair_sweep_moves_a_stranded_binding_only_to_its_owners_r // the connect paths apply, on every tick, and leave alone every shape that rule leaves alone. var world = await SeedWorldAsync(); var (bound, successor) = await SeedStrandedCredentialsAsync(world, shape); - var repoId = await SeedRepositoryAsync(world, bound, shape is StrandedShape.TokenLinkedInTheSameSave or StrandedShape.BoundToALiveCredentialWithANewerSibling ? RepositoryStatus.Active : RepositoryStatus.Error); + var repoId = await SeedRepositoryAsync(world, bound, LeavesTheRepositoryActive(shape) ? RepositoryStatus.Active : RepositoryStatus.Error); var hookId = await SeedConnectionWebhookAsync(world, bound, RepositoryWebhookRegistrationStatus.DeadLettered); using var dispatches = Jobs().ManualExecution(); @@ -239,6 +289,10 @@ public async Task The_repair_sweep_moves_a_stranded_binding_only_to_its_owners_r [InlineData(StrandedShape.ReconnectedByAnotherMember, false)] [InlineData(StrandedShape.ReconnectedThenDisconnectedAgain, false)] [InlineData(StrandedShape.BoundToALiveCredentialWithANewerSibling, false)] + [InlineData(StrandedShape.RefusedThenReconnected, true)] + [InlineData(StrandedShape.RefusedWithOnlyAnOlderLiveCredential, false)] + [InlineData(StrandedShape.RefusedWithASiblingConnectedBeforeItLastWorked, false)] + [InlineData(StrandedShape.RefusedThenReconnectedByAnotherMember, false)] public async Task The_census_counts_exactly_the_shapes_the_sweep_moves_and_falls_as_it_moves_them(StrandedShape shape, bool counted) { // The health report's backlog is only worth reading if it is the sweep's own candidate set. The shapes are the sweep @@ -251,7 +305,7 @@ public async Task The_census_counts_exactly_the_shapes_the_sweep_moves_and_falls var baseline = await CountStrandedAsync(); var (bound, _) = await SeedStrandedCredentialsAsync(world, shape); - await SeedRepositoryAsync(world, bound, shape is StrandedShape.TokenLinkedInTheSameSave or StrandedShape.BoundToALiveCredentialWithANewerSibling ? RepositoryStatus.Active : RepositoryStatus.Error); + await SeedRepositoryAsync(world, bound, LeavesTheRepositoryActive(shape) ? RepositoryStatus.Active : RepositoryStatus.Error); (await CountStrandedAsync()).ShouldBe(baseline + (counted ? 1 : 0), $"{shape}: counted only when the sweep would move it"); @@ -387,6 +441,14 @@ private async Task ShouldStillCloneThroughTheBoundCredentialAsync(World world, G return; } + if (IsRefused(shape)) + { + var refusal = await Should.ThrowAsync(() => ResolveCloneTokenAsync(world, repositoryId)); + + refusal.CredentialId.ShouldBe(bound, $"{shape}: the run names the credential its provider refused, which nobody has reconnected"); + return; + } + var failure = await Should.ThrowAsync(() => ResolveCloneTokenAsync(world, repositoryId)); failure.CredentialId.ShouldBe(bound, $"{shape}: the run names the disconnect it is still bound to"); @@ -419,28 +481,38 @@ private async Task SeedWorldAsync() /// /// The credential rows of one stranded shape, timestamped as a connect path without the carry-forward wrote them: /// (the one the repository and hook are bound to, the one connected around it). The bound one is Revoked with a - /// blank payload, except in , where it is live. + /// blank payload, except in , where it is live, and + /// in the Refused shapes, where it is a token its provider refused: Error, payload kept, last validated + /// some days after it was connected and last written minutes ago — a confirmation of the verdict dates the row afresh every + /// hour, so the row's last-modified date says nothing about when it died. /// private async Task<(Guid Bound, Guid Other)> SeedStrandedCredentialsAsync(World world, StrandedShape shape) { var connected = DateTimeOffset.UtcNow.AddDays(-30); var revoked = connected.AddDays(10); + var lastWorked = connected.AddDays(5); - var otherOwner = shape == StrandedShape.ReconnectedByAnotherMember ? world.MemberId : world.OwnerId; + var otherOwner = shape is StrandedShape.ReconnectedByAnotherMember or StrandedShape.RefusedThenReconnectedByAnotherMember ? world.MemberId : world.OwnerId; var otherConnected = shape switch { StrandedShape.TokenLinkedInTheSameSave => revoked, StrandedShape.OnlyAnOlderLiveCredential => connected.AddDays(-1), StrandedShape.BoundToALiveCredentialWithANewerSibling => connected.AddDays(1), + StrandedShape.RefusedThenReconnected or StrandedShape.RefusedThenReconnectedByAnotherMember => lastWorked.AddDays(7), + StrandedShape.RefusedWithOnlyAnOlderLiveCredential => connected.AddDays(-1), + StrandedShape.RefusedWithASiblingConnectedBeforeItLastWorked => connected.AddDays(2), _ => revoked.AddDays(1) }; using var scope = _fixture.BeginScope(); var db = scope.Resolve(); - var bound = shape == StrandedShape.BoundToALiveCredentialWithANewerSibling - ? StrandedCredential(world, world.OwnerId, AuthType.Pat, CredentialStatus.Active, SealToken(scope, "glpat-bound"), connected, connected) - : StrandedCredential(world, world.OwnerId, AuthType.OAuth, CredentialStatus.Revoked, string.Empty, connected, revoked); + var bound = shape switch + { + StrandedShape.BoundToALiveCredentialWithANewerSibling => StrandedCredential(world, world.OwnerId, AuthType.Pat, CredentialStatus.Active, SealToken(scope, "glpat-bound"), connected, connected), + _ when IsRefused(shape) => RefusedCredential(world, SealToken(scope, "glpat-refused"), connected, lastWorked), + _ => StrandedCredential(world, world.OwnerId, AuthType.OAuth, CredentialStatus.Revoked, string.Empty, connected, revoked) + }; var other = shape == StrandedShape.ReconnectedThenDisconnectedAgain ? StrandedCredential(world, otherOwner, AuthType.OAuth, CredentialStatus.Revoked, string.Empty, otherConnected, otherConnected.AddDays(1)) @@ -454,6 +526,21 @@ private async Task SeedWorldAsync() return (bound.Id, other.Id); } + private static bool IsRefused(StrandedShape shape) => shape is StrandedShape.RefusedThenReconnected or StrandedShape.RefusedWithOnlyAnOlderLiveCredential or StrandedShape.RefusedWithASiblingConnectedBeforeItLastWorked or StrandedShape.RefusedThenReconnectedByAnotherMember; + + /// The shapes whose bound credential was never disconnected, so nothing turned the repository Error: it is Active, on a credential that cannot be used. + private static bool LeavesTheRepositoryActive(StrandedShape shape) => shape is StrandedShape.TokenLinkedInTheSameSave or StrandedShape.BoundToALiveCredentialWithANewerSibling || IsRefused(shape); + + private static Credential RefusedCredential(World world, string encryptedPayload, DateTimeOffset createdDate, DateTimeOffset lastValidatedDate) + { + var refused = StrandedCredential(world, world.OwnerId, AuthType.Pat, CredentialStatus.Error, encryptedPayload, createdDate, DateTimeOffset.UtcNow.AddMinutes(-5)); + + refused.LastError = "token rejected by GitLab"; + refused.LastValidatedDate = lastValidatedDate; + + return refused; + } + private string SealToken(ILifetimeScope scope, string token) => _encryptor.Encrypt(scope.Resolve().Serialize(new PatPayload { Token = token })); private static Credential StrandedCredential(World world, Guid ownerId, AuthType authType, CredentialStatus status, string encryptedPayload, DateTimeOffset createdDate, DateTimeOffset lastModifiedDate) => diff --git a/backend/tests/CodeSpace.IntegrationTests/OAuth/RotatedPairStoreFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/OAuth/RotatedPairStoreFlowTests.cs index ba3623846..bb7b161b0 100644 --- a/backend/tests/CodeSpace.IntegrationTests/OAuth/RotatedPairStoreFlowTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/OAuth/RotatedPairStoreFlowTests.cs @@ -1,3 +1,5 @@ +using System.Collections; +using System.Reflection; using Autofac; using CodeSpace.Core.Persistence.Db; using CodeSpace.Core.Services.Credentials; @@ -7,6 +9,7 @@ using CodeSpace.IntegrationTests.Infrastructure; using CodeSpace.Messages.Credentials; using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Exceptions; using Microsoft.EntityFrameworkCore; using Npgsql; using Shouldly; @@ -110,6 +113,43 @@ public async Task A_read_that_fails_right_after_the_swap_was_lost_does_not_cost_ (await GitLabCredentialSeed.ReadStoredPayloadAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).RefreshToken.ShouldBe("rt-2"); } + [Fact] + public async Task A_second_pods_refusal_of_the_consumed_pair_does_not_strand_the_pair_the_first_pod_kept() + { + // Pod A: GitLab rotates rt-0 and every store fails, so A keeps rt-2 in memory. Pod B has no such memory: it reads the row + // — still the consumed rt-0 — and is refused for good, which marks the credential Error. The credential is not dead: a + // live pair exists, in A. A's next use must store it, not be told to reconnect by the verdict B's refusal wrote. + using var gitlab = new GitLabDoorkeeperStub(); + gitlab.Seed(GitLabDoorkeeperStub.DefaultClientId, "nacho", "at-0", "rt-0"); + var seed = await GitLabCredentialSeed.SeedAsync(_fixture, gitlab.BaseUrl, "at-0", "rt-0").ConfigureAwait(false); + + await Should.ThrowAsync(() => CredentialRefreshHarness.ResolveTokenAsync(_fixture, seed.RepositoryId, writer => new FailingCredentialPayloadWriter(writer, int.MaxValue))).ConfigureAwait(false); + + using var podB = KeptPairs.SetAsideLikeAnotherProcess(seed.CredentialId); + + await Should.ThrowAsync(() => CredentialRefreshHarness.ResolveTokenAsync(_fixture, seed.RepositoryId)).ConfigureAwait(false); + + var verdict = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + verdict.Status.ShouldBe(CredentialStatus.Error, "fixture check: pod B's refusal marked the credential"); + gitlab.Count(StubEvent.Refused).ShouldBe(1, "fixture check: B presented the consumed refresh token and was refused"); + + podB.PodAIsStillRunning(); + + var token = await CredentialRefreshHarness.ResolveTokenAsync(_fixture, seed.RepositoryId).ConfigureAwait(false); + + token.ShouldBe("at-2", "pod A holds the pair GitLab is waiting for, and its next use stores it before anything judges the credential by the row"); + gitlab.Count(StubEvent.Refused).ShouldBe(1, "A presented nothing GitLab had consumed"); + gitlab.Count(StubEvent.Rotated).ShouldBe(1, "storing the kept pair needs no second refresh"); + (await GitLabCredentialSeed.ReadStoredPayloadAsync(_fixture, seed.CredentialId).ConfigureAwait(false)).RefreshToken.ShouldBe("rt-2", "the row holds the pair GitLab is waiting for"); + + var healed = await GitLabCredentialSeed.ReadCredentialAsync(_fixture, seed.CredentialId).ConfigureAwait(false); + + healed.Status.ShouldBe(CredentialStatus.Active, "a live pair stored over the verdict lifts it"); + healed.LastError.ShouldBeNull(); + KeptPairs.IsHeldFor(seed.CredentialId).ShouldBeFalse("stored, so no longer kept"); + } + // ─── Plumbing ────────────────────────────────────────────────────────────── /// One run whose refresh is answered by GitLab and then, before the refresher stores, finds the old pair back in the row. @@ -205,4 +245,49 @@ public async Task RefreshAsync(OAuthRefreshInput input, Canc return answer; } } + + /// + /// The refresher keeps a pair it could not store in a static, per process, so a second pod "has no such memory". Tests run + /// every pod in one process, so the second pod is made by taking the pair out of that static for the time it runs and + /// putting it back when the first pod is alive again. Reflection, because the memory is deliberately not part of the + /// refresher's surface. + /// + private sealed class KeptPairs : IDisposable + { + private static readonly IDictionary Memory = (IDictionary)typeof(OAuthTokenRefresher).GetField("KeptRotations", BindingFlags.NonPublic | BindingFlags.Static)!.GetValue(null)!; + + private readonly Guid _credentialId; + private object? _setAside; + + private KeptPairs(Guid credentialId, object setAside) + { + _credentialId = credentialId; + _setAside = setAside; + } + + public static KeptPairs SetAsideLikeAnotherProcess(Guid credentialId) + { + var kept = Memory[credentialId] ?? throw new ShouldAssertException($"fixture check: no process kept a pair for credential {credentialId}"); + + Memory.Remove(credentialId); + + return new KeptPairs(credentialId, kept); + } + + public static bool IsHeldFor(Guid credentialId) => Memory.Contains(credentialId); + + public void PodAIsStillRunning() + { + Memory[_credentialId] = _setAside; + _setAside = null; + } + + /// A test that fails before pod A is back must not leave its pair in the static for the next test of this process. + public void Dispose() + { + if (_setAside is null) return; + + Memory.Remove(_credentialId); + } + } } diff --git a/backend/tests/CodeSpace.IntegrationTests/Webhooks/SilentProviderHost.cs b/backend/tests/CodeSpace.IntegrationTests/Webhooks/SilentProviderHost.cs new file mode 100644 index 000000000..32144466c --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Webhooks/SilentProviderHost.cs @@ -0,0 +1,66 @@ +using System.Net; +using System.Net.Sockets; + +namespace CodeSpace.IntegrationTests.Webhooks; + +/// +/// A provider that has stopped answering: it accepts connections, reads the request, and says nothing — a GitLab behind a +/// proxy that holds the line, or one that is up and wedged. Not a refusal and not a failure, which is what makes it the case +/// the health sweep must survive: a call to it never returns on its own, and a caller that does not give up holds its thread +/// for the HTTP client's whole timeout. answers one request at a time and always answers, so +/// it cannot stand in for this. +/// +internal sealed class SilentProviderHost : IDisposable +{ + private readonly TcpListener _listener; + private readonly CancellationTokenSource _stopping = new(); + private readonly List _held = new(); + private int _connections; + + public SilentProviderHost() + { + _listener = new TcpListener(IPAddress.Loopback, 0); + _listener.Start(); + + BaseUrl = $"http://127.0.0.1:{((IPEndPoint)_listener.LocalEndpoint).Port}"; + + _ = Task.Run(AcceptAsync); + } + + public string BaseUrl { get; } + + /// How many connections the provider has been called on. + public int Connections => Volatile.Read(ref _connections); + + public void Dispose() + { + _stopping.Cancel(); + _listener.Stop(); + + lock (_held) + { + foreach (var client in _held) client.Dispose(); + } + } + + private async Task AcceptAsync() + { + while (!_stopping.IsCancellationRequested) + { + TcpClient client; + + try + { + client = await _listener.AcceptTcpClientAsync(_stopping.Token).ConfigureAwait(false); + } + catch (Exception) + { + return; // the listener was stopped — the only way out of the accept + } + + Interlocked.Increment(ref _connections); + + lock (_held) _held.Add(client); + } + } +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentPublishCredentialFreshnessFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentPublishCredentialFreshnessFlowTests.cs index d5b9842b4..122d27491 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentPublishCredentialFreshnessFlowTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentPublishCredentialFreshnessFlowTests.cs @@ -157,12 +157,12 @@ public async Task A_repository_deleted_while_the_agent_worked_is_not_pushed_to() } [Fact] - public async Task A_refresh_the_provider_refuses_is_a_workspace_failure_the_push_retry_can_see_not_a_raw_oauth_exception() + public async Task A_refresh_the_provider_refuses_for_good_is_the_typed_rejection_and_the_push_is_not_retried_nor_a_raw_oauth_exception_let_out() { - // The stored refresh token is one the provider no longer honours (rotated elsewhere, grant revoked). Turning that - // into a typed 'reconnect' is the refresher's own arc; what this pins is that the push path stays inside its own - // exception contract — a WorkspaceException — instead of letting an OAuth exception escape the publish and flip the - // run that produced the work. + // The stored refresh token is one the provider no longer honours (rotated elsewhere, grant revoked). That is the + // provider's verdict on the grant: the credential is marked as needing a reconnect, and the push says so as the same + // typed rejection a disconnected credential gets — the push retry only retries a WorkspaceException, and a dead credential + // resolves the same way every time. No OAuth exception escapes the publish to flip the run that produced the work. if (!await CanRunGitAsync()) return; using var ctx = await FreshnessContext.StartAsync(_fixture, liveRefreshToken: "rt-held-by-someone-else"); @@ -171,11 +171,13 @@ public async Task A_refresh_the_provider_refuses_is_a_workspace_failure_the_push await ctx.AgeStoredPairAsync(secondsLeft: 30); await ctx.AgentWritesAsync(handle.Directory); - var failure = await Should.ThrowAsync(() => ((IWorkspacePushHandle)handle).PushChangesAsync(ctx.BranchName, CancellationToken.None)); + var failure = await Should.ThrowAsync(() => ((IWorkspacePushHandle)handle).PushChangesAsync(ctx.BranchName, CancellationToken.None)); - failure.Message.ShouldContain("Could not resolve the git credential to push with"); + failure.Code.ShouldBe(FailureCodes.GitCredentialRejected); + failure.Message.ShouldContain("was refused by its provider"); failure.Message.ShouldContain("invalid_grant", Case.Insensitive); failure.Message.ShouldNotContain("at-0", Case.Insensitive, "the token is never in the message"); + failure.InnerException.ShouldBeOfType(); ctx.Remote.RefusedAuthorizations.ShouldBeEmpty(); } diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/PullRequestTriggerAdmissionFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/PullRequestTriggerAdmissionFlowTests.cs index 8ff5b66c8..3ac8faeb8 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/PullRequestTriggerAdmissionFlowTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/PullRequestTriggerAdmissionFlowTests.cs @@ -154,6 +154,32 @@ public async Task On_a_public_gitlab_project_a_push_by_someone_else_to_a_fork_mr gitlab.Requests.Count(r => r.PathAndQuery.Contains("/members/all/66")).ShouldBe(1, customMessage: "GitLab names the pusher but not their standing, so it is asked like the author's"); } + [Theory] + [InlineData(403, WorkflowRunRequestRejectionReasons.AuthorStandingUnknown, "refused the lookup (HTTP 403)")] + [InlineData(503, WorkflowRunRequestRejectionReasons.AuthorStandingUnknown, "answered HTTP 503 to the lookup")] + [InlineData(401, WorkflowRunRequestRejectionReasons.CredentialRejected, "answered HTTP 401 to the repository's credential")] + public async Task On_a_public_gitlab_project_a_pusher_whose_standing_could_not_be_read_is_refused_for_that_not_as_a_stranger(int status, string reason, string cause) + { + // The author is a known member; whoever else pushed to their fork's head has to be a member too, and asking GitLab + // about them failed. Which refusal that is matters: "their standing is unknown, set Pull requests from to Anyone" sends + // the operator to loosen the trigger for a problem that is the credential's, or the lookup's. + using var gitlab = new StubProviderHost(); + var seed = await SeedAsync(ProviderKind.GitLab, gitlab.BaseUrl); + gitlab.Answer("GET", $"/api/v4/projects/{seed.ExternalId}/members/all/51", 200, """{"id":51,"access_level":40}""") + .Answer("GET", $"/api/v4/projects/{seed.ExternalId}/members/all/66", status, """{"message":"no"}"""); + await SeedActivationAsync(seed, Updated, "{}"); + + await PublishAsync(Event(Updated, seed.RepositoryId, number: 17, new PullRequestOrigin { AuthorExternalId = "51", RepositoryVisibility = RepositoryVisibility.Public, IsFork = true, PusherExternalId = "66" })); + + (await CountRunsAsync(seed.WorkflowId)).ShouldBe(0, customMessage: "a pusher nobody could vouch for is not admitted — Members still means members"); + + var refusal = (await LoadRefusalsAsync(seed.TeamId, reason)).ShouldHaveSingleItem(); + + refusal.ShouldContain("whoever pushed its new commits", customMessage: "the refusal says whose standing could not be read — here the pusher's, the author's being known"); + refusal.ShouldContain(cause); + (await LoadRefusalsAsync(seed.TeamId, WorkflowRunRequestRejectionReasons.AuthorNotMember)).ShouldBeEmpty("nothing is known about the pusher, so no one was refused as a stranger"); + } + [Fact] public async Task One_delivery_asks_gitlab_once_however_many_activations_match_and_an_any_trigger_still_learns_the_standing() { @@ -172,7 +198,7 @@ public async Task One_delivery_asks_gitlab_once_however_many_activations_match_a } [Fact] - public async Task A_lookup_gitlab_could_not_answer_is_asked_once_per_delivery_not_once_per_activation() + public async Task A_lookup_gitlab_could_not_answer_is_asked_once_per_delivery_with_one_retry_not_once_per_activation() { using var gitlab = new StubProviderHost(); var seed = await SeedAsync(ProviderKind.GitLab, gitlab.BaseUrl); @@ -183,7 +209,7 @@ public async Task A_lookup_gitlab_could_not_answer_is_asked_once_per_delivery_no await PublishAsync(Event(Opened, seed.RepositoryId, number: 11, new PullRequestOrigin { AuthorExternalId = "51", RepositoryVisibility = RepositoryVisibility.Public })); - gitlab.Requests.Count(r => r.PathAndQuery.Contains("/members/all/51")).ShouldBe(1, customMessage: "an unanswered lookup costs up to its timeout; every matching activation repeating it would hold the delivery past the provider's own timeout"); + gitlab.Requests.Count(r => r.PathAndQuery.Contains("/members/all/51")).ShouldBe(2, customMessage: "an unanswered lookup costs up to its timeout, and a 5xx is asked once more; every matching activation repeating that would hold the delivery past the provider's own timeout"); (await CountRunsAsync(seed.WorkflowId)).ShouldBe(0); (await CountRunsAsync(sibling.WorkflowId)).ShouldBe(0); } diff --git a/backend/tests/CodeSpace.StorageTestWorker/OAuthRefreshPodWorker.cs b/backend/tests/CodeSpace.StorageTestWorker/OAuthRefreshPodWorker.cs index 576e49dbf..28e07b714 100644 --- a/backend/tests/CodeSpace.StorageTestWorker/OAuthRefreshPodWorker.cs +++ b/backend/tests/CodeSpace.StorageTestWorker/OAuthRefreshPodWorker.cs @@ -119,7 +119,7 @@ public IOAuthTokenRefresher ThisBuildRefresher() { var trackedEntry = new TrackedCredentialEntry(NewDb()); - return new OAuthTokenRefresher(_clients, new CredentialPayloadWriter(Encryptor, Serializer, _settings, trackedEntry), _reader, Resolver, Encryptor, TimeProvider.System, new PostgresAdvisoryLock(_settings), trackedEntry, NullLogger.Instance); + return new OAuthTokenRefresher(_clients, new CredentialPayloadWriter(Encryptor, Serializer, _settings, trackedEntry), _reader, Resolver, Encryptor, TimeProvider.System, new PostgresAdvisoryLock(_settings), trackedEntry, new CredentialHealthWriter(_settings), NullLogger.Instance); } public IOAuthTokenRefresher PreviousBuildRefresher() => diff --git a/backend/tests/CodeSpace.UnitTests/Architecture/FailureTaxonomyTests.cs b/backend/tests/CodeSpace.UnitTests/Architecture/FailureTaxonomyTests.cs index b0ba2e7ac..c96f35c82 100644 --- a/backend/tests/CodeSpace.UnitTests/Architecture/FailureTaxonomyTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Architecture/FailureTaxonomyTests.cs @@ -75,6 +75,7 @@ public void The_wire_codes_are_pinned() FailureCodes.ProviderError.ShouldBe("provider_error"); FailureCodes.RateLimited.ShouldBe("rate_limited"); FailureCodes.CredentialDisconnected.ShouldBe("credential_disconnected"); + FailureCodes.CredentialNeedsReconnect.ShouldBe("credential_needs_reconnect"); FailureCodes.GitCredentialRejected.ShouldBe("git_credential_rejected"); FailureCodes.ModelEndpointKeyRequired.ShouldBe("model_endpoint_key_required"); FailureCodes.ProviderEndpointCredentialsBound.ShouldBe("provider_endpoint_credentials_bound"); diff --git a/backend/tests/CodeSpace.UnitTests/Architecture/RequestAuthorizationInventoryTests.cs b/backend/tests/CodeSpace.UnitTests/Architecture/RequestAuthorizationInventoryTests.cs index aee53154f..4360026ff 100644 --- a/backend/tests/CodeSpace.UnitTests/Architecture/RequestAuthorizationInventoryTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Architecture/RequestAuthorizationInventoryTests.cs @@ -80,6 +80,7 @@ public class RequestAuthorizationInventoryTests ["BackfillStorageNamespaceFingerprintsCommand"] = "puts back, across every team in bounded batches, the namespace fingerprint a storage profile revision's own configuration hashes to as written, where a physical-identity build stamped another; it changes no configuration, route or placement, nothing in this build reads the value to decide anything, and a restored revision is no longer a candidate.", ["RepairDisconnectedCredentialBindingsCommand"] = "moves repositories and group hooks still bound to a disconnected credential onto the credential that credential's own owner connected afterwards on the same provider instance and team, across every team, in bounded batches; it never moves a binding off a live credential or onto another person's, so it only completes a reconnect the owner already made, and a moved binding is no longer a candidate.", ["RetireDepartedMembersCredentialsCommand"] = "retires the personal tokens whose owner — or, for one stored without an owner, whose creator account — has no standing in the team the token was lent to (no membership there and not an instance Admin), across every team, in bounded batches; it ends access the person no longer has standing to lend, exactly as their departure does, never touches a team-service token, the token of anyone who still stands in its team, or one it cannot attribute to an account, and a retired token is no longer a candidate.", + ["ProbeCredentialsCommand"] = "asks each credential's own provider whether the token still works, across every team, in bounded batches, and records the answer on the credential row: a validation date and a static token's real expiry, or — only on a definite 401 — the Error that says it needs reconnecting; it changes no token, binding or repository, an answer that is not an answer (a timeout, a 5xx) changes nothing, and a credential the provider accepts again heals.", ["ShredRetiredSecretsCommand"] = "blanks the ciphertext of secrets that are already retired — deleted variables, revoked or soft-deleted git tokens, deleted connections' client and hook secrets, revoked storage credentials' revisions, deployment-default envelopes no template names — across every team; it never touches a live secret, ends no access anyone still has, and a blanked row is no longer a candidate, so it only finishes a retirement an older build left decryptable.", ["BackfillRunScorecardsCommand"] = "projects the observation-only north-star row for terminal runs that terminalized before the table existed, across every team, in bounded batches; a run that already has a row is not a candidate, so it can only ever add a measurement of a settled run — it authors nothing a user owns and changes no run's outcome.", ["SweepStaleAgentWorkspacesCommand"] = "sweep", diff --git a/backend/tests/CodeSpace.UnitTests/CodeSpace.UnitTests.csproj b/backend/tests/CodeSpace.UnitTests/CodeSpace.UnitTests.csproj index d86de9850..90bd1d29f 100644 --- a/backend/tests/CodeSpace.UnitTests/CodeSpace.UnitTests.csproj +++ b/backend/tests/CodeSpace.UnitTests/CodeSpace.UnitTests.csproj @@ -41,6 +41,8 @@ + + diff --git a/backend/tests/CodeSpace.UnitTests/Credentials/CredentialHealthDerivationTests.cs b/backend/tests/CodeSpace.UnitTests/Credentials/CredentialHealthDerivationTests.cs new file mode 100644 index 000000000..5b30e28ac --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Credentials/CredentialHealthDerivationTests.cs @@ -0,0 +1,114 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using CodeSpace.Core.Services.Credentials; +using CodeSpace.Messages.Dtos.Credentials; +using CodeSpace.Messages.Enums; +using Shouldly; + +namespace CodeSpace.UnitTests.Credentials; + +/// +/// What a credential list says about each credential, from what is recorded: never stored, so it cannot disagree with the +/// columns. Only a static token has an expiry worth showing — an OAuth credential's expiry is its access token's and renews +/// every couple of hours. +/// +[Trait("Category", "Unit")] +public class CredentialHealthDerivationTests +{ + private static readonly DateTimeOffset Now = new(2030, 6, 1, 12, 0, 0, TimeSpan.Zero); + + [Theory] + // status, auth type, days until the recorded expiry (null = none recorded), owner left, expected + [InlineData(CredentialStatus.Active, AuthType.Pat, null, false, CredentialHealth.Active)] + [InlineData(CredentialStatus.Active, AuthType.Pat, 200, false, CredentialHealth.Active)] + [InlineData(CredentialStatus.Active, AuthType.Pat, 31, false, CredentialHealth.Active)] + [InlineData(CredentialStatus.Active, AuthType.Pat, 30, false, CredentialHealth.Expiring)] + [InlineData(CredentialStatus.Active, AuthType.ProjectAccessToken, 3, false, CredentialHealth.Expiring)] + [InlineData(CredentialStatus.Active, AuthType.GroupAccessToken, 1, false, CredentialHealth.Expiring)] + [InlineData(CredentialStatus.Active, AuthType.Pat, -1, false, CredentialHealth.Rejected)] // its provider-side expiry has passed: it will be refused + [InlineData(CredentialStatus.Active, AuthType.OAuth, 0, false, CredentialHealth.Active)] // an access token's expiry is not the credential's + [InlineData(CredentialStatus.Active, AuthType.OAuth, -1, false, CredentialHealth.Active)] + [InlineData(CredentialStatus.Error, AuthType.OAuth, null, false, CredentialHealth.Rejected)] + [InlineData(CredentialStatus.Error, AuthType.Pat, 100, false, CredentialHealth.Rejected)] + [InlineData(CredentialStatus.Expired, AuthType.Pat, null, false, CredentialHealth.Rejected)] + [InlineData(CredentialStatus.Revoked, AuthType.Pat, null, false, CredentialHealth.Revoked)] + [InlineData(CredentialStatus.Revoked, AuthType.OAuth, null, true, CredentialHealth.Revoked)] // disconnected beats everything + [InlineData(CredentialStatus.Active, AuthType.Pat, 100, true, CredentialHealth.OwnerLeft)] + [InlineData(CredentialStatus.Active, AuthType.Pat, 5, true, CredentialHealth.OwnerLeft)] // someone must act on the departure before the expiry + [InlineData(CredentialStatus.Error, AuthType.Pat, null, true, CredentialHealth.Rejected)] // it needs reconnecting whoever holds it + public void Health_is_read_off_the_recorded_columns(CredentialStatus status, AuthType type, int? daysToExpiry, bool ownerLeft, CredentialHealth expected) + { + DateTimeOffset? expires = daysToExpiry is { } days ? Now.AddDays(days).AddHours(-1) : null; // "30" is a token with a little under 30 days left + + CredentialHealthDerivation.Derive(status, type, expires, ownerLeft, Now).ShouldBe(expected); + } + + [Fact] + public void An_expiry_at_this_very_moment_has_passed() + { + CredentialHealthDerivation.Derive(CredentialStatus.Active, AuthType.Pat, Now, ownerLeft: false, Now).ShouldBe(CredentialHealth.Rejected); + } + + [Theory] + [InlineData(AuthType.Pat, 3.5, 3)] + [InlineData(AuthType.ProjectAccessToken, 0.2, 0)] + [InlineData(AuthType.GroupAccessToken, 364.9, 364)] + [InlineData(AuthType.Pat, -4.0, 0)] + public void Days_to_expiry_are_whole_days_left_and_never_negative(AuthType type, double days, int expected) + { + CredentialHealthDerivation.DaysToExpiry(type, Now.AddDays(days), Now).ShouldBe(expected); + } + + [Theory] + [InlineData(AuthType.OAuth)] + [InlineData(AuthType.GitHubApp)] + public void A_credential_that_is_not_a_static_token_has_no_days_to_expiry(AuthType type) + { + CredentialHealthDerivation.DaysToExpiry(type, Now.AddDays(5), Now).ShouldBeNull(); + } + + [Fact] + public void A_static_token_with_no_recorded_expiry_has_no_days_to_expiry() + { + CredentialHealthDerivation.DaysToExpiry(AuthType.Pat, null, Now).ShouldBeNull(); + } + + [Fact] + public void The_warning_window_is_a_month() + { + CredentialHealthDerivation.ExpiringWithin.ShouldBe(TimeSpan.FromDays(30)); + } + + [Fact] + public void The_summary_is_additive_on_the_wire_and_names_its_health_in_the_words_clients_branch_on() + { + // Clients that read the old fields keep reading them; the two new ones are extra, and the health is spelled the way + // the contract says (active | expiring | rejected | revoked | owner_left), not as the C# member names. + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) { Converters = { new JsonStringEnumConverter() } }; + var summary = new CredentialSummary { Id = Guid.NewGuid(), TeamId = Guid.NewGuid(), ProviderInstanceId = Guid.NewGuid(), Ownership = CredentialOwnership.Personal, AuthType = AuthType.Pat, DisplayName = "GitLab · nacho", Status = CredentialStatus.Active, CreatedDate = Now, Health = CredentialHealth.OwnerLeft, DaysToExpiry = 12 }; + + using var json = JsonDocument.Parse(JsonSerializer.Serialize(summary, options)); + var root = json.RootElement; + + foreach (var property in new[] { "id", "teamId", "providerInstanceId", "ownerUserId", "ownerUserName", "ownership", "authType", "displayName", "status", "expiresDate", "lastValidatedDate", "lastError", "createdDate" }) + root.TryGetProperty(property, out _).ShouldBeTrue($"'{property}' was in the summary before and must stay"); + + root.GetProperty("status").GetString().ShouldBe("Active", "existing enums keep their spelling"); + root.GetProperty("health").GetString().ShouldBe("owner_left"); + root.GetProperty("daysToExpiry").GetInt32().ShouldBe(12); + } + + [Theory] + [InlineData(CredentialHealth.Active, "active")] + [InlineData(CredentialHealth.Expiring, "expiring")] + [InlineData(CredentialHealth.Rejected, "rejected")] + [InlineData(CredentialHealth.Revoked, "revoked")] + [InlineData(CredentialHealth.OwnerLeft, "owner_left")] + public void Every_health_has_its_pinned_wire_name(CredentialHealth health, string wire) + { + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) { Converters = { new JsonStringEnumConverter() } }; + + JsonSerializer.Serialize(health, options).ShouldBe($"\"{wire}\"", "a rename is a breaking change to every client that branches on it"); + JsonSerializer.Deserialize($"\"{wire}\"", options).ShouldBe(health); + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Credentials/CredentialResolverTests.cs b/backend/tests/CodeSpace.UnitTests/Credentials/CredentialResolverTests.cs index 90fe025ac..755b5ec54 100644 --- a/backend/tests/CodeSpace.UnitTests/Credentials/CredentialResolverTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Credentials/CredentialResolverTests.cs @@ -43,6 +43,67 @@ public void A_disconnected_credential_names_the_disconnect_instead_of_decrypting failure.Message.ShouldContain("'alice's GitLab' was disconnected"); } + [Fact] + public void A_credential_its_provider_refused_fails_fast_naming_it_and_what_was_recorded_instead_of_presenting_its_token_again() + { + var credential = BuildCredential(CredentialStatus.Error, _encryptor.Encrypt(_serializer.Serialize(new PatPayload { Token = "glpat-dead" }))); + credential.LastError = "needs reconnect: invalid_grant"; + + var failure = Should.Throw(() => new CredentialResolver(_encryptor, _serializer).Resolve(credential)); + + failure.CredentialId.ShouldBe(credential.Id); + failure.DisplayName.ShouldBe("alice's GitLab"); + failure.Code.ShouldBe(FailureCodes.CredentialNeedsReconnect); + failure.Kind.ShouldBe(FailureKind.PreconditionRequired, "a person reconnects it; a retry as-is reads the same refused credential"); + failure.Details.ShouldNotBeNull()["credentialId"].ShouldBe(credential.Id); + failure.Message.ShouldContain("needs reconnect: invalid_grant", customMessage: "what the provider said when it was marked is what an operator reading a run's error needs"); + failure.Message.ShouldNotContain("glpat-dead"); + } + + [Theory] + [InlineData(AuthType.OAuth, 120, false)] // refused over an hour ago: its refresher is asked again, so one that heals recovers on use + [InlineData(AuthType.OAuth, 5, true)] // refused minutes ago: fails from what the row says + [InlineData(AuthType.Pat, 120, true)] // a static token has nothing to refresh: the health sweep re-asks it, and a use would only be refused + [InlineData(AuthType.ProjectAccessToken, 120, true)] + [InlineData(AuthType.GroupAccessToken, 120, true)] + public void An_oauth_credential_refused_over_an_hour_ago_is_asked_again_on_use_and_nothing_else_is(AuthType type, int minutesSinceTheVerdict, bool failsFast) + { + var payload = type == AuthType.OAuth ? (CredentialPayload)new OAuthPayload { AccessToken = "at", RefreshToken = "rt" } : new PatPayload { Token = "glpat-static" }; + var credential = BuildCredential(CredentialStatus.Error, _encryptor.Encrypt(_serializer.Serialize(payload))); + credential.AuthType = type; + credential.LastError = "needs reconnect: invalid_grant"; + credential.LastModifiedDate = DateTimeOffset.UtcNow.AddMinutes(-minutesSinceTheVerdict); + + var resolver = new CredentialResolver(_encryptor, _serializer); + + if (failsFast) Should.Throw(() => resolver.Resolve(credential)); + else resolver.Resolve(credential).ShouldBeOfType().AccessToken.ShouldBe("at", "the row's payload is what the refresher decides from, so a refresh can be attempted and the verdict lifted if the provider accepts it"); + } + + [Fact] + public void The_window_an_oauth_credential_is_refused_for_is_the_one_the_health_sweep_rechecks_after() + { + // One definition of "long enough to ask again": a use and the sweep must agree, or a credential the sweep would + // re-ask stays refused to runs, or the reverse. + var credential = BuildCredential(CredentialStatus.Error, _encryptor.Encrypt(_serializer.Serialize(new OAuthPayload { AccessToken = "at", RefreshToken = "rt" }))); + credential.AuthType = AuthType.OAuth; + var resolver = new CredentialResolver(_encryptor, _serializer); + + credential.LastModifiedDate = DateTimeOffset.UtcNow - CredentialHealthService.ErrorRecheckAfter + TimeSpan.FromSeconds(30); + Should.Throw(() => resolver.Resolve(credential)); + + credential.LastModifiedDate = DateTimeOffset.UtcNow - CredentialHealthService.ErrorRecheckAfter - TimeSpan.FromSeconds(30); + resolver.Resolve(credential).ShouldBeOfType(); + } + + [Fact] + public void An_Error_credential_with_no_recorded_reason_still_says_it_was_refused() + { + var credential = BuildCredential(CredentialStatus.Error, "sealed"); + + Should.Throw(() => new CredentialResolver(_encryptor, _serializer).Resolve(credential)).Message.ShouldContain("refused it"); + } + [Fact] public void The_disconnect_names_the_remedies_a_person_can_perform_today_and_who_may_apply_them() { diff --git a/backend/tests/CodeSpace.UnitTests/Jobs/CredentialHealthProbeDispatchTests.cs b/backend/tests/CodeSpace.UnitTests/Jobs/CredentialHealthProbeDispatchTests.cs new file mode 100644 index 000000000..ce1342219 --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Jobs/CredentialHealthProbeDispatchTests.cs @@ -0,0 +1,120 @@ +using CodeSpace.Core.Handlers.CommandHandlers.Credentials; +using CodeSpace.Core.Jobs.RecurringJobs; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Credentials; +using CodeSpace.Messages.Commands.Credentials; +using CodeSpace.Messages.Dtos.Credentials; +using CodeSpace.Messages.Mediation; +using MediatR; +using Shouldly; + +namespace CodeSpace.UnitTests.Jobs; + +/// +/// 🟢 Unit: the credential health probe is the standard job → command → service chain (Rule 14 + Rule 16) — the job holds no +/// query and calls no provider, the handler holds no logic. Hand-rolled recording doubles, matching the codebase convention. +/// +[Trait("Category", "Unit")] +public class CredentialHealthProbeDispatchTests +{ + [Fact] + public async Task The_job_is_a_thin_dispatcher_of_the_probe_command_every_ten_minutes() + { + var mediator = new RecordingMediator(); + var job = new CredentialHealthProbeRecurringJob(mediator); + + job.JobId.ShouldBe(nameof(CredentialHealthProbeRecurringJob), "Hangfire indexes the schedule by this id — a rename strands the old one"); + job.CronExpression.ShouldBe("*/10 * * * *"); + + await job.Execute(); + + mediator.Sent.ShouldHaveSingleItem().ShouldBeOfType("the job is a thin dispatcher — it only sends the command"); + } + + [Fact] + public async Task The_handler_hands_its_batch_size_to_the_service_and_returns_what_it_probed() + { + var health = new RecordingHealth { ToReturn = 3 }; + + var probed = await new ProbeCredentialsCommandHandler(health).Handle(new ProbeCredentialsCommand { BatchSize = 7 }, CancellationToken.None); + + health.BatchSizes.ShouldHaveSingleItem().ShouldBe(7, "the handler delegates the whole pass to the service (Rule 16)"); + probed.ShouldBe(3); + } + + [Fact] + public void The_command_is_bounded_and_not_transactional() + { + new ProbeCredentialsCommand().BatchSize.ShouldBe(25, "an unbounded pass would let one tick call every provider for every credential in the deployment"); + typeof(INonTransactionalCommand).IsAssignableFrom(typeof(ProbeCredentialsCommand)).ShouldBeTrue("each credential is recorded on its own connection; one transaction would hold their rows across every provider call"); + } + + [Fact] + public void The_intervals_that_decide_how_often_a_provider_is_asked_are_pinned() + { + CredentialHealthService.ValidatedStaleAfter.ShouldBe(TimeSpan.FromHours(6)); + CredentialHealthService.ErrorRecheckAfter.ShouldBe(TimeSpan.FromHours(1)); + CredentialHealthDeadlines.Default.Probe.ShouldBe(TimeSpan.FromSeconds(30)); + CredentialHealthDeadlines.Default.Pass.ShouldBe(TimeSpan.FromMinutes(5)); + } + + [Fact] + public void A_pass_is_over_before_the_next_tick_and_a_probe_cannot_outlast_it() + { + // The job fires every ten minutes; a pass that could still be running at the next tick is what piles up behind a provider that stopped answering. + var interval = TimeSpan.FromMinutes(10); + + new CredentialHealthProbeRecurringJob(new RecordingMediator()).CronExpression.ShouldBe("*/10 * * * *", "the interval the budget is measured against"); + CredentialHealthDeadlines.Default.Pass.ShouldBeLessThan(interval); + CredentialHealthDeadlines.Default.Probe.ShouldBeLessThan(CredentialHealthDeadlines.Default.Pass); + } + + [Fact] + public void The_passs_lock_key_is_pinned_because_every_worker_must_agree_on_it() + { + // Two workers holding different keys would run two passes at once, which is what the lock is there to prevent. + CredentialHealthService.PassLockKey.ShouldBe(0x4352_4544_4845_4C54); + } + + private sealed class RecordingHealth : ICredentialHealthService + { + public List BatchSizes { get; } = new(); + public int ToReturn; + + public Task ProbeBatchAsync(int batchSize, CancellationToken cancellationToken) + { + BatchSizes.Add(batchSize); + return Task.FromResult(ToReturn); + } + + public Task ValidateNewAsync(Credential credential, ProviderInstance instance, CancellationToken cancellationToken) => Task.CompletedTask; + } + + private sealed class RecordingMediator : IMediator + { + public List Sent { get; } = new(); + + public Task Send(IRequest request, CancellationToken cancellationToken = default) + { + Sent.Add(request); + return Task.FromResult(default(TResponse)!); + } + + public Task Send(object request, CancellationToken cancellationToken = default) + { + Sent.Add(request); + return Task.FromResult(null); + } + + public Task Send(TRequest request, CancellationToken cancellationToken = default) where TRequest : IRequest + { + Sent.Add(request!); + return Task.CompletedTask; + } + + public IAsyncEnumerable CreateStream(IStreamRequest request, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public IAsyncEnumerable CreateStream(object request, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public Task Publish(object notification, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public Task Publish(TNotification notification, CancellationToken cancellationToken = default) where TNotification : INotification => throw new NotSupportedException(); + } +} diff --git a/backend/tests/CodeSpace.UnitTests/OAuth/OAuthExchangeExceptionTests.cs b/backend/tests/CodeSpace.UnitTests/OAuth/OAuthExchangeExceptionTests.cs new file mode 100644 index 000000000..d4685b3e7 --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/OAuth/OAuthExchangeExceptionTests.cs @@ -0,0 +1,61 @@ +using CodeSpace.Core.Services.OAuth; +using CodeSpace.Messages.Failures; +using Shouldly; + +namespace CodeSpace.UnitTests.OAuth; + +/// +/// Which of a provider's refusals of a refresh is a verdict on the grant — the one the refresher turns into a credential that +/// needs reconnecting — and the text a refusal carries once it names the credential it was for. +/// +[Trait("Category", "Unit")] +public class OAuthExchangeExceptionTests +{ + [Fact] + public void The_refusals_that_judge_the_grant_or_the_client_are_pinned_member_by_member() + { + // Too wide and a provider that is merely down — or a request CodeSpace got wrong — marks a working credential as + // dead and stops every run through it; too narrow and a dead grant stays Active and fails every run at the provider. + // Either way the drift is silent, so the set is spelled out here as literals rather than compared with its source. + OAuthExchangeException.FinalRefusalErrors.ShouldBe(new[] { "invalid_grant", "invalid_client", "unauthorized_client", "bad_refresh_token", "incorrect_client_credentials" }, ignoreOrder: true, + customMessage: "adding an error code here makes the credential Error on the provider's say-so — state why the code can never be a transient or our own malformed request, and add it to this list deliberately"); + } + + [Theory] + [InlineData("invalid_grant", true)] + [InlineData("invalid_client", true)] + [InlineData("INVALID_GRANT", false)] // the codes are exact: RFC 6749 spells them lower-case, and nothing case-folds a verdict into existence + [InlineData("http_500", false)] + [InlineData("invalid_request", false)] + [InlineData("temporarily_unavailable", false)] + public void A_refusal_is_final_only_when_its_code_is_one_of_the_pinned_set(string error, bool final) + { + new OAuthExchangeException(error, "description", providerBody: null).IsFinalRefusal.ShouldBe(final); + } + + [Fact] + public void Naming_the_credential_keeps_what_the_provider_said_and_adds_which_credential_it_was_for() + { + var credentialId = Guid.NewGuid(); + var original = new OAuthExchangeException("invalid_grant", "The provided authorization grant is invalid", "{\"error\":\"invalid_grant\"}"); + + var named = original.ForCredential(credentialId); + + named.Error.ShouldBe("invalid_grant"); + named.Description.ShouldBe("The provided authorization grant is invalid"); + named.ProviderBody.ShouldBe("{\"error\":\"invalid_grant\"}"); + named.Message.ShouldBe($"{original.Message} (credential {credentialId})"); + named.InnerException.ShouldBeSameAs(original); + named.IsFinalRefusal.ShouldBeTrue(); + named.Code.ShouldBe(FailureCodes.OAuthExchangeFailed); + named.Kind.ShouldBe(original.Kind); + } + + [Fact] + public void The_text_never_carries_more_than_the_provider_and_the_credential_id() + { + var named = new OAuthExchangeException("invalid_client", "Client authentication failed", providerBody: "client_secret=hunter2").ForCredential(Guid.NewGuid()); + + named.Message.ShouldNotContain("hunter2", customMessage: "the provider's body stays on the exception for diagnostics and is never part of its message"); + } +} diff --git a/backend/tests/CodeSpace.UnitTests/OAuth/OAuthTokenRefresherTests.Classification.cs b/backend/tests/CodeSpace.UnitTests/OAuth/OAuthTokenRefresherTests.Classification.cs new file mode 100644 index 000000000..990c23b7e --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/OAuth/OAuthTokenRefresherTests.Classification.cs @@ -0,0 +1,205 @@ +using CodeSpace.Core.Services.Credentials; +using CodeSpace.Core.Services.OAuth; +using CodeSpace.Core.Services.Providers; +using CodeSpace.Messages.Credentials; +using CodeSpace.Messages.Exceptions; +using CodeSpace.Messages.Failures; +using Shouldly; + +namespace CodeSpace.UnitTests.OAuth; + +/// +/// What the refresher does with a refusal that stands, and with an access token the provider refused after it was handed out. +/// +/// A refusal that names the grant or the client, with the row unchanged, is the provider's verdict: the credential is +/// marked, over the row the verdict was judged against, and the caller is told to reconnect it — naming it. Any other +/// refusal says nothing about the credential. After a 401 the refresher adopts what another caller already stored, or +/// rotates once; a token it rotated moments ago is not rotated again. +/// +public partial class OAuthTokenRefresherTests +{ + [Fact] + public async Task A_final_refusal_marks_the_credential_over_the_row_it_judged_and_names_it() + { + var ctx = BuildContext(); + var stale = ExpiredPair(); + var row = new StoredRow(stale); + var health = new RecordingHealthWriter(); + + var verdict = await Should.ThrowAsync(() => RunAsync(stale, ctx, row, RecordingOAuthClient.Refusing(), healthWriter: health)); + + verdict.CredentialId.ShouldBe(ctx.Credential.Id); + verdict.DisplayName.ShouldBe("cred"); + verdict.Reason.ShouldBe("invalid_grant"); + verdict.Message.ShouldContain(ctx.Credential.Id.ToString()); + verdict.Code.ShouldBe(FailureCodes.CredentialNeedsReconnect); + + var recorded = health.Rejections.ShouldHaveSingleItem(); + + recorded.CredentialId.ShouldBe(ctx.Credential.Id); + recorded.LastError.ShouldBe("needs reconnect: invalid_grant"); + recorded.ExpectedPayload.ShouldBe(Serializer.Serialize(stale), "the verdict is written against the row the refusal was judged against, so a reconnect that replaced it meanwhile is never overwritten"); + } + + [Theory] + [InlineData("invalid_grant", true)] + [InlineData("invalid_client", true)] + [InlineData("unauthorized_client", true)] + [InlineData("bad_refresh_token", true)] + [InlineData("incorrect_client_credentials", true)] + [InlineData("http_503", false)] + [InlineData("temporarily_unavailable", false)] + [InlineData("server_error", false)] + [InlineData("invalid_request", false)] + [InlineData("malformed_response", false)] + [InlineData("unknown_error", false)] + public async Task Only_a_refusal_of_the_grant_or_the_client_is_a_verdict(string oauthError, bool isVerdict) + { + var ctx = BuildContext(); + var stale = ExpiredPair(); + var health = new RecordingHealthWriter(); + + var failure = await Should.ThrowAsync(() => RunAsync(stale, ctx, new StoredRow(stale), RecordingOAuthClient.Refusing(error: oauthError), healthWriter: health)); + + if (isVerdict) + { + failure.ShouldBeOfType(); + health.Rejections.ShouldHaveSingleItem().LastError.ShouldBe($"needs reconnect: {oauthError}"); + return; + } + + var exchange = failure.ShouldBeOfType($"'{oauthError}' says nothing about the credential, so it stays the provider's failure"); + exchange.Error.ShouldBe(oauthError); + exchange.Message.ShouldContain(ctx.Credential.Id.ToString(), customMessage: "the provider's answer names no credential; the failure must, or an operator cannot tell which row it was for"); + health.Rejections.ShouldBeEmpty(); + } + + [Fact] + public async Task A_verdict_that_cannot_be_recorded_still_reaches_the_caller() + { + var ctx = BuildContext(); + var stale = ExpiredPair(); + + await Should.ThrowAsync(() => RunAsync(stale, ctx, new StoredRow(stale), RecordingOAuthClient.Refusing(), healthWriter: new RecordingHealthWriter { Fails = true })); + } + + [Fact] + public async Task A_refusal_the_row_moved_past_is_not_a_verdict() + { + // The lock session was lost and the pod that took over stored a new pair: the refusal was about the old one. + var ctx = BuildContext(); + var stale = ExpiredPair(); + var row = new StoredRow(stale); + var health = new RecordingHealthWriter(); + var provider = RecordingOAuthClient.Refusing(onRefusing: () => row.Store(new OAuthPayload { AccessToken = "rotated", RefreshToken = "r-1", ExpiresAt = Now.AddHours(2) })); + + var (result, _, _) = await RunAsync(stale, ctx, row, provider, healthWriter: health); + + result.AccessToken.ShouldBe("rotated"); + health.Rejections.ShouldBeEmpty(); + } + + // ─── After a 401 ─────────────────────────────────────────────────────────── + + private static OAuthPayload LivePair(string accessToken = "a", string? refreshToken = "r-0") => new() { AccessToken = accessToken, RefreshToken = refreshToken, ExpiresAt = Now.AddHours(1) }; + + private static Task RecoverAsync(OAuthPayload entity, string rejected, ProviderContext ctx, StoredRow row, RecordingOAuthClient provider, ICredentialHealthWriter? health = null) => + BuildRefresher(entity, ctx, row, provider, healthWriter: health).Refresher.RecoverAfterRejectionAsync(ctx, rejected, CancellationToken.None); + + [Fact] + public async Task A_token_another_caller_stored_is_adopted_instead_of_rotating() + { + var ctx = BuildContext(); + var entity = LivePair("old", "r-consumed"); + var row = new StoredRow(LivePair("rotated", "r-rotated")); + var provider = new RecordingOAuthClient("unused", null, null); + + var recovered = await RecoverAsync(entity, "old", ctx, row, provider); + + recovered.ShouldNotBeNull().AccessToken.ShouldBe("rotated"); + provider.RefreshCalls.ShouldBe(0, "the provider retired the token because the pair was rotated and stored; rotating again would consume the pair that was stored"); + ResolveEntity(ctx).AccessToken.ShouldBe("rotated", "the caller's entity carries it, so later reads in the scope are not refused the same way"); + } + + [Fact] + public async Task The_token_that_was_refused_is_rotated_once_and_the_new_pair_stored() + { + var ctx = BuildContext(); + var pair = LivePair("a", "r-0"); + var row = new StoredRow(pair); + var provider = new RecordingOAuthClient("b", "r-1", Now.AddHours(2)); + + var recovered = await RecoverAsync(pair, "a", ctx, row, provider); + + recovered.ShouldNotBeNull().AccessToken.ShouldBe("b"); + provider.RefreshCalls.ShouldBe(1); + row.Read().RefreshToken.ShouldBe("r-1", "the pair the provider issued is the credential now"); + } + + [Fact] + public async Task A_token_rotated_moments_ago_that_is_refused_too_is_not_rotated_again() + { + var ctx = BuildContext(); + var row = new StoredRow(LivePair("a", "r-0")); + var provider = new RecordingOAuthClient("b", "r-1", Now.AddHours(2)); + + await RecoverAsync(LivePair("a", "r-0"), "a", ctx, row, provider); + + var again = await RecoverAsync(row.Read(), "b", ctx, row, provider); + + again.ShouldBeNull("a token issued a moment ago that is refused is not aged out: the 401 has another cause, and rotating again would spend a refresh token on it"); + provider.RefreshCalls.ShouldBe(1); + } + + [Fact] + public async Task A_credential_with_no_refresh_token_has_nothing_to_rotate_with() + { + var ctx = BuildContext(); + var pair = LivePair("a", refreshToken: null); + var provider = new RecordingOAuthClient("b", "r-1", Now.AddHours(2)); + + (await RecoverAsync(pair, "a", ctx, new StoredRow(pair), provider)).ShouldBeNull(); + + provider.RefreshCalls.ShouldBe(0); + } + + [Fact] + public async Task A_stored_token_that_is_itself_due_is_refreshed_not_handed_back_to_be_refused_again() + { + var ctx = BuildContext(); + var row = new StoredRow(ExpiredPair("stored", "r-9")); + var provider = new RecordingOAuthClient("b", "r-10", Now.AddHours(2)); + + var recovered = await RecoverAsync(LivePair("old", "r-consumed"), "old", ctx, row, provider); + + recovered.ShouldNotBeNull().AccessToken.ShouldBe("b"); + provider.RefreshCalls.ShouldBe(1); + } + + [Fact] + public async Task A_rotation_the_provider_refuses_for_good_after_a_401_is_a_verdict_too() + { + var ctx = BuildContext(); + var pair = LivePair("a", "r-0"); + var health = new RecordingHealthWriter(); + + await Should.ThrowAsync(() => RecoverAsync(pair, "a", ctx, new StoredRow(pair), RecordingOAuthClient.Refusing(), health)); + + health.Rejections.ShouldHaveSingleItem().LastError.ShouldBe("needs reconnect: invalid_grant"); + } + + [Fact] + public async Task A_credential_revoked_meanwhile_is_not_recovered() + { + var ctx = BuildContext(); + var pair = LivePair(); + var row = new StoredRow(pair); + row.Revoke(); + var provider = new RecordingOAuthClient("b", "r-1", Now.AddHours(2)); + + var failure = await Should.ThrowAsync(() => RecoverAsync(pair, "a", ctx, row, provider)); + + failure.Message.ShouldContain("was revoked"); + provider.RefreshCalls.ShouldBe(0); + } +} diff --git a/backend/tests/CodeSpace.UnitTests/OAuth/OAuthTokenRefresherTests.Rotation.cs b/backend/tests/CodeSpace.UnitTests/OAuth/OAuthTokenRefresherTests.Rotation.cs index 26a052d66..db5cd24f9 100644 --- a/backend/tests/CodeSpace.UnitTests/OAuth/OAuthTokenRefresherTests.Rotation.cs +++ b/backend/tests/CodeSpace.UnitTests/OAuth/OAuthTokenRefresherTests.Rotation.cs @@ -3,6 +3,7 @@ using CodeSpace.Core.Services.Credentials; using CodeSpace.Core.Services.OAuth; using CodeSpace.Messages.Credentials; +using CodeSpace.Messages.Enums; using Microsoft.Extensions.Logging; using Shouldly; @@ -80,6 +81,68 @@ public async Task A_store_that_keeps_failing_is_tried_three_times_then_the_next_ row.Read().RefreshToken.ShouldBe("r-1"); } + [Fact] + public async Task StoreKeptPairAsync_stores_a_pair_the_process_kept_without_asking_the_provider_again_or_waiting_for_a_refresh_to_be_due() + { + // What a strategy does before it reads a credential another process marked refused: the pair this process kept is the + // live one, and storing it is what lifts that verdict (the writer clears an Error with any pair the provider issued). + var ctx = BuildContext(); + var stale = ExpiredPair(); + var row = new StoredRow(stale); + var provider = new RecordingOAuthClient("new", "r-1", Now.AddHours(2)); + + await Should.ThrowAsync(() => RunAsync(stale, ctx, row, provider, decorateWriter: w => new FlakyPayloadWriter(w, failures: int.MaxValue))); + + row.Read().RefreshToken.ShouldBe("r-0", "fixture check: nothing was stored"); + + var (refresher, writer) = BuildRefresher(stale, ctx, row, provider); + + await refresher.StoreKeptPairAsync(ctx, CancellationToken.None); + + row.Read().RefreshToken.ShouldBe("r-1"); + writer.PersistCalls.ShouldBe(1); + provider.RefreshCalls.ShouldBe(1, "the provider consumed r-0 once, in the first call; presenting it again is refused"); + + await refresher.StoreKeptPairAsync(ctx, CancellationToken.None); + + writer.PersistCalls.ShouldBe(1, "a pair that is stored is no longer kept, so the next use stores nothing"); + } + + [Fact] + public async Task StoreKeptPairAsync_does_nothing_for_a_credential_the_process_holds_no_pair_for() + { + var ctx = BuildContext(); + var stale = ExpiredPair(); + var row = new StoredRow(stale); + var provider = new RecordingOAuthClient("new", "r-1", Now.AddHours(2)); + var (refresher, writer) = BuildRefresher(stale, ctx, row, provider); + + await refresher.StoreKeptPairAsync(ctx, CancellationToken.None); + + row.Read().RefreshToken.ShouldBe("r-0"); + writer.PersistCalls.ShouldBe(0); + provider.RefreshCalls.ShouldBe(0, "storing what is kept never refreshes anything"); + } + + [Fact] + public async Task StoreKeptPairAsync_leaves_a_disconnected_credential_to_the_typed_failure_the_resolver_raises() + { + var ctx = BuildContext(); + var stale = ExpiredPair(); + var row = new StoredRow(stale); + var provider = new RecordingOAuthClient("new", "r-1", Now.AddHours(2)); + + await Should.ThrowAsync(() => RunAsync(stale, ctx, row, provider, decorateWriter: w => new FlakyPayloadWriter(w, failures: int.MaxValue))); + + ctx.Credential.Status = CredentialStatus.Revoked; + var (refresher, writer) = BuildRefresher(stale, ctx, row, provider); + + await refresher.StoreKeptPairAsync(ctx, CancellationToken.None); + + writer.PersistCalls.ShouldBe(0, "a disconnected credential is not written to; the resolver says it was disconnected"); + row.Read().RefreshToken.ShouldBe("r-0"); + } + [Fact] public async Task A_kept_pair_is_revoked_at_the_provider_when_the_credential_was_disconnected_meanwhile() { diff --git a/backend/tests/CodeSpace.UnitTests/OAuth/OAuthTokenRefresherTests.cs b/backend/tests/CodeSpace.UnitTests/OAuth/OAuthTokenRefresherTests.cs index 22efd7356..fe7661a03 100644 --- a/backend/tests/CodeSpace.UnitTests/OAuth/OAuthTokenRefresherTests.cs +++ b/backend/tests/CodeSpace.UnitTests/OAuth/OAuthTokenRefresherTests.cs @@ -4,6 +4,7 @@ using CodeSpace.Core.Services.Providers; using CodeSpace.Messages.Credentials; using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Exceptions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Shouldly; @@ -163,14 +164,14 @@ public async Task A_caller_cancelled_after_the_provider_rotated_still_stores_the } [Fact] - public async Task A_refused_refresh_with_nothing_newer_stored_rethrows_the_refusal() + public async Task A_refused_refresh_with_nothing_newer_stored_is_the_providers_verdict_and_the_credential_needs_reconnecting() { var ctx = BuildContext(); var stale = new OAuthPayload { AccessToken = "old", RefreshToken = "r-0", ExpiresAt = Now.AddSeconds(-1) }; - var refusal = await Should.ThrowAsync(() => RunAsync(stale, ctx, new StoredRow(stale), RecordingOAuthClient.Refusing())); + var verdict = await Should.ThrowAsync(() => RunAsync(stale, ctx, new StoredRow(stale), RecordingOAuthClient.Refusing())); - refusal.Error.ShouldBe("invalid_grant"); + verdict.InnerException.ShouldBeOfType().Error.ShouldBe("invalid_grant", "the provider's own answer stays reachable for whoever diagnoses it"); } /// @@ -264,7 +265,16 @@ private static ProviderContext BuildContext() RunAsync(current, ctx, new StoredRow(current), new RecordingOAuthClient(newAccessToken ?? "fresh", newRefreshToken, newExpiresAt)); /// The caller's entity holds — what it loaded; is what the database holds now. - private static async Task<(OAuthPayload Result, int Exchanges, int Persists)> RunAsync(OAuthPayload current, ProviderContext ctx, StoredRow row, RecordingOAuthClient oauthClient, CancellationToken cancellationToken = default, Func? decorateWriter = null, IPayloadEncryptor? encryptor = null, ILogger? logger = null, ITrackedCredentialEntry? trackedEntry = null, Func? decorateReader = null) + private static async Task<(OAuthPayload Result, int Exchanges, int Persists)> RunAsync(OAuthPayload current, ProviderContext ctx, StoredRow row, RecordingOAuthClient oauthClient, CancellationToken cancellationToken = default, Func? decorateWriter = null, IPayloadEncryptor? encryptor = null, ILogger? logger = null, ITrackedCredentialEntry? trackedEntry = null, Func? decorateReader = null, ICredentialHealthWriter? healthWriter = null) + { + var (refresher, writer) = BuildRefresher(current, ctx, row, oauthClient, decorateWriter, encryptor, logger, trackedEntry, decorateReader, healthWriter); + + var result = await refresher.RefreshIfNeededAsync(ctx, current, cancellationToken); + + return (result, oauthClient.RefreshCalls, writer.PersistCalls); + } + + private static (OAuthTokenRefresher Refresher, RecordingPayloadWriter Writer) BuildRefresher(OAuthPayload current, ProviderContext ctx, StoredRow row, RecordingOAuthClient oauthClient, Func? decorateWriter = null, IPayloadEncryptor? encryptor = null, ILogger? logger = null, ITrackedCredentialEntry? trackedEntry = null, Func? decorateReader = null, ICredentialHealthWriter? healthWriter = null) { ctx.Credential.EncryptedPayload = Serializer.Serialize(current); @@ -272,11 +282,9 @@ private static ProviderContext BuildContext() var writer = new RecordingPayloadWriter(row); var resolver = new CredentialResolver(new IdentityEncryptor(), Serializer); - var refresher = new OAuthTokenRefresher(registry, decorateWriter?.Invoke(writer) ?? writer, decorateReader?.Invoke(row) ?? row, resolver, encryptor ?? new IdentityEncryptor(), new FixedClock(Now), new NoopCrossProcessLock(), trackedEntry ?? new NoopTrackedCredentialEntry(), logger ?? NullLogger.Instance); - - var result = await refresher.RefreshIfNeededAsync(ctx, current, cancellationToken); + var refresher = new OAuthTokenRefresher(registry, decorateWriter?.Invoke(writer) ?? writer, decorateReader?.Invoke(row) ?? row, resolver, encryptor ?? new IdentityEncryptor(), new FixedClock(Now), new NoopCrossProcessLock(), trackedEntry ?? new NoopTrackedCredentialEntry(), healthWriter ?? new RecordingHealthWriter(), logger ?? NullLogger.Instance); - return (result, oauthClient.RefreshCalls, writer.PersistCalls); + return (refresher, writer); } private static readonly CredentialPayloadSerializer Serializer = new(); @@ -340,20 +348,22 @@ private sealed class RecordingOAuthClient : IOAuthClient private readonly DateTimeOffset? _expiresAt; private readonly Action? _refuse; private readonly Action? _onRefreshing; + private readonly string _refusalError; public int RefreshCalls; - public RecordingOAuthClient(string accessToken, string? refreshToken, DateTimeOffset? expiresAt, Action? refuse = null, Action? onRefreshing = null) + public RecordingOAuthClient(string accessToken, string? refreshToken, DateTimeOffset? expiresAt, Action? refuse = null, Action? onRefreshing = null, string refusalError = "invalid_grant") { _accessToken = accessToken; _refreshToken = refreshToken; _expiresAt = expiresAt; _refuse = refuse; _onRefreshing = onRefreshing; + _refusalError = refusalError; } - /// A provider that refuses every refresh with invalid_grant, after running . - public static RecordingOAuthClient Refusing(Action? onRefusing = null) => new("unused", null, null, onRefusing ?? (() => { })); + /// A provider that refuses every refresh with (invalid_grant by default), after running . + public static RecordingOAuthClient Refusing(Action? onRefusing = null, string error = "invalid_grant") => new("unused", null, null, onRefusing ?? (() => { }), refusalError: error); public List RevokedTokens { get; } = new(); @@ -373,7 +383,7 @@ public Task RefreshAsync(OAuthRefreshInput input, Cancellati if (_refuse != null) { _refuse(); - throw new OAuthExchangeException("invalid_grant", "The provided authorization grant is invalid", null); + throw new OAuthExchangeException(_refusalError, "The provided authorization grant is invalid", null); } _onRefreshing?.Invoke(); @@ -426,6 +436,25 @@ private sealed class NoopTrackedCredentialEntry : ITrackedCredentialEntry public void MarkPayloadStored(Credential credential) { } } + /// The verdicts the refresher recorded, in order. Answers true — the row was still the one judged. + private sealed class RecordingHealthWriter : ICredentialHealthWriter + { + public List<(Guid CredentialId, string ExpectedPayload, string LastError)> Rejections { get; } = new(); + + public bool Fails { get; init; } + + public Task MarkRejectedAsync(Guid credentialId, string expectedEncryptedPayload, string lastError, CancellationToken cancellationToken) + { + if (Fails) throw new InvalidOperationException("simulated: the store cannot be reached"); + + Rejections.Add((credentialId, expectedEncryptedPayload, lastError)); + + return Task.FromResult(true); + } + + public Task MarkValidatedAsync(Guid credentialId, string expectedEncryptedPayload, DateTimeOffset validatedAt, ExpiryRecord? expiry, CancellationToken cancellationToken) => Task.FromResult(true); + } + private sealed class NoopCrossProcessLock : ICrossProcessLock { public Task AcquireAsync(long key, CancellationToken cancellationToken) => Task.FromResult(NoopHandle.Instance); diff --git a/backend/tests/CodeSpace.UnitTests/Providers/Auth/GitHubAuthStrategyTests.cs b/backend/tests/CodeSpace.UnitTests/Providers/Auth/GitHubAuthStrategyTests.cs index 7b806c385..7aa89e46a 100644 --- a/backend/tests/CodeSpace.UnitTests/Providers/Auth/GitHubAuthStrategyTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Providers/Auth/GitHubAuthStrategyTests.cs @@ -34,7 +34,7 @@ public async Task GitHubPat_throws_when_payload_type_does_not_match() public async Task GitHubOAuth_returns_access_token_and_expiry() { var expires = new DateTimeOffset(2030, 1, 1, 0, 0, 0, TimeSpan.Zero); - var strategy = new GitHubOAuthAuthStrategy(new FakeCredentialResolver(new OAuthPayload { AccessToken = "ghu_abc", ExpiresAt = expires }), new NoopOAuthTokenRefresher()); + var strategy = new GitHubOAuthAuthStrategy(new FakeCredentialResolver(new OAuthPayload { AccessToken = "ghu_abc", ExpiresAt = expires }), new NoopOAuthTokenRefresher(), new NoopOAuthTokenRefresher()); var result = await strategy.ResolveAsync(AuthTestContextFactory.Build(ProviderKind.GitHub, AuthType.OAuth), CancellationToken.None); @@ -42,10 +42,25 @@ public async Task GitHubOAuth_returns_access_token_and_expiry() result.ExpiresAt.ShouldBe(expires); } + [Fact] + public async Task GitHubOAuth_stores_the_pair_its_process_kept_before_reading_a_credential_another_process_marked_refused() + { + var context = AuthTestContextFactory.Build(ProviderKind.GitHub, AuthType.OAuth); + context.Credential.Status = CredentialStatus.Error; + context.Credential.LastError = "needs reconnect: bad_refresh_token"; + var holding = new HoldingKeptPair(); + var strategy = new GitHubOAuthAuthStrategy(new ErrorFailingCredentialResolver(new OAuthPayload { AccessToken = "ghu_kept" }), holding, holding); + + var result = await strategy.ResolveAsync(context, CancellationToken.None); + + result.Token.ShouldBe("ghu_kept", "a live pair this process holds outranks the verdict a refusal of the consumed one wrote on the row"); + holding.Stores.ShouldBe(1); + } + [Fact] public async Task GitHubOAuth_throws_when_payload_type_does_not_match() { - var strategy = new GitHubOAuthAuthStrategy(new FakeCredentialResolver(new PatPayload { Token = "wrong" }), new NoopOAuthTokenRefresher()); + var strategy = new GitHubOAuthAuthStrategy(new FakeCredentialResolver(new PatPayload { Token = "wrong" }), new NoopOAuthTokenRefresher(), new NoopOAuthTokenRefresher()); var act = async () => await strategy.ResolveAsync(AuthTestContextFactory.Build(ProviderKind.GitHub, AuthType.OAuth), CancellationToken.None); @@ -65,7 +80,7 @@ public void GitHubPat_declares_correct_kind_and_authtype() [Fact] public void GitHubOAuth_declares_correct_kind_and_authtype() { - var strategy = new GitHubOAuthAuthStrategy(new FakeCredentialResolver(new OAuthPayload { AccessToken = "t" }), new NoopOAuthTokenRefresher()); + var strategy = new GitHubOAuthAuthStrategy(new FakeCredentialResolver(new OAuthPayload { AccessToken = "t" }), new NoopOAuthTokenRefresher(), new NoopOAuthTokenRefresher()); strategy.Kind.ShouldBe(ProviderKind.GitHub); strategy.AuthType.ShouldBe(AuthType.OAuth); diff --git a/backend/tests/CodeSpace.UnitTests/Providers/Auth/GitLabAuthStrategyTests.cs b/backend/tests/CodeSpace.UnitTests/Providers/Auth/GitLabAuthStrategyTests.cs index 717dcf6c7..56701ea30 100644 --- a/backend/tests/CodeSpace.UnitTests/Providers/Auth/GitLabAuthStrategyTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Providers/Auth/GitLabAuthStrategyTests.cs @@ -1,3 +1,6 @@ +using CodeSpace.Core.Services.OAuth; +using CodeSpace.Core.Services.Providers; +using CodeSpace.Core.Services.Providers.Auth; using CodeSpace.Core.Services.Providers.GitLab.Auth; using CodeSpace.Messages.Credentials; using CodeSpace.Messages.Enums; @@ -42,7 +45,7 @@ public async Task GitLabGroupAccessToken_returns_group_token() public async Task GitLabOAuth_returns_access_token() { var expires = new DateTimeOffset(2030, 1, 1, 0, 0, 0, TimeSpan.Zero); - var strategy = new GitLabOAuthAuthStrategy(new FakeCredentialResolver(new OAuthPayload { AccessToken = "glat_xyz", ExpiresAt = expires }), new NoopOAuthTokenRefresher()); + var strategy = new GitLabOAuthAuthStrategy(new FakeCredentialResolver(new OAuthPayload { AccessToken = "glat_xyz", ExpiresAt = expires }), new NoopOAuthTokenRefresher(), new NoopOAuthTokenRefresher()); var result = await strategy.ResolveAsync(AuthTestContextFactory.Build(ProviderKind.GitLab, AuthType.OAuth), CancellationToken.None); @@ -50,6 +53,68 @@ public async Task GitLabOAuth_returns_access_token() result.ExpiresAt.ShouldBe(expires); } + [Fact] + public async Task GitLabOAuth_hands_back_the_token_recovery_found_and_nothing_when_it_found_none() + { + var expires = new DateTimeOffset(2030, 1, 1, 0, 0, 0, TimeSpan.Zero); + var context = AuthTestContextFactory.Build(ProviderKind.GitLab, AuthType.OAuth); + var found = new GitLabOAuthAuthStrategy(new FakeCredentialResolver(new OAuthPayload { AccessToken = "unused" }), new NoopOAuthTokenRefresher(), new FixedRecovery(new OAuthPayload { AccessToken = "glat_replacement", ExpiresAt = expires })); + var none = new GitLabOAuthAuthStrategy(new FakeCredentialResolver(new OAuthPayload { AccessToken = "unused" }), new NoopOAuthTokenRefresher(), new FixedRecovery(null)); + + var replacement = await found.InvalidateAsync(context, "glat_refused", CancellationToken.None); + + replacement.ShouldNotBeNull().Token.ShouldBe("glat_replacement"); + replacement.ExpiresAt.ShouldBe(expires); + (await none.InvalidateAsync(context, "glat_refused", CancellationToken.None)).ShouldBeNull(); + } + + [Fact] + public async Task GitLabOAuth_stores_the_pair_its_process_kept_before_reading_a_credential_another_process_marked_refused() + { + var context = AuthTestContextFactory.Build(ProviderKind.GitLab, AuthType.OAuth); + context.Credential.Status = CredentialStatus.Error; + context.Credential.LastError = "needs reconnect: invalid_grant"; + var holding = new HoldingKeptPair(); + var strategy = new GitLabOAuthAuthStrategy(new ErrorFailingCredentialResolver(new OAuthPayload { AccessToken = "glat_kept" }), holding, holding); + + var result = await strategy.ResolveAsync(context, CancellationToken.None); + + result.Token.ShouldBe("glat_kept", "a live pair this process holds outranks the verdict a refusal of the consumed one wrote on the row"); + holding.Stores.ShouldBe(1); + } + + [Fact] + public async Task GitLabOAuth_still_fails_fast_on_a_refused_credential_when_it_holds_no_pair() + { + var context = AuthTestContextFactory.Build(ProviderKind.GitLab, AuthType.OAuth); + context.Credential.Status = CredentialStatus.Error; + var strategy = new GitLabOAuthAuthStrategy(new ErrorFailingCredentialResolver(new OAuthPayload { AccessToken = "unused" }), new NoopOAuthTokenRefresher(), new NoopOAuthTokenRefresher()); + + await Should.ThrowAsync(() => strategy.ResolveAsync(context, CancellationToken.None)); + } + + [Fact] + public void Only_the_oauth_strategy_can_recover_a_token_the_static_ones_have_nothing_to_replace_it_with() + { + var resolver = new FakeCredentialResolver(new PatPayload { Token = "t" }); + + (new GitLabOAuthAuthStrategy(resolver, new NoopOAuthTokenRefresher(), new NoopOAuthTokenRefresher()) is IRecoverableProviderAuthStrategy).ShouldBeTrue(); + (new GitLabPatAuthStrategy(resolver) is IRecoverableProviderAuthStrategy).ShouldBeFalse(); + (new GitLabProjectAccessTokenStrategy(resolver) is IRecoverableProviderAuthStrategy).ShouldBeFalse(); + (new GitLabGroupAccessTokenStrategy(resolver) is IRecoverableProviderAuthStrategy).ShouldBeFalse(); + } + + private sealed class FixedRecovery : IOAuthTokenRecovery + { + private readonly OAuthPayload? _payload; + + public FixedRecovery(OAuthPayload? payload) { _payload = payload; } + + public Task RecoverAfterRejectionAsync(ProviderContext context, string rejectedAccessToken, CancellationToken cancellationToken) => Task.FromResult(_payload); + + public Task StoreKeptPairAsync(ProviderContext context, CancellationToken cancellationToken) => Task.CompletedTask; + } + [Theory] [InlineData("Pat")] [InlineData("ProjectAccessToken")] @@ -65,7 +130,7 @@ public async Task Each_strategy_throws_when_payload_type_does_not_match(string s "Pat" => async () => await new GitLabPatAuthStrategy(resolver).ResolveAsync(AuthTestContextFactory.Build(ProviderKind.GitLab, AuthType.Pat), CancellationToken.None), "ProjectAccessToken" => async () => await new GitLabProjectAccessTokenStrategy(resolver).ResolveAsync(AuthTestContextFactory.Build(ProviderKind.GitLab, AuthType.ProjectAccessToken), CancellationToken.None), "GroupAccessToken" => async () => await new GitLabGroupAccessTokenStrategy(resolver).ResolveAsync(AuthTestContextFactory.Build(ProviderKind.GitLab, AuthType.GroupAccessToken), CancellationToken.None), - "OAuth" => async () => await new GitLabOAuthAuthStrategy(resolver, new NoopOAuthTokenRefresher()).ResolveAsync(AuthTestContextFactory.Build(ProviderKind.GitLab, AuthType.OAuth), CancellationToken.None), + "OAuth" => async () => await new GitLabOAuthAuthStrategy(resolver, new NoopOAuthTokenRefresher(), new NoopOAuthTokenRefresher()).ResolveAsync(AuthTestContextFactory.Build(ProviderKind.GitLab, AuthType.OAuth), CancellationToken.None), _ => throw new ArgumentException(strategyName) }; @@ -79,7 +144,7 @@ public void All_GitLab_strategies_declare_GitLab_kind() new GitLabPatAuthStrategy(resolver).Kind.ShouldBe(ProviderKind.GitLab); new GitLabProjectAccessTokenStrategy(resolver).Kind.ShouldBe(ProviderKind.GitLab); new GitLabGroupAccessTokenStrategy(resolver).Kind.ShouldBe(ProviderKind.GitLab); - new GitLabOAuthAuthStrategy(resolver, new NoopOAuthTokenRefresher()).Kind.ShouldBe(ProviderKind.GitLab); + new GitLabOAuthAuthStrategy(resolver, new NoopOAuthTokenRefresher(), new NoopOAuthTokenRefresher()).Kind.ShouldBe(ProviderKind.GitLab); } [Fact] @@ -91,7 +156,7 @@ public void GitLab_strategies_declare_distinct_auth_types() new GitLabPatAuthStrategy(resolver).AuthType, new GitLabProjectAccessTokenStrategy(resolver).AuthType, new GitLabGroupAccessTokenStrategy(resolver).AuthType, - new GitLabOAuthAuthStrategy(resolver, new NoopOAuthTokenRefresher()).AuthType + new GitLabOAuthAuthStrategy(resolver, new NoopOAuthTokenRefresher(), new NoopOAuthTokenRefresher()).AuthType }; authTypes.Distinct().Count().ShouldBe(4); diff --git a/backend/tests/CodeSpace.UnitTests/Providers/Auth/KeptPairDoubles.cs b/backend/tests/CodeSpace.UnitTests/Providers/Auth/KeptPairDoubles.cs new file mode 100644 index 000000000..e8275c48b --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Providers/Auth/KeptPairDoubles.cs @@ -0,0 +1,45 @@ +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Credentials; +using CodeSpace.Core.Services.OAuth; +using CodeSpace.Core.Services.Providers; +using CodeSpace.Messages.Credentials; +using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Exceptions; + +namespace CodeSpace.UnitTests.Providers.Auth; + +/// +/// What the production resolver does with a credential its provider refused for good — fail with the typed +/// credential_needs_reconnect — and nothing else, in front of a payload the test chooses. With +/// it is the pair of doubles that says whether a strategy stores the pair its process kept before +/// it reads the credential: the store lifts the verdict on the row, and only then is the credential readable. +/// +internal sealed class ErrorFailingCredentialResolver : ICredentialResolver +{ + private readonly CredentialPayload _payload; + + public ErrorFailingCredentialResolver(CredentialPayload payload) { _payload = payload; } + + public CredentialPayload Resolve(Credential credential) => + credential.Status == CredentialStatus.Error ? throw new CredentialNeedsReconnectException(credential.Id, credential.DisplayName, credential.LastError ?? "refused") : _payload; +} + +/// A process that holds a rotated pair for the credential: asked to store it, it does, and the verdict a refusal elsewhere wrote on the row is lifted with it. +internal sealed class HoldingKeptPair : IOAuthTokenRefresher, IOAuthTokenRecovery +{ + public int Stores { get; private set; } + + public Task RefreshIfNeededAsync(ProviderContext context, OAuthPayload current, CancellationToken cancellationToken) => Task.FromResult(current); + + public Task RecoverAfterRejectionAsync(ProviderContext context, string rejectedAccessToken, CancellationToken cancellationToken) => Task.FromResult(null); + + public Task StoreKeptPairAsync(ProviderContext context, CancellationToken cancellationToken) + { + Stores++; + + context.Credential.Status = CredentialStatus.Active; + context.Credential.LastError = null; + + return Task.CompletedTask; + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Providers/Auth/NoopOAuthTokenRefresher.cs b/backend/tests/CodeSpace.UnitTests/Providers/Auth/NoopOAuthTokenRefresher.cs index cb4051047..c6c52983c 100644 --- a/backend/tests/CodeSpace.UnitTests/Providers/Auth/NoopOAuthTokenRefresher.cs +++ b/backend/tests/CodeSpace.UnitTests/Providers/Auth/NoopOAuthTokenRefresher.cs @@ -9,8 +9,13 @@ namespace CodeSpace.UnitTests.Providers.Auth; /// OAuthPayload has an ExpiresAt far in the future (no refresh expected) or null /// (refresh impossible) — i.e. when the refresh path is not the subject of the test. /// -internal sealed class NoopOAuthTokenRefresher : IOAuthTokenRefresher +internal sealed class NoopOAuthTokenRefresher : IOAuthTokenRefresher, IOAuthTokenRecovery { public Task RefreshIfNeededAsync(ProviderContext context, OAuthPayload current, CancellationToken cancellationToken) => Task.FromResult(current); + + public Task RecoverAfterRejectionAsync(ProviderContext context, string rejectedAccessToken, CancellationToken cancellationToken) + => Task.FromResult(null); + + public Task StoreKeptPairAsync(ProviderContext context, CancellationToken cancellationToken) => Task.CompletedTask; } diff --git a/backend/tests/CodeSpace.UnitTests/Providers/Auth/ProviderAuthResolverTests.cs b/backend/tests/CodeSpace.UnitTests/Providers/Auth/ProviderAuthResolverTests.cs index fbb0d6915..dab5cbe3f 100644 --- a/backend/tests/CodeSpace.UnitTests/Providers/Auth/ProviderAuthResolverTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Providers/Auth/ProviderAuthResolverTests.cs @@ -1,3 +1,4 @@ +using CodeSpace.Core.Services.Providers; using CodeSpace.Core.Services.Providers.Auth; using CodeSpace.Messages.Enums; using Shouldly; @@ -48,6 +49,64 @@ public void Constructor_throws_when_two_strategies_claim_same_kind_and_auth_type act.ShouldThrow(); } + // ─── After a provider answered 401 to the token it was handed ─── + + [Fact] + public async Task InvalidateAsync_asks_the_strategy_that_can_recover_and_remembers_the_replacement_so_an_echo_of_it_is_struck() + { + var resolver = new ProviderAuthResolver(new IProviderAuthStrategy[] { new RecoverableStrategy(ProviderKind.GitLab, AuthType.OAuth, replacement: "glat-REPLACEMENT-0123456789") }); + var context = AuthTestContextFactory.Build(ProviderKind.GitLab, AuthType.OAuth); + + var replacement = await resolver.InvalidateAsync(context, "glat-refused", CancellationToken.None); + + replacement.ShouldNotBeNull().Token.ShouldBe("glat-REPLACEMENT-0123456789"); + PresentedProviderCredentials.RedactorFor(context.Instance).Redact("the provider said glat-REPLACEMENT-0123456789 was refused").ShouldNotContain("glat-REPLACEMENT-0123456789"); + } + + [Fact] + public async Task InvalidateAsync_answers_null_when_the_strategy_has_nothing_to_replace_the_token_with() + { + var resolver = new ProviderAuthResolver(new IProviderAuthStrategy[] { new RecoverableStrategy(ProviderKind.GitLab, AuthType.OAuth, replacement: null) }); + + (await resolver.InvalidateAsync(AuthTestContextFactory.Build(ProviderKind.GitLab, AuthType.OAuth), "glat-refused", CancellationToken.None)).ShouldBeNull(); + } + + [Fact] + public async Task InvalidateAsync_answers_null_for_a_strategy_that_cannot_recover_a_static_token_is_the_same_token_whenever_it_is_asked_for() + { + var resolver = new ProviderAuthResolver(new IProviderAuthStrategy[] { new StubStrategy(ProviderKind.GitLab, AuthType.Pat, "glpat-token") }); + + (await resolver.InvalidateAsync(AuthTestContextFactory.Build(ProviderKind.GitLab, AuthType.Pat), "glpat-token", CancellationToken.None)).ShouldBeNull(); + } + + [Fact] + public async Task InvalidateAsync_answers_null_rather_than_throwing_when_no_strategy_is_registered() + { + var resolver = new ProviderAuthResolver(new IProviderAuthStrategy[] { new StubStrategy(ProviderKind.GitHub, AuthType.Pat, "token") }); + + (await resolver.InvalidateAsync(AuthTestContextFactory.Build(ProviderKind.GitHub, AuthType.OAuth), "token", CancellationToken.None)).ShouldBeNull("the caller surfaces the 401 it already has; it is not owed a second failure for asking"); + } + + private sealed class RecoverableStrategy : IRecoverableProviderAuthStrategy + { + private readonly string? _replacement; + + public RecoverableStrategy(ProviderKind kind, AuthType authType, string? replacement) + { + Kind = kind; + AuthType = authType; + _replacement = replacement; + } + + public ProviderKind Kind { get; } + public AuthType AuthType { get; } + + public Task ResolveAsync(ProviderContext context, CancellationToken cancellationToken) => Task.FromResult(new ResolvedAuth { Token = "unused" }); + + public Task InvalidateAsync(ProviderContext context, string rejectedToken, CancellationToken cancellationToken) => + Task.FromResult(_replacement is null ? null : new ResolvedAuth { Token = _replacement }); + } + private sealed class StubStrategy : IProviderAuthStrategy { private readonly string _token; diff --git a/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabLoopback.cs b/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabLoopback.cs index f0e03fb20..991df0b9a 100644 --- a/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabLoopback.cs +++ b/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabLoopback.cs @@ -89,12 +89,12 @@ public static void ShouldCarryOnly(RecordedRequest request, AuthType type) public static ProviderContext Context(string baseUrl, AuthType type) => new(new ProviderInstance { Id = Guid.NewGuid(), TeamId = Guid.NewGuid(), Provider = ProviderKind.GitLab, DisplayName = "loopback", BaseUrl = baseUrl }, new Credential { Id = Guid.NewGuid(), AuthType = type, DisplayName = "token", EncryptedPayload = "unused" }); - public static GitLabRepositoryProvider Provider() + public static GitLabRepositoryProvider Provider(IProviderAuthResolver? auth = null) { var resilience = new ExternalCallResilience(new ProviderErrorMapperRegistry(new IProviderErrorMapper[] { new GitLabErrorMapper() }), NullLogger.Instance); var normalizer = new GitLabEventNormalizer(new ProviderEventSubscriptionRegistry(Array.Empty())); - return new GitLabRepositoryProvider(new StaticTokenAuth(), resilience, new GitLabSignatureVerifier(), normalizer, new GitLabWebhookRepositoryIdentifier()); + return new GitLabRepositoryProvider(auth ?? new StaticTokenAuth(), resilience, new GitLabSignatureVerifier(), normalizer, new GitLabWebhookRepositoryIdentifier()); } private sealed class StaticTokenAuth : IProviderAuthResolver diff --git a/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabMemberStandingLookupTests.cs b/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabMemberStandingLookupTests.cs new file mode 100644 index 000000000..7ecd2fe2a --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabMemberStandingLookupTests.cs @@ -0,0 +1,139 @@ +using CodeSpace.Core.Services.Providers; +using CodeSpace.Core.Services.Providers.Auth; +using CodeSpace.IntegrationTests.Webhooks; +using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Exceptions; +using Shouldly; +using static CodeSpace.IntegrationTests.Webhooks.StubProviderHost; +using static CodeSpace.UnitTests.Providers.GitLab.GitLabLoopback; + +namespace CodeSpace.UnitTests.Providers.GitLab; + +/// +/// The member-standing lookup says only what GitLab said about the author: a role (200) or none (404). Everything else — +/// the credential refused, GitLab erroring — is not an answer about the author, and was returned as Unknown, which a +/// members-only trigger reads as "not a member": a dead token or one 503 refused the author's own merge request, with no log +/// line and a reason that told the operator to open the trigger to everyone. It is now thrown with its status, after one more +/// try for a 429 or 5xx. +/// +[Trait("Category", "Unit")] +public sealed class GitLabMemberStandingLookupTests : IDisposable +{ + private const string Member = """{"id":7,"access_level":30}"""; + private const string StandingPath = "/members/all/7"; + + private readonly List _gitlabs = new(); + + public void Dispose() + { + foreach (var gitlab in _gitlabs) gitlab.Dispose(); + } + + [Theory] + [InlineData(200, Member, PullRequestAuthorAssociation.Member)] + [InlineData(200, """{"id":7,"access_level":20}""", PullRequestAuthorAssociation.None)] + [InlineData(404, """{"message":"404 Not found"}""", PullRequestAuthorAssociation.None)] + public async Task Only_a_role_or_the_absence_of_one_is_an_answer_about_the_author(int status, string body, PullRequestAuthorAssociation expected) + { + var gitlab = Gitlab(_ => new StubReply(status, body)); + + (await Standing(gitlab, AuthType.Pat)).ShouldBe(expected); + + gitlab.Requests.Count.ShouldBe(1); + } + + [Theory] + [InlineData(500)] + [InlineData(502)] + [InlineData(503)] + [InlineData(429)] + public async Task A_429_or_5xx_is_asked_again_once_and_the_second_answer_is_used(int firstStatus) + { + var calls = 0; + var gitlab = Gitlab(_ => Interlocked.Increment(ref calls) == 1 ? new StubReply(firstStatus, """{"message":"try later"}""") : new StubReply(200, Member)); + + (await Standing(gitlab, AuthType.Pat)).ShouldBe(PullRequestAuthorAssociation.Member); + + gitlab.Requests.Count.ShouldBe(2); + } + + [Theory] + [InlineData(500)] + [InlineData(503)] + [InlineData(429)] + public async Task A_second_failure_is_thrown_with_its_status_and_the_lookup_is_not_asked_a_third_time(int status) + { + var gitlab = Gitlab(_ => new StubReply(status, """{"message":"still down"}""")); + + var failure = await Should.ThrowAsync(() => Standing(gitlab, AuthType.Pat)); + + failure.StatusCode.ShouldBe(status); + failure.ProviderKind.ShouldBe(ProviderKind.GitLab); + gitlab.Requests.Count.ShouldBe(2); + } + + [Theory] + [InlineData(403)] + [InlineData(401)] + [InlineData(400)] + public async Task A_refusal_is_thrown_at_once_and_never_asked_again(int status) + { + var gitlab = Gitlab(_ => new StubReply(status, """{"message":"refused"}""")); + + var failure = await Should.ThrowAsync(() => Standing(gitlab, AuthType.Pat)); + + failure.StatusCode.ShouldBe(status); + gitlab.Requests.Count.ShouldBe(1, "the same credential would be refused the same way"); + } + + [Fact] + public async Task What_the_failing_response_said_is_never_carried_into_the_failure() + { + var gitlab = Gitlab(_ => new StubReply(403, """{"message":"token glpat-ECHOED-BACK-0123456789 may not read members"}""")); + + var failure = await Should.ThrowAsync(() => Standing(gitlab, AuthType.Pat)); + + failure.Message.ShouldNotContain("glpat-ECHOED-BACK"); + failure.ProviderMessage.ShouldNotContain("glpat-ECHOED-BACK"); + } + + [Fact] + public async Task An_OAuth_credential_refused_with_a_stale_token_is_replaced_and_the_lookup_answers() + { + var gitlab = Gitlab(request => request.AuthorizationHeader == "Bearer stale-token-0123456789" ? new StubReply(401, """{"message":"401 Unauthorized"}""") : new StubReply(200, Member)); + + var standing = await Provider(new ReplacingAuth("stale-token-0123456789", "fresh-token-0123456789")).GetMemberStandingAsync(Context(gitlab.BaseUrl, AuthType.OAuth), Repository, "7", CancellationToken.None); + + standing.ShouldBe(PullRequestAuthorAssociation.Member); + gitlab.Requests.Select(r => r.AuthorizationHeader).ShouldBe(new[] { "Bearer stale-token-0123456789", "Bearer fresh-token-0123456789" }); + } + + private async Task Standing(StubProviderHost gitlab, AuthType type) => + await Provider().GetMemberStandingAsync(Context(gitlab.BaseUrl, type), Repository, "7", CancellationToken.None); + + private StubProviderHost Gitlab(Func answer) + { + var gitlab = new StubProviderHost().Answer("GET", StandingPath, answer); + + _gitlabs.Add(gitlab); + + return gitlab; + } + + private sealed class ReplacingAuth : IProviderAuthResolver + { + private readonly string _stale; + private readonly string _fresh; + + public ReplacingAuth(string stale, string fresh) + { + _stale = stale; + _fresh = fresh; + } + + public Task ResolveAsync(ProviderContext context, CancellationToken cancellationToken) => Task.FromResult(new ResolvedAuth { Token = _stale }); + + public Task InvalidateAsync(ProviderContext context, string rejectedToken, CancellationToken cancellationToken) => + Task.FromResult(rejectedToken == _stale ? new ResolvedAuth { Token = _fresh } : null); + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabProbeTests.cs b/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabProbeTests.cs new file mode 100644 index 000000000..bbc81ea35 --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabProbeTests.cs @@ -0,0 +1,302 @@ +using CodeSpace.Core.Services.Providers; +using CodeSpace.Core.Services.Providers.Auth; +using CodeSpace.IntegrationTests.Webhooks; +using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Exceptions; +using Shouldly; +using static CodeSpace.IntegrationTests.Webhooks.StubProviderHost; +using static CodeSpace.UnitTests.Providers.GitLab.GitLabLoopback; + +namespace CodeSpace.UnitTests.Providers.GitLab; + +/// +/// What a probe tells the caller about a credential: who it is and what it may do, now also when a static token expires and +/// whether GitLab calls it active — and, apart from "not valid", whether GitLab ANSWERED and refused it. Only a 401 is +/// evidence against a credential; a 5xx or a dropped connection says nothing about it, and the health sweep must not mark a +/// working token as dead because GitLab had a bad minute. +/// +[Trait("Category", "Unit")] +public sealed class GitLabProbeTests : IDisposable +{ + private const string User = """{"id":7,"username":"nacho"}"""; + + private readonly List _gitlabs = new(); + + public void Dispose() + { + foreach (var gitlab in _gitlabs) gitlab.Dispose(); + } + + [Fact] + public async Task A_live_token_reports_its_user_scopes_expiry_and_that_it_is_active() + { + var gitlab = Gitlab(user: new StubReply(200, User), self: new StubReply(200, """{"scopes":["api","read_user"],"expires_at":"2030-07-04","active":true,"revoked":false}""")); + + var probe = await Provider().ProbeCredentialAsync(Context(gitlab.BaseUrl, AuthType.Pat), CancellationToken.None); + + probe.IsValid.ShouldBeTrue(); + probe.IsRejected.ShouldBeFalse(); + probe.AuthenticatedUserExternalId.ShouldBe("7"); + probe.AuthenticatedUserName.ShouldBe("nacho"); + probe.GrantedScopes.ShouldBe(new[] { "api", "read_user" }); + probe.ExpiresAt.ShouldBe(new DateTimeOffset(2030, 7, 4, 0, 0, 0, TimeSpan.Zero)); + probe.Active.ShouldBe(true); + } + + [Fact] + public async Task A_token_that_never_expires_is_described_with_no_expiry() + { + var gitlab = Gitlab(user: new StubReply(200, User), self: new StubReply(200, """{"scopes":["api"],"expires_at":null,"active":true}""")); + + var probe = await Provider().ProbeCredentialAsync(Context(gitlab.BaseUrl, AuthType.Pat), CancellationToken.None); + + probe.IsValid.ShouldBeTrue(); + probe.ExpiresAt.ShouldBeNull(); + probe.Active.ShouldBe(true, "described, so null means none rather than unknown"); + } + + [Theory] + [InlineData(404)] + [InlineData(403)] + [InlineData(401)] + public async Task A_token_the_self_endpoint_will_not_describe_is_valid_with_nothing_known_about_its_expiry(int selfStatus) + { + // OAuth and impersonation tokens, and GitLab versions without the endpoint for project and group tokens. + var gitlab = Gitlab(user: new StubReply(200, User), self: new StubReply(selfStatus, """{"message":"no"}""")); + + var probe = await Provider().ProbeCredentialAsync(Context(gitlab.BaseUrl, AuthType.Pat), CancellationToken.None); + + probe.IsValid.ShouldBeTrue(); + probe.ExpiresAt.ShouldBeNull(); + probe.Active.ShouldBeNull("not described: the caller must not read this as 'never expires'"); + probe.GrantedScopes.ShouldBeNull(); + } + + [Theory] + [InlineData(AuthType.Pat)] + [InlineData(AuthType.ProjectAccessToken)] + [InlineData(AuthType.GroupAccessToken)] + public async Task A_401_is_a_rejection(AuthType type) + { + var gitlab = Gitlab(user: new StubReply(401, """{"message":"401 Unauthorized"}"""), self: new StubReply(401, "{}")); + + var probe = await Provider().ProbeCredentialAsync(Context(gitlab.BaseUrl, type), CancellationToken.None); + + probe.IsValid.ShouldBeFalse(); + probe.IsRejected.ShouldBeTrue("GitLab answered, and refused the token"); + probe.Error.ShouldNotBeNullOrEmpty(); + } + + [Theory] + [InlineData(403)] + [InlineData(404)] + public async Task A_refusal_that_is_not_about_the_token_is_not_a_rejection(int status) + { + var gitlab = Gitlab(user: new StubReply(status, """{"message":"nope"}"""), self: new StubReply(200, "{}")); + + var probe = await Provider().ProbeCredentialAsync(Context(gitlab.BaseUrl, AuthType.Pat), CancellationToken.None); + + probe.IsValid.ShouldBeFalse(); + probe.IsRejected.ShouldBeFalse("a 403 is a missing scope or role — the token is not dead, and marking it so would stop every run through it"); + } + + [Fact] + public async Task A_403_tagged_insufficient_scope_names_the_scope_and_is_not_a_rejection_of_the_token() + { + var gitlab = Gitlab(user: new StubReply(403, """{"error":"insufficient_scope","error_description":"The request requires higher privileges than provided by the access token.","scope":"read_user"}"""), self: new StubReply(200, "{}")); + + var probe = await Provider().ProbeCredentialAsync(Context(gitlab.BaseUrl, AuthType.Pat), CancellationToken.None); + + probe.IsValid.ShouldBeFalse(); + probe.IsRejected.ShouldBeFalse("the token works; it lacks a scope, which is what the person is told"); + probe.Error.ShouldNotBeNull().ShouldContain("read_user"); + } + + [Theory] + [InlineData("""login""")] + [InlineData("""{"id":"seven","username":"nacho"}""")] + [InlineData("""{"id":7}""")] + [InlineData("""[]""")] + public async Task An_answer_that_is_not_a_user_is_no_answer_about_the_token(string body) + { + // A 200 from a proxy's login page says nothing about whether the token works: it is neither a validation nor a rejection. + var gitlab = Gitlab(user: new StubReply(200, body), self: new StubReply(200, "{}")); + + var probe = await Provider().ProbeCredentialAsync(Context(gitlab.BaseUrl, AuthType.Pat), CancellationToken.None); + + probe.IsValid.ShouldBeFalse(); + probe.IsRejected.ShouldBeFalse(); + } + + [Fact] + public async Task A_server_error_is_asked_again_before_it_is_given_up_on() + { + var calls = 0; + var gitlab = new StubProviderHost().Answer("GET", "/api/v4/user", _ => ++calls == 1 ? new StubReply(503, """{"message":"maintenance"}""") : new StubReply(200, User)).Answer("GET", "/api/v4/personal_access_tokens/self", _ => new StubReply(200, "{}")); + _gitlabs.Add(gitlab); + + var probe = await Provider().ProbeCredentialAsync(Context(gitlab.BaseUrl, AuthType.Pat), CancellationToken.None); + + probe.IsValid.ShouldBeTrue("one 503 is a bad second, not a verdict"); + calls.ShouldBe(2); + } + + [Fact] + public async Task A_401_is_not_asked_again() + { + var gitlab = Gitlab(user: new StubReply(401, """{"message":"401 Unauthorized"}"""), self: new StubReply(401, "{}")); + + await Provider().ProbeCredentialAsync(Context(gitlab.BaseUrl, AuthType.Pat), CancellationToken.None); + + gitlab.Requests.Count(r => r.PathAndQuery.StartsWith("/api/v4/user", StringComparison.Ordinal)).ShouldBe(1, "a refusal of the token is the answer; asking again would only spend quota on the same one"); + } + + [Fact] + public async Task A_server_error_is_no_answer_about_the_token() + { + var gitlab = Gitlab(user: new StubReply(503, """{"message":"maintenance"}"""), self: new StubReply(200, "{}")); + + var probe = await Provider().ProbeCredentialAsync(Context(gitlab.BaseUrl, AuthType.Pat), CancellationToken.None); + + probe.IsValid.ShouldBeFalse(); + probe.IsRejected.ShouldBeFalse(); + } + + [Fact] + public async Task A_provider_that_cannot_be_reached_is_no_answer_about_the_token() + { + var closed = new StubProviderHost(); + var url = closed.BaseUrl; + closed.Dispose(); + + var probe = await Provider().ProbeCredentialAsync(Context(url, AuthType.Pat), CancellationToken.None); + + probe.IsValid.ShouldBeFalse(); + probe.IsRejected.ShouldBeFalse(); + } + + [Fact] + public async Task A_provider_that_never_answers_is_given_up_on_when_the_callers_deadline_passes() + { + // The identity call used to be NGitLab's synchronous one: it ignored the token, blocked its thread for the HTTP + // client's five minutes, and retried a connection that never answered up to three times — so a probe's own deadline + // never took effect, and one such credential held the health sweep's worker for a quarter of an hour. + using var silent = new SilentProviderHost(); + using var deadline = new CancellationTokenSource(TimeSpan.FromMilliseconds(500)); + var provider = Provider(); + var context = Context(silent.BaseUrl, AuthType.Pat); + + // On a thread of its own: a probe that ignores the deadline must fail this test, not hang it. + var probing = Task.Run(() => provider.ProbeCredentialAsync(context, deadline.Token)); + var finished = await Task.WhenAny(probing, Task.Delay(TimeSpan.FromSeconds(15))) == probing; + + finished.ShouldBeTrue("the probe must stop when its deadline passes, not when the HTTP client's own timeout does"); + + var probe = await probing; + + probe.IsValid.ShouldBeFalse(); + probe.IsRejected.ShouldBeFalse("no answer says nothing about the token: marking it dead on a deadline would stop every run through it"); + silent.Connections.ShouldBe(1, "a call that was given up on is not sent again"); + } + + [Theory] + [InlineData("""{"active":false,"expires_at":"2020-01-01"}""")] + [InlineData("""{"active":true,"revoked":true}""")] + public async Task A_token_gitlab_calls_inactive_or_revoked_is_rejected_even_when_the_user_call_was_answered(string self) + { + var gitlab = Gitlab(user: new StubReply(200, User), self: new StubReply(200, self)); + + var probe = await Provider().ProbeCredentialAsync(Context(gitlab.BaseUrl, AuthType.Pat), CancellationToken.None); + + probe.IsValid.ShouldBeFalse(); + probe.IsRejected.ShouldBeTrue(); + probe.Active.ShouldBe(false); + } + + [Fact] + public async Task An_OAuth_probe_refused_with_a_stale_token_is_replaced_and_valid() + { + var gitlab = new StubProviderHost().Answer("GET", "/api/v4/user", request => request.AuthorizationHeader == "Bearer stale-0123456789" ? new StubReply(401, "{}") : new StubReply(200, User)).Answer("GET", "/api/v4/personal_access_tokens/self", _ => new StubReply(404, "{}")); + _gitlabs.Add(gitlab); + + var probe = await Provider(new ReplacingAuth()).ProbeCredentialAsync(Context(gitlab.BaseUrl, AuthType.OAuth), CancellationToken.None); + + probe.IsValid.ShouldBeTrue(); + probe.AuthenticatedUserName.ShouldBe("nacho"); + } + + [Fact] + public async Task An_oauth_token_answered_401_by_the_token_only_self_endpoint_is_not_taken_for_a_stale_token() + { + // GitLab answers an OAuth token 401 at personal_access_tokens/self by design. That says nothing about the token, and + // asking the resolver for a replacement would rotate a working pair on every probe of every OAuth credential. + var gitlab = new StubProviderHost().Answer("GET", "/api/v4/user", _ => new StubReply(200, User)).Answer("GET", "/api/v4/personal_access_tokens/self", _ => new StubReply(401, """{"message":"401 Unauthorized"}""")); + _gitlabs.Add(gitlab); + var auth = new CountingAuth(); + + var probe = await Provider(auth).ProbeCredentialAsync(Context(gitlab.BaseUrl, AuthType.OAuth), CancellationToken.None); + + probe.IsValid.ShouldBeTrue(); + probe.Active.ShouldBeNull(); + auth.Invalidations.ShouldBe(0, "the self endpoint's 401 is not a reason to rotate"); + gitlab.Requests.Count(r => r.PathAndQuery.Contains("personal_access_tokens/self")).ShouldBe(1, "and it was not sent again either"); + } + + [Fact] + public async Task A_credential_marked_as_needing_a_reconnect_is_a_rejection_without_asking_the_provider() + { + var gitlab = Gitlab(user: new StubReply(200, User), self: new StubReply(200, "{}")); + + var probe = await Provider(new RefusingAuth()).ProbeCredentialAsync(Context(gitlab.BaseUrl, AuthType.OAuth), CancellationToken.None); + + probe.IsRejected.ShouldBeTrue(); + probe.IsValid.ShouldBeFalse(); + gitlab.Requests.ShouldBeEmpty(); + } + + [Fact] + public void A_probe_failure_is_a_rejection_only_for_a_refusal_of_the_credential() + { + CodeSpace.Messages.Dtos.Providers.CredentialProbeResult.FromFailure(new ProviderApiException(ProviderKind.GitLab, 401, "op", "no", new Exception())).IsRejected.ShouldBeTrue(); + CodeSpace.Messages.Dtos.Providers.CredentialProbeResult.FromFailure(new CredentialNeedsReconnectException(Guid.NewGuid(), "c", "invalid_grant")).IsRejected.ShouldBeTrue(); + CodeSpace.Messages.Dtos.Providers.CredentialProbeResult.FromFailure(new ProviderApiException(ProviderKind.GitLab, 403, "op", "no", new Exception())).IsRejected.ShouldBeFalse(); + CodeSpace.Messages.Dtos.Providers.CredentialProbeResult.FromFailure(new ProviderApiException(ProviderKind.GitLab, 503, "op", "no", new Exception())).IsRejected.ShouldBeFalse(); + CodeSpace.Messages.Dtos.Providers.CredentialProbeResult.FromFailure(new HttpRequestException("down")).IsRejected.ShouldBeFalse(); + CodeSpace.Messages.Dtos.Providers.CredentialProbeResult.FromFailure(new CredentialDisconnectedException(Guid.NewGuid(), "c")).IsRejected.ShouldBeFalse(); + } + + private StubProviderHost Gitlab(StubReply user, StubReply self) + { + var gitlab = new StubProviderHost().Answer("GET", "/api/v4/user", _ => user).Answer("GET", "/api/v4/personal_access_tokens/self", _ => self); + + _gitlabs.Add(gitlab); + + return gitlab; + } + + private sealed class ReplacingAuth : IProviderAuthResolver + { + public Task ResolveAsync(ProviderContext context, CancellationToken cancellationToken) => Task.FromResult(new ResolvedAuth { Token = "stale-0123456789" }); + + public Task InvalidateAsync(ProviderContext context, string rejectedToken, CancellationToken cancellationToken) => Task.FromResult(new ResolvedAuth { Token = "fresh-0123456789" }); + } + + private sealed class CountingAuth : IProviderAuthResolver + { + public int Invalidations; + + public Task ResolveAsync(ProviderContext context, CancellationToken cancellationToken) => Task.FromResult(new ResolvedAuth { Token = "oauth-token-0123456789" }); + + public Task InvalidateAsync(ProviderContext context, string rejectedToken, CancellationToken cancellationToken) + { + Invalidations++; + + return Task.FromResult(new ResolvedAuth { Token = "replacement-0123456789" }); + } + } + + private sealed class RefusingAuth : IProviderAuthResolver + { + public Task ResolveAsync(ProviderContext context, CancellationToken cancellationToken) => throw new CredentialNeedsReconnectException(context.Credential.Id, "GitLab · nacho", "invalid_grant"); + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabTokenSelfTests.cs b/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabTokenSelfTests.cs new file mode 100644 index 000000000..af7630d6f --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabTokenSelfTests.cs @@ -0,0 +1,66 @@ +using CodeSpace.Core.Services.Providers.GitLab; +using Shouldly; + +namespace CodeSpace.UnitTests.Providers.GitLab; + +/// +/// What GET /personal_access_tokens/self says about a token beyond its scopes: when it expires and whether it is +/// still live. GitLab gives every new personal, project and group token an expiry and a token that reaches it answers 401 +/// to everything, yet CodeSpace discarded both fields. Each part is null when the body does not say it — a caller must +/// never read "does not say" as "never expires" or "not active". +/// +[Trait("Category", "Unit")] +public class GitLabTokenSelfTests +{ + [Fact] + public void A_dated_expiry_is_midnight_UTC_at_its_start_and_the_token_is_live() + { + var self = GitLabRepositoryProvider.ParseTokenSelf("""{"id":1,"name":"codespace","scopes":["api","read_user"],"expires_at":"2030-07-04","active":true,"revoked":false}"""); + + self.ExpiresAt.ShouldBe(new DateTimeOffset(2030, 7, 4, 0, 0, 0, TimeSpan.Zero)); + self.Active.ShouldBe(true); + self.Scopes.ShouldBe(new[] { "api", "read_user" }); + } + + [Theory] + [InlineData("""{"expires_at":"2030-07-04T10:30:00.000Z","active":true}""", 2030, 7, 4, 10, 30)] + [InlineData("""{"expires_at":"2030-07-04T10:30:00+08:00","active":true}""", 2030, 7, 4, 2, 30)] + public void A_timestamped_expiry_is_read_as_the_instant_it_names(string json, int year, int month, int day, int hour, int minute) + { + GitLabRepositoryProvider.ParseTokenSelf(json).ExpiresAt.ShouldBe(new DateTimeOffset(year, month, day, hour, minute, 0, TimeSpan.Zero)); + } + + [Fact] + public void A_token_that_never_expires_has_no_expiry_but_is_still_described() + { + var self = GitLabRepositoryProvider.ParseTokenSelf("""{"scopes":["api"],"expires_at":null,"active":true}"""); + + self.ExpiresAt.ShouldBeNull(); + self.Active.ShouldBe(true, "the body described the token — null expiry here means none, and Active is what says so"); + } + + [Theory] + [InlineData("""{"active":false}""")] + [InlineData("""{"active":true,"revoked":true}""")] + [InlineData("""{"revoked":true}""")] + public void A_revoked_or_inactive_token_is_not_active(string json) + { + GitLabRepositoryProvider.ParseTokenSelf(json).Active.ShouldBe(false); + } + + [Theory] + [InlineData("{}")] + [InlineData("""{"scopes":["api"]}""")] + [InlineData("""{"expires_at":"not a date","active":"yes"}""")] + [InlineData("""{"expires_at":20300704}""")] + [InlineData("not json")] + [InlineData("")] + [InlineData("[1,2]")] + public void What_the_body_does_not_say_is_unknown_not_a_default(string json) + { + var self = GitLabRepositoryProvider.ParseTokenSelf(json); + + self.ExpiresAt.ShouldBeNull(); + self.Active.ShouldBeNull(); + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabUnauthorizedRetryTests.cs b/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabUnauthorizedRetryTests.cs new file mode 100644 index 000000000..d8674e816 --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Providers/GitLab/GitLabUnauthorizedRetryTests.cs @@ -0,0 +1,351 @@ +using System.Net; +using CodeSpace.Core.Services.Providers; +using CodeSpace.Core.Services.Providers.Auth; +using CodeSpace.Core.Services.Providers.GitLab; +using CodeSpace.IntegrationTests.Webhooks; +using CodeSpace.Messages.Dtos.Providers; +using CodeSpace.Messages.Enums; +using CodeSpace.Messages.Exceptions; +using NGitLab.Models; +using Shouldly; +using static CodeSpace.IntegrationTests.Webhooks.StubProviderHost; +using static CodeSpace.UnitTests.Providers.GitLab.GitLabLoopback; + +namespace CodeSpace.UnitTests.Providers.GitLab; + +/// +/// A 401 to an OAuth credential is sent once more with the token the resolver says replaces it. GitLab retires an access token +/// the moment its pair is refreshed — by another run, another pod — however much of its lifetime is left, so a token read +/// from the row an instant ago can be refused; before, that surfaced as a 422 "provider_unauthorized" for up to two hours. +/// Every GitLab call path is covered: NGitLab's typed client, its raw paged reader, and each raw HTTP call (the matrix is the +/// loopback suite's own list of calls, so a new one joins it). A static token has nothing to replace it with and is not +/// retried. The loopback GitLab refuses the stale token and answers the fresh one. +/// +[Trait("Category", "Unit")] +public sealed class GitLabUnauthorizedRetryTests : IDisposable +{ + private const string Stale = "oauth-stale-token-0123456789"; + private const string Fresh = "oauth-fresh-token-0123456789"; + + private readonly List _gitlabs = new(); + + public void Dispose() + { + foreach (var gitlab in _gitlabs) gitlab.Dispose(); + } + + public static TheoryData EveryCall() + { + var data = new TheoryData(); + + foreach (var operation in Operations.Keys) data.Add(operation); + + return data; + } + + [Theory] + [MemberData(nameof(EveryCall))] + public async Task A_call_refused_with_a_stale_token_is_sent_again_with_the_replacement_and_the_replacement_stays_for_the_rest_of_the_call(string operation) + { + var gitlab = GitLabThatRefuses(Stale); + var auth = new RecoveringAuth(Stale, replacement: Fresh); + + await Operations[operation](Provider(auth), Context(gitlab.BaseUrl, AuthType.OAuth)); + + auth.Rejected.ShouldHaveSingleItem($"'{operation}' was refused once and asked once: the replacement is what every later request of the call carries"); + auth.Rejected[0].ShouldBe(Stale); + + var refused = gitlab.Requests.Where(r => r.AuthorizationHeader == $"Bearer {Stale}").ToList(); + var served = gitlab.Requests.Where(r => r.AuthorizationHeader == $"Bearer {Fresh}").ToList(); + + refused.ShouldHaveSingleItem("only the first request of the call is ever refused"); + served.ShouldContain(r => r.Method == refused[0].Method && r.PathAndQuery == refused[0].PathAndQuery && r.Body == refused[0].Body, $"{refused[0].Method} {refused[0].PathAndQuery} was sent again unchanged"); + gitlab.Requests.ShouldAllBe(r => r.PrivateTokenHeader == null, "an OAuth token is never a PRIVATE-TOKEN, replayed or not"); + } + + [Theory] + [MemberData(nameof(EveryCall))] + public async Task A_replacement_that_is_refused_too_is_the_answer_and_the_stale_token_is_never_sent_twice(string operation) + { + var gitlab = GitLabThatRefuses(Stale, Fresh); + var auth = new RecoveringAuth(Stale, replacement: Fresh); + + var failure = await Record.ExceptionAsync(() => Operations[operation](Provider(auth), Context(gitlab.BaseUrl, AuthType.OAuth))); + + // Some calls degrade a refusal to an empty result by design (a languages tab, an actor's access); the rest throw it. + if (failure is not null) IsUnauthorized(failure).ShouldBeTrue($"the second 401 is what the caller is shown — {failure.GetType().Name}: {failure.Message}"); + + gitlab.Requests.Count(r => r.AuthorizationHeader == $"Bearer {Stale}").ShouldBe(1, "the refused token is sent once, however the call is retried above the transport"); + auth.Rejected.Count(token => token == Stale).ShouldBe(1, "the resolver is asked about the refused token once; the replacement it hands out is not asked about again"); + } + + [Theory] + [InlineData(AuthType.Pat)] + [InlineData(AuthType.ProjectAccessToken)] + [InlineData(AuthType.GroupAccessToken)] + public async Task A_static_token_has_nothing_to_replace_it_with_so_its_401_is_not_retried(AuthType type) + { + var gitlab = Track(new StubProviderHost().Answer("GET", "/api/v4/", _ => new StubReply(401, """{"message":"401 Unauthorized"}"""))); + var auth = new RecoveringAuth(Stale, replacement: Fresh); + + await Should.ThrowAsync(() => Provider(auth).GetByExternalIdAsync(Context(gitlab.BaseUrl, type), "42", CancellationToken.None)); + + gitlab.Requests.Count.ShouldBe(1, "the same token would be refused the same way"); + auth.Rejected.ShouldBeEmpty(); + } + + [Fact] + public async Task A_resolver_with_nothing_to_offer_leaves_the_401_as_it_was() + { + var gitlab = GitLabThatRefuses(Stale); + var auth = new RecoveringAuth(Stale, replacement: null); + + var failure = await Should.ThrowAsync(() => Provider(auth).GetByExternalIdAsync(Context(gitlab.BaseUrl, AuthType.OAuth), "42", CancellationToken.None)); + + failure.StatusCode.ShouldBe(401); + gitlab.Requests.Count.ShouldBe(1); + auth.Rejected.ShouldHaveSingleItem(); + } + + [Theory] + [MemberData(nameof(EveryCall))] + public async Task A_credential_the_provider_refuses_for_good_is_what_the_caller_is_told_instead_of_the_401(string operation) + { + var gitlab = GitLabThatRefuses(Stale); + var dead = new CredentialNeedsReconnectException(Guid.NewGuid(), "GitLab · nacho", "invalid_grant"); + var auth = new RecoveringAuth(Stale, replacement: null) { OnInvalidate = () => throw dead }; + + var failure = await Record.ExceptionAsync(() => Operations[operation](Provider(auth), Context(gitlab.BaseUrl, AuthType.OAuth))); + + // Some calls degrade any failure to an empty result by design; the rest say why the credential cannot be used. + if (failure is not null) Chain(failure).ShouldContain(dead, $"{failure.GetType().Name}: {failure.Message}"); + + auth.Rejected.ShouldNotBeEmpty("the refusal was taken to the resolver"); + } + + [Fact] + public async Task A_request_that_changes_something_through_the_typed_client_cannot_be_sent_again_but_the_credential_is_still_put_right() + { + // NGitLab has written the body to a stream it does not hand back, so a write is not replayed. Asking still settles the + // credential — the stored token is the replacement afterwards — so the next call carries it, and this one's 401 stands. + var gitlab = GitLabThatRefuses(Stale); + var recoveries = 0; + var credential = new GitLabCredential(Stale, AuthType.OAuth, (_, _) => + { + recoveries++; + return Task.FromResult(Fresh); + }); + var client = GitLabHttp.NewClient(gitlab.BaseUrl, credential); + + Should.Throw(() => client.Issues.Create(new IssueCreate { ProjectId = 42, Title = "t", Description = "d" })); + + gitlab.Requests.Count.ShouldBe(1, "the write was sent once"); + recoveries.ShouldBe(1); + credential.Token.ShouldBe(Fresh, "the credential carries the replacement from then on, so the call's next request is not refused"); + } + + public static TheoryData EveryTypedWrite() + { + var data = new TheoryData(); + + foreach (var operation in TypedWrites.Keys) data.Add(operation); + + return data; + } + + [Theory] + [MemberData(nameof(EveryTypedWrite))] + public async Task A_write_through_the_typed_client_that_gitlab_refused_with_a_stale_token_is_sent_again_once_with_the_replacement(string operation) + { + // NGitLab cannot send a write again (it wrote the body to a stream it does not hand back), but a 401 is GitLab's answer + // before it acts, and the refusal has already put the replacement on the credential — so the provider repeats the call: + // the first merge, comment or issue after another consumer rotated the pair is not lost to the very 401 this recovers from. + var gitlab = GitLabThatRefusesWrites(Stale); + var auth = new RecoveringAuth(Stale, replacement: Fresh); + + await TypedWrites[operation](Provider(auth), Context(gitlab.BaseUrl, AuthType.OAuth)); + + auth.Rejected.ShouldHaveSingleItem($"'{operation}' was refused once and asked once"); + + var refused = gitlab.Requests.Where(r => r.AuthorizationHeader == $"Bearer {Stale}").ToList(); + var served = gitlab.Requests.Where(r => r.AuthorizationHeader == $"Bearer {Fresh}").ToList(); + + refused.ShouldHaveSingleItem("only the first request of the call is ever refused"); + refused[0].Method.ShouldNotBe("GET", "the refused request is the write itself: nothing before it settled the token"); + served.ShouldContain(r => r.Method == refused[0].Method && r.PathAndQuery == refused[0].PathAndQuery && r.Body == refused[0].Body, $"{refused[0].Method} {refused[0].PathAndQuery} was sent again, with the same body"); + gitlab.Requests.ShouldAllBe(r => r.PrivateTokenHeader == null, "an OAuth token is never a PRIVATE-TOKEN, replayed or not"); + } + + [Theory] + [MemberData(nameof(EveryTypedWrite))] + public async Task A_typed_write_whose_replacement_is_refused_too_is_the_answer_and_is_not_sent_a_third_time(string operation) + { + var gitlab = GitLabThatRefusesWrites(Stale, Fresh); + var auth = new RecoveringAuth(Stale, replacement: Fresh); + + var failure = await Should.ThrowAsync(() => TypedWrites[operation](Provider(auth), Context(gitlab.BaseUrl, AuthType.OAuth))); + + failure.StatusCode.ShouldBe(401); + + // Counted over the writes: a failed open also reads the project's merge requests, to see whether it was a repeat of one already open. + var writes = gitlab.Requests.Where(r => r.Method != "GET").ToList(); + + writes.Count(r => r.AuthorizationHeader == $"Bearer {Stale}").ShouldBe(1, "the refused token is sent once"); + writes.Count(r => r.AuthorizationHeader == $"Bearer {Fresh}").ShouldBe(1, "and the replacement once: a second refusal is the answer"); + + // That read builds a client of its own from this resolver, which still hands out the stale token (the real one hands out + // what it stored), so the refusal of it is asked about once more. + auth.Rejected.Count(token => token == Stale).ShouldBe(operation == "open a merge request" ? 2 : 1); + } + + [Theory] + [MemberData(nameof(EveryTypedWrite))] + public async Task A_typed_write_whose_401_has_no_replacement_is_not_sent_again(string operation) + { + var gitlab = GitLabThatRefusesWrites(Stale); + var auth = new RecoveringAuth(Stale, replacement: null); + + var failure = await Should.ThrowAsync(() => TypedWrites[operation](Provider(auth), Context(gitlab.BaseUrl, AuthType.OAuth))); + + failure.StatusCode.ShouldBe(401); + gitlab.Requests.Count(r => r.Method != "GET").ShouldBe(1, "the same token would be refused the same way"); + } + + [Theory] + [InlineData(AuthType.Pat)] + [InlineData(AuthType.ProjectAccessToken)] + [InlineData(AuthType.GroupAccessToken)] + public async Task A_static_tokens_typed_write_is_not_sent_again_after_a_401(AuthType type) + { + var gitlab = Track(new StubProviderHost().Answer("POST", "/api/v4/", _ => new StubReply(401, """{"message":"401 Unauthorized"}"""))); + var auth = new RecoveringAuth(Stale, replacement: Fresh); + + await Should.ThrowAsync(() => Provider(auth).CommentIssueAsync(Context(gitlab.BaseUrl, type), Repository, 7, "hello", CancellationToken.None)); + + gitlab.Requests.ShouldHaveSingleItem("the same token would be refused the same way"); + auth.Rejected.ShouldBeEmpty(); + } + + [Fact] + public async Task A_replayed_raw_write_carries_its_body_again() + { + var gitlab = GitLabThatRefuses(Stale); + var auth = new RecoveringAuth(Stale, replacement: Fresh); + + var rendered = await Provider(auth).RenderMarkdownAsync(Context(gitlab.BaseUrl, AuthType.OAuth), Repository, "# hello", CancellationToken.None); + + rendered.Html.ShouldBe("

x

"); + gitlab.Requests.Select(r => r.Method).ShouldBe(new[] { "POST", "POST" }); + gitlab.Requests[0].Body.ShouldContain("# hello"); + gitlab.Requests[1].Body.ShouldBe(gitlab.Requests[0].Body, "a 401 happens before GitLab acts on a request, so repeating it repeats nothing"); + gitlab.Requests[1].AuthorizationHeader.ShouldBe($"Bearer {Fresh}"); + } + + /// A refusal as the call sites surface it: the typed provider failure itself, or — for the webhook calls, which wrap what they catch in their own exception and say the status in its text — that wrapper. + private static bool IsUnauthorized(Exception failure) => Chain(failure).Any(e => e is ProviderApiException { StatusCode: 401 } or NGitLab.GitLabException { StatusCode: HttpStatusCode.Unauthorized } || (e is ProviderWebhookRegistrationException && e.Message.Contains("401", StringComparison.Ordinal))); + + private static IEnumerable Chain(Exception failure) + { + for (var e = failure; e is not null; e = e.InnerException) yield return e; + } + + /// A GitLab that answers to any token but the refused ones, which it answers 401. + private StubProviderHost GitLabThatRefuses(params string[] refusedTokens) + { + StubReply Respond(RecordedRequest request) => + refusedTokens.Any(token => request.AuthorizationHeader == $"Bearer {token}") ? new StubReply(401, """{"message":"401 Unauthorized"}""") : AnswerWithMergeRequestNotes(request); + + return Track(new StubProviderHost().Answer("GET", "/api/v4/", Respond).Answer("POST", "/api/v4/", Respond)); + } + + /// A GitLab that refuses the given tokens and otherwise answers the typed writes (and every route knows). + private StubProviderHost GitLabThatRefusesWrites(params string[] refusedTokens) + { + StubReply Respond(RecordedRequest request) => + refusedTokens.Any(token => request.AuthorizationHeader == $"Bearer {token}") ? new StubReply(401, """{"message":"401 Unauthorized"}""") : AnswerWrites(request); + + return Track(new StubProviderHost().Answer("GET", "/api/v4/", Respond).Answer("POST", "/api/v4/", Respond).Answer("PUT", "/api/v4/", Respond).Answer("DELETE", "/api/v4/", Respond)); + } + + private const string MergeRequestJson = """{"id":71,"iid":7,"project_id":42,"title":"t","description":"d","state":"opened","source_branch":"feature","target_branch":"main","source_project_id":42,"target_project_id":42,"web_url":"http://gitlab.test/g/r/-/merge_requests/7","sha":"abc","labels":[],"author":{"id":7,"username":"nacho"},"created_at":"2030-01-01T00:00:00Z","updated_at":"2030-01-01T00:00:00Z"}"""; + + private const string IssueJson = """{"id":51,"iid":7,"project_id":42,"title":"t","description":"d","state":"opened","labels":[],"web_url":"http://gitlab.test/g/r/-/issues/7","author":{"id":7,"username":"nacho"},"created_at":"2030-01-01T00:00:00Z","updated_at":"2030-01-01T00:00:00Z"}"""; + + private const string NoteJson = """{"id":9,"body":"b","author":{"id":7,"username":"nacho"},"created_at":"2030-01-01T00:00:00Z"}"""; + + /// What GitLab answers each typed write; anything else falls through to the loopback's own routes. + private static StubReply AnswerWrites(RecordedRequest request) + { + var path = request.PathAndQuery.Split('?')[0]; + + return (request.Method, path) switch + { + ("POST", ProjectPath + "/merge_requests") => new StubReply(201, MergeRequestJson), + ("PUT", ProjectPath + "/merge_requests/7/merge") => new StubReply(200, MergeRequestJson.Replace("\"opened\"", "\"merged\"", StringComparison.Ordinal)), + ("POST", ProjectPath + "/issues") => new StubReply(201, IssueJson), + ("POST", ProjectPath + "/issues/7/notes") => new StubReply(201, NoteJson), + ("PUT", ProjectPath + "/issues/7") => new StubReply(200, IssueJson.Replace("\"opened\"", "\"closed\"", StringComparison.Ordinal)), + ("POST", ProjectPath + "/merge_requests/7/notes") => new StubReply(201, NoteJson), + ("DELETE", ProjectPath + "/hooks/9") => new StubReply(204, string.Empty), + ("GET", ProjectPath + "/merge_requests") => new StubReply(200, "[]"), + ("GET", ProjectPath + "/merge_requests/7") => new StubReply(200, MergeRequestJson), + ("GET", ProjectPath + "/issues") => new StubReply(200, "[]"), + ("GET", ProjectPath + "/issues/7/notes") => new StubReply(200, "[]"), + ("GET", ProjectPath + "/merge_requests/7/notes") => new StubReply(200, "[]"), + _ => AnswerWithMergeRequestNotes(request) + }; + } + + /// The typed writes: the provider's calls that change something through NGitLab, the first request of each being the write itself. + private static readonly IReadOnlyDictionary> TypedWrites = new Dictionary> + { + ["open a merge request"] = (p, c) => p.OpenPullRequestAsync(c, Repository, new OpenPullRequestInput { Title = "t", SourceBranch = "feature", TargetBranch = "main", Body = "d" }, CancellationToken.None), + ["merge a merge request"] = (p, c) => p.MergePullRequestAsync(c, Repository, 7, new MergePullRequestInput(), CancellationToken.None), + ["create an issue"] = (p, c) => p.CreateIssueAsync(c, Repository, new CreateIssueInput { Title = "t", Body = "d" }, CancellationToken.None), + ["comment on an issue"] = (p, c) => p.CommentIssueAsync(c, Repository, 7, "hello", CancellationToken.None), + ["close an issue"] = (p, c) => p.CloseIssueAsync(c, Repository, 7, CancellationToken.None), + ["comment on a merge request"] = (p, c) => p.PostCommentAsync(c, Repository, 7, "hello", CancellationToken.None), + ["delete a project hook"] = (p, c) => p.DeleteWebhookAsync(c, Repository, "9", CancellationToken.None), + }; + + /// The loopback's answers, and the merge request notes a review leaves after the unapprove. + private static StubReply AnswerWithMergeRequestNotes(RecordedRequest request) => request.PathAndQuery.Contains("/merge_requests/7/notes", StringComparison.Ordinal) + ? request.Method == "GET" ? new StubReply(200, "[]") : new StubReply(201, """{"id":9,"body":"b","author":{"id":7,"username":"nacho"},"created_at":"2030-01-01T00:00:00Z"}""") + : Answer(request); + + private StubProviderHost Track(StubProviderHost gitlab) + { + _gitlabs.Add(gitlab); + + return gitlab; + } + + /// Hands out and, when asked about it, — what the real resolver does when another caller already rotated the pair. + private sealed class RecoveringAuth : IProviderAuthResolver + { + private readonly string _token; + private readonly string? _replacement; + + public RecoveringAuth(string token, string? replacement) + { + _token = token; + _replacement = replacement; + } + + public Action? OnInvalidate { get; init; } + + public List Rejected { get; } = new(); + + public Task ResolveAsync(ProviderContext context, CancellationToken cancellationToken) => Task.FromResult(new ResolvedAuth { Token = _token }); + + public Task InvalidateAsync(ProviderContext context, string rejectedToken, CancellationToken cancellationToken) + { + lock (Rejected) Rejected.Add(rejectedToken); + + OnInvalidate?.Invoke(); + + // Like the real resolver: it replaces the token it handed out, and has nothing further for the replacement itself (a pair rotated moments ago is not rotated again). + return Task.FromResult(_replacement is null || rejectedToken != _token ? null : new ResolvedAuth { Token = _replacement }); + } + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/LocalGitWorkspaceProviderCredentialTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/LocalGitWorkspaceProviderCredentialTests.cs index 9de0de902..8260c420f 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/LocalGitWorkspaceProviderCredentialTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/LocalGitWorkspaceProviderCredentialTests.cs @@ -232,6 +232,23 @@ public async Task A_credential_disconnected_while_the_run_worked_is_typed_and_no publish.Runner.RemoteSpecs.ShouldBeEmpty("nothing was put to the remote"); } + [Fact] + public async Task A_credential_its_provider_refused_while_the_run_worked_is_typed_the_same_way_and_nothing_reaches_the_remote() + { + // The probe or a refresh marked the credential as needing a reconnect while the agent ran: resolving it again can + // only fail the same way, so the push is not retried and the run is told what to do. + var source = new ScriptedSource { Answer = _ => throw new CredentialNeedsReconnectException(Guid.NewGuid(), "GitLab · someone", "invalid_grant") }; + await using var publish = await PrepareAsync(new ScriptedRunner(), source); + + var failure = await Should.ThrowAsync(() => publish.PushAsync()); + + failure.Code.ShouldBe(FailureCodes.GitCredentialRejected); + failure.Message.ShouldContain("was refused by its provider"); + failure.Message.ShouldContain("Reconnect the provider"); + failure.InnerException.ShouldBeOfType(); + publish.Runner.RemoteSpecs.ShouldBeEmpty("nothing was put to the remote"); + } + [Fact] public async Task A_failure_to_resolve_the_credential_is_a_plain_workspace_failure_the_push_retry_can_retry() { diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/WorkflowRunRequestRejectionReasonsTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/WorkflowRunRequestRejectionReasonsTests.cs index f307beb6c..e0371b3fe 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/WorkflowRunRequestRejectionReasonsTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/WorkflowRunRequestRejectionReasonsTests.cs @@ -24,6 +24,8 @@ public class WorkflowRunRequestRejectionReasonsTests [InlineData("delivery_id_missing", nameof(WorkflowRunRequestRejectionReasons.DeliveryIdMissing))] [InlineData("delivery_replayed", nameof(WorkflowRunRequestRejectionReasons.DeliveryReplayed))] [InlineData("author_not_member", nameof(WorkflowRunRequestRejectionReasons.AuthorNotMember))] + [InlineData("author_standing_unknown", nameof(WorkflowRunRequestRejectionReasons.AuthorStandingUnknown))] + [InlineData("credential_rejected", nameof(WorkflowRunRequestRejectionReasons.CredentialRejected))] [InlineData("pull_request_debounced", nameof(WorkflowRunRequestRejectionReasons.PullRequestDebounced))] public void Rejection_reason_string_form_is_pinned(string expected, string constantName) { diff --git a/backend/tests/TESTING.md b/backend/tests/TESTING.md index 4cd02e556..915250155 100644 --- a/backend/tests/TESTING.md +++ b/backend/tests/TESTING.md @@ -51,9 +51,15 @@ shared `PostgresFixture` type (`CodeSpace.E2ETests/Infrastructure/PostgresCollec Provider-credential tests share two more pieces, both in `CodeSpace.IntegrationTests/Infrastructure` (and reused by E2E): - `GitLabDoorkeeperStub` — a loopback GitLab whose OAuth server is modelled on Doorkeeper: refresh tokens are single use, a - refresh revokes the access token it replaces, and the REST and git smart-HTTP endpoints honour only live tokens. Its - counters (`Count(StubEvent.Refused)`, …) are how a test says "GitLab was never shown a consumed refresh token". + refresh revokes the access token it replaces, and the REST and git smart-HTTP endpoints honour only live tokens. It also + stands in for personal, project and group access tokens (`AddAccessToken` / `RevokeAccessToken`: `PRIVATE-TOKEN`, a date-valued + `expires_at` at `personal_access_tokens/self`, 401 once revoked or expired) and can script the members lookup to fail + (`FailMembers(status, times)`). Its counters (`Count(StubEvent.Refused)`, …) are how a test says "GitLab was never shown a consumed refresh token". `GitLabDoorkeeperStubTests` pins what it does, so a red test built on it is the product's and not the stub's. +- `SilentProviderHost` (`CodeSpace.IntegrationTests/Webhooks`, linked into the unit project like `StubProviderHost`) — a loopback provider that + accepts the connection and never answers, for what a hung GitLab costs the caller. A call to it only returns when the caller gives + up, so a test of it runs the call on a thread of its own and bounds the wait, and shortens the sweep's deadlines by registering a + `CredentialHealthDeadlines` instead of waiting out the real ones. - `RefreshPodHarness` — pods as real OS processes (`CodeSpace.StorageTestWorker`'s `oauth-refresh-pod` mode) running the production refresher, or the pinned previous build's (`StorageTestWorker/PreviousBuild`), against the fixture's Postgres. One process shares one refresh semaphore, so only separate processes exercise the cross-pod advisory lock.