Show build, role and credential plumbing in a health report

backend/src/CodeSpace.Api/Startup.cs

@@ -6,6 +6,8 @@
 using CodeSpace.Api.Filters;
 using CodeSpace.Api.Logging;
 using CodeSpace.Core.Services.Credentials;
+using CodeSpace.Core.Services.Health;
+using CodeSpace.Core.Services.Health.Checks;
 using CodeSpace.Core.Services.OutboundHttp;
 using CodeSpace.Core.Services.Identity;
 using CodeSpace.Core.Settings;
@@ -49,7 +51,17 @@ public void ConfigureServices(IServiceCollection services)
         // the pipeline responds, so a transient DB blip doesn't trigger a pod RESTART. /health/ready (readiness)
         // runs the "ready"-tagged checks — here a DbContext ping — so a pod is pulled from the Service's endpoints
         // (stops receiving traffic) while its DB is unreachable, without being killed. Mapped anonymous below.
-        services.AddHealthChecks().AddDbContextCheck<CodeSpaceDbContext>("db", tags: new[] { "ready" });
+        //
+        // /health is the report for a person or a script, not a probe: this process's build and role plus every check. The
+        // three checks after the database are deployment facts that otherwise take a query or a shell on the pod to learn —
+        // whether the credential refresh lock is exclusive across connections, whether git can carry a clone token, whether the
+        // credential repair sweep is scheduled. They carry NO "ready" tag, so /health/ready is exactly what it was and none
+        // of them can pull a pod from rotation or restart it: they degrade the report and say why.
+        services.AddHealthChecks()
+            .AddDbContextCheck<CodeSpaceDbContext>("db", tags: new[] { "ready" })
+            .AddCheck<RefreshLockHealthCheck>(RefreshLockHealthCheck.Name)
+            .AddCheck<GitVersionHealthCheck>(GitVersionHealthCheck.Name)
+            .AddCheck<CredentialRepairSweepHealthCheck>(CredentialRepairSweepHealthCheck.Name);
 
         // CORS — only matters when the SPA calls the backend directly (VITE_API_URL=http://localhost:5099) rather
         // than through Vite's same-origin /api proxy. The policy, its origins setting and the Development-only
@@ -134,13 +146,22 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
         // so the filter can read HttpContext.User.
         app.UseCodeSpaceHangfire(Configuration);
 
+        // Once the host is serving: one line naming the build, role and git, and the refresh-lock self-test. Describes the pod; never
+        // refuses it.
+        app.ApplicationServices.GetRequiredService<BootPostureReporter>().ReportWhenStarted();
+
         app.UseEndpoints(endpoints =>
         {
             // k8s liveness/readiness probes — AllowAnonymous because the global FallbackPolicy requires auth and a
             // probe is unauthenticated. Live = process up (no checks); Ready = the "ready"-tagged checks (DB ping).
             endpoints.MapHealthChecks("/health/live", new HealthCheckOptions { Predicate = _ => false }).AllowAnonymous();
             endpoints.MapHealthChecks("/health/ready", new HealthCheckOptions { Predicate = check => check.Tags.Contains("ready") }).AllowAnonymous();
 
+            // The report, as JSON. Anonymous like the probes, so what anyone reads is the build, the role and each check's status; a
+            // signed-in global Admin also reads each check's description and data (see HealthResponseWriter). Nothing an exception
+            // message could carry is printed to either.
+            endpoints.MapHealthChecks("/health", new HealthCheckOptions { ResponseWriter = HealthResponseWriter.WriteAsync }).AllowAnonymous();
+
             endpoints.MapControllers();
             if (env.IsDevelopment()) endpoints.MapOpenApi();
 

backend/src/CodeSpace.Core/Services/Agents/PackCloneFetcher.cs

@@ -91,9 +91,18 @@ private async Task CloneAsync(string url, string? reference, string dir, Cancell
         var result = await _runners.Resolve(SandboxKinds.Local).RunAsync(BuildCloneSpec(url, reference, dir), cancellationToken).ConfigureAwait(false);
 
         if (result.Status != SandboxStatus.Success)
-            throw new PackImportException(CloneFailedMessage(url, result));
+            throw OldGitFailure(url, result) ?? new PackImportException(CloneFailedMessage(url, result));
     }
 
+    /// <summary>
+    /// The clone failure when the cause is a host git older than <see cref="TokenedGitCommand.MinimumGit"/>: a pasted credential
+    /// travels through <c>GIT_CONFIG_COUNT</c>, which that git ignores, so the remote's request for it went unanswered. The
+    /// message names the URL without its credential and the actionable cause, in place of git's "could not read Username"; the
+    /// typed exception is kept as the inner one. Null whenever an old git is not the explanation.
+    /// </summary>
+    private static PackImportException? OldGitFailure(string url, SandboxResult result) =>
+        TokenedGitCommand.OldGitFailure(PastedSecret(url) is not null, result.Stderr) is { } oldGit ? new PackImportException($"git clone of '{WithoutPastedCredential(url)}' failed: {oldGit.Message}", oldGit) : null;
+
     /// <summary>
     /// The import walks this checkout (a worker that dies mid-import leaves it on disk for the janitor), so its origin must
     /// hold no credential. The clone already named the URL without it; as a belt, rewrite origin to that URL through the

backend/src/CodeSpace.Core/Services/Agents/Workspace/GitCredentialHelperUnsupportedException.cs

@@ -0,0 +1,25 @@
+using CodeSpace.Messages.Health;
+
+namespace CodeSpace.Core.Services.Agents.Workspace;
+
+/// <summary>
+/// A tokened git command was refused for want of a credential on a host whose git is older than
+/// <see cref="TokenedGitCommand.MinimumGit"/>: the helper that would have answered is configured through
+/// <c>GIT_CONFIG_COUNT</c>, which that git ignores. Still a <see cref="WorkspaceException"/> — the run executor, the
+/// integrate nodes and the acceptance grader already handle that type — and it keeps its failure code; what changes is that
+/// the message names the cause and the fix, where git says only that it "could not read Username".
+/// </summary>
+public sealed class GitCredentialHelperUnsupportedException : WorkspaceException
+{
+    public GitCredentialHelperUnsupportedException(GitVersion version) : base(Describe(version))
+    {
+        Version = version;
+    }
+
+    /// <summary>The release of git this host runs.</summary>
+    public GitVersion Version { get; }
+
+    private static string Describe(GitVersion version) =>
+        $"This host's git is {version}, older than {TokenedGitCommand.MinimumGit}. The credential helper that hands git its token is configured through GIT_CONFIG_COUNT, " +
+        $"which git ignores before {TokenedGitCommand.MinimumGit}, so the remote's request for a credential went unanswered. Upgrade the git in this image to {TokenedGitCommand.MinimumGit} or later.";
+}

backend/src/CodeSpace.Core/Services/Agents/Workspace/HostGit.cs

@@ -0,0 +1,69 @@
+using CodeSpace.Messages.Health;
+using System.Diagnostics;
+
+namespace CodeSpace.Core.Services.Agents.Workspace;
+
+/// <summary>
+/// The <c>git</c> this host runs, asked once per process: every clone, fetch, push and launch-time <c>ls-remote</c> spawns
+/// the same binary from the same PATH, so one <c>git --version</c> answers for all of them. The answer is what the
+/// credential helper's 2.31 floor (<see cref="TokenedGitCommand.MinimumGit"/>) is measured against — at boot, in the health
+/// report, and when a tokened command fails for want of a credential.
+///
+/// <para>A probe that fails — no git on PATH, a hung or unreadable answer — is a <see cref="GitReading"/> with no version,
+/// never an exception: this is read from a boot path and a health endpoint, neither of which may fail because of it.</para>
+/// </summary>
+public static class HostGit
+{
+    private const int ProbeTimeoutMs = 10_000;
+
+    private static readonly Lazy<GitReading> Probed = new(Probe, LazyThreadSafetyMode.ExecutionAndPublication);
+
+    private static readonly AsyncLocal<GitReading?> Stand = new();
+
+    /// <summary>The host's git as it answered the first time it was asked.</summary>
+    public static GitReading Current => Stand.Value ?? Probed.Value;
+
+    /// <summary>
+    /// Answer as <paramref name="reading"/> for the current async flow until the returned scope is disposed — the seam that
+    /// lets a test put an old git in front of a code path without installing one. Scoped to the flow, so tests running in
+    /// parallel never see each other's.
+    /// </summary>
+    internal static IDisposable Use(GitReading reading)
+    {
+        var previous = Stand.Value;
+        Stand.Value = reading;
+
+        return new Restore(previous);
+    }
+
+    private static GitReading Probe()
+    {
+        try
+        {
+            var start = new ProcessStartInfo { FileName = "git", UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true };
+            start.ArgumentList.Add("--version");
+
+            using var process = Process.Start(start) ?? throw new InvalidOperationException("the process did not start");
+
+            if (!process.WaitForExit(ProbeTimeoutMs))
+            {
+                try { process.Kill(entireProcessTree: true); } catch { /* best-effort */ }
+
+                return new GitReading(null, $"git could not be run: `git --version` did not answer within {ProbeTimeoutMs / 1000}s");
+            }
+
+            var output = process.StandardOutput.ReadToEnd().Trim();
+
+            return process.ExitCode == 0 ? new GitReading(GitVersion.Parse(output), output) : new GitReading(null, $"git could not be run: `git --version` exited {process.ExitCode}");
+        }
+        catch (Exception ex)
+        {
+            return new GitReading(null, $"git could not be run: {ex.GetType().Name}");
+        }
+    }
+
+    private sealed class Restore(GitReading? previous) : IDisposable
+    {
+        public void Dispose() => Stand.Value = previous;
+    }
+}

backend/src/CodeSpace.Core/Services/Agents/Workspace/Integrators/LocalGitBranchIntegrator.cs

@@ -216,7 +216,7 @@ private async Task CloneAsync(IntegrationRequest request, string directory, Canc
         var result = await RunTokenedGitAsync(request, new[] { "clone", url, directory }, directory, cancellationToken).ConfigureAwait(false);
 
         if (result.Status != SandboxStatus.Success)
-            throw new WorkspaceException($"git clone failed (exit {result.ExitCode}): {LocalGitWorkspaceProvider.Redact(Summarize(result.Stderr), request.Token)}");
+            throw TokenedGitCommand.OldGitFailure(Remote(request).IsTokened, result.Stderr) ?? new WorkspaceException($"git clone failed (exit {result.ExitCode}): {LocalGitWorkspaceProvider.Redact(Summarize(result.Stderr), request.Token)}");
     }
 
     /// <summary>Check out the EXACT shared base (detached) so every <c>apply --3way</c> resolves the pre-image against the commit the agents saw. False when the SHA is not in history (a bad base).</summary>
@@ -457,7 +457,7 @@ private async Task CommitAsync(string directory, int count, CancellationToken ca
             return "remote integration branch advanced — refusing to clobber it";
 
         // A genuine infrastructure failure (auth / network / repo gone) — surface it (token redacted).
-        throw new WorkspaceException($"git push failed (exit {result.ExitCode}): {LocalGitWorkspaceProvider.Redact(Summarize(result.Stderr), request.Token)}");
+        throw TokenedGitCommand.OldGitFailure(Remote(request).IsTokened, result.Stderr) ?? new WorkspaceException($"git push failed (exit {result.ExitCode}): {LocalGitWorkspaceProvider.Redact(Summarize(result.Stderr), request.Token)}");
     }
 
     private async Task ResetToBaseAsync(string directory, IntegrationRequest request, CancellationToken cancellationToken)

backend/src/CodeSpace.Core/Services/Agents/Workspace/Providers/LocalGitWorkspaceProvider.cs

@@ -341,7 +341,7 @@ private async Task CloneAsync(WorkspaceRequest request, RepositoryCommandContext
         var result = await RunGitAsync(args, context, cancellationToken).ConfigureAwait(false);
 
         if (result.Status != SandboxStatus.Success)
-            throw new WorkspaceException($"git clone failed (exit {result.ExitCode}): {Redact(Summarize(result.Stderr), request.Token)}");
+            throw TokenedGitCommand.OldGitFailure(context.Remote.IsTokened, result.Stderr) ?? new WorkspaceException($"git clone failed (exit {result.ExitCode}): {Redact(Summarize(result.Stderr), request.Token)}");
 
         if (!string.IsNullOrWhiteSpace(request.PinnedSha))
             await MaterializePinAsync(request, context, cancellationToken).ConfigureAwait(false);
@@ -934,7 +934,7 @@ private Task<SandboxResult> RunPublishGitAsync(MaterializedRepo repo, string pub
             ExecuteGitAsync(repo, args, PublishGitSpec(publishDir, args, network, timeoutSeconds), cancellationToken);
 
         private Task<string> RunTokenedPublishGitOrThrowAsync(MaterializedRepo repo, string publishDir, TokenedGitCommand.Remote remote, IReadOnlyList<string> args, CancellationToken cancellationToken) =>
-            EnsureSuccessAsync(repo, args, RunTokenedPublishGitAsync(repo, publishDir, remote, args, cancellationToken));
+            EnsureSuccessAsync(repo, args, RunTokenedPublishGitAsync(repo, publishDir, remote, args, cancellationToken), remote.IsTokened);
 
         /// <summary>Run a publish-repo command that reaches <paramref name="remote"/> — the LFS upload, the push, the readback — as a <see cref="TokenedGitCommand"/>, with the network and the push budget.</summary>
         private Task<SandboxResult> RunTokenedPublishGitAsync(MaterializedRepo repo, string publishDir, TokenedGitCommand.Remote remote, IReadOnlyList<string> args, CancellationToken cancellationToken) =>
@@ -1029,13 +1029,17 @@ private async Task<SandboxResult> ExecuteGitAsync(MaterializedRepo repo, IReadOn
             }
         }
 
-        /// <summary>Await a git result and throw a redacted <see cref="WorkspaceException"/> on a non-success status; the argv is redacted too, as a belt.</summary>
-        private static async Task<string> EnsureSuccessAsync(MaterializedRepo repo, IReadOnlyList<string> args, Task<SandboxResult> run)
+        /// <summary>
+        /// Await a git result and throw a redacted <see cref="WorkspaceException"/> on a non-success status; the argv is redacted too, as a belt.
+        /// <paramref name="tokened"/> is whether the command ran against a remote that carried a credential (<see cref="TokenedGitCommand.Remote.IsTokened"/>),
+        /// the only commands an old git can fail for want of one: it is the remote's own answer, not the run's clone token, so the two cannot disagree.
+        /// </summary>
+        private static async Task<string> EnsureSuccessAsync(MaterializedRepo repo, IReadOnlyList<string> args, Task<SandboxResult> run, bool tokened = false)
         {
             var result = await run.ConfigureAwait(false);
 
             if (result.Status != SandboxStatus.Success)
-                throw new WorkspaceException($"git {string.Join(' ', RedactArgs(args, repo.Token))} failed (exit {result.ExitCode}): {Redact(result.Stderr.Trim(), repo.Token)}");
+                throw TokenedGitCommand.OldGitFailure(tokened, result.Stderr) ?? new WorkspaceException($"git {string.Join(' ', RedactArgs(args, repo.Token))} failed (exit {result.ExitCode}): {Redact(result.Stderr.Trim(), repo.Token)}");
 
             SandboxOutputCompleteness.RequireStdout(result);
             return result.Stdout;

backend/src/CodeSpace.Core/Services/Agents/Workspace/RemoteTipResolver.cs

@@ -93,7 +93,7 @@ public sealed class RemoteTipResolver : IRemoteTipResolver, ISingletonDependency
         }
 
         if (result.Status != SandboxStatus.Success)
-            throw new WorkspaceException($"git ls-remote failed (exit {result.ExitCode}) resolving the launch base for {SanitizeUrl(request.RepositoryUrl)}: {LocalGitWorkspaceProvider.Redact(Truncate(result.Stderr), request.Token)} — the launch base pin requires a reachable remote; the clone would fail the same way later");
+            throw TokenedGitCommand.OldGitFailure(remote.IsTokened, result.Stderr) ?? new WorkspaceException($"git ls-remote failed (exit {result.ExitCode}) resolving the launch base for {SanitizeUrl(request.RepositoryUrl)}: {LocalGitWorkspaceProvider.Redact(Truncate(result.Stderr), request.Token)} — the launch base pin requires a reachable remote; the clone would fail the same way later");
 
         // sha1 (40) and sha256 (64) object formats both count — a sha256 remote must not read as "ref missing".
         return result.Stdout

backend/src/CodeSpace.Core/Services/Agents/Workspace/TokenedGitCommand.cs

@@ -1,4 +1,5 @@
 using CodeSpace.Messages.Agents;
+using CodeSpace.Messages.Health;
 
 namespace CodeSpace.Core.Services.Agents.Workspace;
 
@@ -64,6 +65,32 @@ internal static class TokenedGitCommand
     /// <summary>The username a token is sent under when its provider names none: GitHub's.</summary>
     internal const string DefaultTokenUsername = "x-access-token";
 
+    /// <summary>The first git that reads <c>GIT_CONFIG_COUNT</c>, which the whole credential helper is configured through. An older git ignores the variable, so a tokened command finds no credential.</summary>
+    internal static readonly GitVersion MinimumGit = new(2, 31, 0);
+
+    /// <summary>
+    /// What a tokened command that git refused for want of a credential becomes on a host whose git is older than
+    /// <see cref="MinimumGit"/>: a <see cref="GitCredentialHelperUnsupportedException"/> naming that cause, in place of git's own
+    /// "could not read Username". Null whenever the old git is not the explanation — the command was not tokened (no helper was
+    /// ever going to be configured), git's complaint is some other one (an old git clones a public repository fine), or the
+    /// host's release is unknown or new enough, in which case the credential really was missing or refused and git's words stand.
+    /// The caller throws it in place of its own failure, with the result's redacted detail unchanged when this is null.
+    /// </summary>
+    internal static WorkspaceException? OldGitFailure(bool tokened, string stderr) =>
+        tokened && AskedForACredential(stderr) ? OldGitFailure(tokened, stderr, HostGit.Current) : null;
+
+    /// <summary><see cref="OldGitFailure(bool, string)"/> against a given reading of the host's git.</summary>
+    internal static WorkspaceException? OldGitFailure(bool tokened, string stderr, GitReading git)
+    {
+        if (!tokened || !AskedForACredential(stderr)) return null;
+
+        return git.Version is { } version && !version.IsAtLeast(MinimumGit) ? new GitCredentialHelperUnsupportedException(version) : null;
+    }
+
+    /// <summary>git's non-interactive words for a remote that wanted a credential nothing supplied.</summary>
+    private static bool AskedForACredential(string stderr) =>
+        stderr.Contains("could not read Username", StringComparison.Ordinal) || stderr.Contains("could not read Password", StringComparison.Ordinal);
+
     /// <summary>
     /// The credential helper a tokened command names: a shell function that answers <c>get</c> from the two variables until
     /// an <c>erase</c> records in the <see cref="StateVariable"/> directory that the remote refused the credential, then says

backend/src/CodeSpace.Core/Services/Agents/Workspace/WorkspaceException.cs

@@ -5,8 +5,11 @@ namespace CodeSpace.Core.Services.Agents.Workspace;
 /// <summary>
 /// Thrown when a workspace can't be prepared — a failed clone, a missing ref, or git being unavailable.
 /// The executor maps it to a clean agent-run failure (the run never starts the harness).
+///
+/// <para>Not sealed so a failure with a known cause can say so (<see cref="GitCredentialHelperUnsupportedException"/>) while
+/// every catch site that handles this type keeps handling it.</para>
 /// </summary>
-public sealed class WorkspaceException : Exception, IFailure
+public class WorkspaceException : Exception, IFailure
 {
     public FailureKind Kind => FailureKind.Unprocessable;