Keep the legacy storage fingerprint for one release

backend/src/CodeSpace.Core/Handlers/CommandHandlers/Storage/BackfillStorageNamespaceFingerprintsCommandHandler.cs

@@ -0,0 +1,21 @@
+using CodeSpace.Core.Services.Workflows.Artifacts.Profiles;
+using CodeSpace.Messages.Commands.Storage;
+using MediatR;
+
+namespace CodeSpace.Core.Handlers.CommandHandlers.Storage;
+
+/// <summary>Thin dispatcher (Rule 16) — the production caller of <see cref="IStorageNamespaceFingerprintBackfillService.BackfillAsync"/>.</summary>
+public sealed class BackfillStorageNamespaceFingerprintsCommandHandler : IRequestHandler<BackfillStorageNamespaceFingerprintsCommand, int>
+{
+    private readonly IStorageNamespaceFingerprintBackfillService _backfill;
+
+    public BackfillStorageNamespaceFingerprintsCommandHandler(IStorageNamespaceFingerprintBackfillService backfill)
+    {
+        _backfill = backfill;
+    }
+
+    public async Task<int> Handle(BackfillStorageNamespaceFingerprintsCommand request, CancellationToken cancellationToken)
+    {
+        return await _backfill.BackfillAsync(request.BatchSize, cancellationToken).ConfigureAwait(false);
+    }
+}

backend/src/CodeSpace.Core/Jobs/RecurringJobs/StorageNamespaceFingerprintBackfillRecurringJob.cs

@@ -0,0 +1,17 @@
+using CodeSpace.Messages.Commands.Storage;
+using MediatR;
+
+namespace CodeSpace.Core.Jobs.RecurringJobs;
+
+/// <summary>Every ten minutes: put the namespace fingerprint the previous build compares back on any storage profile revision a physical-identity build stamped otherwise — the revisions it admitted before this release, and any an older pod admits during the rollout (thin Rule-14 dispatcher). A restored revision is no longer a candidate, so an idle tick is one small read. Deleting this class must also remove its recurring entry: see <c>StorageProfileRules.StoredNamespaceFingerprint</c>.</summary>
+public sealed class StorageNamespaceFingerprintBackfillRecurringJob : IRecurringJob
+{
+    private readonly IMediator _mediator;
+
+    public StorageNamespaceFingerprintBackfillRecurringJob(IMediator mediator) { _mediator = mediator; }
+
+    public string JobId => nameof(StorageNamespaceFingerprintBackfillRecurringJob);
+    public string CronExpression => "*/10 * * * *";
+
+    public async Task Execute() => await _mediator.Send(new BackfillStorageNamespaceFingerprintsCommand()).ConfigureAwait(false);
+}

backend/src/CodeSpace.Core/Services/Agents/AgentRunLogging/AgentRunLogStorageReadiness.cs

@@ -74,11 +74,11 @@ public async Task EnsureDefaultRouteAsync(Guid teamId, CancellationToken cancell
         }
     }
 
-    /// <summary>The same derivation admission uses, so this bootstrap's revision names its namespace exactly as a Settings-created one would.</summary>
+    /// <summary>The value admission stores (<see cref="StorageProfileRules.StoredNamespaceFingerprint"/>), so this bootstrap's revision carries exactly what a Settings-created one for the same root would.</summary>
     private string NamespaceFingerprint(string canonicalConfig)
     {
         using var document = JsonDocument.Parse(canonicalConfig);
-        return StorageProfileRules.NamespaceFingerprint(_modules.Require(LocalRwxArtifactStorageDriverFactory.TypeKey), document.RootElement);
+        return StorageProfileRules.StoredNamespaceFingerprint(_modules.Require(LocalRwxArtifactStorageDriverFactory.TypeKey), document.RootElement);
     }
 
     private static StorageProfile BuildProfile(Guid teamId, string canonicalConfig, string namespaceFingerprint, DateTimeOffset now)

backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/IStorageNamespaceFingerprintBackfillService.cs

@@ -0,0 +1,17 @@
+namespace CodeSpace.Core.Services.Workflows.Artifacts.Profiles;
+
+/// <summary>
+/// Restores <see cref="StorageProfileRules.StoredNamespaceFingerprint"/> on revisions that a build storing the
+/// PHYSICAL namespace identity already wrote — the builds between namespace canonicalization and this release, and any
+/// such pod still admitting during this rollout. The previous build's retention reaper compares stored fingerprints,
+/// so those revisions must carry the value it would have written for the same configuration before a rollback can
+/// bring it back.
+///
+/// <para>This build decides nothing on the column, so the rewrite changes nothing it does. It runs only where Hangfire
+/// processes jobs.</para>
+/// </summary>
+public interface IStorageNamespaceFingerprintBackfillService
+{
+    /// <summary>Restore up to <paramref name="batchSize"/> revisions. Idempotent and safe on several workers at once: a restored revision is no longer a candidate, and each write is conditional on the row still holding the value the pass read. Returns how many revisions this pass restored.</summary>
+    Task<int> BackfillAsync(int batchSize, CancellationToken cancellationToken);
+}

backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageNamespaceFingerprintBackfillService.cs

@@ -0,0 +1,96 @@
+using System.Text.Json;
+using CodeSpace.Core.DependencyInjection;
+using CodeSpace.Core.Persistence.Db;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace CodeSpace.Core.Services.Workflows.Artifacts.Profiles;
+
+/// <summary>
+/// The bounded pass behind <see cref="IStorageNamespaceFingerprintBackfillService"/>. A candidate is a revision whose
+/// stored fingerprint is not <see cref="StorageProfileRules.LegacyNamespaceFingerprint"/> of its own stored
+/// configuration. Every build that predates physical namespaces wrote exactly that value, so the only revisions that
+/// differ are the ones a physical-identity build wrote for a configuration not already in canonical form; after the
+/// rewrite they leave the set, so the pass is self-terminating.
+///
+/// <para>Revisions are immutable under a trigger, so the pass suspends it inside its own transaction and restores it
+/// before commit: no other session ever sees the table without it, and a pass that fails rolls the switch back with
+/// its writes. The switch locks out concurrent admissions for the length of one short pass, and the lock wait is
+/// bounded so a pass queued behind a long transaction gives up rather than stall them. Each row is rewritten by an
+/// UPDATE conditional on its id AND the value the pass read, so a second worker restoring the same row matches
+/// nothing.</para>
+/// </summary>
+public sealed class StorageNamespaceFingerprintBackfillService : IStorageNamespaceFingerprintBackfillService, IScopedDependency
+{
+    private const string BoundLockWait = "SET LOCAL lock_timeout = '5s'";
+    private const string SuspendImmutability = "ALTER TABLE storage_profile_revision DISABLE TRIGGER storage_profile_revision_enforce_immutability";
+    private const string ResumeImmutability = "ALTER TABLE storage_profile_revision ENABLE TRIGGER storage_profile_revision_enforce_immutability";
+
+    private readonly CodeSpaceDbContext _db;
+    private readonly ILogger<StorageNamespaceFingerprintBackfillService> _logger;
+
+    public StorageNamespaceFingerprintBackfillService(CodeSpaceDbContext db, ILogger<StorageNamespaceFingerprintBackfillService> logger)
+    {
+        _db = db;
+        _logger = logger;
+    }
+
+    public async Task<int> BackfillAsync(int batchSize, CancellationToken cancellationToken)
+    {
+        var candidates = await LoadCandidatesAsync(batchSize, cancellationToken).ConfigureAwait(false);
+
+        if (candidates.Count == 0) return 0;
+
+        var restored = await RestoreAsync(candidates, cancellationToken).ConfigureAwait(false);
+
+        _logger.LogInformation("Restored the stored namespace fingerprint of {Restored} storage profile revision(s) to the value the previous build compares", restored);
+
+        return restored;
+    }
+
+    /// <summary>The oldest revisions whose stored fingerprint is not the one their configuration hashes to as written. The configuration table is small, so it is read whole and judged in memory by the writer's own rule.</summary>
+    private async Task<IReadOnlyList<Candidate>> LoadCandidatesAsync(int batchSize, CancellationToken cancellationToken)
+    {
+        var revisions = await _db.StorageProfileRevision.AsNoTracking()
+            .OrderBy(revision => revision.CreatedDate).ThenBy(revision => revision.Id)
+            .Select(revision => new { revision.Id, revision.ProviderTypeKey, revision.NonSecretConfigJson, revision.NamespaceFingerprint })
+            .ToListAsync(cancellationToken).ConfigureAwait(false);
+
+        return revisions
+            .Select(revision => new Candidate(revision.Id, revision.NamespaceFingerprint, Legacy(revision.ProviderTypeKey, revision.NonSecretConfigJson)))
+            .Where(candidate => candidate.Stored != candidate.Legacy)
+            .Take(batchSize)
+            .ToList();
+    }
+
+    private async Task<int> RestoreAsync(IReadOnlyList<Candidate> candidates, CancellationToken cancellationToken)
+    {
+        await using var transaction = await _db.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
+
+        await _db.Database.ExecuteSqlRawAsync(BoundLockWait, cancellationToken).ConfigureAwait(false);
+        await _db.Database.ExecuteSqlRawAsync(SuspendImmutability, cancellationToken).ConfigureAwait(false);
+
+        var restored = 0;
+
+        foreach (var candidate in candidates) restored += await RestoreOneAsync(candidate, cancellationToken).ConfigureAwait(false);
+
+        await _db.Database.ExecuteSqlRawAsync(ResumeImmutability, cancellationToken).ConfigureAwait(false);
+        await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
+
+        return restored;
+    }
+
+    /// <summary>Rewrites the fingerprint alone, and only while the row still holds the value this pass read. 0 when another worker restored it first.</summary>
+    private Task<int> RestoreOneAsync(Candidate candidate, CancellationToken cancellationToken) =>
+        _db.StorageProfileRevision
+            .Where(revision => revision.Id == candidate.Id && revision.NamespaceFingerprint == candidate.Stored)
+            .ExecuteUpdateAsync(set => set.SetProperty(revision => revision.NamespaceFingerprint, candidate.Legacy), cancellationToken);
+
+    private static string Legacy(string providerTypeKey, string nonSecretConfigJson)
+    {
+        using var configuration = JsonDocument.Parse(nonSecretConfigJson);
+        return StorageProfileRules.LegacyNamespaceFingerprint(providerTypeKey, configuration.RootElement);
+    }
+
+    private sealed record Candidate(Guid Id, string Stored, string Legacy);
+}

backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileRules.cs

@@ -46,9 +46,9 @@ public static string NamespaceFingerprint(string providerTypeKey, JsonElement na
 
     /// <summary>
     /// The identity of the PHYSICAL namespace a configuration reaches: the provider's own canonical projection of it,
-    /// hashed. The one derivation behind the fingerprint a revision stores and behind the retention reaper's later
-    /// comparison of two revisions' objects (<see cref="TryObjectIdentity"/>), so the two cannot disagree about what
-    /// "the same namespace" means.
+    /// hashed. The derivation behind the retention reaper's comparison of two revisions' objects
+    /// (<see cref="TryObjectIdentity"/>), made at sweep time from each revision's configuration. Not yet what a revision
+    /// stores: see <see cref="StoredNamespaceFingerprint"/>.
     /// </summary>
     public static string NamespaceFingerprint(IStorageProviderModule module, JsonElement nonSecretConfiguration)
     {
@@ -59,6 +59,48 @@ public static string NamespaceFingerprint(IStorageProviderModule module, JsonEle
         return NamespaceFingerprint(module.TypeKey, namespaceConfig);
     }
 
+    /// <summary>
+    /// The fingerprint every build before physical namespaces stored: the hash of the configuration exactly as written,
+    /// which is what their provider modules' default <c>GetNamespaceConfiguration</c> projected. Needs no module, so
+    /// it can be recomputed for any stored revision.
+    /// </summary>
+    public static string LegacyNamespaceFingerprint(string providerTypeKey, JsonElement canonicalConfig)
+    {
+        if (canonicalConfig.ValueKind != JsonValueKind.Object)
+            throw new InvalidOperationException($"Storage provider '{providerTypeKey}' namespace configuration projection must be a JSON object.");
+
+        return NamespaceFingerprint(providerTypeKey, canonicalConfig);
+    }
+
+    /// <summary>
+    /// The fingerprint a NEW revision stores, the one value admission, the agent-run-log bootstrap and the
+    /// deployment-default materializer write.
+    ///
+    /// <para>For this release it is <see cref="LegacyNamespaceFingerprint"/>, not the physical identity. The previous
+    /// build's retention reaper decides whether one object key is one object across teams by comparing these STORED
+    /// values, and a rolling deploy or a rollback runs it over revisions this build admits: a stored physical identity
+    /// would differ from the hash that build stored for the same text, and it would collect another team's live
+    /// bytes. This build never decides on the column (its reaper derives <see cref="NamespaceFingerprint(IStorageProviderModule, JsonElement)"/>
+    /// at sweep time), so the legacy value costs it nothing.</para>
+    ///
+    /// <para>The physical projection is still taken, so a configuration that reaches no namespace is refused exactly
+    /// as before. Switching the stored value to the physical identity is a later release, once no build that reads the
+    /// column can run against this database; that change also deletes the
+    /// <see cref="IStorageNamespaceFingerprintBackfillService"/> pass, which restores this value.</para>
+    ///
+    /// <para>Deleting the pass must also remove its schedule. The worker registrar only adds recurring entries, so a
+    /// deleted job class leaves <c>StorageNamespaceFingerprintBackfillRecurringJob</c> registered, and every ten-minute
+    /// tick then fails to load its type. Remove the entry at worker boot with
+    /// <c>ICodeSpaceBackgroundJobClient.RemoveRecurringJobIfExists("StorageNamespaceFingerprintBackfillRecurringJob")</c>,
+    /// or keep the job class as a no-op for one more release and remove both then.</para>
+    /// </summary>
+    public static string StoredNamespaceFingerprint(IStorageProviderModule module, JsonElement canonicalConfig)
+    {
+        _ = NamespaceFingerprint(module, canonicalConfig);
+
+        return LegacyNamespaceFingerprint(module.TypeKey, canonicalConfig);
+    }
+
     /// <summary>
     /// WHICH physical object a STORED placement's key reaches, derived now from its revision's configuration rather
     /// than read from the fingerprint the revision stored, or null when it cannot be derived: the provider is not

backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileService.cs

@@ -270,9 +270,9 @@ private async Task<PreparedRevision> PrepareRevisionAsync(Guid teamId, string pr
 
         var normalizedCredentialRef = await ValidateCredentialAsync(teamId, module.TypeKey, credentialRef, cancellationToken).ConfigureAwait(false);
 
-        // The fingerprint names the PHYSICAL namespace the provider says this configuration reaches, not its text, so two
-        // spellings of one destination store one fingerprint. The configuration itself is persisted as written.
-        var fingerprint = ExecuteRule(() => StorageProfileRules.NamespaceFingerprint(module, canonicalElement));
+        // For this release the stored fingerprint is still the hash of the configuration as written, the value the previous
+        // build's reaper compares; see StorageProfileRules.StoredNamespaceFingerprint. The configuration is persisted as written.
+        var fingerprint = ExecuteRule(() => StorageProfileRules.StoredNamespaceFingerprint(module, canonicalElement));
         return new PreparedRevision(module.TypeKey, canonicalConfig, normalizedCredentialRef, fingerprint);
     }
 

backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/IStorageProviderModule.cs

@@ -22,11 +22,11 @@ public interface IStorageProviderModule
     JsonElement SecretSchema { get; }
 
     /// <summary>
-    /// Project the non-secret configuration onto the PHYSICAL namespace it reaches, spelled one way. The control plane
-    /// canonicalizes and hashes this server-owned projection into the revision's fingerprint (clients can never submit
-    /// one), and the retention reaper re-derives it from stored revisions to decide whether one object key is one
-    /// physical object across teams — unless the provider names that object itself
-    /// (<see cref="IStorageProviderPhysicalObjects"/>), because its keys can leave the namespace its root names.
+    /// Project the non-secret configuration onto the PHYSICAL namespace it reaches, spelled one way. The retention
+    /// reaper re-derives it from stored revisions to decide whether one object key is one physical object across teams
+    /// — unless the provider names that object itself (<see cref="IStorageProviderPhysicalObjects"/>), because its keys
+    /// can leave the namespace its root names. Admission takes it too, to refuse a configuration that reaches nothing,
+    /// but for one release still stores the hash of the configuration as written (<c>StorageProfileRules.StoredNamespaceFingerprint</c>).
     ///
     /// <para>So two configurations that reach the same namespace MUST project identically — a root with and without a
     /// trailing separator, a host and its alias — and where a provider cannot tell, it projects them together: a

backend/src/CodeSpace.Messages/Commands/Storage/BackfillStorageNamespaceFingerprintsCommand.cs

@@ -0,0 +1,19 @@
+using CodeSpace.Messages.Mediation;
+
+namespace CodeSpace.Messages.Commands.Storage;
+
+/// <summary>
+/// Put back, on every storage profile revision, the namespace fingerprint the previous build stores — the hash of the
+/// configuration as written — where a build that stored the physical namespace identity wrote another value (see
+/// <c>IStorageNamespaceFingerprintBackfillService</c>). Idempotent and bounded: a restored revision is no longer a
+/// candidate, so the backlog only shrinks. NOT tenant-scoped: a system-wide repair that runs without an actor context
+/// (mirrors <c>BackfillPackCloneUrlsCommand</c>).
+///
+/// <para>NOT transactional (<see cref="INonTransactionalCommand"/>): the service opens the one transaction its trigger
+/// switch has to live inside, and a command transaction around it would refuse a second.</para>
+/// </summary>
+public sealed record BackfillStorageNamespaceFingerprintsCommand : ICommand<int>, INonTransactionalCommand
+{
+    /// <summary>Revisions restored per tick — bounds how long one pass holds the revision table's write lock.</summary>
+    public int BatchSize { get; init; } = 200;
+}