From a0eaee143ee79a1e7501fe998b19951baaf3b5a8 Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Tue, 13 Oct 2026 11:34:31 +0800 Subject: [PATCH] Keep the legacy storage fingerprint for one release Since namespaces were derived physically, admission, the agent-run-log bootstrap and the deployment-default materializer stored the hash of the provider's physical projection rather than the hash of the configuration as written. This build never decides on that column, but the previous build's retention reaper does: it treats one object key as one object across teams only when the revisions' stored fingerprints match. During a rolling deploy, or after a rollback, that reaper saw a revision this build admitted for 'R/' (or an OSS -internal or https endpoint, or an explicit region) as another namespace than one it had admitted for the same text, and collected another team's live bytes. New revisions store the legacy value again for one release, through StorageProfileRules.StoredNamespaceFingerprint. The physical projection is still taken, so a configuration that reaches no namespace is refused as before, and the reaper keeps deriving the physical object at sweep time. A recurring backfill restores the legacy value on revisions a physical-identity build already wrote, and on any an older pod writes during the rollout. Revisions are immutable under a trigger, so the pass switches it off inside its own transaction and back on before it commits; no migration is needed. Storing the physical identity is a follow-up release, once no build that compares the column can run against the database. That change also deletes the backfill, and must remove its schedule with it: the worker registrar only adds recurring entries, so a deleted job class leaves StorageNamespaceFingerprintBackfillRecurringJob registered and every ten-minute tick fails to load its type. Remove the entry at worker boot with ICodeSpaceBackgroundJobClient.RemoveRecurringJobIfExists, or keep the job class as a no-op for one more release. --- ...rageNamespaceFingerprintsCommandHandler.cs | 21 ++ ...amespaceFingerprintBackfillRecurringJob.cs | 17 ++ .../AgentRunLogStorageReadiness.cs | 4 +- ...rageNamespaceFingerprintBackfillService.cs | 17 ++ ...rageNamespaceFingerprintBackfillService.cs | 96 +++++++ .../Artifacts/Profiles/StorageProfileRules.cs | 48 +++- .../Profiles/StorageProfileService.cs | 6 +- .../Providers/IStorageProviderModule.cs | 10 +- ...fillStorageNamespaceFingerprintsCommand.cs | 19 ++ ...ageNamespaceFingerprintBackfillE2ETests.cs | 244 ++++++++++++++++++ .../StorageNamespaceSharingE2ETests.cs | 27 ++ .../Workflows/AgentRunLogRuntimeTests.cs | 25 +- ...factRetentionReaperFlowTests.Namespaces.cs | 31 ++- ...tRetentionReaperFlowTests.PreviousBuild.cs | 127 +++++++++ ...geNamespaceFingerprintBackfillFlowTests.cs | 159 ++++++++++++ .../RequestAuthorizationInventoryTests.cs | 1 + .../ScopedTransactionInventoryTests.cs | 1 + ...mespaceFingerprintBackfillDispatchTests.cs | 95 +++++++ .../StorageNamespaceLegacyFingerprintTests.cs | 94 +++++++ 19 files changed, 1012 insertions(+), 30 deletions(-) create mode 100644 backend/src/CodeSpace.Core/Handlers/CommandHandlers/Storage/BackfillStorageNamespaceFingerprintsCommandHandler.cs create mode 100644 backend/src/CodeSpace.Core/Jobs/RecurringJobs/StorageNamespaceFingerprintBackfillRecurringJob.cs create mode 100644 backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/IStorageNamespaceFingerprintBackfillService.cs create mode 100644 backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageNamespaceFingerprintBackfillService.cs create mode 100644 backend/src/CodeSpace.Messages/Commands/Storage/BackfillStorageNamespaceFingerprintsCommand.cs create mode 100644 backend/tests/CodeSpace.E2ETests/Storage/StorageNamespaceFingerprintBackfillE2ETests.cs create mode 100644 backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.PreviousBuild.cs create mode 100644 backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/Profiles/StorageNamespaceFingerprintBackfillFlowTests.cs create mode 100644 backend/tests/CodeSpace.UnitTests/Jobs/StorageNamespaceFingerprintBackfillDispatchTests.cs create mode 100644 backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Profiles/StorageNamespaceLegacyFingerprintTests.cs diff --git a/backend/src/CodeSpace.Core/Handlers/CommandHandlers/Storage/BackfillStorageNamespaceFingerprintsCommandHandler.cs b/backend/src/CodeSpace.Core/Handlers/CommandHandlers/Storage/BackfillStorageNamespaceFingerprintsCommandHandler.cs new file mode 100644 index 000000000..e1e844de2 --- /dev/null +++ b/backend/src/CodeSpace.Core/Handlers/CommandHandlers/Storage/BackfillStorageNamespaceFingerprintsCommandHandler.cs @@ -0,0 +1,21 @@ +using CodeSpace.Core.Services.Workflows.Artifacts.Profiles; +using CodeSpace.Messages.Commands.Storage; +using MediatR; + +namespace CodeSpace.Core.Handlers.CommandHandlers.Storage; + +/// Thin dispatcher (Rule 16) — the production caller of . +public sealed class BackfillStorageNamespaceFingerprintsCommandHandler : IRequestHandler +{ + private readonly IStorageNamespaceFingerprintBackfillService _backfill; + + public BackfillStorageNamespaceFingerprintsCommandHandler(IStorageNamespaceFingerprintBackfillService backfill) + { + _backfill = backfill; + } + + public async Task Handle(BackfillStorageNamespaceFingerprintsCommand request, CancellationToken cancellationToken) + { + return await _backfill.BackfillAsync(request.BatchSize, cancellationToken).ConfigureAwait(false); + } +} diff --git a/backend/src/CodeSpace.Core/Jobs/RecurringJobs/StorageNamespaceFingerprintBackfillRecurringJob.cs b/backend/src/CodeSpace.Core/Jobs/RecurringJobs/StorageNamespaceFingerprintBackfillRecurringJob.cs new file mode 100644 index 000000000..a8afbbb46 --- /dev/null +++ b/backend/src/CodeSpace.Core/Jobs/RecurringJobs/StorageNamespaceFingerprintBackfillRecurringJob.cs @@ -0,0 +1,17 @@ +using CodeSpace.Messages.Commands.Storage; +using MediatR; + +namespace CodeSpace.Core.Jobs.RecurringJobs; + +/// Every ten minutes: put the namespace fingerprint the previous build compares back on any storage profile revision a physical-identity build stamped otherwise — the revisions it admitted before this release, and any an older pod admits during the rollout (thin Rule-14 dispatcher). A restored revision is no longer a candidate, so an idle tick is one small read. Deleting this class must also remove its recurring entry: see StorageProfileRules.StoredNamespaceFingerprint. +public sealed class StorageNamespaceFingerprintBackfillRecurringJob : IRecurringJob +{ + private readonly IMediator _mediator; + + public StorageNamespaceFingerprintBackfillRecurringJob(IMediator mediator) { _mediator = mediator; } + + public string JobId => nameof(StorageNamespaceFingerprintBackfillRecurringJob); + public string CronExpression => "*/10 * * * *"; + + public async Task Execute() => await _mediator.Send(new BackfillStorageNamespaceFingerprintsCommand()).ConfigureAwait(false); +} diff --git a/backend/src/CodeSpace.Core/Services/Agents/AgentRunLogging/AgentRunLogStorageReadiness.cs b/backend/src/CodeSpace.Core/Services/Agents/AgentRunLogging/AgentRunLogStorageReadiness.cs index bd960517b..6cd519bf7 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/AgentRunLogging/AgentRunLogStorageReadiness.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/AgentRunLogging/AgentRunLogStorageReadiness.cs @@ -74,11 +74,11 @@ public async Task EnsureDefaultRouteAsync(Guid teamId, CancellationToken cancell } } - /// The same derivation admission uses, so this bootstrap's revision names its namespace exactly as a Settings-created one would. + /// The value admission stores (), so this bootstrap's revision carries exactly what a Settings-created one for the same root would. private string NamespaceFingerprint(string canonicalConfig) { using var document = JsonDocument.Parse(canonicalConfig); - return StorageProfileRules.NamespaceFingerprint(_modules.Require(LocalRwxArtifactStorageDriverFactory.TypeKey), document.RootElement); + return StorageProfileRules.StoredNamespaceFingerprint(_modules.Require(LocalRwxArtifactStorageDriverFactory.TypeKey), document.RootElement); } private static StorageProfile BuildProfile(Guid teamId, string canonicalConfig, string namespaceFingerprint, DateTimeOffset now) diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/IStorageNamespaceFingerprintBackfillService.cs b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/IStorageNamespaceFingerprintBackfillService.cs new file mode 100644 index 000000000..f061cf676 --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/IStorageNamespaceFingerprintBackfillService.cs @@ -0,0 +1,17 @@ +namespace CodeSpace.Core.Services.Workflows.Artifacts.Profiles; + +/// +/// Restores on revisions that a build storing the +/// PHYSICAL namespace identity already wrote — the builds between namespace canonicalization and this release, and any +/// such pod still admitting during this rollout. The previous build's retention reaper compares stored fingerprints, +/// so those revisions must carry the value it would have written for the same configuration before a rollback can +/// bring it back. +/// +/// This build decides nothing on the column, so the rewrite changes nothing it does. It runs only where Hangfire +/// processes jobs. +/// +public interface IStorageNamespaceFingerprintBackfillService +{ + /// Restore up to revisions. Idempotent and safe on several workers at once: a restored revision is no longer a candidate, and each write is conditional on the row still holding the value the pass read. Returns how many revisions this pass restored. + Task BackfillAsync(int batchSize, CancellationToken cancellationToken); +} diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageNamespaceFingerprintBackfillService.cs b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageNamespaceFingerprintBackfillService.cs new file mode 100644 index 000000000..bcbc04472 --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageNamespaceFingerprintBackfillService.cs @@ -0,0 +1,96 @@ +using System.Text.Json; +using CodeSpace.Core.DependencyInjection; +using CodeSpace.Core.Persistence.Db; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace CodeSpace.Core.Services.Workflows.Artifacts.Profiles; + +/// +/// The bounded pass behind . A candidate is a revision whose +/// stored fingerprint is not of its own stored +/// configuration. Every build that predates physical namespaces wrote exactly that value, so the only revisions that +/// differ are the ones a physical-identity build wrote for a configuration not already in canonical form; after the +/// rewrite they leave the set, so the pass is self-terminating. +/// +/// Revisions are immutable under a trigger, so the pass suspends it inside its own transaction and restores it +/// before commit: no other session ever sees the table without it, and a pass that fails rolls the switch back with +/// its writes. The switch locks out concurrent admissions for the length of one short pass, and the lock wait is +/// bounded so a pass queued behind a long transaction gives up rather than stall them. Each row is rewritten by an +/// UPDATE conditional on its id AND the value the pass read, so a second worker restoring the same row matches +/// nothing. +/// +public sealed class StorageNamespaceFingerprintBackfillService : IStorageNamespaceFingerprintBackfillService, IScopedDependency +{ + private const string BoundLockWait = "SET LOCAL lock_timeout = '5s'"; + private const string SuspendImmutability = "ALTER TABLE storage_profile_revision DISABLE TRIGGER storage_profile_revision_enforce_immutability"; + private const string ResumeImmutability = "ALTER TABLE storage_profile_revision ENABLE TRIGGER storage_profile_revision_enforce_immutability"; + + private readonly CodeSpaceDbContext _db; + private readonly ILogger _logger; + + public StorageNamespaceFingerprintBackfillService(CodeSpaceDbContext db, ILogger logger) + { + _db = db; + _logger = logger; + } + + public async Task BackfillAsync(int batchSize, CancellationToken cancellationToken) + { + var candidates = await LoadCandidatesAsync(batchSize, cancellationToken).ConfigureAwait(false); + + if (candidates.Count == 0) return 0; + + var restored = await RestoreAsync(candidates, cancellationToken).ConfigureAwait(false); + + _logger.LogInformation("Restored the stored namespace fingerprint of {Restored} storage profile revision(s) to the value the previous build compares", restored); + + return restored; + } + + /// The oldest revisions whose stored fingerprint is not the one their configuration hashes to as written. The configuration table is small, so it is read whole and judged in memory by the writer's own rule. + private async Task> LoadCandidatesAsync(int batchSize, CancellationToken cancellationToken) + { + var revisions = await _db.StorageProfileRevision.AsNoTracking() + .OrderBy(revision => revision.CreatedDate).ThenBy(revision => revision.Id) + .Select(revision => new { revision.Id, revision.ProviderTypeKey, revision.NonSecretConfigJson, revision.NamespaceFingerprint }) + .ToListAsync(cancellationToken).ConfigureAwait(false); + + return revisions + .Select(revision => new Candidate(revision.Id, revision.NamespaceFingerprint, Legacy(revision.ProviderTypeKey, revision.NonSecretConfigJson))) + .Where(candidate => candidate.Stored != candidate.Legacy) + .Take(batchSize) + .ToList(); + } + + private async Task RestoreAsync(IReadOnlyList candidates, CancellationToken cancellationToken) + { + await using var transaction = await _db.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + + await _db.Database.ExecuteSqlRawAsync(BoundLockWait, cancellationToken).ConfigureAwait(false); + await _db.Database.ExecuteSqlRawAsync(SuspendImmutability, cancellationToken).ConfigureAwait(false); + + var restored = 0; + + foreach (var candidate in candidates) restored += await RestoreOneAsync(candidate, cancellationToken).ConfigureAwait(false); + + await _db.Database.ExecuteSqlRawAsync(ResumeImmutability, cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + + return restored; + } + + /// Rewrites the fingerprint alone, and only while the row still holds the value this pass read. 0 when another worker restored it first. + private Task RestoreOneAsync(Candidate candidate, CancellationToken cancellationToken) => + _db.StorageProfileRevision + .Where(revision => revision.Id == candidate.Id && revision.NamespaceFingerprint == candidate.Stored) + .ExecuteUpdateAsync(set => set.SetProperty(revision => revision.NamespaceFingerprint, candidate.Legacy), cancellationToken); + + private static string Legacy(string providerTypeKey, string nonSecretConfigJson) + { + using var configuration = JsonDocument.Parse(nonSecretConfigJson); + return StorageProfileRules.LegacyNamespaceFingerprint(providerTypeKey, configuration.RootElement); + } + + private sealed record Candidate(Guid Id, string Stored, string Legacy); +} diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileRules.cs b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileRules.cs index 58cf6ef95..347947fd4 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileRules.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileRules.cs @@ -46,9 +46,9 @@ public static string NamespaceFingerprint(string providerTypeKey, JsonElement na /// /// The identity of the PHYSICAL namespace a configuration reaches: the provider's own canonical projection of it, - /// hashed. The one derivation behind the fingerprint a revision stores and behind the retention reaper's later - /// comparison of two revisions' objects (), so the two cannot disagree about what - /// "the same namespace" means. + /// hashed. The derivation behind the retention reaper's comparison of two revisions' objects + /// (), made at sweep time from each revision's configuration. Not yet what a revision + /// stores: see . /// public static string NamespaceFingerprint(IStorageProviderModule module, JsonElement nonSecretConfiguration) { @@ -59,6 +59,48 @@ public static string NamespaceFingerprint(IStorageProviderModule module, JsonEle return NamespaceFingerprint(module.TypeKey, namespaceConfig); } + /// + /// The fingerprint every build before physical namespaces stored: the hash of the configuration exactly as written, + /// which is what their provider modules' default GetNamespaceConfiguration projected. Needs no module, so + /// it can be recomputed for any stored revision. + /// + public static string LegacyNamespaceFingerprint(string providerTypeKey, JsonElement canonicalConfig) + { + if (canonicalConfig.ValueKind != JsonValueKind.Object) + throw new InvalidOperationException($"Storage provider '{providerTypeKey}' namespace configuration projection must be a JSON object."); + + return NamespaceFingerprint(providerTypeKey, canonicalConfig); + } + + /// + /// The fingerprint a NEW revision stores, the one value admission, the agent-run-log bootstrap and the + /// deployment-default materializer write. + /// + /// For this release it is , not the physical identity. The previous + /// build's retention reaper decides whether one object key is one object across teams by comparing these STORED + /// values, and a rolling deploy or a rollback runs it over revisions this build admits: a stored physical identity + /// would differ from the hash that build stored for the same text, and it would collect another team's live + /// bytes. This build never decides on the column (its reaper derives + /// at sweep time), so the legacy value costs it nothing. + /// + /// The physical projection is still taken, so a configuration that reaches no namespace is refused exactly + /// as before. Switching the stored value to the physical identity is a later release, once no build that reads the + /// column can run against this database; that change also deletes the + /// pass, which restores this value. + /// + /// Deleting the pass must also remove its schedule. The worker registrar only adds recurring entries, so a + /// deleted job class leaves StorageNamespaceFingerprintBackfillRecurringJob registered, and every ten-minute + /// tick then fails to load its type. Remove the entry at worker boot with + /// ICodeSpaceBackgroundJobClient.RemoveRecurringJobIfExists("StorageNamespaceFingerprintBackfillRecurringJob"), + /// or keep the job class as a no-op for one more release and remove both then. + /// + public static string StoredNamespaceFingerprint(IStorageProviderModule module, JsonElement canonicalConfig) + { + _ = NamespaceFingerprint(module, canonicalConfig); + + return LegacyNamespaceFingerprint(module.TypeKey, canonicalConfig); + } + /// /// WHICH physical object a STORED placement's key reaches, derived now from its revision's configuration rather /// than read from the fingerprint the revision stored, or null when it cannot be derived: the provider is not diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileService.cs b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileService.cs index 5ee04d154..669545ce4 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileService.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileService.cs @@ -270,9 +270,9 @@ private async Task PrepareRevisionAsync(Guid teamId, string pr var normalizedCredentialRef = await ValidateCredentialAsync(teamId, module.TypeKey, credentialRef, cancellationToken).ConfigureAwait(false); - // The fingerprint names the PHYSICAL namespace the provider says this configuration reaches, not its text, so two - // spellings of one destination store one fingerprint. The configuration itself is persisted as written. - var fingerprint = ExecuteRule(() => StorageProfileRules.NamespaceFingerprint(module, canonicalElement)); + // For this release the stored fingerprint is still the hash of the configuration as written, the value the previous + // build's reaper compares; see StorageProfileRules.StoredNamespaceFingerprint. The configuration is persisted as written. + var fingerprint = ExecuteRule(() => StorageProfileRules.StoredNamespaceFingerprint(module, canonicalElement)); return new PreparedRevision(module.TypeKey, canonicalConfig, normalizedCredentialRef, fingerprint); } diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/IStorageProviderModule.cs b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/IStorageProviderModule.cs index e5d1340a7..5fa78fd7d 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/IStorageProviderModule.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/IStorageProviderModule.cs @@ -22,11 +22,11 @@ public interface IStorageProviderModule JsonElement SecretSchema { get; } /// - /// Project the non-secret configuration onto the PHYSICAL namespace it reaches, spelled one way. The control plane - /// canonicalizes and hashes this server-owned projection into the revision's fingerprint (clients can never submit - /// one), and the retention reaper re-derives it from stored revisions to decide whether one object key is one - /// physical object across teams — unless the provider names that object itself - /// (), because its keys can leave the namespace its root names. + /// Project the non-secret configuration onto the PHYSICAL namespace it reaches, spelled one way. The retention + /// reaper re-derives it from stored revisions to decide whether one object key is one physical object across teams + /// — unless the provider names that object itself (), because its keys + /// can leave the namespace its root names. Admission takes it too, to refuse a configuration that reaches nothing, + /// but for one release still stores the hash of the configuration as written (StorageProfileRules.StoredNamespaceFingerprint). /// /// So two configurations that reach the same namespace MUST project identically — a root with and without a /// trailing separator, a host and its alias — and where a provider cannot tell, it projects them together: a diff --git a/backend/src/CodeSpace.Messages/Commands/Storage/BackfillStorageNamespaceFingerprintsCommand.cs b/backend/src/CodeSpace.Messages/Commands/Storage/BackfillStorageNamespaceFingerprintsCommand.cs new file mode 100644 index 000000000..c95721e05 --- /dev/null +++ b/backend/src/CodeSpace.Messages/Commands/Storage/BackfillStorageNamespaceFingerprintsCommand.cs @@ -0,0 +1,19 @@ +using CodeSpace.Messages.Mediation; + +namespace CodeSpace.Messages.Commands.Storage; + +/// +/// Put back, on every storage profile revision, the namespace fingerprint the previous build stores — the hash of the +/// configuration as written — where a build that stored the physical namespace identity wrote another value (see +/// IStorageNamespaceFingerprintBackfillService). Idempotent and bounded: a restored revision is no longer a +/// candidate, so the backlog only shrinks. NOT tenant-scoped: a system-wide repair that runs without an actor context +/// (mirrors BackfillPackCloneUrlsCommand). +/// +/// NOT transactional (): the service opens the one transaction its trigger +/// switch has to live inside, and a command transaction around it would refuse a second. +/// +public sealed record BackfillStorageNamespaceFingerprintsCommand : ICommand, INonTransactionalCommand +{ + /// Revisions restored per tick — bounds how long one pass holds the revision table's write lock. + public int BatchSize { get; init; } = 200; +} diff --git a/backend/tests/CodeSpace.E2ETests/Storage/StorageNamespaceFingerprintBackfillE2ETests.cs b/backend/tests/CodeSpace.E2ETests/Storage/StorageNamespaceFingerprintBackfillE2ETests.cs new file mode 100644 index 000000000..1d38a6f82 --- /dev/null +++ b/backend/tests/CodeSpace.E2ETests/Storage/StorageNamespaceFingerprintBackfillE2ETests.cs @@ -0,0 +1,244 @@ +using System.Collections.Concurrent; +using System.Text.Json; +using CodeSpace.Api.Extensions.Hangfire; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Workflows.Artifacts.Profiles; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers.Local; +using CodeSpace.E2ETests.Infrastructure; +using CodeSpace.Messages.Enums; +using Hangfire; +using Hangfire.PostgreSql; +using Hangfire.PostgreSql.Factories; +using Hangfire.States; +using Hangfire.Storage; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Npgsql; +using Serilog.Core; +using Serilog.Events; +using Shouldly; + +namespace CodeSpace.E2ETests.Storage; + +/// +/// The storage namespace fingerprint backfill, fired the way a worker pod fires it, over a revision a build that stored +/// the PHYSICAL namespace identity wrote. +/// +/// Tier: 🟢 High-fidelity (Rule 12) — the real worker host (HangfireHosting=Worker) registers the +/// schedule, and the test triggers THAT registered entry by its id. The tick therefore runs the serialized +/// IJobSafeRunner.Run(jobId, jobType) through a real control-pool server, the job's own DI scope, the mediator +/// pipeline and the production against real Postgres — +/// including the switch of the revision table's immutability trigger. Nothing on the job path is substituted; the +/// host-level substitutions are named on . +/// +/// Must hold: the physical revision ends holding , +/// the value the previous build's reaper compares; a revision that already holds it is left alone; and the table is +/// immutable again after the pass. Each run owns a GUID-named database that the fixture drops on every path, and the +/// local root is never created on disk. +/// +/// Not covered here: what the previous build's reaper decides over the restored values +/// (ArtifactRetentionReaperFlowTests.PreviousBuild), and the per-pass edge cases — idempotence, concurrent +/// workers (StorageNamespaceFingerprintBackfillFlowTests). +/// +[Trait("Category", "E2E")] +[Trait("Surface", "Worker")] +public sealed class StorageNamespaceFingerprintBackfillE2ETests +{ + /// The recurring entry's id as Hangfire holds it, spelled out rather than taken from nameof: it is the id a release that deletes the job must remove from the schedule. + private const string JobId = "StorageNamespaceFingerprintBackfillRecurringJob"; + + private const string ImmutabilityTrigger = "storage_profile_revision_enforce_immutability"; + + /// One tick restores a fresh database's revisions; the slack absorbs a tick the cron scheduler fired at the same moment, and a job that never restores the row fails instead of hanging. + private const int MaxTicks = 3; + + /// Bound on fetch + execution of one tick, sized for a cold CI runner like the sibling smoke test. + private static readonly TimeSpan TickBudget = TimeSpan.FromMinutes(3); + + private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(250); + + [Fact] + public async Task A_tick_of_the_registered_job_restores_the_legacy_fingerprint_and_leaves_the_revisions_immutable() + { + var sink = new CapturedLogLines(); + + await using var factory = new RecurringJobWorkerHostFactory(sink); + await factory.InitializeAsync(); + + _ = factory.Services; // builds and starts the worker host, which registers every recurring schedule + + var typed = RootConfig($"/srv/codespace-backfill-e2e-{Guid.NewGuid():N}/"); + PhysicalFingerprint(typed).ShouldNotBe(LegacyFingerprint(typed), "fixture check: a trailing separator is where the two identities part, so the seeded row is a real candidate"); + + var physical = await SeedRevisionAsync(factory, typed, PhysicalFingerprint(typed)); + var previousBuild = await SeedRevisionAsync(factory, typed, LegacyFingerprint(typed)); + + await TickUntilRestoredAsync(factory, sink, physical, LegacyFingerprint(typed)); + + (await StoredAsync(factory, physical)).ShouldBe(new Stored(typed, LegacyFingerprint(typed)), "only the fingerprint moves; the configuration stays as written"); + (await StoredAsync(factory, previousBuild)).ShouldBe(new Stored(typed, LegacyFingerprint(typed)), "a revision that already holds the legacy value is not rewritten"); + + (await TriggerStateAsync(factory)).ShouldBe("O", $"the pass must switch {ImmutabilityTrigger} back on before it commits; check `SELECT tgenabled FROM pg_trigger WHERE tgname = '{ImmutabilityTrigger}'` ('D' = left disabled)"); + await ShouldRefuseAnUpdateAsync(factory, physical); + } + + // ─── Ticks ─────────────────────────────────────────────────────────────── + + /// Triggers the registered schedule until the owned row holds . Every tick must succeed; the loop is bounded and its failure names the row and the tick. + private static async Task TickUntilRestoredAsync(RecurringJobWorkerHostFactory factory, CapturedLogLines sink, Guid revisionId, string expected) + { + var storage = OpenStorageOnTheFixtureDatabase(factory); + + for (var tick = 0; tick < MaxTicks; tick++) + { + var state = await AwaitTerminalStateAsync(storage, Trigger(storage)); + + (state?.Name).ShouldBe(SucceededState.StateName, $"tick {tick + 1} of {JobId} ended '{state?.Name ?? ""}'{Cause(state)}. Host warnings: {string.Join(" | ", sink.Lines())}"); + + if ((await StoredAsync(factory, revisionId)).Fingerprint == expected) return; + } + + throw new ShouldAssertException($"revision {revisionId} still lacks the legacy fingerprint after {MaxTicks} successful ticks of {JobId}: the tick ran but restored nothing. Check that the job still sends BackfillStorageNamespaceFingerprintsCommand, then run StorageNamespaceFingerprintBackfillFlowTests, which drive the same service without Hangfire"); + } + + /// A storage handle the test owns over the fixture's database, with the production options — the same independent observation RecurringJobWorkerSmokeE2ETests makes, rather than the process-wide JobStorage.Current. + private static JobStorage OpenStorageOnTheFixtureDatabase(RecurringJobWorkerHostFactory factory) + { + var options = HangfireRegistrarBase.BuildStorageOptions(); + + return new PostgreSqlStorage(new NpgsqlConnectionFactory(factory.ConnectionString, options), options); + } + + private static string Trigger(JobStorage storage) => + new RecurringJobManager(storage).TriggerJob(JobId) + ?? throw new ShouldAssertException($"the worker registered no recurring entry '{JobId}'. A renamed job class mints a new id and strands this one; a deleted one must also remove its schedule (ICodeSpaceBackgroundJobClient.RemoveRecurringJobIfExists)"); + + /// Polls the triggered background job until it is terminal or the budget runs out. The state starts unobserved, so an empty read never ends the wait. + private static async Task AwaitTerminalStateAsync(JobStorage storage, string backgroundJobId) + { + var deadline = DateTime.UtcNow + TickBudget; + StateData? state = null; + + while (!IsTerminal(state) && DateTime.UtcNow < deadline) + { + await Task.Delay(PollInterval); + + using var connection = storage.GetConnection(); + state = connection.GetStateData(backgroundJobId); + } + + return state; + } + + private static bool IsTerminal(StateData? state) => + state != null && (state.Name == SucceededState.StateName || state.Name == FailedState.StateName || state.Name == DeletedState.StateName); + + private static string Cause(StateData? state) + { + if (state?.Data == null) return string.Empty; + + state.Data.TryGetValue("ExceptionType", out var type); + state.Data.TryGetValue("ExceptionMessage", out var message); + + return message == null ? $" ({state.Reason})" : $": {type} — {message}"; + } + + // ─── Rows ──────────────────────────────────────────────────────────────── + + private sealed record Stored(string Config, string Fingerprint); + + /// A revision exactly as admission writes one, stamped with : inserting is what the immutability trigger allows. + private static async Task SeedRevisionAsync(RecurringJobWorkerHostFactory factory, string config, string fingerprint) + { + var actorId = Guid.NewGuid(); + var teamId = Guid.NewGuid(); + var profileId = Guid.NewGuid(); + var revisionId = Guid.NewGuid(); + var now = DateTimeOffset.UtcNow; + + using var scope = factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + db.User.Add(new User { Id = actorId, Email = $"fingerprint-{actorId:N}@test.local", Name = $"fingerprint-{actorId:N}" }); + db.Team.Add(new Team { Id = teamId, Slug = $"fingerprint-{teamId:N}", Name = "Fingerprint Backfill Team", Kind = TeamKind.Workspace }); + db.TeamMembership.Add(new TeamMembership { Id = Guid.NewGuid(), TeamId = teamId, UserId = actorId, Role = TeamRole.Owner }); + var profile = new StorageProfile + { + Id = profileId, TeamId = teamId, StableName = $"fingerprint-{profileId:N}", CurrentRevision = 1, + State = StorageProfileState.Active, CreatedDate = now, CreatedBy = actorId, LastModifiedDate = now, LastModifiedBy = actorId, + }; + profile.Revisions.Add(new StorageProfileRevision + { + Id = revisionId, TeamId = teamId, StorageProfileId = profileId, Revision = 1, ProviderTypeKey = LocalRwxArtifactStorageDriverFactory.TypeKey, + NonSecretConfigJson = config, NamespaceFingerprint = fingerprint, CreatedDate = now, CreatedBy = actorId, + }); + db.StorageProfile.Add(profile); + await db.SaveChangesAsync(); + + return revisionId; + } + + private static async Task StoredAsync(RecurringJobWorkerHostFactory factory, Guid revisionId) + { + using var scope = factory.Services.CreateScope(); + + var row = await scope.ServiceProvider.GetRequiredService().StorageProfileRevision.AsNoTracking() + .Where(revision => revision.Id == revisionId).Select(revision => new { revision.NonSecretConfigJson, revision.NamespaceFingerprint }).SingleAsync(); + + using var config = JsonDocument.Parse(row.NonSecretConfigJson); + return new Stored(StorageProfileRules.CanonicalJson(config.RootElement), row.NamespaceFingerprint); + } + + private static async Task TriggerStateAsync(RecurringJobWorkerHostFactory factory) + { + await using var connection = new NpgsqlConnection(factory.ConnectionString); + await connection.OpenAsync(); + await using var command = new NpgsqlCommand("SELECT tgenabled::text FROM pg_trigger WHERE tgname = @name", connection); + command.Parameters.AddWithValue("name", ImmutabilityTrigger); + + return (string)(await command.ExecuteScalarAsync()).ShouldNotBeNull($"trigger {ImmutabilityTrigger} must exist in database {factory.DatabaseName}"); + } + + private static async Task ShouldRefuseAnUpdateAsync(RecurringJobWorkerHostFactory factory, Guid revisionId) + { + await using var connection = new NpgsqlConnection(factory.ConnectionString); + await connection.OpenAsync(); + await using var command = new NpgsqlCommand("UPDATE storage_profile_revision SET namespace_fingerprint = 'tampered' WHERE id = @id", connection); + command.Parameters.AddWithValue("id", revisionId); + + var refused = await Should.ThrowAsync(() => command.ExecuteNonQueryAsync(), $"revision {revisionId} accepted an UPDATE after the backfill: the pass left {ImmutabilityTrigger} off"); + refused.MessageText.ShouldContain("storage_profile_revision is immutable", Case.Sensitive); + } + + // ─── Fingerprints ──────────────────────────────────────────────────────── + + /// The canonical text admission persists for a local root. + private static string RootConfig(string rootPath) + { + using var document = JsonDocument.Parse(JsonSerializer.Serialize(new { rootPath })); + return StorageProfileRules.CanonicalJson(document.RootElement); + } + + /// What a build that stored the physical identity wrote for this configuration. + private static string PhysicalFingerprint(string config) + { + using var document = JsonDocument.Parse(config); + return StorageProfileRules.NamespaceFingerprint(new LocalRwxStorageProviderModule(), document.RootElement); + } + + private static string LegacyFingerprint(string config) + { + using var document = JsonDocument.Parse(config); + return StorageProfileRules.LegacyNamespaceFingerprint(LocalRwxArtifactStorageDriverFactory.TypeKey, document.RootElement); + } + + /// Keeps what the host's logger lets through (Warning and above), so a failed tick's message can quote it. + private sealed class CapturedLogLines : ILogEventSink + { + private readonly ConcurrentQueue _events = new(); + + public void Emit(LogEvent logEvent) => _events.Enqueue(logEvent); + + public IReadOnlyList Lines() => _events.Select(logEvent => logEvent.RenderMessage()).ToList(); + } +} diff --git a/backend/tests/CodeSpace.E2ETests/Storage/StorageNamespaceSharingE2ETests.cs b/backend/tests/CodeSpace.E2ETests/Storage/StorageNamespaceSharingE2ETests.cs index 77391648d..fd13b7433 100644 --- a/backend/tests/CodeSpace.E2ETests/Storage/StorageNamespaceSharingE2ETests.cs +++ b/backend/tests/CodeSpace.E2ETests/Storage/StorageNamespaceSharingE2ETests.cs @@ -123,6 +123,33 @@ public async Task A_victim_destination_admitted_before_namespaces_were_canonical (await download.Content.ReadAsByteArrayAsync()).ShouldBe(payload); } + [Fact] + public async Task A_destination_created_now_for_a_root_typed_as_an_earlier_one_stores_the_fingerprint_the_previous_build_compares() + { + // A rolling deploy or a rollback runs the previous build's reaper over destinations this build creates, and that + // reaper decides "one object" by comparing STORED fingerprints. So a root typed exactly as a destination that + // build admitted must store what that build stored for it; this build's own sweep keeps the object either way. + var world = await SeedTwoTeamsAsync(); + var root = Root(); + var payload = Payload("victim deliverable under a root two builds admitted alike"); + + await SeedDestinationAdmittedBeforeCanonicalAsync(world.Victim, root + "/"); + var victimArtifactId = await PutRoutedAsync(world.Victim.TeamId, payload); + await CreateDestinationAsync(world.Collector, root + "/"); + var collectorArtifactId = await PutDeclaredAsync(world.Collector.TeamId, payload); + + (await StoredFingerprintsAsync(world)).Distinct().Count().ShouldBe(1, "one root typed alike must store one fingerprint, or the previous build's reaper reads two namespaces and collects the victim's object"); + + await SweepUntilSettledAsync(collectorArtifactId); + + var declaration = (await DeclarationAsync(collectorArtifactId)).ShouldNotBeNull("this build's sweep collected an object two teams reach"); + declaration.LastErrorCode.ShouldBe("artifact-routed-object-shared"); + + var download = await DownloadAsync(world.Victim, victimArtifactId); + download.StatusCode.ShouldBe(HttpStatusCode.OK, await download.Content.ReadAsStringAsync()); + (await download.Content.ReadAsByteArrayAsync()).ShouldBe(payload); + } + [Fact] public async Task A_second_team_in_a_root_of_its_own_still_has_its_unreferenced_bytes_collected() { diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunLogRuntimeTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunLogRuntimeTests.cs index 6802a55de..387f27c88 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunLogRuntimeTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunLogRuntimeTests.cs @@ -8,6 +8,7 @@ using CodeSpace.Core.Services.Agents.AgentRunLogging; using CodeSpace.Core.Services.Identity; using CodeSpace.Core.Services.Workflows.Artifacts.Profiles; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers.Local; using CodeSpace.Core.Services.Workflows.Artifacts.Runtime; using CodeSpace.Core.Settings; using CodeSpace.IntegrationTests.Infrastructure; @@ -213,11 +214,12 @@ public async Task First_resolution_creates_one_idempotent_settings_visible_local } [Fact] - public async Task The_bootstrap_destination_names_its_namespace_exactly_as_settings_admission_names_that_directory() + public async Task The_bootstrap_destination_stores_the_fingerprint_settings_admission_stores_for_that_root() { - // The bootstrap writes its revision without going through admission, so it must derive the namespace the way - // admission does. Reached through a link here, the deployment root's TEXT differs from the directory an operator - // types into Settings, while the directory itself is one: a fingerprint of the text would name two namespaces. + // The bootstrap writes its revision without going through admission, so it must derive the stored fingerprint the + // way admission does: for this release the hash of the configuration as written, which is the value the previous + // build's retention reaper compares. Reached through a link here, so the directory's physical identity is a + // different value and a bootstrap that stored it would split from that build's own revisions of this root. if (OperatingSystem.IsWindows()) return; var world = await SeedWorldAsync(); @@ -231,13 +233,14 @@ public async Task The_bootstrap_destination_names_its_namespace_exactly_as_setti using var scope = _fixture.BeginScope(); var bootstrap = (await scope.Resolve().ResolveAsync(world.TeamId, CancellationToken.None)).ShouldBeOfType(); - using var typed = JsonDocument.Parse(JsonSerializer.Serialize(new { rootPath = real + "/" })); - var admitted = await scope.Resolve().CreateAsync(world.TeamId, world.ActorId, new CreateStorageProfileCommand { StableName = $"admitted-{Guid.NewGuid():N}", ProviderTypeKey = "local-rwx/v1", NonSecretConfig = typed.RootElement }, CancellationToken.None); - - var bootstrapFingerprint = await scope.Resolve().StorageProfileRevision.AsNoTracking() - .Where(value => value.StorageProfileId == bootstrap.StorageProfileId).Select(value => value.NamespaceFingerprint).SingleAsync(); - - bootstrapFingerprint.ShouldBe(admitted.Revisions.ShouldHaveSingleItem().NamespaceFingerprint, $"'{link}' and '{real}/' are one directory, so the bootstrap and admission must name one namespace"); + var stored = await scope.Resolve().StorageProfileRevision.AsNoTracking() + .Where(value => value.StorageProfileId == bootstrap.StorageProfileId).Select(value => new { value.NonSecretConfigJson, value.NamespaceFingerprint }).SingleAsync(); + using var config = JsonDocument.Parse(stored.NonSecretConfigJson); + var admitted = await scope.Resolve().CreateAsync(world.TeamId, world.ActorId, new CreateStorageProfileCommand { StableName = $"admitted-{Guid.NewGuid():N}", ProviderTypeKey = "local-rwx/v1", NonSecretConfig = config.RootElement }, CancellationToken.None); + + StorageProfileRules.NamespaceFingerprint(new LocalRwxStorageProviderModule(), config.RootElement).ShouldNotBe(stored.NamespaceFingerprint, $"the precondition: '{link}' is a link, so its physical identity is not the hash of its text"); + stored.NamespaceFingerprint.ShouldBe(admitted.Revisions.ShouldHaveSingleItem().NamespaceFingerprint, "the bootstrap and admission must store one fingerprint for one root as written"); + stored.NamespaceFingerprint.ShouldBe(StorageProfileRules.LegacyNamespaceFingerprint("local-rwx/v1", config.RootElement), "what the previous build stored for this configuration"); } finally { diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.Namespaces.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.Namespaces.cs index 1dac29ea6..6fa714bdb 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.Namespaces.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.Namespaces.cs @@ -254,18 +254,17 @@ private static string StoredBeforeCanonicalAdmission(string providerTypeKey, str /// pointing row, and an Available location at the production key. Each team owns its own object row for the same /// bytes, exactly as two independent routed writes produce. /// - private async Task SeedPlacementAsync(World world, byte[] bytes, string providerTypeKey, string config) + private async Task SeedPlacementAsync(World world, byte[] bytes, string providerTypeKey, string config) => + await SeedPlacementUnderAsync(world, bytes, await SeedRevisionAdmittedBeforeCanonicalAsync(world, providerTypeKey, config)); + + /// An Active profile whose one revision carries what admission stored before namespaces were canonical. + private async Task SeedRevisionAdmittedBeforeCanonicalAsync(World world, string providerTypeKey, string config) { using var scope = _fixture.BeginScope(); var db = scope.Resolve(); var now = DateTimeOffset.UtcNow; - var digest = SHA256.HashData(bytes); var profileId = Guid.NewGuid(); var revisionId = Guid.NewGuid(); - var objectId = Guid.NewGuid(); - var artifactId = Guid.NewGuid(); - var locationId = Guid.NewGuid(); - var key = ObjectKeyOf(bytes); db.StorageProfile.Add(new StorageProfile { @@ -281,6 +280,26 @@ private async Task SeedPlacementAsync(World world, byte[] bytes }, }, }); + await db.SaveChangesAsync(); + + return revisionId; + } + + /// + /// One team's routed artifact under an existing revision of its own: the team's own artifact_object for the + /// digest, the pointing row, and an Available location at the production key. + /// + private async Task SeedPlacementUnderAsync(World world, byte[] bytes, Guid revisionId) + { + using var scope = _fixture.BeginScope(); + var db = scope.Resolve(); + var now = DateTimeOffset.UtcNow; + var digest = SHA256.HashData(bytes); + var objectId = Guid.NewGuid(); + var artifactId = Guid.NewGuid(); + var locationId = Guid.NewGuid(); + var key = ObjectKeyOf(bytes); + db.ArtifactObject.Add(new ArtifactObject { Id = objectId, TeamId = world.TeamId, DigestAlgorithm = ArtifactDigestAlgorithm.Sha256, Digest = digest, diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.PreviousBuild.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.PreviousBuild.cs new file mode 100644 index 000000000..5379072e3 --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.PreviousBuild.cs @@ -0,0 +1,127 @@ +using System.Text.Json; +using Autofac; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Workflows.Artifacts.Profiles; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers.Local; +using CodeSpace.Messages.Artifacts; +using CodeSpace.Messages.Commands.Storage; +using Microsoft.EntityFrameworkCore; +using Shouldly; + +namespace CodeSpace.IntegrationTests.Workflows; + +/// +/// The routed sharing guard as the PREVIOUS build asks it, over revisions THIS build admits — the mixed fleet a rolling +/// deploy or a rollback runs. That build decided "one object" by comparing the namespace fingerprints revisions STORE, +/// so the bytes it keeps depend on what this build writes into that column. +/// +/// Fidelity: medium-mirror. The previous build's question is reproduced below verbatim from its source +/// (583efb0df ArtifactRetentionReaper.RoutedPurgePathAsync) and run against the real database; admission and +/// this build's reaper are the production classes. The mirrored source is a released build that can no longer change, +/// so there is nothing for a drift detector to compare it with: what can drift is this build's writes, which is what +/// these tests pin. +/// +public sealed partial class ArtifactRetentionReaperFlowTests +{ + [Fact] + public async Task A_root_admitted_now_and_typed_as_the_previous_build_admitted_it_is_one_object_to_both_reapers() + { + // A victim's destination admitted by the previous build stored the hash of 'R/' as typed. A second team types + // the same 'R/' and THIS build admits it. If this build stored the physical identity, the previous build's + // reaper — on a worker not yet replaced, or back after a rollback — would see two namespaces and collect the + // second team's object, deleting the victim's bytes with it. + var victim = await SeedWorldAsync(); + var collector = await SeedWorldAsync(); + var root = NamespaceRoot(); + var typed = RootConfig(root + "/"); + var bytes = RoutedBytes("one payload under one root typed alike by two builds"); + var file = PlaceObject(root, bytes); + + var victimPlacement = await SeedPlacementAsync(victim, bytes, LocalRwxArtifactStorageDriverFactory.TypeKey, typed); + var collectorPlacement = await SeedPlacementUnderAsync(collector, bytes, await AdmitAsync(collector, typed)); + await DeclarePlacementAsync(collector, collectorPlacement.ArtifactId); + + (await StoredFingerprintAsync(collectorPlacement)).ShouldNotBe(PhysicalFingerprint(typed), + "the precondition that makes this a regression test: the physical identity of 'R/' differs from the hash the previous build stored for it"); + (await PreviousBuildFindsSharedAsync(collectorPlacement.ArtifactId)).ShouldBeTrue( + "the previous build's reaper compares stored fingerprints: a revision admitted now must store what that build stored for the same text, or that reaper collects the victim's object"); + + await SweepToSettlementAsync(collectorPlacement.ArtifactId); + + var declaration = (await DeclarationAsync(collectorPlacement.ArtifactId)).ShouldNotBeNull("this build's reaper collected an object two teams reach"); + declaration.State.ShouldBe(ArtifactRetentionState.Indeterminate); + declaration.LastErrorCode.ShouldBe("artifact-routed-object-shared", "this build's reaper still derives the object at sweep time and still keeps it"); + File.Exists(file).ShouldBeTrue($"the victim's object at {file} is gone; check `ls {Path.GetDirectoryName(file)}`"); + (await LocationStateAsync(victimPlacement.LocationId)).ShouldBe(ArtifactLocationState.Available); + } + + [Fact] + public async Task The_previous_builds_question_still_tells_physically_distinct_roots_apart() + { + // The positive control for the mirror: content addressing gives both teams the same key, so a question that + // answered "shared" for distinct directories would make the test above pass for the wrong reason. + var victim = await SeedWorldAsync(); + var collector = await SeedWorldAsync(); + var bytes = RoutedBytes("one payload in two directories, both admitted now"); + + await SeedPlacementUnderAsync(victim, bytes, await AdmitAsync(victim, RootConfig(NamespaceRoot()))); + var collectorPlacement = await SeedPlacementUnderAsync(collector, bytes, await AdmitAsync(collector, RootConfig(NamespaceRoot() + "/"))); + + (await PreviousBuildFindsSharedAsync(collectorPlacement.ArtifactId)).ShouldBeFalse(); + } + + // ─── Previous-build helpers ────────────────────────────────────────────── + + /// A destination admitted by THIS build, through the production admission path a Settings save takes. + private async Task AdmitAsync(World world, string config) + { + using var scope = _fixture.BeginScope(); + using var document = JsonDocument.Parse(config); + + var admitted = await scope.Resolve().CreateAsync(world.TeamId, world.ActorId, new CreateStorageProfileCommand + { + StableName = $"admitted-now-{Guid.NewGuid():N}", ProviderTypeKey = LocalRwxArtifactStorageDriverFactory.TypeKey, NonSecretConfig = document.RootElement, + }, CancellationToken.None); + + return admitted.Revisions.ShouldHaveSingleItem().Id; + } + + private static string PhysicalFingerprint(string config) + { + using var document = JsonDocument.Parse(config); + return StorageProfileRules.NamespaceFingerprint(new LocalRwxStorageProviderModule(), document.RootElement); + } + + /// + /// The previous build's routed sharing question, verbatim: does another object's placement sit at one of this + /// object's keys under a revision with the same STORED namespace fingerprint? True is that build's + /// RoutedObjectShared (keep), false its Routed (collect). + /// + private async Task PreviousBuildFindsSharedAsync(Guid artifactId) + { + using var scope = _fixture.BeginScope(); + var db = scope.Resolve(); + var objectId = await db.WorkflowArtifact.AsNoTracking().Where(artifact => artifact.Id == artifactId).Select(artifact => artifact.CasArtifactObjectId!.Value).SingleAsync(); + + var mine = await db.ArtifactLocation.AsNoTracking() + .Where(location => location.ArtifactObjectId == objectId) + .Join(db.StorageProfileRevision.AsNoTracking(), location => location.StorageProfileRevisionId, revision => revision.Id, + (location, revision) => new PreviousBuildPlacedAt(location.ObjectKey, revision.NamespaceFingerprint)) + .Distinct() + .ToListAsync(); + + var keys = mine.Select(placed => placed.ObjectKey).Distinct().ToList(); + + var collisions = await db.ArtifactLocation.AsNoTracking() + .Where(location => location.ArtifactObjectId != objectId && keys.Contains(location.ObjectKey)) + .Join(db.StorageProfileRevision.AsNoTracking(), location => location.StorageProfileRevisionId, revision => revision.Id, + (location, revision) => new PreviousBuildPlacedAt(location.ObjectKey, revision.NamespaceFingerprint)) + .Distinct() + .ToListAsync(); + + return collisions.Any(mine.Contains); + } + + private sealed record PreviousBuildPlacedAt(string ObjectKey, string NamespaceFingerprint); +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/Profiles/StorageNamespaceFingerprintBackfillFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/Profiles/StorageNamespaceFingerprintBackfillFlowTests.cs new file mode 100644 index 000000000..a92819867 --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/Profiles/StorageNamespaceFingerprintBackfillFlowTests.cs @@ -0,0 +1,159 @@ +using System.Text.Json; +using Autofac; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Workflows.Artifacts.Profiles; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers.Local; +using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.Messages.Enums; +using Microsoft.EntityFrameworkCore; +using Npgsql; +using Shouldly; + +namespace CodeSpace.IntegrationTests.Workflows.Artifacts.Profiles; + +/// +/// Revisions a build that stored the PHYSICAL namespace identity already wrote, put back to the value the previous +/// build stores — so that build's reaper, after a rollback or beside a pod not yet replaced, compares like with like. +/// +/// The backfill sweeps the whole table, and other suites seed revisions with fingerprints no admission ever +/// produced, so every test here waits on the rows it owns rather than on a pass's tally. +/// +[Collection(PostgresCollection.Name)] +[Trait("Category", "Integration")] +public sealed class StorageNamespaceFingerprintBackfillFlowTests +{ + /// Enough passes to drain every leftover revision of a full suite run ahead of this test's own rows; a pass that never reaches them fails instead of hanging. + private const int MaxPasses = 500; + + private readonly PostgresFixture _fixture; + + public StorageNamespaceFingerprintBackfillFlowTests(PostgresFixture fixture) => _fixture = fixture; + + [Fact] + public async Task A_revision_stamped_with_the_physical_identity_is_restored_to_what_the_previous_build_stores() + { + var typed = RootConfig($"/srv/codespace-backfill-{Guid.NewGuid():N}/"); + var physical = await SeedRevisionAsync(typed, PhysicalFingerprint(typed)); + var previousBuild = await SeedRevisionAsync(typed, LegacyFingerprint(typed)); + + PhysicalFingerprint(typed).ShouldNotBe(LegacyFingerprint(typed), "the precondition: a trailing separator is where the two identities part"); + + await BackfillUntilAsync(physical, LegacyFingerprint(typed)); + + (await StoredAsync(physical)).ShouldBe(new Stored(typed, LegacyFingerprint(typed)), "only the fingerprint moves; the configuration is the revision's record and stays as written"); + (await StoredAsync(physical)).Fingerprint.ShouldBe((await StoredAsync(previousBuild)).Fingerprint, "one root typed alike is one namespace to the previous build's reaper again"); + (await StoredAsync(previousBuild)).ShouldBe(new Stored(typed, LegacyFingerprint(typed)), "a revision the previous build wrote is already right and is never rewritten"); + + await BackfillPassAsync(); + + (await StoredAsync(physical)).Fingerprint.ShouldBe(LegacyFingerprint(typed), "a restored revision is no longer a candidate: the pass is idempotent"); + } + + [Fact] + public async Task The_revision_table_is_immutable_again_once_a_pass_has_restored_rows() + { + var typed = RootConfig($"/srv/codespace-backfill-{Guid.NewGuid():N}/."); + var revisionId = await SeedRevisionAsync(typed, PhysicalFingerprint(typed)); + + await BackfillUntilAsync(revisionId, LegacyFingerprint(typed)); + + using var scope = _fixture.BeginScope(); + var db = scope.Resolve(); + + var refused = await Should.ThrowAsync(() => db.Database.ExecuteSqlInterpolatedAsync($"UPDATE storage_profile_revision SET namespace_fingerprint = {PhysicalFingerprint(typed)} WHERE id = {revisionId}")); + + refused.MessageText.ShouldContain("storage_profile_revision is immutable", Case.Sensitive, "the pass switches the immutability trigger off only inside its own transaction"); + } + + [Fact] + public async Task Passes_on_several_workers_at_once_restore_a_revision_once_and_all_finish() + { + var typed = RootConfig($"/srv/codespace-backfill-{Guid.NewGuid():N}//"); + var revisionId = await SeedRevisionAsync(typed, PhysicalFingerprint(typed)); + + for (var pass = 0; pass < MaxPasses && (await StoredAsync(revisionId)).Fingerprint != LegacyFingerprint(typed); pass++) + await Task.WhenAll(BackfillPassAsync(), BackfillPassAsync()); + + (await StoredAsync(revisionId)).Fingerprint.ShouldBe(LegacyFingerprint(typed), $"two concurrent workers never restored revision {revisionId}; query storage_profile_revision for it and check pg_locks for a stuck pass"); + } + + // ─── Helpers ───────────────────────────────────────────────────────────── + + private sealed record Stored(string Config, string Fingerprint); + + /// Runs passes until holds , bounded so a pass that never reaches it fails with the row named. + private async Task BackfillUntilAsync(Guid revisionId, string expected) + { + for (var pass = 0; pass < MaxPasses && (await StoredAsync(revisionId)).Fingerprint != expected; pass++) await BackfillPassAsync(); + + (await StoredAsync(revisionId)).Fingerprint.ShouldBe(expected, $"{MaxPasses} passes never restored revision {revisionId}; run the backfill command by hand and read its log"); + } + + private async Task BackfillPassAsync() + { + using var scope = _fixture.BeginScope(); + await scope.Resolve().BackfillAsync(200, CancellationToken.None); + } + + /// A revision exactly as admission wrote it, stamped with : inserting is what the immutability trigger allows. + private async Task SeedRevisionAsync(string config, string fingerprint) + { + var actorId = Guid.NewGuid(); + var teamId = Guid.NewGuid(); + var profileId = Guid.NewGuid(); + var revisionId = Guid.NewGuid(); + var now = DateTimeOffset.UtcNow; + + using var scope = _fixture.BeginScope(); + var db = scope.Resolve(); + db.User.Add(new User { Id = actorId, Email = $"fingerprint-{actorId:N}@test.local", Name = $"fingerprint-{actorId:N}" }); + db.Team.Add(new Team { Id = teamId, Slug = $"fingerprint-{teamId:N}", Name = "Fingerprint Backfill Team", Kind = TeamKind.Workspace }); + db.TeamMembership.Add(new TeamMembership { Id = Guid.NewGuid(), TeamId = teamId, UserId = actorId, Role = TeamRole.Owner }); + var profile = new StorageProfile + { + Id = profileId, TeamId = teamId, StableName = $"fingerprint-{profileId:N}", CurrentRevision = 1, + State = StorageProfileState.Active, CreatedDate = now, CreatedBy = actorId, LastModifiedDate = now, LastModifiedBy = actorId, + }; + profile.Revisions.Add(new StorageProfileRevision + { + Id = revisionId, TeamId = teamId, StorageProfileId = profileId, Revision = 1, ProviderTypeKey = LocalRwxArtifactStorageDriverFactory.TypeKey, + NonSecretConfigJson = config, NamespaceFingerprint = fingerprint, CreatedDate = now, CreatedBy = actorId, + }); + db.StorageProfile.Add(profile); + await db.SaveChangesAsync(); + + return revisionId; + } + + private async Task StoredAsync(Guid revisionId) + { + using var scope = _fixture.BeginScope(); + + var row = await scope.Resolve().StorageProfileRevision.AsNoTracking() + .Where(revision => revision.Id == revisionId).Select(revision => new { revision.NonSecretConfigJson, revision.NamespaceFingerprint }).SingleAsync(); + + using var config = JsonDocument.Parse(row.NonSecretConfigJson); + return new Stored(StorageProfileRules.CanonicalJson(config.RootElement), row.NamespaceFingerprint); + } + + /// The canonical text admission persists for a local root. + private static string RootConfig(string rootPath) + { + using var document = JsonDocument.Parse(JsonSerializer.Serialize(new { rootPath })); + return StorageProfileRules.CanonicalJson(document.RootElement); + } + + /// What a build that stored the physical identity wrote for this configuration. + private static string PhysicalFingerprint(string config) + { + using var document = JsonDocument.Parse(config); + return StorageProfileRules.NamespaceFingerprint(new LocalRwxStorageProviderModule(), document.RootElement); + } + + private static string LegacyFingerprint(string config) + { + using var document = JsonDocument.Parse(config); + return StorageProfileRules.LegacyNamespaceFingerprint(LocalRwxArtifactStorageDriverFactory.TypeKey, document.RootElement); + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Architecture/RequestAuthorizationInventoryTests.cs b/backend/tests/CodeSpace.UnitTests/Architecture/RequestAuthorizationInventoryTests.cs index 9c42f0368..aee53154f 100644 --- a/backend/tests/CodeSpace.UnitTests/Architecture/RequestAuthorizationInventoryTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Architecture/RequestAuthorizationInventoryTests.cs @@ -77,6 +77,7 @@ public class RequestAuthorizationInventoryTests ["SweepBudgetSettlementCommand"] = "sweep", ["SweepCompletionShadowCommand"] = "sweep", ["BackfillPackCloneUrlsCommand"] = "seals the clone URL of pack rows that still hold a pasted token in plaintext, across every team, in bounded batches; it changes where a pack's existing credential is kept, never which source a pack clones or what any user can reach, and a sealed row is no longer a candidate.", + ["BackfillStorageNamespaceFingerprintsCommand"] = "puts back, across every team in bounded batches, the namespace fingerprint a storage profile revision's own configuration hashes to as written, where a physical-identity build stamped another; it changes no configuration, route or placement, nothing in this build reads the value to decide anything, and a restored revision is no longer a candidate.", ["RepairDisconnectedCredentialBindingsCommand"] = "moves repositories and group hooks still bound to a disconnected credential onto the credential that credential's own owner connected afterwards on the same provider instance and team, across every team, in bounded batches; it never moves a binding off a live credential or onto another person's, so it only completes a reconnect the owner already made, and a moved binding is no longer a candidate.", ["RetireDepartedMembersCredentialsCommand"] = "retires the personal tokens whose owner — or, for one stored without an owner, whose creator account — has no standing in the team the token was lent to (no membership there and not an instance Admin), across every team, in bounded batches; it ends access the person no longer has standing to lend, exactly as their departure does, never touches a team-service token, the token of anyone who still stands in its team, or one it cannot attribute to an account, and a retired token is no longer a candidate.", ["ShredRetiredSecretsCommand"] = "blanks the ciphertext of secrets that are already retired — deleted variables, revoked or soft-deleted git tokens, deleted connections' client and hook secrets, revoked storage credentials' revisions, deployment-default envelopes no template names — across every team; it never touches a live secret, ends no access anyone still has, and a blanked row is no longer a candidate, so it only finishes a retirement an older build left decryptable.", diff --git a/backend/tests/CodeSpace.UnitTests/Architecture/ScopedTransactionInventoryTests.cs b/backend/tests/CodeSpace.UnitTests/Architecture/ScopedTransactionInventoryTests.cs index e8f631f42..dfa0b3a72 100644 --- a/backend/tests/CodeSpace.UnitTests/Architecture/ScopedTransactionInventoryTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Architecture/ScopedTransactionInventoryTests.cs @@ -71,6 +71,7 @@ public class ScopedTransactionInventoryTests ["Supervisor/Observation/SupervisorPlanObservationLeafReader.cs"] = (1, "already joins, same RepeatableRead snapshot."), ["Tasks/RoutePreview/TaskRouteSnapshotService.cs"] = (1, "already joins through a savepoint branch taken before this line; this arm runs only when nothing is ambient."), ["Workflows/Artifacts/Defaults/StorageDefaultMaterializer.cs"] = (1, "already joins, but through a SAVEPOINT: a refused destination must not leave the profile and credential it wrote behind in the caller's commit."), + ["Workflows/Artifacts/Profiles/StorageNamespaceFingerprintBackfillService.cs"] = (1, "switches the revision immutability trigger off for its own writes and back on before its own commit; joined, the trigger would stay off for the rest of the caller's transaction and the table's write lock would be held until the caller commits."), ["Workflows/Artifacts/Retention/ArtifactRetentionReaper.cs"] = (3, "the same private CreateDb() context; nothing ambient can reach it."), ["Workflows/Artifacts/Runtime/ArtifactCasRuntimeCoordinator.cs"] = (1, "the same private CreateDb() context; nothing ambient can reach it."), ["Workflows/Artifacts/Runtime/ArtifactCasRuntimeCoordinator.Purge.cs"] = (1, "the same private CreateDb() context; nothing ambient can reach it."), diff --git a/backend/tests/CodeSpace.UnitTests/Jobs/StorageNamespaceFingerprintBackfillDispatchTests.cs b/backend/tests/CodeSpace.UnitTests/Jobs/StorageNamespaceFingerprintBackfillDispatchTests.cs new file mode 100644 index 000000000..329d27310 --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Jobs/StorageNamespaceFingerprintBackfillDispatchTests.cs @@ -0,0 +1,95 @@ +using CodeSpace.Core.Handlers.CommandHandlers.Storage; +using CodeSpace.Core.Jobs.RecurringJobs; +using CodeSpace.Core.Services.Workflows.Artifacts.Profiles; +using CodeSpace.Messages.Commands.Storage; +using CodeSpace.Messages.Mediation; +using MediatR; +using Shouldly; + +namespace CodeSpace.UnitTests.Jobs; + +/// +/// 🟢 Unit: the storage namespace fingerprint backfill is the standard job → command → service chain (Rule 14 + Rule +/// 16) — the job holds no query, the handler holds no logic. Hand-rolled recording doubles, matching the codebase +/// convention. +/// +[Trait("Category", "Unit")] +public class StorageNamespaceFingerprintBackfillDispatchTests +{ + [Fact] + public async Task The_backfill_job_dispatches_its_command_every_ten_minutes_and_holds_no_logic() + { + var mediator = new RecordingMediator(); + var job = new StorageNamespaceFingerprintBackfillRecurringJob(mediator); + + job.JobId.ShouldBe(nameof(StorageNamespaceFingerprintBackfillRecurringJob), "Hangfire indexes the schedule by this id — a rename strands the old one"); + job.CronExpression.ShouldBe("*/10 * * * *", "a committed cadence: a revision an older pod stamps during the rollout is restored within ten minutes"); + + await job.Execute(); + + mediator.Sent.ShouldHaveSingleItem().ShouldBeOfType("the job is a thin dispatcher — it only sends the command"); + } + + [Fact] + public async Task The_backfill_handler_forwards_the_batch_size_and_returns_the_restored_count() + { + var backfill = new RecordingBackfill { ToReturn = 3 }; + var handler = new BackfillStorageNamespaceFingerprintsCommandHandler(backfill); + + var restored = await handler.Handle(new BackfillStorageNamespaceFingerprintsCommand { BatchSize = 7 }, CancellationToken.None); + + backfill.BatchSizes.ShouldHaveSingleItem().ShouldBe(7, "the handler delegates the whole pass to the service (Rule 16)"); + restored.ShouldBe(3, "the handler surfaces the service's count verbatim"); + } + + [Fact] + public void The_backfill_command_is_bounded_and_owns_no_command_transaction() + { + new BackfillStorageNamespaceFingerprintsCommand().BatchSize.ShouldBe(200, "an unbounded pass would hold the revision table's write lock for as long as the backlog is long"); + + typeof(INonTransactionalCommand).IsAssignableFrom(typeof(BackfillStorageNamespaceFingerprintsCommand)) + .ShouldBeTrue("the service opens the one transaction its trigger switch must live inside; a command transaction around it would refuse a second"); + } + + /// Records the requests sent through the mediator; the rest of the surface is unreachable in these tests. + private sealed class RecordingMediator : IMediator + { + public List Sent { get; } = new(); + + public Task Send(IRequest request, CancellationToken cancellationToken = default) + { + Sent.Add(request); + return Task.FromResult(default(TResponse)!); + } + + public Task Send(object request, CancellationToken cancellationToken = default) + { + Sent.Add(request); + return Task.FromResult(null); + } + + public Task Send(TRequest request, CancellationToken cancellationToken = default) where TRequest : IRequest + { + Sent.Add(request!); + return Task.CompletedTask; + } + + public IAsyncEnumerable CreateStream(IStreamRequest request, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public IAsyncEnumerable CreateStream(object request, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public Task Publish(object notification, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public Task Publish(TNotification notification, CancellationToken cancellationToken = default) where TNotification : INotification => throw new NotSupportedException(); + } + + /// Records the batch sizes asked for + returns a canned count, so the handler test asserts pure delegation. + private sealed class RecordingBackfill : IStorageNamespaceFingerprintBackfillService + { + public List BatchSizes { get; } = new(); + public int ToReturn; + + public Task BackfillAsync(int batchSize, CancellationToken cancellationToken) + { + BatchSizes.Add(batchSize); + return Task.FromResult(ToReturn); + } + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Profiles/StorageNamespaceLegacyFingerprintTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Profiles/StorageNamespaceLegacyFingerprintTests.cs new file mode 100644 index 000000000..8742806e7 --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Profiles/StorageNamespaceLegacyFingerprintTests.cs @@ -0,0 +1,94 @@ +using System.Text.Json; +using CodeSpace.Core.Services.Workflows.Artifacts.Profiles; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers.AliyunOss; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers.Local; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers.Local.Legacy; +using Shouldly; + +namespace CodeSpace.UnitTests.Workflows.Artifacts.Profiles; + +/// +/// The fingerprint a revision STORES, held to the value the previous build stored for the same configuration. +/// +/// The previous build's retention reaper decides whether one object key is one object across two teams by +/// comparing stored fingerprints, and a rolling deploy or a rollback runs that reaper over revisions this build +/// admitted. So for one release this build stores exactly what that build stored — the hash of the configuration as +/// written — or a second team's placement under the same typed root reads to that reaper as another namespace, and it +/// deletes the first team's live bytes. +/// +/// The expected values were produced by the previous build itself (583efb0df: its own admission arithmetic, +/// GetNamespaceConfiguration then NamespaceFingerprint, loaded from its compiled assembly and run on +/// these configurations), never by this build's code, so a change to canonicalization, to the hash input or to which +/// value admission stores fails here. +/// +[Trait("Category", "Unit")] +public sealed class StorageNamespaceLegacyFingerprintTests +{ + private static readonly IStorageProviderModule[] Modules = [new LocalRwxStorageProviderModule(), new LocalLegacyStorageProviderModule(), new AliyunOssStorageProviderModule()]; + + public static TheoryData PreviousBuildFingerprints => new() + { + { "local-rwx/v1", """{"rootPath":"/srv/shared-bucket"}""", "sha256:093f1366bc44ad1ab2d3f00d3e3ec39cbb739d823c3938a37ab735250ec4a9e4" }, + { "local-rwx/v1", """{"rootPath":"/srv/shared-bucket/"}""", "sha256:a2d502242c785d4dfe940fde7d7f526933cbf0f7267a802e379fdb72847585a3" }, + { "local-rwx/v1", """{"rootPath":"/srv/shared-bucket/."}""", "sha256:c96998794d1302bd05a8938754c029d563895f1c8cd373610ced5e48808ed979" }, + { "local-rwx/v1", """{"rootPath":"/srv//shared-bucket"}""", "sha256:225b05ee64045303133a910b9db6661f9aa7ef8ac64f287e3018c107da504fe0" }, + { "local-rwx/v1", """{"rootPath":"/var/lib/codespace/artifacts"}""", "sha256:cb775836c24c2af3263152dd023982abffc39fc989ef5b68f0ad556ef2c31b6c" }, + { "local-rwx/v1", """{"rootPath":"/tmp/codespace-store"}""", "sha256:7039c5008170b4b1dc438ae9097259d0b5aa82f38774f8a2bb30db32bcc9fe6d" }, + { "local-rwx/v1", """{"rootPath":"/srv/Shared-Bucket"}""", "sha256:d76573877d14a4005892967e72e3c26839a7d4210a277bdb6830ab0d57c6dce0" }, + { "local-rwx/v1", """{"rootPath":"/srv/café"}""", "sha256:23bcf70b1b16176638d65aa5d5f1d0250b3ea712fed74741ee3a1efef35eb4d7" }, + { "local-rwx/v1", """{ "rootPath" : "/srv/shared-bucket/" }""", "sha256:a2d502242c785d4dfe940fde7d7f526933cbf0f7267a802e379fdb72847585a3" }, + { "local-legacy/v1", """{"rootPath":"/var/lib/codespace/artifacts"}""", "sha256:248d13d1e0dd50d09c3f4482550e28dab37aba148c43cdefa437e8b7c3288e25" }, + { "local-legacy/v1", """{"rootPath":"/var/lib/codespace/artifacts/"}""", "sha256:fc5f46c23028c33a9716394dd8f82f15150e86d469a909c7cedd8b221564e19d" }, + { "aliyun-oss/v1", """{"bucket":"acme-artifacts","endpoint":"oss-cn-hangzhou.aliyuncs.com"}""", "sha256:c4f957bc326598c5be60c0cb477d88fc8206a44a711d75b836514daa55233c56" }, + { "aliyun-oss/v1", """{"endpoint":"oss-cn-hangzhou.aliyuncs.com","bucket":"acme-artifacts"}""", "sha256:c4f957bc326598c5be60c0cb477d88fc8206a44a711d75b836514daa55233c56" }, + { "aliyun-oss/v1", """{"bucket":"acme-artifacts","endpoint":"oss-cn-hangzhou-internal.aliyuncs.com"}""", "sha256:c53891fdffda0aaab403e48377feb988edd9f7e3cfbbd8992f040f6d64c113fa" }, + { "aliyun-oss/v1", """{"bucket":"acme-artifacts","endpoint":"https://oss-cn-hangzhou.aliyuncs.com","keyPrefix":"codespace/"}""", "sha256:cfceea5c7919d60c0b3de79fac8f1146137cf9be4f5813d5b20c267c12956dcd" }, + { "aliyun-oss/v1", """{"bucket":"acme-artifacts","endpoint":"oss-cn-hangzhou.aliyuncs.com","region":"cn-hangzhou"}""", "sha256:8948c27532243bb3b55f8193b5fbf1f47a324dc7427b006b9d2b1a7427801af9" }, + { "aliyun-oss/v1", """{"bucket":"acme-artifacts","endpoint":"oss-cn-hangzhou.aliyuncs.com","keyPrefix":"team-7/"}""", "sha256:5fb52e872c5e3d1e286775a1095767457b8636131268f6aadd2ff50210606015" }, + }; + + [Theory] + [MemberData(nameof(PreviousBuildFingerprints))] + public void The_legacy_fingerprint_is_the_one_the_previous_build_stored(string providerTypeKey, string config, string previousBuild) + { + using var document = JsonDocument.Parse(config); + + StorageProfileRules.LegacyNamespaceFingerprint(providerTypeKey, document.RootElement).ShouldBe(previousBuild); + } + + [Theory] + [MemberData(nameof(PreviousBuildFingerprints))] + public void Admission_stores_what_the_previous_build_stored_for_every_shipped_module(string providerTypeKey, string config, string previousBuild) + { + // Admission hands the provider the CANONICAL configuration, so this does too. + using var typed = JsonDocument.Parse(config); + using var canonical = JsonDocument.Parse(StorageProfileRules.CanonicalJson(typed.RootElement)); + + StorageProfileRules.StoredNamespaceFingerprint(Module(providerTypeKey), canonical.RootElement).ShouldBe(previousBuild, + "a revision admitted now must compare equal, in the previous build's reaper, to one that build admitted from the same text"); + } + + [Fact] + public void The_stored_value_is_not_the_physical_identity_for_a_configuration_written_another_way() + { + // The precondition that makes the golden theory above a regression test: for a trailing separator the physical + // identity and the stored one differ, so storing the physical one would fail it. + using var config = JsonDocument.Parse("""{"rootPath":"/srv/shared-bucket/"}"""); + var module = Module("local-rwx/v1"); + + StorageProfileRules.StoredNamespaceFingerprint(module, config.RootElement).ShouldNotBe(StorageProfileRules.NamespaceFingerprint(module, config.RootElement)); + } + + [Fact] + public void A_configuration_that_reaches_no_namespace_is_still_refused_rather_than_given_a_legacy_identity() + { + // Admission has refused a root that names no directory since namespaces were derived physically. Storing the + // legacy value must not quietly re-admit it. + using var blank = JsonDocument.Parse("""{"rootPath":" "}"""); + + Should.Throw(() => StorageProfileRules.StoredNamespaceFingerprint(Module("local-rwx/v1"), blank.RootElement)); + } + + private static IStorageProviderModule Module(string providerTypeKey) => Modules.Single(module => module.TypeKey == providerTypeKey); +}