From d31034d46b1f00ae032ab36a6bc995ae74139217 Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Mon, 12 Oct 2026 14:06:35 +0800 Subject: [PATCH] Compare storage namespaces by physical location The retention reaper decided whether a routed object key was shared with another team by comparing namespace fingerprints, and a fingerprint was the hash of the configuration exactly as a team typed it. Object keys carry no team segment, so 'R', 'R/' and 'R/.' of one local-rwx root counted as three namespaces, and one team's sweep deleted another team's live bytes. Each provider module now projects its configuration onto the physical namespace it reaches: a local root is made absolute, trimmed of trailing separators, resolved through existing symlinks the way realpath(3) resolves them (a link target is walked component by component, so 'L -> a/..' climbs from where 'a' lands, not from L's directory) and case-folded where the platform's filesystem ignores case; an OSS configuration reduces to bucket, region and key prefix, merging every host that can only reach that bucket. Admission, the agent-run-log bootstrap and the deployment-default materializer all store that projection's hash, and a configuration already written canonically keeps the fingerprint it had. Existing revisions are immutable and their fingerprints still name the typed text, so the reaper no longer reads them: it re-derives the object each colliding placement reaches at sweep time. A namespace is not enough for a filesystem, because the driver opens /objects/ and the OS follows a link anywhere along it; a root the collector owns outright can link 'objects' (or any directory below it) into the victim's. So local-rwx, through the new IStorageProviderPhysicalObjects capability, names the file a delete of the key would unlink, with every directory on the way resolved and the final name left as written (unlink removes a link, not its target). Other providers are identified by namespace plus key. An object the reaper cannot derive (an unregistered provider, an unreadable configuration, a key no driver accepts) counts as shared, which keeps the bytes. The verifier flow tests asserted after one bounded pass over the whole table. Rows behind roots that earlier tests deleted never move, and the new two-team fixtures pushed their count past the batch, so those tests now sweep at the verifier's ceiling and wait on their own row. --- .../AgentRunLogStorageReadiness.cs | 19 +- .../Artifacts/Profiles/StorageProfileRules.cs | 47 ++ .../Profiles/StorageProfileService.cs | 8 +- .../AliyunOssStorageProviderModule.cs | 6 + .../Providers/AliyunOss/AliyunOssTarget.cs | 27 ++ .../Providers/IStorageProviderModule.cs | 15 +- .../IStorageProviderPhysicalObjects.cs | 22 + .../LocalLegacyStorageProviderModule.cs | 3 + .../Providers/Local/LocalRootPath.cs | 159 +++++++ .../Local/LocalRwxStorageProviderModule.cs | 14 +- .../Retention/ArtifactRetentionReaper.cs | 77 ++- .../StorageNamespaceSharingE2ETests.cs | 443 ++++++++++++++++++ .../Workflows/AgentRunLogRuntimeTests.cs | 35 ++ ...factRetentionReaperFlowTests.Namespaces.cs | 359 ++++++++++++++ .../ArtifactRetentionReaperFlowTests.cs | 5 +- ...ArtifactStoreRoutedDestinationFlowTests.cs | 6 +- .../Artifacts/LegacyPlacementAdoptionTests.cs | 6 +- .../ArtifactLocationVerifierFlowTests.cs | 46 +- .../Artifacts/ArtifactRetentionPolicyTests.cs | 7 +- .../Profiles/StorageNamespaceIdentityTests.cs | 130 +++++ .../AliyunOssStorageNamespaceTests.cs | 90 ++++ .../Local/LocalStorageNamespaceTests.cs | 372 +++++++++++++++ 22 files changed, 1831 insertions(+), 65 deletions(-) create mode 100644 backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/IStorageProviderPhysicalObjects.cs create mode 100644 backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/Local/LocalRootPath.cs create mode 100644 backend/tests/CodeSpace.E2ETests/Storage/StorageNamespaceSharingE2ETests.cs create mode 100644 backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.Namespaces.cs create mode 100644 backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Profiles/StorageNamespaceIdentityTests.cs create mode 100644 backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Providers/AliyunOss/AliyunOssStorageNamespaceTests.cs create mode 100644 backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Providers/Local/LocalStorageNamespaceTests.cs diff --git a/backend/src/CodeSpace.Core/Services/Agents/AgentRunLogging/AgentRunLogStorageReadiness.cs b/backend/src/CodeSpace.Core/Services/Agents/AgentRunLogging/AgentRunLogStorageReadiness.cs index b9be53d91..bd960517b 100644 --- a/backend/src/CodeSpace.Core/Services/Agents/AgentRunLogging/AgentRunLogStorageReadiness.cs +++ b/backend/src/CodeSpace.Core/Services/Agents/AgentRunLogging/AgentRunLogStorageReadiness.cs @@ -3,6 +3,7 @@ using CodeSpace.Core.Persistence.Entities; using CodeSpace.Core.Services.Workflows.Artifacts; using CodeSpace.Core.Services.Workflows.Artifacts.Profiles; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers; using CodeSpace.Core.Services.Workflows.Artifacts.Providers.Local; using CodeSpace.Core.Settings; using CodeSpace.Messages.Constants; @@ -22,11 +23,13 @@ public sealed class AgentRunLogStorageReadiness : IAgentRunLogStorageReadiness internal const string DefaultProfileStableName = "codespace-agent-run-log-default"; private readonly CodeSpaceDbContext _db; private readonly TimeProvider _clock; + private readonly IStorageProviderModuleCatalog _modules; - public AgentRunLogStorageReadiness(CodeSpaceDbContext db, TimeProvider clock) + public AgentRunLogStorageReadiness(CodeSpaceDbContext db, TimeProvider clock, IStorageProviderModuleCatalog modules) { _db = db; _clock = clock; + _modules = modules; } public async Task EnsureDefaultRouteAsync(Guid teamId, CancellationToken cancellationToken) @@ -53,7 +56,7 @@ public async Task EnsureDefaultRouteAsync(Guid teamId, CancellationToken cancell return; } - var profile = BuildProfile(teamId, canonicalConfig, _clock.GetUtcNow()); + var profile = BuildProfile(teamId, canonicalConfig, NamespaceFingerprint(canonicalConfig), _clock.GetUtcNow()); _db.StorageProfile.Add(profile); var route = BuildRoute(teamId, profile.Id, _clock.GetUtcNow()); _db.StorageRoute.Add(route); @@ -71,7 +74,14 @@ public async Task EnsureDefaultRouteAsync(Guid teamId, CancellationToken cancell } } - private static StorageProfile BuildProfile(Guid teamId, string canonicalConfig, DateTimeOffset now) + /// The same derivation admission uses, so this bootstrap's revision names its namespace exactly as a Settings-created one would. + private string NamespaceFingerprint(string canonicalConfig) + { + using var document = JsonDocument.Parse(canonicalConfig); + return StorageProfileRules.NamespaceFingerprint(_modules.Require(LocalRwxArtifactStorageDriverFactory.TypeKey), document.RootElement); + } + + private static StorageProfile BuildProfile(Guid teamId, string canonicalConfig, string namespaceFingerprint, DateTimeOffset now) { var profile = new StorageProfile { @@ -79,12 +89,11 @@ private static StorageProfile BuildProfile(Guid teamId, string canonicalConfig, State = StorageProfileState.Active, CreatedDate = now, CreatedBy = SystemUsers.SeederId, LastModifiedDate = now, LastModifiedBy = SystemUsers.SeederId, }; - using var document = JsonDocument.Parse(canonicalConfig); profile.Revisions.Add(new StorageProfileRevision { Id = Guid.NewGuid(), TeamId = teamId, StorageProfileId = profile.Id, Revision = 1, ProviderTypeKey = LocalRwxArtifactStorageDriverFactory.TypeKey, NonSecretConfigJson = canonicalConfig, - CredentialRef = null, NamespaceFingerprint = StorageProfileRules.NamespaceFingerprint(LocalRwxArtifactStorageDriverFactory.TypeKey, document.RootElement), + CredentialRef = null, NamespaceFingerprint = namespaceFingerprint, CreatedDate = now, CreatedBy = SystemUsers.SeederId, }); return profile; diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileRules.cs b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileRules.cs index 7fd406406..58cf6ef95 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileRules.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileRules.cs @@ -44,6 +44,53 @@ public static string NamespaceFingerprint(string providerTypeKey, JsonElement na return "sha256:" + Convert.ToHexString(SHA256.HashData(input)).ToLowerInvariant(); } + /// + /// The identity of the PHYSICAL namespace a configuration reaches: the provider's own canonical projection of it, + /// hashed. The one derivation behind the fingerprint a revision stores and behind the retention reaper's later + /// comparison of two revisions' objects (), so the two cannot disagree about what + /// "the same namespace" means. + /// + public static string NamespaceFingerprint(IStorageProviderModule module, JsonElement nonSecretConfiguration) + { + var namespaceConfig = module.GetNamespaceConfiguration(nonSecretConfiguration); + if (namespaceConfig.ValueKind != JsonValueKind.Object) + throw new InvalidOperationException($"Storage provider '{module.TypeKey}' namespace configuration projection must be a JSON object."); + + return NamespaceFingerprint(module.TypeKey, namespaceConfig); + } + + /// + /// WHICH physical object a STORED placement's key reaches, derived now from its revision's configuration rather + /// than read from the fingerprint the revision stored, or null when it cannot be derived: the provider is not + /// registered in this build, the configuration no longer parses, or the provider cannot place the key. + /// + /// A provider that can name the object itself () is asked for it, + /// because for a filesystem a link beneath the root can carry a key out of the namespace its root names. Every other + /// provider's object is its namespace and the key together. + /// + /// Null is "unknown", never "different". A caller comparing objects must treat it as possibly the same object + /// as anything at that key, which is why every failure lands here instead of propagating: the comparison this feeds + /// decides whether bytes are deleted, and there only a kept byte is a safe mistake. + /// + public static string? TryObjectIdentity(IStorageProviderModuleCatalog catalog, string providerTypeKey, string nonSecretConfigJson, string objectKey) + { + try + { + var module = catalog.Get(providerTypeKey); + if (module == null) return null; + + using var configuration = JsonDocument.Parse(nonSecretConfigJson); + return ObjectIdentity(module, configuration.RootElement, objectKey); + } + catch (Exception) + { + return null; + } + } + + private static string ObjectIdentity(IStorageProviderModule module, JsonElement configuration, string objectKey) => + module is IStorageProviderPhysicalObjects physical ? physical.PhysicalObject(configuration, objectKey) : $"{NamespaceFingerprint(module, configuration)}/{objectKey}"; + public static bool TryParseCredentialRef(string? value, out StorageProfileCredentialReference reference) { reference = default; diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileService.cs b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileService.cs index 45b99e1bc..5ee04d154 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileService.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Profiles/StorageProfileService.cs @@ -269,10 +269,10 @@ private async Task PrepareRevisionAsync(Guid teamId, string pr ExecuteRule(() => module.EnsureConfigurationReadable(canonicalElement)); var normalizedCredentialRef = await ValidateCredentialAsync(teamId, module.TypeKey, credentialRef, cancellationToken).ConfigureAwait(false); - var namespaceConfig = module.GetNamespaceConfiguration(canonicalElement); - if (namespaceConfig.ValueKind != JsonValueKind.Object) - throw new InvalidOperationException($"Storage provider '{module.TypeKey}' namespace configuration projection must be a JSON object."); - var fingerprint = ExecuteRule(() => StorageProfileRules.NamespaceFingerprint(module.TypeKey, namespaceConfig)); + + // The fingerprint names the PHYSICAL namespace the provider says this configuration reaches, not its text, so two + // spellings of one destination store one fingerprint. The configuration itself is persisted as written. + var fingerprint = ExecuteRule(() => StorageProfileRules.NamespaceFingerprint(module, canonicalElement)); return new PreparedRevision(module.TypeKey, canonicalConfig, normalizedCredentialRef, fingerprint); } diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/AliyunOss/AliyunOssStorageProviderModule.cs b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/AliyunOss/AliyunOssStorageProviderModule.cs index a76d3d46f..db28da15c 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/AliyunOss/AliyunOssStorageProviderModule.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/AliyunOss/AliyunOssStorageProviderModule.cs @@ -120,6 +120,12 @@ public sealed class AliyunOssStorageProviderModule : IStorageProviderModule, ISt /// public void EnsureConfigurationReadable(JsonElement nonSecretConfiguration) => AliyunOssTarget.Parse(nonSecretConfiguration); + /// + /// The bucket and prefix the configuration reaches, read through the same parser activation uses: a configuration + /// the driver could never open has no namespace to name, so it is refused rather than given an identity. + /// + public JsonElement GetNamespaceConfiguration(JsonElement nonSecretConfiguration) => AliyunOssTarget.Parse(nonSecretConfiguration).NamespaceConfiguration(); + private static JsonElement ParseSchema(string json) { using var document = JsonDocument.Parse(json); diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/AliyunOss/AliyunOssTarget.cs b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/AliyunOss/AliyunOssTarget.cs index d9e392ec9..b0aaf4817 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/AliyunOss/AliyunOssTarget.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/AliyunOss/AliyunOssTarget.cs @@ -44,6 +44,33 @@ public static AliyunOssTarget Parse(JsonElement configuration) }; } + /// + /// The bucket namespace this target addresses, spelled one way: the canonical configuration every spelling that + /// reaches the same bucket and prefix projects to. + /// + /// The host is only a ROUTE to a bucket: the public, -internal VPC and accelerate hosts, with or + /// without the https:// scheme, all reach the bucket of one region. A custom domain or proxy cannot say + /// which bucket it fronts, and an identity that cannot tell two routes apart must merge them rather than split + /// them; OSS bucket names are unique service-wide and every request is signed for the bucket's region, so the one + /// OSS bucket any route can reach is that region's bucket of that name. So the identity names the region's public + /// host whatever host was written. An empty key prefix is the bucket root and is spelled by leaving it out. A + /// configuration already written this way projects to exactly itself, which keeps the fingerprint every such + /// revision already stored. + /// + public JsonElement NamespaceConfiguration() + { + var namespaceConfiguration = new SortedDictionary(StringComparer.Ordinal) + { + ["bucket"] = Bucket, + ["endpoint"] = $"{ServiceHostPrefix}{Region}{ServiceDomainSuffix}", + }; + + if (KeyPrefix.Length > 0) namespaceConfiguration["keyPrefix"] = KeyPrefix; + + using var document = JsonDocument.Parse(JsonSerializer.Serialize(namespaceConfiguration)); + return document.RootElement.Clone(); + } + /// /// Projects a caller's object key onto a bucket key, refusing anything that could escape the profile's prefix. /// Traversal and empty segments are rejected rather than normalized so two different caller keys can never diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/IStorageProviderModule.cs b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/IStorageProviderModule.cs index 8f13be9db..e5d1340a7 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/IStorageProviderModule.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/IStorageProviderModule.cs @@ -22,9 +22,18 @@ public interface IStorageProviderModule JsonElement SecretSchema { get; } /// - /// Project the non-secret configuration fields that identify the provider namespace. The control plane canonicalizes - /// and hashes this server-owned projection; clients can never submit a fingerprint. Providers with operational - /// tuning fields should override the conservative default so those fields do not create a new namespace identity. + /// Project the non-secret configuration onto the PHYSICAL namespace it reaches, spelled one way. The control plane + /// canonicalizes and hashes this server-owned projection into the revision's fingerprint (clients can never submit + /// one), and the retention reaper re-derives it from stored revisions to decide whether one object key is one + /// physical object across teams — unless the provider names that object itself + /// (), because its keys can leave the namespace its root names. + /// + /// So two configurations that reach the same namespace MUST project identically — a root with and without a + /// trailing separator, a host and its alias — and where a provider cannot tell, it projects them together: a + /// merged identity only keeps bytes, a split one lets one team's purge delete another's. A configuration that + /// reaches no namespace at all throws. Operational tuning fields belong outside the projection. The default is the + /// raw configuration, which is right only for a provider whose configuration has exactly one spelling; every + /// shipped module overrides it. /// JsonElement GetNamespaceConfiguration(JsonElement nonSecretConfiguration) => nonSecretConfiguration.Clone(); diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/IStorageProviderPhysicalObjects.cs b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/IStorageProviderPhysicalObjects.cs new file mode 100644 index 000000000..daa801bdb --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/IStorageProviderPhysicalObjects.cs @@ -0,0 +1,22 @@ +using System.Text.Json; + +namespace CodeSpace.Core.Services.Workflows.Artifacts.Providers; + +/// +/// A provider that can name the PHYSICAL object one of its keys reaches, because its namespace alone cannot: a +/// filesystem destination resolves every key through the operating system, which follows a link anywhere beneath the +/// configured root, so two roots that are different directories can still lead one key to one file. +/// +/// A sibling of (Rule 7): for an object store the bucket and prefix the +/// namespace projection names, plus the key, already ARE the object, and such a provider must not have to say so. +/// +/// The retention reaper asks this at sweep time, for every placement that shares a key with the one it is about +/// to delete, because object keys carry no team segment and the answer decides whether a delete removes another team's +/// bytes. So two keys that reach one object MUST answer identically, and an object that cannot be established throws: +/// the caller reads a throw as "possibly the same as anything", which only ever keeps a byte. +/// +public interface IStorageProviderPhysicalObjects +{ + /// The object a delete of would remove under this configuration, spelled one way, as it stands now. + string PhysicalObject(JsonElement nonSecretConfiguration, string objectKey); +} diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/Local/Legacy/LocalLegacyStorageProviderModule.cs b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/Local/Legacy/LocalLegacyStorageProviderModule.cs index 761c51d61..8dfba3fc2 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/Local/Legacy/LocalLegacyStorageProviderModule.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/Local/Legacy/LocalLegacyStorageProviderModule.cs @@ -52,6 +52,9 @@ public sealed class LocalLegacyStorageProviderModule : IStorageProviderModule, I public StorageProviderCapabilities Capabilities => StorageProviderCapabilities.HealthProbe | StorageProviderCapabilities.StreamingRead; public Type FactoryType => typeof(LocalLegacyArtifactStorageDriverFactory); + /// The directory the root reaches, not the text that names it: see . + public JsonElement GetNamespaceConfiguration(JsonElement nonSecretConfiguration) => LocalRootPath.NamespaceConfiguration(nonSecretConfiguration); + public string? ResolveLegacyObjectKey(JsonElement nonSecretConfiguration, string sha256, string recordedLocator) { var key = LegacyLocalObjectKeys.For(sha256); diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/Local/LocalRootPath.cs b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/Local/LocalRootPath.cs new file mode 100644 index 000000000..8724a5eef --- /dev/null +++ b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/Local/LocalRootPath.cs @@ -0,0 +1,159 @@ +using System.Text; +using System.Text.Json; + +namespace CodeSpace.Core.Services.Workflows.Artifacts.Providers.Local; + +/// +/// The physical identity of a filesystem namespace: the one directory a configured rootPath actually reaches, +/// spelled one way. +/// +/// Every local driver opens its root with and lets the operating system +/// follow whatever links lie along it, so R, R/, R/., R//, a relative spelling of +/// R, a symbolic link to R and — on a case-insensitive filesystem — r are all one directory. +/// Object keys carry no team segment, so two profiles naming that directory share every object they both store; a +/// comparison that saw two spellings as two namespaces would let one team's purge delete the other team's bytes. +/// +/// Where the comparison is unsure it errs towards SAME: the folding below may merge directories a case-sensitive +/// volume keeps apart, and that only ever keeps bytes. Splitting one directory into two is the failure that loses +/// them. +/// +/// Links are resolved in the order the operating system resolves them — one component at a time, a link's target +/// spliced in where the link stood, and .. applied to the directory reached so far rather than to the text — so +/// L -> a/.. beside a -> R/objects is R, as it is to open(2), not the directory +/// holding L. A root is also not the only place a link can stand: resolves the whole +/// path a driver opens for one key. +/// +internal static class LocalRootPath +{ + /// The configuration property every local provider names its root with. + public const string Property = "rootPath"; + + /// The same bound realpath(3) uses; a longer chain is a loop or an attack, and neither has an identity. + private const int MaxSymbolicLinkHops = 40; + + /// The provider's namespace projection: {"rootPath": <canonical>}, so a canonical configuration projects to itself. + public static JsonElement NamespaceConfiguration(JsonElement nonSecretConfiguration) + { + var canonical = Canonicalize(Read(nonSecretConfiguration)); + + using var document = JsonDocument.Parse(JsonSerializer.Serialize(new Dictionary { [Property] = canonical })); + return document.RootElement.Clone(); + } + + /// Absolute, lexically normalised, links resolved where they exist, then folded as this platform's filesystem compares names. + public static string Canonicalize(string rootPath) => FoldForPlatform(Physical(rootPath)); + + /// + /// The file a local driver removes when it deletes : every directory from the root down + /// to it resolved as the operating system resolves them, the file's own name as written. + /// + /// Every directory, because a link beneath the root redirects a key exactly as a link in the root does, so a + /// collector that owns its root outright can still reach a victim's file through objects or any directory + /// below it. Not the name itself, because a delete unlinks the name it is given: a key that is a link removes the + /// link and leaves its target alone. + /// + public static string ObjectPath(JsonElement nonSecretConfiguration, string objectsDirectory, string objectKey) + { + var segments = KeySegments(objectKey); + var directory = Physical(Path.Combine([Read(nonSecretConfiguration), objectsDirectory, .. segments[..^1]])); + + return FoldForPlatform(Path.Combine(directory, segments[^1])); + } + + private static string Physical(string path) => ResolveSymbolicLinks(Path.TrimEndingDirectorySeparator(Path.GetFullPath(path))); + + /// The key exactly as a local driver splits it, refused exactly where a driver refuses it: a key no driver opens names no object. + private static string[] KeySegments(string objectKey) + { + if (string.IsNullOrWhiteSpace(objectKey) || objectKey.Contains('\0') || Path.IsPathRooted(objectKey)) + throw new ArgumentException("An object key must be a non-empty relative key.", nameof(objectKey)); + + var segments = objectKey.Replace('\\', '/').Split('/', StringSplitOptions.RemoveEmptyEntries); + + if (segments.Length == 0 || segments.Any(segment => segment is "." or "..")) + throw new ArgumentException($"Object key '{objectKey}' cannot contain traversal segments.", nameof(objectKey)); + + return segments; + } + + private static string Read(JsonElement configuration) + { + if (configuration.ValueKind == JsonValueKind.Object && configuration.TryGetProperty(Property, out var root) + && root.ValueKind == JsonValueKind.String && !string.IsNullOrWhiteSpace(root.GetString())) + return root.GetString()!; + + throw new ArgumentException($"A local storage namespace requires a non-empty string '{Property}'.", nameof(configuration)); + } + + /// + /// with every existing symbolic link along it replaced by what it points at, the way + /// realpath(3) does it: the path is walked one component at a time from the root, a link's target is spliced + /// in where the link stood and walked in its turn, and .. steps up from the directory reached so far — which + /// holds no links by then, so stepping up from it is exactly what the operating system does. + /// + /// Components that do not exist yet stay as written: there is nothing there to follow, and the directory a + /// later write creates is exactly that spelling beneath the part already resolved. + /// + private static string ResolveSymbolicLinks(string fullPath) + { + var resolved = Path.GetPathRoot(fullPath)!; + var pending = new Stack(); + var hops = 0; + + PushComponents(pending, fullPath); + + while (pending.TryPop(out var component)) + { + var target = component is "." or ".." ? null : LinkTarget(Path.Combine(resolved, component)); + + if (target == null) + { + resolved = Step(resolved, component); + continue; + } + + if (++hops > MaxSymbolicLinkHops) + throw new ArgumentException($"'{Property}' '{fullPath}' passes through more than {MaxSymbolicLinkHops} symbolic links, so the directory it names cannot be established."); + + if (Path.IsPathRooted(target)) resolved = Path.GetPathRoot(target)!; + + PushComponents(pending, target); + } + + return resolved; + } + + /// One component down from a directory with no links left in it — or, for .., one up. + private static string Step(string resolved, string component) => component switch + { + "." => resolved, + ".." => Path.GetDirectoryName(resolved) ?? resolved, + _ => Path.Combine(resolved, component), + }; + + /// Queues the components of below its root so that its first component is walked next. + private static void PushComponents(Stack pending, string path) + { + var components = path[Path.GetPathRoot(path)!.Length..].Split([Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar], StringSplitOptions.RemoveEmptyEntries); + + for (var index = components.Length - 1; index >= 0; index--) pending.Push(components[index]); + } + + /// + /// A component this process cannot inspect is treated as no link. Resolving it is not possible from here, and + /// whatever this process cannot reach it cannot write to or delete from either. + /// + private static string? LinkTarget(string path) + { + try { return new FileInfo(path).LinkTarget; } + catch (IOException) { return null; } + catch (UnauthorizedAccessException) { return null; } + } + + /// + /// Windows and macOS filesystems compare names case-insensitively by default, and macOS also ignores Unicode + /// normalisation, so on those platforms both are folded. Linux compares bytes and keeps the path as resolved. + /// + private static string FoldForPlatform(string path) => + OperatingSystem.IsWindows() || OperatingSystem.IsMacOS() ? path.Normalize(NormalizationForm.FormC).ToLowerInvariant() : path; +} diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/Local/LocalRwxStorageProviderModule.cs b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/Local/LocalRwxStorageProviderModule.cs index 20f5e6808..804a9dd29 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/Local/LocalRwxStorageProviderModule.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Providers/Local/LocalRwxStorageProviderModule.cs @@ -7,8 +7,14 @@ namespace CodeSpace.Core.Services.Workflows.Artifacts.Providers.Local; /// driver or activate it through : that store continues to receive the convention-registered /// exactly as before until the profile/runtime cutover lands. /// -public sealed class LocalRwxStorageProviderModule : IStorageProviderModule, IStorageProviderTeamNamespace +public sealed class LocalRwxStorageProviderModule : IStorageProviderModule, IStorageProviderTeamNamespace, IStorageProviderPhysicalObjects { + /// + /// The directory beneath the root that keeps every object in + /// (TryResolveObjectPath); restated here, and pinned against the real driver by a unit test. + /// + private const string ObjectsDirectory = "objects"; + private static readonly JsonElement Config = ParseSchema(""" { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -53,6 +59,12 @@ public sealed class LocalRwxStorageProviderModule : IStorageProviderModule, ISto | StorageProviderCapabilities.HealthProbe; public Type FactoryType => typeof(LocalRwxArtifactStorageDriverFactory); + /// The directory the root reaches, not the text that names it: see . + public JsonElement GetNamespaceConfiguration(JsonElement nonSecretConfiguration) => LocalRootPath.NamespaceConfiguration(nonSecretConfiguration); + + /// The file the driver's delete of that key would unlink, every link on the way to it followed: see . + public string PhysicalObject(JsonElement nonSecretConfiguration, string objectKey) => LocalRootPath.ObjectPath(nonSecretConfiguration, ObjectsDirectory, objectKey); + private static JsonElement ParseSchema(string json) { using var document = JsonDocument.Parse(json); diff --git a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Retention/ArtifactRetentionReaper.cs b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Retention/ArtifactRetentionReaper.cs index 68aa49081..dc9f686fd 100644 --- a/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Retention/ArtifactRetentionReaper.cs +++ b/backend/src/CodeSpace.Core/Services/Workflows/Artifacts/Retention/ArtifactRetentionReaper.cs @@ -1,5 +1,7 @@ using CodeSpace.Core.Persistence.Db; using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Workflows.Artifacts.Profiles; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers; using CodeSpace.Core.Services.Workflows.Artifacts.Runtime; using CodeSpace.Messages.Artifacts; using CodeSpace.Messages.Constants; @@ -60,11 +62,14 @@ public sealed class ArtifactRetentionReaper : IArtifactRetentionReaper private readonly IArtifactBlobPurge? _purge; private readonly IArtifactCasPurgeCoordinator _routedPurge; + /// The provider modules a routed placement's object is derived through when its key collides with another's. + private readonly IStorageProviderModuleCatalog _modules; + private readonly ILogger _logger; private readonly ArtifactRetentionReaperOptions _options; public ArtifactRetentionReaper(DbContextOptions dbOptions, IArtifactReferenceOracle oracle, IArtifactBlobBackend blobs, - IArtifactCasPurgeCoordinator routedPurge, ILogger logger) : this(new ArtifactRetentionReaperServices(dbOptions, oracle, blobs, routedPurge, logger), Defaults) { } + IArtifactCasPurgeCoordinator routedPurge, IStorageProviderModuleCatalog modules, ILogger logger) : this(new ArtifactRetentionReaperServices(dbOptions, oracle, blobs, routedPurge, modules, logger), Defaults) { } internal ArtifactRetentionReaper(ArtifactRetentionReaperServices services, ArtifactRetentionReaperOptions options) { @@ -75,6 +80,7 @@ internal ArtifactRetentionReaper(ArtifactRetentionReaperServices services, Artif _oracle = services.Oracle; _purge = services.Blobs as IArtifactBlobPurge; _routedPurge = services.RoutedPurge; + _modules = services.Modules; _logger = services.Logger; _options = options; } @@ -447,14 +453,24 @@ private static ArtifactRetentionDecision ClaimRejection(ArtifactCasProblem probl /// The routed arm of the same sharing question the local arm asks below, and it has to be asked for the same /// reason: ArtifactStore.ObjectKeyFor builds workflow-artifacts/{aa}/{bb}/{sha256} with NO team /// segment, so two objects holding identical content land on the same key. Whether that key is the same PHYSICAL - /// object then depends entirely on whether the two profile revisions name the same namespace — which is what - /// storage_profile_revision.namespace_fingerprint identifies, and the only thing that identifies it. + /// object then depends on where each profile revision's key actually leads. + /// + /// That object is re-derived HERE, from each revision's configuration through its own provider module + /// (), and never read from + /// storage_profile_revision.namespace_fingerprint. The stored column cannot answer: revisions are immutable, + /// and every one admitted before namespaces were canonical hashed the configuration exactly as typed, so a root + /// spelled R by one team and R/ by another stored two fingerprints for one directory — and this probe, + /// reading them, let one team's sweep delete the other team's live bytes. Nor can any namespace, for a filesystem: + /// a link beneath a root the collector owns outright (objects, or any directory below it) carries its key + /// into the victim's directory, so a local provider is asked for the file the key reaches, every link on the way + /// followed as the operating system follows it, at the moment of the sweep. /// - /// Comparing the object key ALONE would be unsound in the safe direction but useless in practice: every team - /// storing the same bytes shares a key, so the reaper would refuse to collect any deduplicated content even when - /// the two teams are in different buckets. Comparing the fingerprint alone would be unsound in the other - /// direction. Both together are the question "is this the same object", which is the one the local arm answers - /// with a storage_url comparison because there the url already carries both. + /// A placement whose object cannot be derived — a provider this build does not register, a configuration + /// it can no longer read — is "unknown", and unknown collides with everything at its key. Comparing the object key + /// ALONE would be unsound in the safe direction but useless in practice: every team storing the same bytes shares + /// a key, so the reaper would refuse to collect any deduplicated content even when the two teams are in different + /// buckets. The key and the namespace together are the question "is this the same object", which is the one the + /// local arm answers with a storage_url comparison because there the url already carries both. /// /// Deliberately NOT team-scoped, and deliberately not filtered by . A /// location whose bytes are already Purged cannot really be harmed, so including it only ever costs a kept @@ -466,31 +482,46 @@ private static ArtifactRetentionDecision ClaimRejection(ArtifactCasProblem probl /// commits. Closing it would need the placement path to hold a lock across its transfer and its location insert, /// which this lane does not change. /// - private static async Task RoutedPurgePathAsync(CodeSpaceDbContext db, Guid objectId, CancellationToken cancellationToken) + private async Task RoutedPurgePathAsync(CodeSpaceDbContext db, Guid objectId, CancellationToken cancellationToken) { - var mine = await db.ArtifactLocation.AsNoTracking() - .Where(location => location.ArtifactObjectId == objectId) - .Join(db.StorageProfileRevision.AsNoTracking(), location => location.StorageProfileRevisionId, revision => revision.Id, - (location, revision) => new PlacedAt(location.ObjectKey, revision.NamespaceFingerprint)) - .Distinct() - .ToListAsync(cancellationToken).ConfigureAwait(false); + var mine = await StoredPlacementsAsync(db, db.ArtifactLocation.AsNoTracking().Where(location => location.ArtifactObjectId == objectId), cancellationToken).ConfigureAwait(false); if (mine.Count == 0) return ArtifactPurgePath.Routed; var keys = mine.Select(placed => placed.ObjectKey).Distinct().ToList(); + var collisions = await StoredPlacementsAsync(db, db.ArtifactLocation.AsNoTracking().Where(location => location.ArtifactObjectId != objectId && keys.Contains(location.ObjectKey)), cancellationToken).ConfigureAwait(false); + + if (collisions.Count == 0) return ArtifactPurgePath.Routed; + + return SharesAnObject(Identify(mine), Identify(collisions)) ? ArtifactPurgePath.RoutedObjectShared : ArtifactPurgePath.Routed; + } - var collisions = await db.ArtifactLocation.AsNoTracking() - .Where(location => location.ArtifactObjectId != objectId && keys.Contains(location.ObjectKey)) + private static async Task> StoredPlacementsAsync(CodeSpaceDbContext db, IQueryable locations, CancellationToken cancellationToken) => + await locations .Join(db.StorageProfileRevision.AsNoTracking(), location => location.StorageProfileRevisionId, revision => revision.Id, - (location, revision) => new PlacedAt(location.ObjectKey, revision.NamespaceFingerprint)) + (location, revision) => new StoredPlacement(location.ObjectKey, revision.ProviderTypeKey, revision.NonSecretConfigJson)) .Distinct() .ToListAsync(cancellationToken).ConfigureAwait(false); - return collisions.Any(mine.Contains) ? ArtifactPurgePath.RoutedObjectShared : ArtifactPurgePath.Routed; - } + /// Each placement's key beside the object it reaches today, derived through the provider that owns it. + private List Identify(List placements) => + placements.Select(placed => new PlacedAt(placed.ObjectKey, StorageProfileRules.TryObjectIdentity(_modules, placed.ProviderTypeKey, placed.NonSecretConfigJson, placed.ObjectKey))).ToList(); + + private static bool SharesAnObject(List mine, List collisions) => collisions.Any(other => mine.Any(own => own.IsOnePhysicalObjectWith(other))); - /// One physical destination: the key, plus the namespace identity that says WHICH store that key is inside. - private sealed record PlacedAt(string ObjectKey, string NamespaceFingerprint); + /// A location's key and its revision's configuration, as stored: the raw material of an object identity, not the identity itself. + private sealed record StoredPlacement(string ObjectKey, string ProviderTypeKey, string NonSecretConfigJson); + + /// + /// One physical destination: the key, plus the identity of the object that key reaches. A null identity is one that + /// could not be derived, and it is treated as every object at that key at once: the only cost of that reading is a + /// kept byte. + /// + private sealed record PlacedAt(string ObjectKey, string? Object) + { + public bool IsOnePhysicalObjectWith(PlacedAt other) => + ObjectKey == other.ObjectKey && (Object == null || other.Object == null || Object == other.Object); + } /// /// Applies the outcome under a proven claim. The COLLECT arm additionally re-asks the reference question inside this @@ -729,6 +760,6 @@ public void Record(SweepSettlement settlement) } internal sealed record ArtifactRetentionReaperServices(DbContextOptions DbOptions, IArtifactReferenceOracle Oracle, - IArtifactBlobBackend Blobs, IArtifactCasPurgeCoordinator RoutedPurge, ILogger Logger); + IArtifactBlobBackend Blobs, IArtifactCasPurgeCoordinator RoutedPurge, IStorageProviderModuleCatalog Modules, ILogger Logger); internal sealed record ArtifactRetentionReaperOptions(int BatchSize, int ClaimSize, int MaxAttempts, TimeSpan LeaseDuration, TimeSpan OperationTimeout, TimeSpan RetryDelay); diff --git a/backend/tests/CodeSpace.E2ETests/Storage/StorageNamespaceSharingE2ETests.cs b/backend/tests/CodeSpace.E2ETests/Storage/StorageNamespaceSharingE2ETests.cs new file mode 100644 index 000000000..77391648d --- /dev/null +++ b/backend/tests/CodeSpace.E2ETests/Storage/StorageNamespaceSharingE2ETests.cs @@ -0,0 +1,443 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Workflows.Artifacts; +using CodeSpace.Core.Services.Workflows.Artifacts.Profiles; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers.Local; +using CodeSpace.Core.Services.Workflows.Artifacts.Retention; +using CodeSpace.E2ETests.Infrastructure; +using CodeSpace.Messages.Artifacts; +using CodeSpace.Messages.Constants; +using CodeSpace.Messages.Enums; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Shouldly; + +namespace CodeSpace.E2ETests.Storage; + +/// +/// Two teams, one physical directory, two spellings of it — and the retention sweep of one team must not delete the +/// other team's live bytes. +/// +/// The shape an audit proved destructive: a victim team's admin points workflow-artifact/v1 at a local root +/// R and stores an artifact; a member of that team who owns a Personal team points their own destination at +/// R/ (or R/., or any other spelling of R) and stores the same bytes, unreferenced. Object keys are +/// content-addressed with no team segment, so both teams' locations name ONE file. The sharing guard compared a hash of +/// the configuration as typed, saw two namespaces, and collected the second team's object — deleting the file the +/// victim's download then answered 502 for. +/// +/// A spelling need not name the victim's root at all. The driver opens <root>/objects/<key> +/// and the operating system follows a link anywhere along it, so a collector root that is a directory of its own, with +/// objects (or a directory below it) linked into the victim's, reaches the same file — and so does a link whose +/// target climbs out through another link (L -> a/..), which the operating system resolves before it +/// climbs. +/// +/// Fidelity: high. Every destination is created over real HTTP through admission, where the namespace +/// fingerprint is derived; bytes land through the production routed write path into the real local-rwx driver on real +/// directories; the real reaper sweeps the real Postgres database; and the victim reads back over real HTTP. The only +/// non-production steps are the two that have no HTTP surface or would take a week: bytes are put through the in-process +/// and seams (no endpoint uploads an artifact; runs +/// do, through these same classes), and the declaration is aged in SQL past its age floor and quarantine window. +/// +[Trait("Category", "E2E")] +[Trait("Surface", "Http")] +public sealed class StorageNamespaceSharingE2ETests : IClassFixture, IDisposable +{ + /// Two sweeps decide any declaration here; the slack absorbs a sweep another class's rows crowd, and a loop that stops progressing fails instead of hanging. + private const int MaxSweeps = 6; + + private readonly TaskLaunchApiFactory _factory; + private readonly List _paths = []; + + public StorageNamespaceSharingE2ETests(TaskLaunchApiFactory factory) { _factory = factory; } + + [Theory] + [InlineData("as-written")] + [InlineData("trailing-slash")] + [InlineData("dot-segment")] + [InlineData("doubled-slash")] + [InlineData("relative")] + [InlineData("symlink")] + [InlineData("dotdot-through-link")] + [InlineData("objects-link")] + [InlineData("nested-link")] + public async Task A_second_team_reaching_the_victims_root_through_any_spelling_cannot_collect_the_victims_bytes(string spelling) + { + if (spelling.Contains("link") && OperatingSystem.IsWindows()) return; + + var world = await SeedTwoTeamsAsync(); + var root = Root(); + var payload = Payload($"victim deliverable {spelling}"); + + await CreateDestinationAsync(world.Victim, root); + var victimArtifactId = await PutRoutedAsync(world.Victim.TeamId, payload); + var spelled = Spell(root, spelling); + await CreateDestinationAsync(world.Collector, spelled); + var collectorArtifactId = await PutDeclaredAsync(world.Collector.TeamId, payload); + var file = ObjectPath(root, payload); + + File.Exists(file).ShouldBeTrue($"the precondition: both teams' writes landed on ONE file at {file}"); + File.Exists(ObjectPath(spelled, payload)).ShouldBeTrue($"the precondition: the operating system opens that file through '{spelled}'"); + + await SweepUntilSettledAsync(collectorArtifactId); + + var declaration = (await DeclarationAsync(collectorArtifactId)).ShouldNotBeNull( + $"the collector's artifact was COLLECTED through '{spelled}', which reaches '{root}' — so the victim's bytes went with it"); + declaration.State.ShouldBe(ArtifactRetentionState.Indeterminate); + declaration.LastErrorCode.ShouldBe("artifact-routed-object-shared", "the kept row must say WHY, not merely survive"); + File.Exists(file).ShouldBeTrue($"the victim's object is gone from disk; `ls {Path.GetDirectoryName(file)}` shows what the sweep left"); + + var download = await DownloadAsync(world.Victim, victimArtifactId); + download.StatusCode.ShouldBe(HttpStatusCode.OK, $"the victim's own artifact must still download after the other team's sweep: {await download.Content.ReadAsStringAsync()}"); + (await download.Content.ReadAsByteArrayAsync()).ShouldBe(payload); + } + + [Fact] + public async Task A_victim_destination_admitted_before_namespaces_were_canonical_is_still_protected() + { + // Every destination already in a database was admitted when the fingerprint hashed the configuration as typed, + // and revisions are immutable, so those stored values never change. A victim whose root was typed 'R/' carries + // that text's hash; a collector admitted now at 'R' carries the directory's. Only a sweep that re-derives both + // namespaces from their configurations sees that they are one directory. + var world = await SeedTwoTeamsAsync(); + var root = Root(); + var payload = Payload("victim deliverable from before canonical admission"); + + await SeedDestinationAdmittedBeforeCanonicalAsync(world.Victim, root + "/"); + var victimArtifactId = await PutRoutedAsync(world.Victim.TeamId, payload); + await CreateDestinationAsync(world.Collector, root); + var collectorArtifactId = await PutDeclaredAsync(world.Collector.TeamId, payload); + + (await StoredFingerprintsAsync(world)).Distinct().Count().ShouldBe(2, "the precondition: the stored fingerprints disagree, so they cannot be what keeps these bytes"); + + await SweepUntilSettledAsync(collectorArtifactId); + + var declaration = (await DeclarationAsync(collectorArtifactId)).ShouldNotBeNull("the sweep trusted the victim's stored fingerprint and collected the shared object"); + declaration.LastErrorCode.ShouldBe("artifact-routed-object-shared"); + + var download = await DownloadAsync(world.Victim, victimArtifactId); + download.StatusCode.ShouldBe(HttpStatusCode.OK, await download.Content.ReadAsStringAsync()); + (await download.Content.ReadAsByteArrayAsync()).ShouldBe(payload); + } + + [Fact] + public async Task A_second_team_in_a_root_of_its_own_still_has_its_unreferenced_bytes_collected() + { + // The positive control: content addressing gives both teams the same key, so if a different directory also + // counted as shared, deduplicated content would never be collected and every theory case above would pass + // for the wrong reason. + var world = await SeedTwoTeamsAsync(); + var victimRoot = Root(); + var collectorRoot = Root(); + var payload = Payload("one payload in two separate directories"); + + await CreateDestinationAsync(world.Victim, victimRoot); + var victimArtifactId = await PutRoutedAsync(world.Victim.TeamId, payload); + await CreateDestinationAsync(world.Collector, collectorRoot + "/"); + var collectorArtifactId = await PutDeclaredAsync(world.Collector.TeamId, payload); + + await SweepUntilSettledAsync(collectorArtifactId); + + (await DeclarationAsync(collectorArtifactId)).ShouldBeNull("the collector's directory is its own, so nothing else reaches its bytes and they are collected"); + File.Exists(ObjectPath(collectorRoot, payload)).ShouldBeFalse("collection removes the bytes the collected row named"); + File.Exists(ObjectPath(victimRoot, payload)).ShouldBeTrue("a different directory holding the same key is a different object"); + + var download = await DownloadAsync(world.Victim, victimArtifactId); + download.StatusCode.ShouldBe(HttpStatusCode.OK, await download.Content.ReadAsStringAsync()); + (await download.Content.ReadAsByteArrayAsync()).ShouldBe(payload); + } + + // ─── World + helpers ───────────────────────────────────────────────────── + + private sealed record Member(Guid UserId, Guid TeamId); + + private sealed record World(Member Victim, Member Collector); + + /// + /// The victim: Owner of a workspace team. The collector: a Viewer of that team — so it can read the team's bytes + /// but has no storage rights there — and Owner of its own Personal team, where it has every storage right. + /// + private async Task SeedTwoTeamsAsync() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var suffix = Guid.NewGuid().ToString("N")[..8]; + var victim = new Member(Guid.NewGuid(), Guid.NewGuid()); + var collector = new Member(Guid.NewGuid(), Guid.NewGuid()); + + db.User.AddRange( + new User { Id = victim.UserId, SecurityStamp = TestToken.SeedStamp, Email = $"namespace-victim-{suffix}@test.local", Name = "Victim", CreatedBy = SystemUsers.SeederId, LastModifiedBy = SystemUsers.SeederId }, + new User { Id = collector.UserId, SecurityStamp = TestToken.SeedStamp, Email = $"namespace-collector-{suffix}@test.local", Name = "Collector", CreatedBy = SystemUsers.SeederId, LastModifiedBy = SystemUsers.SeederId }); + db.Team.AddRange( + new Team { Id = victim.TeamId, Slug = $"namespace-victim-{suffix}", Name = "Victim", Kind = TeamKind.Workspace, CreatedBy = SystemUsers.SeederId, LastModifiedBy = SystemUsers.SeederId }, + new Team { Id = collector.TeamId, Slug = $"namespace-personal-{suffix}", Name = "Personal", Kind = TeamKind.Personal, PersonalForUserId = collector.UserId, CreatedBy = SystemUsers.SeederId, LastModifiedBy = SystemUsers.SeederId }); + db.TeamMembership.AddRange( + new TeamMembership { Id = Guid.NewGuid(), TeamId = victim.TeamId, UserId = victim.UserId, Role = TeamRole.Owner, CreatedBy = SystemUsers.SeederId, LastModifiedBy = SystemUsers.SeederId }, + new TeamMembership { Id = Guid.NewGuid(), TeamId = victim.TeamId, UserId = collector.UserId, Role = TeamRole.Viewer, CreatedBy = SystemUsers.SeederId, LastModifiedBy = SystemUsers.SeederId }, + new TeamMembership { Id = Guid.NewGuid(), TeamId = collector.TeamId, UserId = collector.UserId, Role = TeamRole.Owner, CreatedBy = SystemUsers.SeederId, LastModifiedBy = SystemUsers.SeederId }); + await db.SaveChangesAsync(); + + return new World(victim, collector); + } + + /// The Settings "add destination" call: admits the profile, activates it and routes workflow-artifact/v1 at it in one step. + private async Task CreateDestinationAsync(Member member, string rootPath) + { + var response = await SendAsync(member, HttpMethod.Post, "/api/storage/destinations", new + { + name = $"namespace-{Guid.NewGuid():N}"[..24], + providerTypeKey = "local-rwx/v1", + nonSecretConfig = new { rootPath }, + dataClassTypeKeys = new[] { "workflow-artifact/v1" }, + }); + + response.StatusCode.ShouldBe(HttpStatusCode.OK, $"creating a destination at '{rootPath}': {await response.Content.ReadAsStringAsync()}"); + } + + /// + /// A destination exactly as admission recorded it before namespaces were canonical: its fingerprint is the hash of + /// the configuration as typed. Seeded rather than created, because no endpoint can produce that row any more. + /// + private async Task SeedDestinationAdmittedBeforeCanonicalAsync(Member member, string rootPath) + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var config = JsonSerializer.Serialize(new { rootPath }); + var now = DateTimeOffset.UtcNow; + var profileId = Guid.NewGuid(); + var routeId = Guid.NewGuid(); + using var typed = JsonDocument.Parse(config); + + db.StorageProfile.Add(new StorageProfile + { + Id = profileId, TeamId = member.TeamId, StableName = $"pre-canonical-{profileId:N}", CurrentRevision = 1, + State = StorageProfileState.Active, CreatedDate = now, CreatedBy = member.UserId, LastModifiedDate = now, LastModifiedBy = member.UserId, + Revisions = + { + new StorageProfileRevision + { + Id = Guid.NewGuid(), TeamId = member.TeamId, StorageProfileId = profileId, Revision = 1, ProviderTypeKey = LocalRwxArtifactStorageDriverFactory.TypeKey, + NonSecretConfigJson = config, CredentialRef = null, NamespaceFingerprint = StorageProfileRules.NamespaceFingerprint(LocalRwxArtifactStorageDriverFactory.TypeKey, typed.RootElement), + CreatedDate = now, CreatedBy = member.UserId, + }, + }, + }); + db.StorageRoute.Add(new StorageRoute + { + Id = routeId, TeamId = member.TeamId, DataClassTypeKey = "workflow-artifact/v1", CurrentRevision = 1, + State = StorageRouteState.Draft, CreatedDate = now, CreatedBy = member.UserId, LastModifiedDate = now, LastModifiedBy = member.UserId, + Revisions = + { + new StorageRouteRevision + { + Id = Guid.NewGuid(), TeamId = member.TeamId, StorageRouteId = routeId, Revision = 1, StorageProfileId = profileId, + ProfileRevisionMode = StorageProfileRevisionMode.CurrentAtWrite, PinnedProfileRevision = null, CreatedDate = now, CreatedBy = member.UserId, + }, + }, + }); + await db.SaveChangesAsync(); + await db.Database.ExecuteSqlInterpolatedAsync($"UPDATE storage_route SET state = 'Active' WHERE id = {routeId}"); + } + + private async Task> StoredFingerprintsAsync(World world) + { + using var scope = _factory.Services.CreateScope(); + + return await scope.ServiceProvider.GetRequiredService().StorageProfileRevision.AsNoTracking() + .Where(revision => revision.TeamId == world.Victim.TeamId || revision.TeamId == world.Collector.TeamId) + .Select(revision => revision.NamespaceFingerprint).ToListAsync(); + } + + private async Task PutRoutedAsync(Guid teamId, byte[] payload) + { + using var scope = _factory.Services.CreateScope(); + var artifactId = await scope.ServiceProvider.GetRequiredService().PutAsync(teamId, payload, "application/octet-stream", CancellationToken.None); + + await EnsureRoutedAsync(artifactId); + return artifactId; + } + + /// The declaring write ArtifactManifestStore makes, before its manifest row exists — the unreferenced shape retention collects. + private async Task PutDeclaredAsync(Guid teamId, byte[] payload) + { + using var scope = _factory.Services.CreateScope(); + var write = await scope.ServiceProvider.GetRequiredService().PutDeclaredAsync( + new ArtifactRetentionWriteRequest(teamId, payload, "application/octet-stream", ArtifactRetentionClass.ArtifactManifestContent, "artifact_manifest", Guid.NewGuid()), CancellationToken.None); + + write.Declared.ShouldBeTrue("a first routed write must mint the declaration this test sweeps"); + await EnsureRoutedAsync(write.ArtifactId); + return write.ArtifactId; + } + + private async Task EnsureRoutedAsync(Guid artifactId) + { + using var scope = _factory.Services.CreateScope(); + var row = await scope.ServiceProvider.GetRequiredService().WorkflowArtifact.AsNoTracking().SingleAsync(value => value.Id == artifactId); + + row.CasArtifactObjectId.ShouldNotBeNull("a write that stayed inline or fell back to local disk proves nothing about a routed namespace"); + } + + /// + /// Ages the declaration past its class's age floor, then sweeps until it settles — collected, or kept terminally — + /// re-ageing an opened quarantine between sweeps. + /// + private async Task SweepUntilSettledAsync(Guid artifactId) + { + await AgeAsync(artifactId); + + for (var sweep = 0; sweep < MaxSweeps; sweep++) + { + using (var scope = _factory.Services.CreateScope()) + await scope.ServiceProvider.GetRequiredService().SweepAsync(CancellationToken.None); + + var declaration = await DeclarationAsync(artifactId); + if (declaration == null || declaration.State == ArtifactRetentionState.Indeterminate) return; + + await AgeQuarantineAsync(artifactId); + } + + var stuck = await DeclarationAsync(artifactId); + throw new ShouldAssertException($"declaration {artifactId} did not settle within {MaxSweeps} sweeps (state {stuck?.State}, last error '{stuck?.LastErrorCode}'); " + + $"inspect it with `SELECT * FROM workflow_artifact_retention WHERE artifact_id = '{artifactId}'`"); + } + + private async Task AgeAsync(Guid artifactId) + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + // workflow_artifact rejects UPDATE (migration 0016). The trigger is disabled inside ONE transaction, which is + // what scopes it: DISABLE TRIGGER is a catalog change every session sees until it is re-enabled. + await using (var transaction = await db.Database.BeginTransactionAsync()) + { + await db.Database.ExecuteSqlRawAsync("ALTER TABLE workflow_artifact DISABLE TRIGGER workflow_artifact_enforce_immutability"); + await db.Database.ExecuteSqlInterpolatedAsync($"UPDATE workflow_artifact SET created_at = clock_timestamp() - interval '30 days' WHERE id = {artifactId}"); + await db.Database.ExecuteSqlRawAsync("ALTER TABLE workflow_artifact ENABLE TRIGGER workflow_artifact_enforce_immutability"); + await transaction.CommitAsync(); + } + + await db.Database.ExecuteSqlInterpolatedAsync($""" + UPDATE workflow_artifact_retention + SET declared_at = clock_timestamp() - interval '30 days', next_sweep_at = clock_timestamp() - interval '30 days', last_modified_at = clock_timestamp() + WHERE artifact_id = {artifactId} + """); + } + + private async Task AgeQuarantineAsync(Guid artifactId) + { + using var scope = _factory.Services.CreateScope(); + await scope.ServiceProvider.GetRequiredService().Database.ExecuteSqlInterpolatedAsync($""" + UPDATE workflow_artifact_retention + SET quarantined_at = clock_timestamp() - interval '2 days', next_sweep_at = clock_timestamp() - interval '2 days', last_modified_at = clock_timestamp() + WHERE artifact_id = {artifactId} AND state = 'Quarantined' + """); + } + + private async Task DeclarationAsync(Guid artifactId) + { + using var scope = _factory.Services.CreateScope(); + + return await scope.ServiceProvider.GetRequiredService().WorkflowArtifactRetention.AsNoTracking().SingleOrDefaultAsync(row => row.ArtifactId == artifactId); + } + + private async Task DownloadAsync(Member member, Guid artifactId) => await SendAsync(member, HttpMethod.Get, $"/api/artifacts/{artifactId}"); + + private async Task SendAsync(Member member, HttpMethod method, string path, object? body = null) + { + var request = new HttpRequestMessage(method, path); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", TestToken.Mint(member.UserId, TestToken.SeedStamp)); + request.Headers.Add("X-Team-Id", member.TeamId.ToString()); + if (body != null) request.Content = JsonContent.Create(body); + + return await _factory.CreateClient().SendAsync(request); + } + + /// A real directory of this test's own, GUID-named so repeated and concurrent runs never meet. + private string Root() + { + var root = Path.Combine(Path.GetTempPath(), "codespace-namespace-e2e", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + _paths.Add(root); + return root; + } + + /// The spelling under test; the local-rwx driver opens every one of them as itself. + private string Spell(string root, string spelling) => spelling switch + { + "as-written" => root, + "trailing-slash" => root + "/", + "dot-segment" => root + "/.", + "doubled-slash" => root + "//", + "relative" => Path.GetRelativePath(Environment.CurrentDirectory, root), + "symlink" => LinkTo(root), + "dotdot-through-link" => DotDotThroughLink(root), + "objects-link" => LinkBeneath(root, "objects"), + "nested-link" => LinkBeneath(root, Path.Combine("objects", "workflow-artifacts")), + _ => throw new ArgumentOutOfRangeException(nameof(spelling), spelling, null), + }; + + /// + /// holder/L -> a/.. with holder/a -> root/objects: the operating system follows a before + /// it climbs, so holder/L is root — while collapsing the target as text would call it holder. + /// + private string DotDotThroughLink(string root) + { + var holder = Root(); + Directory.CreateSymbolicLink(Path.Combine(holder, "a"), Path.Combine(root, "objects")); + Directory.CreateSymbolicLink(Path.Combine(holder, "L"), Path.Combine("a", "..")); + return Path.Combine(holder, "L"); + } + + /// A root of the collector's own whose is a link into : the root itself names nothing shared. + private string LinkBeneath(string root, string beneath) + { + var collectorRoot = Root(); + var link = Path.Combine(collectorRoot, beneath); + Directory.CreateDirectory(Path.GetDirectoryName(link)!); + Directory.CreateSymbolicLink(link, Path.Combine(root, beneath)); + return collectorRoot; + } + + private string LinkTo(string root) + { + var link = Path.Combine(Path.GetTempPath(), "codespace-namespace-e2e", $"link-{Guid.NewGuid():N}"); + Directory.CreateSymbolicLink(link, root); + _paths.Add(link); + return link; + } + + /// Where the local-rwx driver keeps these bytes: <root>/objects/workflow-artifacts/{aa}/{bb}/{sha256}, the key with no team in it. + private static string ObjectPath(string root, byte[] payload) + { + var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(payload)); + return Path.Combine(root, "objects", "workflow-artifacts", sha[..2], sha.Substring(2, 2), sha); + } + + /// 32 KiB, past the inline threshold, so the bytes must leave the row for the destination. + private static byte[] Payload(string content) + { + var head = System.Text.Encoding.UTF8.GetBytes($"{content} {Guid.NewGuid():N}\n"); + var bytes = new byte[32 * 1024]; + head.CopyTo(bytes, 0); + for (var i = head.Length; i < bytes.Length; i++) bytes[i] = (byte)('a' + (i % 26)); + + return bytes; + } + + public void Dispose() + { + foreach (var path in Enumerable.Reverse(_paths)) + { + try + { + if (new FileInfo(path).LinkTarget != null) File.Delete(path); + else if (Directory.Exists(path)) Directory.Delete(path, recursive: true); + } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + } + } +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunLogRuntimeTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunLogRuntimeTests.cs index 41fe1898f..6802a55de 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunLogRuntimeTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/AgentRunLogRuntimeTests.cs @@ -11,6 +11,7 @@ using CodeSpace.Core.Services.Workflows.Artifacts.Runtime; using CodeSpace.Core.Settings; using CodeSpace.IntegrationTests.Infrastructure; +using CodeSpace.Messages.Commands.Storage; using CodeSpace.Messages.Constants; using CodeSpace.Messages.Dtos.Agents; using CodeSpace.Messages.Enums; @@ -211,6 +212,40 @@ public async Task First_resolution_creates_one_idempotent_settings_visible_local config.RootElement.GetProperty("rootPath").GetString().ShouldBe(Path.GetFullPath(rootPath)); } + [Fact] + public async Task The_bootstrap_destination_names_its_namespace_exactly_as_settings_admission_names_that_directory() + { + // The bootstrap writes its revision without going through admission, so it must derive the namespace the way + // admission does. Reached through a link here, the deployment root's TEXT differs from the directory an operator + // types into Settings, while the directory itself is one: a fingerprint of the text would name two namespaces. + if (OperatingSystem.IsWindows()) return; + + var world = await SeedWorldAsync(); + var real = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "codespace-agent-log-namespace", Guid.NewGuid().ToString("N"))).FullName; + var link = real + "-link"; + Directory.CreateSymbolicLink(link, real); + + try + { + using var settings = RuntimeSettings.Override(current => current with { ArtifactStoreDirectory = link, ArtifactLocalRwxShared = true }); + using var scope = _fixture.BeginScope(); + + var bootstrap = (await scope.Resolve().ResolveAsync(world.TeamId, CancellationToken.None)).ShouldBeOfType(); + using var typed = JsonDocument.Parse(JsonSerializer.Serialize(new { rootPath = real + "/" })); + var admitted = await scope.Resolve().CreateAsync(world.TeamId, world.ActorId, new CreateStorageProfileCommand { StableName = $"admitted-{Guid.NewGuid():N}", ProviderTypeKey = "local-rwx/v1", NonSecretConfig = typed.RootElement }, CancellationToken.None); + + var bootstrapFingerprint = await scope.Resolve().StorageProfileRevision.AsNoTracking() + .Where(value => value.StorageProfileId == bootstrap.StorageProfileId).Select(value => value.NamespaceFingerprint).SingleAsync(); + + bootstrapFingerprint.ShouldBe(admitted.Revisions.ShouldHaveSingleItem().NamespaceFingerprint, $"'{link}' and '{real}/' are one directory, so the bootstrap and admission must name one namespace"); + } + finally + { + File.Delete(link); + Directory.Delete(real, recursive: true); + } + } + [Fact] public async Task Reserved_profile_collision_is_not_hijacked_and_missing_route_stays_explicit() { diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.Namespaces.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.Namespaces.cs new file mode 100644 index 000000000..1dac29ea6 --- /dev/null +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.Namespaces.cs @@ -0,0 +1,359 @@ +using System.Security.Cryptography; +using System.Text.Json; +using Autofac; +using CodeSpace.Core.Persistence.Db; +using CodeSpace.Core.Persistence.Entities; +using CodeSpace.Core.Services.Workflows.Artifacts.Profiles; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers.Local; +using CodeSpace.Messages.Artifacts; +using Microsoft.EntityFrameworkCore; +using Shouldly; + +namespace CodeSpace.IntegrationTests.Workflows; + +/// +/// The routed sharing guard against REAL namespaces: two teams, real directories, a real file at the shared key, and +/// the real local-rwx driver the purge would delete it through. +/// +/// Every revision here carries the fingerprint pre-canonical admission stored — the hash of the configuration +/// exactly as typed — because that is what every revision already in a database carries, and revisions are +/// immutable. So a guard that read the stored column would see two spellings of one root as two namespaces, and these +/// tests fail; only a guard that re-derives the physical namespace from the configuration at sweep time passes. +/// +/// "Spelling" includes links the collector plants beneath a root it owns outright: the driver opens +/// <root>/objects/<key> and the operating system follows a link anywhere along that path, so only a +/// guard that resolves the object a key reaches — not merely the root a configuration names — sees those as shared. +/// +public sealed partial class ArtifactRetentionReaperFlowTests : IDisposable +{ + private readonly List _namespaceRoots = []; + + [Theory] + [InlineData("as-written")] + [InlineData("trailing-slash")] + [InlineData("dot-segment")] + [InlineData("doubled-slash")] + [InlineData("relative")] + [InlineData("symlink")] + [InlineData("dotdot-through-link")] + [InlineData("objects-link")] + [InlineData("nested-link")] + public async Task A_routed_object_another_team_reaches_through_another_spelling_of_one_root_is_kept(string spelling) + { + if (spelling.Contains("link") && OperatingSystem.IsWindows()) return; + + var victim = await SeedWorldAsync(); + var collector = await SeedWorldAsync(); + var root = NamespaceRoot(); + var bytes = RoutedBytes("one payload two teams both stored"); + var file = PlaceObject(root, bytes); + var spelled = Spell(root, spelling); + + File.Exists(Path.Combine([spelled, "objects", .. ObjectKeyOf(bytes).Split('/')])).ShouldBeTrue($"the precondition: the operating system opens {file} through '{spelled}'"); + + var victimPlacement = await SeedPlacementAsync(victim, bytes, LocalRwxArtifactStorageDriverFactory.TypeKey, RootConfig(root)); + var collectorPlacement = await SeedPlacementAsync(collector, bytes, LocalRwxArtifactStorageDriverFactory.TypeKey, RootConfig(spelled)); + await DeclarePlacementAsync(collector, collectorPlacement.ArtifactId); + + (await StoredFingerprintAsync(victimPlacement) == await StoredFingerprintAsync(collectorPlacement)).ShouldBe(spelling == "as-written", + "the precondition that makes this a regression test: the stored fingerprints disagree for every spelling but the literal one"); + + await SweepToSettlementAsync(collectorPlacement.ArtifactId); + + var declaration = (await DeclarationAsync(collectorPlacement.ArtifactId)).ShouldNotBeNull( + $"the artifact was COLLECTED: '{spelled}' reaches the file '{root}' holds, so its delete removed the victim's bytes"); + declaration.State.ShouldBe(ArtifactRetentionState.Indeterminate); + declaration.LastErrorCode.ShouldBe("artifact-routed-object-shared", "the kept row must say WHY, not merely survive"); + File.Exists(file).ShouldBeTrue($"the victim's object at {file} is the collector's object too; check `ls {Path.GetDirectoryName(file)}`"); + (await File.ReadAllBytesAsync(file)).ShouldBe(bytes); + (await LocationStateAsync(victimPlacement.LocationId)).ShouldBe(ArtifactLocationState.Available); + } + + [Fact] + public async Task A_routed_object_whose_key_another_team_holds_in_a_physically_distinct_root_is_still_collected() + { + // The positive control for the theory above: content addressing gives every team storing these bytes the same + // key, so if distinct directories also counted as shared, deduplicated content would never be collected. + var victim = await SeedWorldAsync(); + var collector = await SeedWorldAsync(); + var victimRoot = NamespaceRoot(); + var collectorRoot = NamespaceRoot(); + var bytes = RoutedBytes("one payload in two separate directories"); + var victimFile = PlaceObject(victimRoot, bytes); + var collectorFile = PlaceObject(collectorRoot, bytes); + + var victimPlacement = await SeedPlacementAsync(victim, bytes, LocalRwxArtifactStorageDriverFactory.TypeKey, RootConfig(victimRoot)); + var collectorPlacement = await SeedPlacementAsync(collector, bytes, LocalRwxArtifactStorageDriverFactory.TypeKey, RootConfig(collectorRoot + "/")); + await DeclarePlacementAsync(collector, collectorPlacement.ArtifactId); + + await SweepToSettlementAsync(collectorPlacement.ArtifactId); + + (await ArtifactExistsAsync(collectorPlacement.ArtifactId)).ShouldBeFalse("the collector's directory is its own, so nothing else reaches these bytes"); + File.Exists(collectorFile).ShouldBeFalse("collection removes the bytes the collected row named"); + File.Exists(victimFile).ShouldBeTrue("a different directory holding the same key is a different object"); + (await LocationStateAsync(victimPlacement.LocationId)).ShouldBe(ArtifactLocationState.Available); + } + + [Fact] + public async Task A_routed_object_whose_key_is_a_link_to_another_teams_file_is_collected_and_only_the_link_goes() + { + // The other edge of following links: a delete unlinks the NAME it is given, so a key that is itself a link + // removes the link and never the file it points at. Counting it as shared would only stall the sweep; proving + // the unlink here is what makes leaving the last component unresolved safe. + if (OperatingSystem.IsWindows()) return; + + var victim = await SeedWorldAsync(); + var collector = await SeedWorldAsync(); + var victimRoot = NamespaceRoot(); + var collectorRoot = NamespaceRoot(); + var bytes = RoutedBytes("one payload one team links to"); + var victimFile = PlaceObject(victimRoot, bytes); + var link = Path.Combine([collectorRoot, "objects", .. ObjectKeyOf(bytes).Split('/')]); + Directory.CreateDirectory(Path.GetDirectoryName(link)!); + File.CreateSymbolicLink(link, victimFile); + + var victimPlacement = await SeedPlacementAsync(victim, bytes, LocalRwxArtifactStorageDriverFactory.TypeKey, RootConfig(victimRoot)); + var collectorPlacement = await SeedPlacementAsync(collector, bytes, LocalRwxArtifactStorageDriverFactory.TypeKey, RootConfig(collectorRoot)); + await DeclarePlacementAsync(collector, collectorPlacement.ArtifactId); + + await SweepToSettlementAsync(collectorPlacement.ArtifactId); + + (await ArtifactExistsAsync(collectorPlacement.ArtifactId)).ShouldBeFalse("the collector's key is a name of its own, so its object is collected"); + new FileInfo(link).LinkTarget.ShouldBeNull("collection removes the link the collected row named"); + File.Exists(victimFile).ShouldBeTrue($"the unlink must not reach through to {victimFile}"); + (await File.ReadAllBytesAsync(victimFile)).ShouldBe(bytes); + (await LocationStateAsync(victimPlacement.LocationId)).ShouldBe(ArtifactLocationState.Available); + } + + [Theory] + [InlineData("retired-provider/v1", """{"rootPath":""}""")] + [InlineData("local-rwx/v1", """{"unexpected":""}""")] + [InlineData("local-rwx/v1", """{"rootPath":42}""")] + public async Task A_key_collision_with_a_revision_whose_namespace_cannot_be_derived_is_kept(string providerTypeKey, string neighbourConfig) + { + // A provider this build no longer registers, or a configuration it can no longer read, names SOME namespace — + // this build just cannot say which. Reading that as "a different one" would delete on a guess. + var neighbour = await SeedWorldAsync(); + var collector = await SeedWorldAsync(); + var root = NamespaceRoot(); + var bytes = RoutedBytes($"one payload beside an unreadable revision {providerTypeKey} {neighbourConfig}"); + var file = PlaceObject(root, bytes); + + await SeedPlacementAsync(neighbour, bytes, providerTypeKey, neighbourConfig.Replace("", JsonEncodedText.Encode(root).ToString())); + var collectorPlacement = await SeedPlacementAsync(collector, bytes, LocalRwxArtifactStorageDriverFactory.TypeKey, RootConfig(root)); + await DeclarePlacementAsync(collector, collectorPlacement.ArtifactId); + + await SweepToSettlementAsync(collectorPlacement.ArtifactId); + + var declaration = (await DeclarationAsync(collectorPlacement.ArtifactId)).ShouldNotBeNull("an unknown namespace was read as a different one and the object was collected"); + declaration.State.ShouldBe(ArtifactRetentionState.Indeterminate); + declaration.LastErrorCode.ShouldBe("artifact-routed-object-shared"); + File.Exists(file).ShouldBeTrue(); + } + + // ─── Namespace helpers ─────────────────────────────────────────────────── + + private sealed record SeededPlacement(Guid ArtifactId, Guid LocationId, Guid RevisionId); + + /// Ages the declaration past both waits and sweeps twice: the first sweep quarantines, the second decides. + private async Task SweepToSettlementAsync(Guid artifactId) + { + await AgeDeclarationAsync(artifactId, TimeSpan.FromDays(30)); + await SweepAsync(); + await AgeQuarantineAsync(artifactId, TimeSpan.FromDays(2)); + await SweepAsync(); + } + + /// A real directory of this test's own, removed on dispose. + private string NamespaceRoot() + { + var root = Path.Combine(Path.GetTempPath(), "codespace-namespace-reaper", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + _namespaceRoots.Add(root); + return root; + } + + /// The spelling under test, each of which the local-rwx driver opens as itself. + private string Spell(string root, string spelling) => spelling switch + { + "as-written" => root, + "trailing-slash" => root + "/", + "dot-segment" => root + "/.", + "doubled-slash" => root + "//", + "relative" => Path.GetRelativePath(Environment.CurrentDirectory, root), + "symlink" => LinkTo(root), + "dotdot-through-link" => DotDotThroughLink(root), + "objects-link" => LinkBeneath(root, "objects"), + "nested-link" => LinkBeneath(root, Path.Combine("objects", "workflow-artifacts")), + _ => throw new ArgumentOutOfRangeException(nameof(spelling), spelling, null), + }; + + /// + /// holder/L -> a/.. with holder/a -> root/objects: the operating system follows a before + /// it climbs, so holder/L is root — while collapsing the target as text would call it holder. + /// + private string DotDotThroughLink(string root) + { + var holder = NamespaceRoot(); + Directory.CreateSymbolicLink(Path.Combine(holder, "a"), Path.Combine(root, "objects")); + Directory.CreateSymbolicLink(Path.Combine(holder, "L"), Path.Combine("a", "..")); + return Path.Combine(holder, "L"); + } + + /// A root of the collector's own whose is a link into : the root itself names nothing shared. + private string LinkBeneath(string root, string beneath) + { + var collectorRoot = NamespaceRoot(); + var link = Path.Combine(collectorRoot, beneath); + Directory.CreateDirectory(Path.GetDirectoryName(link)!); + Directory.CreateSymbolicLink(link, Path.Combine(root, beneath)); + return collectorRoot; + } + + private string LinkTo(string root) + { + var link = Path.Combine(Path.GetTempPath(), "codespace-namespace-reaper", $"link-{Guid.NewGuid():N}"); + Directory.CreateSymbolicLink(link, root); + _namespaceRoots.Add(link); + return link; + } + + /// Content unique to the call, past nothing in particular: a routed row's placement is what is under test, not its size. + private static byte[] RoutedBytes(string content) => System.Text.Encoding.UTF8.GetBytes($"{content} {Guid.NewGuid():N}"); + + /// The key production gives these bytes (ArtifactStore.ObjectKeyFor): content-addressed, with no team segment. + private static string ObjectKeyOf(byte[] bytes) + { + var sha = Convert.ToHexStringLower(SHA256.HashData(bytes)); + return $"workflow-artifacts/{sha[..2]}/{sha.Substring(2, 2)}/{sha}"; + } + + /// Writes the object where the local-rwx driver keeps it: <root>/objects/<key>. + private static string PlaceObject(string root, byte[] bytes) + { + var path = Path.Combine([root, "objects", .. ObjectKeyOf(bytes).Split('/')]); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllBytes(path, bytes); + return path; + } + + private static string RootConfig(string rootPath) => JsonSerializer.Serialize(new { rootPath }); + + /// + /// What admission stored before namespaces were canonical: the hash of the configuration as typed. Every revision + /// already in a database carries this, so it is the fixture the sweep has to be right about. + /// + private static string StoredBeforeCanonicalAdmission(string providerTypeKey, string config) + { + using var document = JsonDocument.Parse(config); + return StorageProfileRules.NamespaceFingerprint(providerTypeKey, document.RootElement); + } + + /// + /// One team's routed artifact: an Active profile revision, the team's own artifact_object for the digest, the + /// pointing row, and an Available location at the production key. Each team owns its own object row for the same + /// bytes, exactly as two independent routed writes produce. + /// + private async Task SeedPlacementAsync(World world, byte[] bytes, string providerTypeKey, string config) + { + using var scope = _fixture.BeginScope(); + var db = scope.Resolve(); + var now = DateTimeOffset.UtcNow; + var digest = SHA256.HashData(bytes); + var profileId = Guid.NewGuid(); + var revisionId = Guid.NewGuid(); + var objectId = Guid.NewGuid(); + var artifactId = Guid.NewGuid(); + var locationId = Guid.NewGuid(); + var key = ObjectKeyOf(bytes); + + db.StorageProfile.Add(new StorageProfile + { + Id = profileId, TeamId = world.TeamId, StableName = $"namespace-probe-{profileId:N}", CurrentRevision = 1, + State = StorageProfileState.Active, CreatedDate = now, CreatedBy = world.ActorId, LastModifiedDate = now, LastModifiedBy = world.ActorId, + Revisions = + { + new StorageProfileRevision + { + Id = revisionId, TeamId = world.TeamId, StorageProfileId = profileId, Revision = 1, ProviderTypeKey = providerTypeKey, + NonSecretConfigJson = config, CredentialRef = null, NamespaceFingerprint = StoredBeforeCanonicalAdmission(providerTypeKey, config), + CreatedDate = now, CreatedBy = world.ActorId, + }, + }, + }); + db.ArtifactObject.Add(new ArtifactObject + { + Id = objectId, TeamId = world.TeamId, DigestAlgorithm = ArtifactDigestAlgorithm.Sha256, Digest = digest, + SizeBytes = bytes.Length, CreatedDate = now, CreatedBy = world.ActorId, + }); + db.WorkflowArtifact.Add(new WorkflowArtifact + { + Id = artifactId, TeamId = world.TeamId, Sha256 = Convert.ToHexStringLower(digest), ContentType = "application/octet-stream", + SizeBytes = bytes.Length, InlineBytes = null, StorageUrl = null, CasArtifactObjectId = objectId, CreatedAt = now, + }); + await db.SaveChangesAsync(); + + // The location and its event go in ONE SaveChanges: every location revision needs its byte-identical event. + db.ArtifactLocationEvent.Add(new ArtifactLocationEvent + { + Id = Guid.NewGuid(), TeamId = world.TeamId, ArtifactLocationId = locationId, Revision = 1, + EventType = ArtifactLocationEventType.Created, State = ArtifactLocationState.Available, ObservedAt = now, + ProviderChecksumAlgorithm = "Sha256", ProviderChecksum = digest, ObservedSizeBytes = bytes.Length, VerifiedAt = now, CreatedBy = world.ActorId, + }); + db.ArtifactLocation.Add(new ArtifactLocation + { + Id = locationId, TeamId = world.TeamId, ArtifactObjectId = objectId, StorageProfileRevisionId = revisionId, + Locator = key, ObjectKey = key, State = ArtifactLocationState.Available, Revision = 1, + ObservedSizeBytes = bytes.Length, ProviderChecksumAlgorithm = "Sha256", ProviderChecksum = digest, VerifiedAt = now, + CreatedDate = now, CreatedBy = world.ActorId, LastModifiedDate = now, LastModifiedBy = world.ActorId, + }); + await db.SaveChangesAsync(); + + return new SeededPlacement(artifactId, locationId, revisionId); + } + + /// The declaration IArtifactRetentionWriter mints on a first routed write, planted beside a seeded row. + private async Task DeclarePlacementAsync(World world, Guid artifactId) + { + using var scope = _fixture.BeginScope(); + var db = scope.Resolve(); + var now = DateTimeOffset.UtcNow; + db.WorkflowArtifactRetention.Add(new WorkflowArtifactRetention + { + ArtifactId = artifactId, TeamId = world.TeamId, RetentionClass = nameof(ArtifactRetentionClass.ArtifactManifestContent), + HolderKind = "artifact_manifest", HolderId = world.AgentRunId, State = ArtifactRetentionState.Declared, + DeclaredAt = now, NextSweepAt = now, Revision = 1, LastModifiedAt = now, + }); + await db.SaveChangesAsync(); + } + + private async Task StoredFingerprintAsync(SeededPlacement placement) + { + using var scope = _fixture.BeginScope(); + + return await scope.Resolve().StorageProfileRevision.AsNoTracking() + .Where(revision => revision.Id == placement.RevisionId).Select(revision => revision.NamespaceFingerprint).SingleAsync(); + } + + private async Task LocationStateAsync(Guid locationId) + { + using var scope = _fixture.BeginScope(); + + return await scope.Resolve().ArtifactLocation.AsNoTracking() + .Where(location => location.Id == locationId).Select(location => location.State).SingleAsync(); + } + + public void Dispose() + { + foreach (var path in Enumerable.Reverse(_namespaceRoots)) + { + try + { + if (new FileInfo(path).LinkTarget != null) File.Delete(path); + else if (Directory.Exists(path)) Directory.Delete(path, recursive: true); + } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + } + } +} diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.cs index 1e163ef79..6226896c6 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/ArtifactRetentionReaperFlowTests.cs @@ -14,6 +14,7 @@ using Shouldly; using CodeSpace.Core.Services.Workflows.Artifacts.Providers.Local; using CodeSpace.Core.Services.Workflows.Artifacts.Profiles; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers; using System.Text.Json; namespace CodeSpace.IntegrationTests.Workflows; @@ -29,7 +30,7 @@ namespace CodeSpace.IntegrationTests.Workflows; /// [Collection(PostgresCollection.Name)] [Trait("Category", "Integration")] -public sealed class ArtifactRetentionReaperFlowTests +public sealed partial class ArtifactRetentionReaperFlowTests { private readonly PostgresFixture _fixture; @@ -670,7 +671,7 @@ private async Task SweepWithRefusedPurgeAsync() { using var scope = _fixture.BeginScope(); var reaper = new ArtifactRetentionReaper(scope.Resolve>(), scope.Resolve(), - new RefusingPurgeBackend(scope.Resolve()), scope.Resolve(), NullLogger.Instance); + new RefusingPurgeBackend(scope.Resolve()), scope.Resolve(), scope.Resolve(), NullLogger.Instance); return await reaper.SweepAsync(CancellationToken.None); } diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/ArtifactStoreRoutedDestinationFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/ArtifactStoreRoutedDestinationFlowTests.cs index 8de3290c3..e80b2faf9 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/ArtifactStoreRoutedDestinationFlowTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/ArtifactStoreRoutedDestinationFlowTests.cs @@ -838,7 +838,7 @@ public async Task A_reference_committed_after_the_location_claim_is_seen_before_ var routed = new ReferenceAfterClaimPurgeCoordinator(sweepScope.Resolve(), token => ReferenceFromAgentRunAsync(agentRunId, artifactId, token)); var reaper = new ArtifactRetentionReaper(sweepScope.Resolve>(), sweepScope.Resolve(), - sweepScope.Resolve(), routed, NullLogger.Instance); + sweepScope.Resolve(), routed, sweepScope.Resolve(), NullLogger.Instance); summary = await reaper.SweepAsync(CancellationToken.None); } @@ -923,7 +923,7 @@ public async Task An_uncertain_provider_delete_stays_live_and_the_next_sweep_rec { var routed = new UncertainDeletePurgeCoordinator(uncertainScope.Resolve()); var reaper = new ArtifactRetentionReaper(uncertainScope.Resolve>(), uncertainScope.Resolve(), - uncertainScope.Resolve(), routed, NullLogger.Instance); + uncertainScope.Resolve(), routed, uncertainScope.Resolve(), NullLogger.Instance); (await reaper.SweepAsync(CancellationToken.None)).Retried.ShouldBeGreaterThanOrEqualTo(1); routed.ReleaseCalls.ShouldBe(0, "an effect-uncertain result must never be relabeled Available"); } @@ -1549,7 +1549,7 @@ private async Task OrphanThePhysicalClaimAsync(Guid teamId, Guid objectId, /// The shipped reaper over a substituted purge coordinator, with a short attempt budget so exhausting it is two sweeps rather than eight. private static ArtifactRetentionReaper BudgetedReaper(ILifetimeScope scope, IArtifactCasPurgeCoordinator routed, int attempts) => new(new ArtifactRetentionReaperServices(scope.Resolve>(), scope.Resolve(), - scope.Resolve(), routed, NullLogger.Instance), + scope.Resolve(), routed, scope.Resolve(), NullLogger.Instance), new ArtifactRetentionReaperOptions(BatchSize: 200, ClaimSize: 25, MaxAttempts: attempts, LeaseDuration: TimeSpan.FromSeconds(60), OperationTimeout: TimeSpan.FromSeconds(15), RetryDelay: TimeSpan.FromMinutes(30))); diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/LegacyPlacementAdoptionTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/LegacyPlacementAdoptionTests.cs index 2ab85bb0d..a0f12132b 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/LegacyPlacementAdoptionTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/LegacyPlacementAdoptionTests.cs @@ -121,7 +121,7 @@ private async Task RetentionClosesSidecarAsync() var blobs = new CountingBlobBackend(scope.Resolve(), originalUrl); var routed = new TrackingCasPurgeCoordinator(scope.Resolve(), placement.ArtifactObjectId); var reaper = new ArtifactRetentionReaper(scope.Resolve>(), scope.Resolve(), - blobs, routed, NullLogger.Instance); + blobs, routed, scope.Resolve(), NullLogger.Instance); await SweepUntilAsync(reaper, retained.ArtifactId, ArtifactRetentionState.Quarantined); await AgeQuarantineAsync(retained.ArtifactId, TimeSpan.FromDays(2)); @@ -645,7 +645,7 @@ private async Task RetentionRaceCoreAsync() { var backend = new BlockingAfterDeleteBlobBackend(reaperScope.Resolve(), deletion); var reaper = new ArtifactRetentionReaper(reaperScope.Resolve>(), reaperScope.Resolve(), - backend, reaperScope.Resolve(), NullLogger.Instance); + backend, reaperScope.Resolve(), reaperScope.Resolve(), NullLogger.Instance); sweep = reaper.SweepAsync(CancellationToken.None); await deletion.Started.Task.WaitAsync(TimeSpan.FromSeconds(10)); File.Exists(new Uri(retained.StorageUrl).LocalPath).ShouldBeFalse("the reaper has removed bytes while its retention UPDATE lock is still held"); @@ -760,7 +760,7 @@ private async Task WitnessRetentionRaceCoreAsync() { var backend = new BlockingAfterDeleteBlobBackend(reaperScope.Resolve(), deletion); var reaper = new ArtifactRetentionReaper(reaperScope.Resolve>(), reaperScope.Resolve(), - backend, reaperScope.Resolve(), NullLogger.Instance); + backend, reaperScope.Resolve(), reaperScope.Resolve(), NullLogger.Instance); sweep = reaper.SweepAsync(CancellationToken.None); await deletion.Started.Task.WaitAsync(TimeSpan.FromSeconds(10)); File.Exists(new Uri(retained.StorageUrl).LocalPath).ShouldBeFalse("retention removed the behind-cursor witness while its declaration lock remains held"); diff --git a/backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/Runtime/ArtifactLocationVerifierFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/Runtime/ArtifactLocationVerifierFlowTests.cs index b4e04b9df..d14b8daf6 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/Runtime/ArtifactLocationVerifierFlowTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Workflows/Artifacts/Runtime/ArtifactLocationVerifierFlowTests.cs @@ -42,10 +42,8 @@ public async Task An_object_that_is_still_there_is_confirmed_and_its_verified_at var world = await SeedStoredArtifactAsync(); var before = (await LocationAsync(world)).VerifiedAt.ShouldNotBeNull(); - var summary = await VerifyAsync(); + var location = await VerifyUntilAsync(world, seen => seen.VerifiedAt > before, "re-verified"); - summary.Checked.ShouldBeGreaterThanOrEqualTo(1); - var location = await LocationAsync(world); location.State.ShouldBe(ArtifactLocationState.Available); location.VerifiedAt.ShouldNotBeNull().ShouldBeGreaterThan(before); } @@ -58,9 +56,8 @@ public async Task An_object_the_destination_no_longer_holds_is_marked_missing_an var world = await SeedStoredArtifactAsync(); DeleteStoredObject(world); - await VerifyAsync(); + var location = await VerifyUntilAsync(world, ArtifactLocationState.Missing); - var location = await LocationAsync(world); location.State.ShouldBe(ArtifactLocationState.Missing); location.LastErrorCode.ShouldBe("location-object-missing"); location.LastErrorMessage.ShouldNotBeNullOrWhiteSpace("a demoted location must say what the destination answered"); @@ -79,9 +76,8 @@ public async Task An_object_replaced_by_something_else_is_marked_corrupt_rather_ var world = await SeedStoredArtifactAsync(); OverwriteStoredObject(world, "a different object entirely"); - await VerifyAsync(); + var location = await VerifyUntilAsync(world, ArtifactLocationState.Corrupt); - var location = await LocationAsync(world); location.State.ShouldBe(ArtifactLocationState.Corrupt); location.LastErrorCode.ShouldBe("location-object-mismatch"); } @@ -133,8 +129,7 @@ public async Task A_location_wrongly_demoted_by_a_destination_fault_comes_back_w // Moved rather than rewritten, so the object comes back byte-identical AND metadata-identical — the shape of a // destination that was briefly serving an incomplete view of itself, not of someone editing the bytes. File.Move(path, stashed); - await VerifyAsync(); - (await LocationAsync(world)).State.ShouldBe(ArtifactLocationState.Missing); + await VerifyUntilAsync(world, ArtifactLocationState.Missing); File.Move(stashed, path); @@ -155,9 +150,7 @@ public async Task A_corrupt_location_is_not_swept_back_into_circulation() // Missing it carries no risk of being a false demotion, and re-reading it would only invite flapping. var world = await SeedStoredArtifactAsync(); OverwriteStoredObject(world, "a different object entirely"); - await VerifyAsync(); - (await LocationAsync(world)).State.ShouldBe(ArtifactLocationState.Corrupt); - var demoted = await LocationAsync(world); + var demoted = await VerifyUntilAsync(world, ArtifactLocationState.Corrupt); await VerifyAsync(); @@ -351,30 +344,45 @@ await scope.Resolve().Database.ExecuteSqlInterpolatedAsync( /// would be asserting that this row won a race it has no reason to win — and would get harder to satisfy as the /// suite grows. /// - private async Task VerifyUntilAsync(StoredArtifact world, ArtifactLocationState state, TimeSpan? within = null) + private Task VerifyUntilAsync(StoredArtifact world, ArtifactLocationState state, TimeSpan? within = null) => VerifyUntilAsync(world, seen => seen.State == state, $"in state {state}", within); + + /// + /// Sweeps until THIS test's location satisfies . A row written moments ago is freshly + /// verified, so every location an earlier test left Available sorts ahead of it; one pass reaches it only while + /// fewer of those exist than the batch holds, which is a property of the suite, not of the verifier. + /// + private async Task VerifyUntilAsync(StoredArtifact world, Func settled, string expectation, TimeSpan? within = null) { var deadline = DateTimeOffset.UtcNow + (within ?? TimeSpan.FromSeconds(10)); ArtifactLocation? seen = null; + ArtifactLocationVerificationSummary? pass = null; while (DateTimeOffset.UtcNow < deadline) { - await VerifyAsync(); + pass = await VerifyAsync(); seen = await LocationAsync(world); - if (seen.State == state) return seen; + if (settled(seen)) return seen; } throw new Xunit.Sdk.XunitException( - $"The location for artifact {world.ArtifactId} never reached {state} (last seen {seen?.State.ToString() ?? "absent"}, " - + $"last error {seen?.LastErrorCode ?? "none"}). The sweep is deployment-wide and its recovery share is bounded, " - + "so check whether earlier tests left more Missing rows than that share holds."); + $"The location for artifact {world.ArtifactId} was never {expectation} (last seen {seen?.State.ToString() ?? "absent"}, " + + $"verified at {seen?.VerifiedAt?.ToString("O") ?? "never"}, last error {seen?.LastErrorCode ?? "none"}). The last pass checked " + + $"{pass?.Checked}: {pass?.Confirmed} confirmed, {pass?.Inconclusive} inconclusive, {pass?.Skipped} skipped. The sweep is " + + "deployment-wide and bounded, so check whether earlier tests left more rows ahead of this one than a batch holds."); } + /// + /// One pass at the verifier's own ceiling. A freshly written row sorts behind every row an earlier test left + /// Available, and a row whose destination that test deleted on cleanup never answers, so it never moves and holds + /// its place ahead of this one in every pass. The suite leaves dozens of those behind; a pass smaller than that + /// never reaches this test's row however often it runs. + /// private async Task VerifyAsync() { using var scope = _fixture.BeginScope(); - return await scope.Resolve().VerifyStaleAsync(100, CancellationToken.None); + return await scope.Resolve().VerifyStaleAsync(500, CancellationToken.None); } /// Runs the real verifier with an observer around only this test's exact destination. diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/ArtifactRetentionPolicyTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/ArtifactRetentionPolicyTests.cs index 56b8eba91..cf3719bb6 100644 --- a/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/ArtifactRetentionPolicyTests.cs +++ b/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/ArtifactRetentionPolicyTests.cs @@ -5,6 +5,7 @@ using CodeSpace.Core.Services.Agents.Publish; using CodeSpace.Core.Services.Workflows.Artifacts; using CodeSpace.Core.Services.Workflows.Artifacts.Backends; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers; using CodeSpace.Core.Services.Workflows.Artifacts.Retention; using CodeSpace.Core.Services.Workflows.Artifacts.Runtime; using CodeSpace.Core.Services.Workflows.Runtime; @@ -67,13 +68,13 @@ public void An_unbounded_reaper_configuration_is_rejected_at_construction(int ba var options = new ArtifactRetentionReaperOptions(batch, claim, attempts, TimeSpan.FromSeconds(leaseSeconds), TimeSpan.FromSeconds(operationSeconds), TimeSpan.FromMinutes(retryMinutes)); Should.Throw(() => new ArtifactRetentionReaper(new ArtifactRetentionReaperServices( - DbOptions(), Oracle(), Blobs(), Purge(), NullLogger.Instance), options)); + DbOptions(), Oracle(), Blobs(), Purge(), Modules(), NullLogger.Instance), options)); } [Fact] public void The_shipped_reaper_configuration_is_accepted() { - Should.NotThrow(() => new ArtifactRetentionReaper(DbOptions(), Oracle(), Blobs(), Purge(), NullLogger.Instance)); + Should.NotThrow(() => new ArtifactRetentionReaper(DbOptions(), Oracle(), Blobs(), Purge(), Modules(), NullLogger.Instance)); } [Fact] @@ -172,6 +173,8 @@ private static DbContextOptions DbOptions() => private static IArtifactCasPurgeCoordinator Purge() => new RefusingCasPurgeCoordinator(); + private static IStorageProviderModuleCatalog Modules() => new StorageProviderModuleCatalog([]); + private static string FindRepoRoot() { var directory = new DirectoryInfo(AppContext.BaseDirectory); diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Profiles/StorageNamespaceIdentityTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Profiles/StorageNamespaceIdentityTests.cs new file mode 100644 index 000000000..e82a1f65b --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Profiles/StorageNamespaceIdentityTests.cs @@ -0,0 +1,130 @@ +using System.Text.Json; +using Autofac; +using CodeSpace.Core; +using CodeSpace.Core.Services.Workflows.Artifacts.Profiles; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers.AliyunOss; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers.Local; +using Microsoft.Extensions.Configuration; +using Serilog; +using Shouldly; + +namespace CodeSpace.UnitTests.Workflows.Artifacts.Profiles; + +/// +/// How the object a stored placement reaches is identified when something has to COMPARE two of them — the retention +/// reaper deciding whether one object key, under two teams' profile revisions, is one physical object. +/// +/// The rule under test is the direction of every failure. An identity that cannot be derived is not "different +/// from everything": it is unknown, and the caller must read unknown as "possibly the same", because the only cost of +/// that reading is a kept byte and the cost of the other is another team's deleted one. +/// +[Trait("Category", "Unit")] +public sealed class StorageNamespaceIdentityTests +{ + private static readonly IStorageProviderModuleCatalog Catalog = new StorageProviderModuleCatalog([new LocalRwxStorageProviderModule(), new AliyunOssStorageProviderModule(), new ThrowingProjectionModule(), new ScalarProjectionModule()]); + + private const string Key = "workflow-artifacts/ab/cd/abcd0000000000000000000000000000000000000000000000000000000000ff"; + + [Fact] + public void A_filesystem_revision_derives_the_physical_object_its_key_reaches() + { + using var config = JsonDocument.Parse("""{"rootPath":"/srv/codespace/artifacts/"}"""); + + StorageProfileRules.TryObjectIdentity(Catalog, "local-rwx/v1", config.RootElement.GetRawText(), Key) + .ShouldBe(new LocalRwxStorageProviderModule().PhysicalObject(config.RootElement, Key)); + } + + [Fact] + public void A_revision_whose_provider_cannot_name_the_object_is_identified_by_its_namespace_and_the_key() + { + // An object store's key IS the object once the bucket and prefix are known, so the namespace admission + // fingerprints and the key together name it. + const string config = """{"endpoint":"oss-cn-hangzhou.aliyuncs.com","bucket":"codespace-artifacts","keyPrefix":"team-7/"}"""; + using var document = JsonDocument.Parse(config); + + StorageProfileRules.TryObjectIdentity(Catalog, "aliyun-oss/v1", config, Key) + .ShouldBe($"{StorageProfileRules.NamespaceFingerprint(new AliyunOssStorageProviderModule(), document.RootElement)}/{Key}"); + } + + [Fact] + public void Two_revisions_that_spell_one_root_differently_derive_one_identity() => + StorageProfileRules.TryObjectIdentity(Catalog, "local-rwx/v1", """{"rootPath":"/srv/codespace/artifacts/."}""", Key) + .ShouldBe(StorageProfileRules.TryObjectIdentity(Catalog, "local-rwx/v1", """{"rootPath":"/srv/codespace/artifacts"}""", Key)); + + [Theory] + [InlineData("retired-provider/v1", """{"rootPath":"/srv/codespace/artifacts"}""", Key)] + [InlineData("local-rwx/v1", """{"rootPath":""", Key)] + [InlineData("local-rwx/v1", """{"unexpected":"/srv/codespace/artifacts"}""", Key)] + [InlineData("local-rwx/v1", """{"rootPath":42}""", Key)] + [InlineData("local-rwx/v1", """{"rootPath":"/srv/codespace/artifacts"}""", "workflow-artifacts/../../escape")] + [InlineData("aliyun-oss/v1", """{"endpoint":"oss-accelerate.aliyuncs.com","bucket":"codespace-artifacts"}""", Key)] + [InlineData("throwing-projection/v1", """{}""", Key)] + [InlineData("scalar-projection/v1", """{}""", Key)] + [InlineData("", """{}""", Key)] + public void A_revision_whose_identity_cannot_be_derived_has_none_rather_than_an_invented_one(string providerTypeKey, string config, string objectKey) => + StorageProfileRules.TryObjectIdentity(Catalog, providerTypeKey, config, objectKey).ShouldBeNull(); + + [Fact] + public void Every_shipped_storage_module_owns_its_namespace_projection() + { + // The interface's default projection is the raw configuration, which is exactly the comparison that let two + // spellings of one root read as two namespaces. A provider that ships without its own projection inherits that + // defect, so adding one is required to be a decision rather than an omission. + using var container = Container(); + + var inheriting = container.Resolve().Modules + .Where(module => ProjectionOwner(module) == typeof(IStorageProviderModule)) + .Select(module => module.TypeKey) + .ToList(); + + inheriting.ShouldBeEmpty("these modules compare namespaces by the text an operator typed, not the place it reaches"); + } + + private static Type ProjectionOwner(IStorageProviderModule module) + { + var map = module.GetType().GetInterfaceMap(typeof(IStorageProviderModule)); + var slot = Array.FindIndex(map.InterfaceMethods, method => method.Name == nameof(IStorageProviderModule.GetNamespaceConfiguration)); + + return map.TargetMethods[slot].DeclaringType!; + } + + private static IContainer Container() + { + var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + ["ASPNETCORE_ENVIRONMENT"] = "Development", + ["CodeSpaceStore:ConnectionString"] = "Host=unused;Database=unused" + }).Build(); + var builder = new ContainerBuilder(); + builder.RegisterModule(new CodeSpaceModule(new LoggerConfiguration().CreateLogger(), configuration)); + return builder.Build(); + } + + /// A provider whose projection fails for a reason no caller can name in advance. + private sealed class ThrowingProjectionModule : TestModule + { + public override string TypeKey => "throwing-projection/v1"; + public override JsonElement GetNamespaceConfiguration(JsonElement nonSecretConfiguration) => throw new IOException("the namespace could not be inspected"); + } + + /// A provider whose projection breaks the contract's shape: an identity must be an object. + private sealed class ScalarProjectionModule : TestModule + { + public override string TypeKey => "scalar-projection/v1"; + public override JsonElement GetNamespaceConfiguration(JsonElement nonSecretConfiguration) => JsonDocument.Parse("\"flat\"").RootElement.Clone(); + } + + private abstract class TestModule : IStorageProviderModule + { + private static readonly JsonElement Schema = JsonDocument.Parse("""{"type":"object"}""").RootElement.Clone(); + + public abstract string TypeKey { get; } + public string DisplayName => TypeKey; + public JsonElement ConfigSchema => Schema; + public JsonElement SecretSchema => Schema; + public StorageProviderCapabilities Capabilities => StorageProviderCapabilities.StreamingRead; + public Type FactoryType => typeof(LocalRwxArtifactStorageDriverFactory); + public abstract JsonElement GetNamespaceConfiguration(JsonElement nonSecretConfiguration); + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Providers/AliyunOss/AliyunOssStorageNamespaceTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Providers/AliyunOss/AliyunOssStorageNamespaceTests.cs new file mode 100644 index 000000000..9507a774f --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Providers/AliyunOss/AliyunOssStorageNamespaceTests.cs @@ -0,0 +1,90 @@ +using System.Text.Json; +using CodeSpace.Core.Services.Workflows.Artifacts.Profiles; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers.AliyunOss; +using Shouldly; + +namespace CodeSpace.UnitTests.Workflows.Artifacts.Providers.AliyunOss; + +/// +/// The namespace identity of an OSS destination is the bucket and key prefix it reaches, not the route it reaches them +/// by. The public host, the VPC host and the accelerate host of one region all address the same bucket, so two profiles +/// that spell the route differently still share every object key — and object keys carry no team segment. +/// +[Trait("Category", "Unit")] +public sealed class AliyunOssStorageNamespaceTests +{ + private const string Canonical = """{"bucket":"codespace-artifacts","endpoint":"oss-cn-hangzhou.aliyuncs.com","keyPrefix":"team-7/"}"""; + + [Theory] + [InlineData("""{"endpoint":"oss-cn-hangzhou.aliyuncs.com","bucket":"codespace-artifacts","keyPrefix":"team-7/"}""")] + [InlineData("""{"endpoint":"https://oss-cn-hangzhou.aliyuncs.com","bucket":"codespace-artifacts","keyPrefix":"team-7/"}""")] + [InlineData("""{"endpoint":"oss-cn-hangzhou-internal.aliyuncs.com","bucket":"codespace-artifacts","keyPrefix":"team-7/"}""")] + [InlineData("""{"endpoint":"https://oss-cn-hangzhou-internal.aliyuncs.com","bucket":"codespace-artifacts","keyPrefix":"team-7/"}""")] + [InlineData("""{"endpoint":"oss-accelerate.aliyuncs.com","region":"cn-hangzhou","bucket":"codespace-artifacts","keyPrefix":"team-7/"}""")] + [InlineData("""{"endpoint":"oss-accelerate-overseas.aliyuncs.com","region":"cn-hangzhou","bucket":"codespace-artifacts","keyPrefix":"team-7/"}""")] + [InlineData("""{"endpoint":"oss-cn-hangzhou.aliyuncs.com","region":"cn-hangzhou","bucket":"codespace-artifacts","keyPrefix":"team-7/"}""")] + public void Every_route_to_one_bucket_and_prefix_projects_to_one_namespace(string spelling) => + Projection(spelling).ShouldBe(Canonical); + + [Fact] + public void An_empty_key_prefix_is_the_bucket_root_and_so_is_no_key_prefix_at_all() => + Projection("""{"endpoint":"oss-cn-hangzhou.aliyuncs.com","bucket":"codespace-artifacts","keyPrefix":""}""") + .ShouldBe(Projection("""{"endpoint":"oss-cn-hangzhou.aliyuncs.com","bucket":"codespace-artifacts"}""")); + + [Theory] + [InlineData("""{"endpoint":"https://files.example.com","region":"cn-hangzhou","bucket":"codespace-artifacts","keyPrefix":"team-7/"}""")] + [InlineData("""{"endpoint":"oss.example.internal","region":"cn-hangzhou","bucket":"codespace-artifacts","keyPrefix":"team-7/"}""")] + public void A_host_outside_the_service_domain_is_one_namespace_with_the_bucket_of_the_region_it_signs_for(string spelling) => + // A custom domain or proxy cannot say which bucket it fronts, and where the identity cannot tell two routes + // apart it must merge them: OSS bucket names are unique service-wide and every request is signed for the + // bucket's own region, so the only OSS bucket this route can reach is that region's bucket of the same name. + Projection(spelling).ShouldBe(Canonical); + + [Theory] + [InlineData("""{"endpoint":"oss-cn-hangzhou.aliyuncs.com","bucket":"other-bucket","keyPrefix":"team-7/"}""")] + [InlineData("""{"endpoint":"oss-cn-hangzhou.aliyuncs.com","bucket":"codespace-artifacts","keyPrefix":"team-9/"}""")] + [InlineData("""{"endpoint":"oss-cn-hangzhou.aliyuncs.com","bucket":"codespace-artifacts","keyPrefix":"team-7/nested/"}""")] + [InlineData("""{"endpoint":"oss-cn-hangzhou.aliyuncs.com","bucket":"codespace-artifacts"}""")] + [InlineData("""{"endpoint":"oss-cn-shanghai.aliyuncs.com","bucket":"codespace-artifacts","keyPrefix":"team-7/"}""")] + public void A_different_bucket_prefix_or_region_stays_a_different_namespace(string other) => + // The guard on the guard: an identity that merged these would keep every deduplicated byte forever. + Projection(other).ShouldNotBe(Canonical); + + [Theory] + [InlineData("a//b/")] + [InlineData("./a/")] + [InlineData("a/./b/")] + [InlineData("a/../b/")] + [InlineData("/a/")] + [InlineData("a")] + public void A_key_prefix_the_driver_would_refuse_has_no_identity_rather_than_an_invented_one(string keyPrefix) + { + // OSS keys are literal, so 'a//b/' and 'a/b/' are different objects; the activation parser refuses both kinds of + // ambiguity outright rather than normalising them. The projection reuses that parser, so a revision carrying + // one has no identity at all — and a caller comparing identities must read that as "possibly the same". + using var config = JsonDocument.Parse(JsonSerializer.Serialize(new { endpoint = "oss-cn-hangzhou.aliyuncs.com", bucket = "codespace-artifacts", keyPrefix })); + + Should.Throw(() => Module.GetNamespaceConfiguration(config.RootElement)); + } + + [Fact] + public void A_configuration_already_written_canonically_keeps_the_fingerprint_its_revision_stored() + { + // Hard-pinned: the value an aliyun-oss revision written this way has carried since the fingerprint was + // introduced. Revisions are immutable, so a change here silently re-means every one of them. + using var config = JsonDocument.Parse("""{"keyPrefix":"team-7/","bucket":"codespace-artifacts","endpoint":"oss-cn-hangzhou.aliyuncs.com"}"""); + + StorageProfileRules.NamespaceFingerprint(Module, config.RootElement).ShouldBe(StorageProfileRules.NamespaceFingerprint(Module.TypeKey, config.RootElement)); + StorageProfileRules.NamespaceFingerprint(Module, config.RootElement).ShouldBe("sha256:a1c39ca50714a2497bf4b94f95aaae1fa2eace6c1674c11163def0f8c8b03261"); + } + + private static IStorageProviderModule Module { get; } = new AliyunOssStorageProviderModule(); + + private static string Projection(string config) + { + using var document = JsonDocument.Parse(config); + + return StorageProfileRules.CanonicalJson(Module.GetNamespaceConfiguration(document.RootElement)); + } +} diff --git a/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Providers/Local/LocalStorageNamespaceTests.cs b/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Providers/Local/LocalStorageNamespaceTests.cs new file mode 100644 index 000000000..868654e6a --- /dev/null +++ b/backend/tests/CodeSpace.UnitTests/Workflows/Artifacts/Providers/Local/LocalStorageNamespaceTests.cs @@ -0,0 +1,372 @@ +using System.Text.Json; +using CodeSpace.Core.Services.Workflows.Artifacts.Profiles; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers.Local; +using CodeSpace.Core.Services.Workflows.Artifacts.Providers.Local.Legacy; +using Shouldly; + +namespace CodeSpace.UnitTests.Workflows.Artifacts.Providers.Local; + +/// +/// The namespace identity of a filesystem destination is the directory its root REACHES, not the text an operator +/// typed. Every local driver opens its root through Path.GetFullPath and the operating system, so every spelling +/// below lands on one directory — and object keys carry no team segment, so two profiles naming that directory share +/// every object they both store. An identity that told two of these spellings apart let one team's retention sweep +/// delete another team's live bytes. +/// +[Trait("Category", "Unit")] +public sealed class LocalStorageNamespaceTests : IDisposable +{ + private readonly string _scratch = Path.Combine(Path.GetTempPath(), "codespace-namespace-unit", Guid.NewGuid().ToString("N")); + + public LocalStorageNamespaceTests() => Directory.CreateDirectory(_scratch); + + public static TheoryData ModulesAndSpellings() + { + var data = new TheoryData(); + foreach (var module in new[] { "local-rwx", "local-legacy" }) + foreach (var spelling in new[] { "as-written", "trailing-slash", "dot-segment", "doubled-slash", "doubled-inner-slash", "parent-segment", "relative" }) + data.Add(module, spelling); + return data; + } + + [Theory] + [MemberData(nameof(ModulesAndSpellings))] + public void Every_spelling_of_an_existing_root_projects_to_one_namespace(string module, string spelling) + { + var root = Path.Combine(_scratch, "victim-root"); + Directory.CreateDirectory(root); + + Projection(module, Spell(root, spelling)).ShouldBe(Projection(module, root), + $"'{Spell(root, spelling)}' opens the very directory '{root}' does, so it must be the same namespace"); + } + + [Theory] + [MemberData(nameof(ModulesAndSpellings))] + public void Every_spelling_of_a_root_that_does_not_exist_yet_projects_to_one_namespace(string module, string spelling) + { + // The directory the first write creates is the same whichever spelling created it, so the identity cannot wait + // for the directory to exist before it agrees with itself. + var root = Path.Combine(_scratch, "not-created-yet", "artifacts"); + + Projection(module, Spell(root, spelling)).ShouldBe(Projection(module, root)); + } + + [Fact] + public void A_symbolic_link_to_a_root_is_that_root() + { + if (OperatingSystem.IsWindows()) return; + + var root = Directory.CreateDirectory(Path.Combine(_scratch, "real-root")).FullName; + var link = Path.Combine(_scratch, "alias-root"); + Directory.CreateSymbolicLink(link, root); + + Projection("local-rwx", link).ShouldBe(Projection("local-rwx", root), "every byte written through the link lands in the directory it points at"); + } + + [Fact] + public void A_directory_beneath_a_linked_parent_that_does_not_exist_yet_is_beneath_the_link_target() + { + if (OperatingSystem.IsWindows()) return; + + var parent = Directory.CreateDirectory(Path.Combine(_scratch, "real-parent")).FullName; + var link = Path.Combine(_scratch, "alias-parent"); + Directory.CreateSymbolicLink(link, parent); + + Projection("local-rwx", Path.Combine(link, "team-a")).ShouldBe(Projection("local-rwx", Path.Combine(parent, "team-a"))); + } + + [Fact] + public void A_relative_link_resolves_against_the_directory_that_holds_it() + { + if (OperatingSystem.IsWindows()) return; + + var root = Directory.CreateDirectory(Path.Combine(_scratch, "relative-target")).FullName; + var holder = Directory.CreateDirectory(Path.Combine(_scratch, "holder")).FullName; + var link = Path.Combine(holder, "alias"); + Directory.CreateSymbolicLink(link, Path.Combine("..", "relative-target")); + + Projection("local-rwx", link).ShouldBe(Projection("local-rwx", root)); + } + + [Fact] + public void A_link_target_that_climbs_out_of_another_link_climbs_from_where_that_link_landed() + { + // The operating system follows 'a' BEFORE it applies the '..' after it, so 'holder/L' -> 'a/..' with + // 'holder/a' -> 'victim-root/objects' opens 'victim-root'. Collapsing 'a/..' as text first would name 'holder' + // instead — a directory nothing is written to — and let a sweep through L delete the victim's bytes. + if (OperatingSystem.IsWindows()) return; + + var victim = Directory.CreateDirectory(Path.Combine(_scratch, "victim-root", "objects")).Parent!.FullName; + var holder = Directory.CreateDirectory(Path.Combine(_scratch, "holder")).FullName; + Directory.CreateSymbolicLink(Path.Combine(holder, "a"), Path.Combine(victim, "objects")); + var link = Path.Combine(holder, "L"); + Directory.CreateSymbolicLink(link, Path.Combine("a", "..")); + + File.WriteAllText(Path.Combine(victim, "marker"), "reached"); + File.Exists(Path.Combine(link, "marker")).ShouldBeTrue("the precondition: the operating system opens the victim's root through L"); + + Projection("local-rwx", link).ShouldBe(Projection("local-rwx", victim)); + } + + [Fact] + public void A_link_cycle_has_no_identity_rather_than_an_invented_one() + { + // A root that never resolves cannot be compared with anything. Refusing it is what lets every caller fall back + // to treating it as possibly the same as anything else. + if (OperatingSystem.IsWindows()) return; + + var first = Path.Combine(_scratch, "loop-a"); + var second = Path.Combine(_scratch, "loop-b"); + Directory.CreateSymbolicLink(first, second); + Directory.CreateSymbolicLink(second, first); + + Should.Throw(() => Projection("local-rwx", Path.Combine(first, "artifacts"))); + } + + [Fact] + public void Two_different_directories_stay_two_namespaces() + { + // The guard on the guard: an identity that merged everything would keep every deduplicated byte forever. + Projection("local-rwx", Path.Combine(_scratch, "team-a")).ShouldNotBe(Projection("local-rwx", Path.Combine(_scratch, "team-b"))); + } + + [Fact] + public void Letter_case_is_folded_exactly_where_the_platform_filesystem_ignores_it() + { + var lower = Projection("local-rwx", Path.Combine(_scratch, "casing", "artifacts")); + var upper = Projection("local-rwx", Path.Combine(_scratch, "casing", "ARTIFACTS")); + + if (OperatingSystem.IsWindows() || OperatingSystem.IsMacOS()) upper.ShouldBe(lower, "a case-insensitive filesystem opens one directory for both"); + else upper.ShouldNotBe(lower, "Linux compares names byte for byte, so these are two directories"); + } + + [Theory] + [InlineData("local-rwx")] + [InlineData("local-legacy")] + public void The_projection_carries_the_root_and_nothing_else(string module) + { + var projection = JsonDocument.Parse(Projection(module, Path.Combine(_scratch, "only-root"))).RootElement; + + projection.EnumerateObject().Select(property => property.Name).ShouldBe(["rootPath"]); + } + + [Theory] + [InlineData("local-rwx")] + [InlineData("local-legacy")] + public void A_root_already_written_canonically_keeps_the_fingerprint_its_revision_stored(string module) + { + // Revisions are immutable and every one written before this projection existed stored the hash of its raw + // configuration. Projecting a canonical root to itself is what keeps those stored values true. + if (OperatingSystem.IsWindows()) return; + + var subject = Module(module); + var canonical = Projection(module, Path.Combine(_scratch, "stable")); + using var stored = JsonDocument.Parse(canonical); + + StorageProfileRules.NamespaceFingerprint(subject, stored.RootElement).ShouldBe(StorageProfileRules.NamespaceFingerprint(subject.TypeKey, stored.RootElement)); + } + + [Fact] + public void The_fingerprint_of_a_canonical_production_root_is_byte_identical_to_the_one_already_stored() + { + // Hard-pinned: this is the value a local-rwx revision naming /srv/codespace/artifacts has carried since the + // fingerprint was introduced. A change here is a change to every stored revision's meaning, not a refactor. + if (OperatingSystem.IsWindows()) return; + + using var config = JsonDocument.Parse("""{"rootPath":"/srv/codespace/artifacts"}"""); + + StorageProfileRules.NamespaceFingerprint(new LocalRwxStorageProviderModule(), config.RootElement) + .ShouldBe("sha256:d8df91c032f0a9ea898c31eee26d176f30992458f399d57623dafe52e78c6a24"); + } + + [Theory] + [InlineData("""{}""")] + [InlineData("""{"rootPath":42}""")] + [InlineData("""{"rootPath":" "}""")] + [InlineData("""[]""")] + public void A_configuration_with_no_readable_root_has_no_identity(string config) + { + using var document = JsonDocument.Parse(config); + + Should.Throw(() => Module("local-rwx").GetNamespaceConfiguration(document.RootElement)); + Should.Throw(() => Module("local-legacy").GetNamespaceConfiguration(document.RootElement)); + } + + // ─── The physical object a key reaches ─────────────────────────────────── + + public static TheoryData ObjectReachingShapes() => ["as-written", "trailing-slash", "symlink-root", "dotdot-through-link", "objects-link", "nested-link"]; + + [Theory] + [MemberData(nameof(ObjectReachingShapes))] + public void Every_path_the_operating_system_follows_to_one_file_is_one_object(string shape) + { + // The driver opens /objects/ and the operating system follows a link ANYWHERE along it, so a root + // the collector owns outright can still reach the victim's file through a link planted beneath it. + if (OperatingSystem.IsWindows()) return; + + var victim = Directory.CreateDirectory(Path.Combine(_scratch, "victim-root")).FullName; + var file = PlaceObject(victim); + var collector = Reach(victim, shape); + + File.Exists(Path.Combine([collector, "objects", .. ObjectKey.Split('/')])).ShouldBeTrue($"the precondition: the operating system opens {file} through '{collector}'"); + + ObjectIdentity(collector).ShouldBe(ObjectIdentity(victim), $"'{collector}' reaches the very file '{victim}' does, so deleting either deletes both"); + } + + [Fact] + public void Every_spelling_of_a_root_whose_object_does_not_exist_yet_reaches_one_object() + { + var root = Path.Combine(_scratch, "not-created-yet", "artifacts"); + + ObjectIdentity(root + "/.").ShouldBe(ObjectIdentity(root)); + ObjectIdentity(Path.GetRelativePath(Environment.CurrentDirectory, root)).ShouldBe(ObjectIdentity(root)); + } + + [Fact] + public void Two_roots_hold_two_objects_at_one_key() + { + // The guard on the guard: content addressing gives every team storing these bytes this key, so merging two real + // directories would keep every deduplicated byte forever. + var first = Directory.CreateDirectory(Path.Combine(_scratch, "team-a")).FullName; + var second = Directory.CreateDirectory(Path.Combine(_scratch, "team-b")).FullName; + PlaceObject(first); + PlaceObject(second); + + ObjectIdentity(first).ShouldNotBe(ObjectIdentity(second)); + } + + [Fact] + public void A_key_that_is_itself_a_link_is_its_own_object() + { + // A delete unlinks the NAME it is given, so a key that is a link to the victim's file removes the link and + // leaves the file. Following it would merge two objects the driver keeps apart — the safe direction, but a + // reaper that could never collect a planted link would be a reaper anyone can stall. + if (OperatingSystem.IsWindows()) return; + + var victim = Directory.CreateDirectory(Path.Combine(_scratch, "victim-root")).FullName; + var collector = Directory.CreateDirectory(Path.Combine(_scratch, "collector-root")).FullName; + var file = PlaceObject(victim); + var name = Path.Combine([collector, "objects", .. ObjectKey.Split('/')]); + Directory.CreateDirectory(Path.GetDirectoryName(name)!); + File.CreateSymbolicLink(name, file); + + ObjectIdentity(collector).ShouldNotBe(ObjectIdentity(victim)); + } + + [Fact] + public async Task The_object_identity_names_the_file_the_real_driver_writes_and_deletes() + { + // Drift detector: the identity re-states the driver's layout (/objects/). If the driver ever moves + // its objects, this is the test that notices before a sweep compares two paths neither of which is real. + if (OperatingSystem.IsWindows()) return; + + var victim = Directory.CreateDirectory(Path.Combine(_scratch, "victim-root", "objects")).Parent!.FullName; + var collector = Reach(victim, "objects-link"); + await using var driver = new LocalRwxArtifactStorageDriver(collector); + await using var content = new MemoryStream("driver-written"u8.ToArray()); + + var put = await driver.PutAsync(new ArtifactStoragePutRequest(ObjectKey, content) { Condition = ArtifactStorageWriteCondition.CreateOnly }, CancellationToken.None); + + put.IsSuccess.ShouldBeTrue(put.Error?.Message); + + File.Exists(ObjectIdentity(victim)).ShouldBeTrue("the identity must name the file the driver wrote, through the link, into the victim's root"); + (await driver.DeleteAsync(new ArtifactStorageDeleteRequest(ObjectKey), CancellationToken.None)).IsSuccess.ShouldBeTrue(); + File.Exists(ObjectIdentity(victim)).ShouldBeFalse("and a delete through the collector's root removes exactly that file"); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("/workflow-artifacts/aa/bb/object")] + [InlineData("workflow-artifacts/../../escape")] + [InlineData("./workflow-artifacts/object")] + [InlineData("workflow-artifacts/a\0b")] + public void A_key_the_driver_would_refuse_has_no_identity(string objectKey) + { + using var config = JsonDocument.Parse(JsonSerializer.Serialize(new { rootPath = _scratch })); + + Should.Throw(() => new LocalRwxStorageProviderModule().PhysicalObject(config.RootElement, objectKey)); + } + + private const string ObjectKey = "workflow-artifacts/ab/cd/abcd0000000000000000000000000000000000000000000000000000000000ff"; + + /// Writes the object where the local-rwx driver keeps it: <root>/objects/<key>. + private static string PlaceObject(string root) + { + var file = Path.Combine([root, "objects", .. ObjectKey.Split('/')]); + Directory.CreateDirectory(Path.GetDirectoryName(file)!); + File.WriteAllText(file, "victim bytes"); + return file; + } + + /// A root the collector could name that the operating system resolves into 's objects. + private string Reach(string victim, string shape) + { + var collector = Path.Combine(_scratch, $"collector-{shape}"); + + switch (shape) + { + case "as-written": return victim; + case "trailing-slash": return victim + "/"; + case "symlink-root": + Directory.CreateSymbolicLink(collector, victim); + return collector; + case "dotdot-through-link": + Directory.CreateDirectory(collector); + Directory.CreateSymbolicLink(Path.Combine(collector, "a"), Path.Combine(victim, "objects")); + Directory.CreateSymbolicLink(Path.Combine(collector, "L"), Path.Combine("a", "..")); + return Path.Combine(collector, "L"); + case "objects-link": + Directory.CreateDirectory(collector); + Directory.CreateSymbolicLink(Path.Combine(collector, "objects"), Path.Combine(victim, "objects")); + return collector; + case "nested-link": + Directory.CreateDirectory(Path.Combine(collector, "objects")); + Directory.CreateSymbolicLink(Path.Combine(collector, "objects", "workflow-artifacts"), Path.Combine(victim, "objects", "workflow-artifacts")); + return collector; + default: throw new ArgumentOutOfRangeException(nameof(shape), shape, null); + } + } + + private static string ObjectIdentity(string rootPath) + { + using var config = JsonDocument.Parse(JsonSerializer.Serialize(new { rootPath })); + + return new LocalRwxStorageProviderModule().PhysicalObject(config.RootElement, ObjectKey); + } + + private static string Spell(string root, string spelling) => spelling switch + { + "as-written" => root, + "trailing-slash" => root + "/", + "dot-segment" => root + "/.", + "doubled-slash" => root + "//", + "doubled-inner-slash" => Path.GetDirectoryName(root) + "//" + Path.GetFileName(root), + "parent-segment" => Path.Combine(root, "..", Path.GetFileName(root)), + "relative" => Path.GetRelativePath(Environment.CurrentDirectory, root), + _ => throw new ArgumentOutOfRangeException(nameof(spelling), spelling, null), + }; + + private static IStorageProviderModule Module(string module) => module switch + { + "local-rwx" => new LocalRwxStorageProviderModule(), + "local-legacy" => new LocalLegacyStorageProviderModule(), + _ => throw new ArgumentOutOfRangeException(nameof(module), module, null), + }; + + /// The canonical JSON of the module's namespace projection — the exact string the fingerprint hashes. + private static string Projection(string module, string rootPath) + { + using var config = JsonDocument.Parse(JsonSerializer.Serialize(new { rootPath })); + + return StorageProfileRules.CanonicalJson(Module(module).GetNamespaceConfiguration(config.RootElement)); + } + + public void Dispose() + { + try { Directory.Delete(_scratch, recursive: true); } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + } +}