From 734cb7e27e330f8f1543ff9a9b9a3b6be741afef Mon Sep 17 00:00:00 2001 From: Christian Dupuis Date: Fri, 20 Nov 2026 12:42:36 +0000 Subject: [PATCH 01/10] feat(spec,frontend): state provides from the package that owns a path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A workload inherits tools at versions its author never sees. §9.6 publishes them as deb/ and apk/ entries, but those carry the distribution's names: a mixin that needs Python has to require deb/python-3.14, which is gone when the base moves to 3.15 and which no Alpine workload provides. A stable name such as python is the author's to choose, yet §9.2 will not publish it without a version, and under the version: fallback it becomes python@1.0.0, the kit's release number under a tool's name. Add providesFrom, a map from capability name to the absolute path of a file in the kit's own filesystem. Publishing finds the installed package whose file list names the path and states name@version in provides, using that package's version as §9.6 renders it. It reads /var/lib/dpkg/info/*.list and the F:/R: records of the apk database directly, because an image assembled from packages keeps the lists without carrying dpkg. A path no package lists is followed through symbolic links, which covers update-alternatives and recipe-made links, and paths are compared through top-level links so both spellings of a merged-/usr path resolve. A path that is absent, a directory, unowned, owned by two packages, at an unnameable version, or at different versions across platforms fails the build: the author asked for the entry by name. The field is authored form only and workload only. The published descriptor carries the resulting provides entries and not the field, and ValidatePublished refuses one that still has it. That is what allows a top-level addition under schemaVersion 3: runtimes decode published descriptors strictly and never see it. A frontend built before the field refuses an authored descriptor that uses it. RELEASES.md records the in-place addition. The lookup lives in the spec package behind a two-method filesystem interface so that other frontends can vendor it; cmd/frontend adapts a BuildKit reference to it. The ubuntu-docker example states its Docker and shell tools this way. Signed-off-by: Christian Dupuis Sandbox-Id: spiffe://sbx.docker.com/org/57b45934-74c0-11e4-bea4-0242ac11001b/user/1580b874-0d59-41d8-bd81-82a10abc4100/sandbox/74f8ada1-8c4b-4fd6-9802-da94ba278db2 Sandbox-Kit: sha256:dfcafd4d7957b8b6ac4574528241772465187a778e16bdf809d9c85f43e8fed7 Co-authored-by: sbx docker/cdupuis/74f8ada1 --- RELEASES.md | 11 + cmd/frontend/build.go | 20 + cmd/frontend/packages.go | 51 +++ cmd/frontend/provides_from_test.go | 118 ++++++ docs/spec/SPEC-v3.md | 111 ++++- examples/ubuntu-docker/ubuntu-docker.yaml | 16 + schema/kit.schema.json | 36 ++ skills/create-kit-v3/SKILL.md | 29 +- skills/migrate-kit-to-v3/FIELD-MAPPING.md | 6 + spec/provides_from.go | 495 ++++++++++++++++++++++ spec/provides_from_test.go | 294 +++++++++++++ spec/schema_test.go | 29 ++ spec/types.go | 15 + spec/validate.go | 8 + tck/sandbox/coverage_test.go | 14 + 15 files changed, 1249 insertions(+), 4 deletions(-) create mode 100644 cmd/frontend/provides_from_test.go create mode 100644 spec/provides_from.go create mode 100644 spec/provides_from_test.go diff --git a/RELEASES.md b/RELEASES.md index c60fc867..16e51c1a 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -142,6 +142,17 @@ new field; runtimes that previously loaded all `@2` mixin guidance inline need the matching draft implementation, even when the field is omitted. `agent-context@1` and `schemaVersion` are unchanged. +The authored-only `providesFrom` field +([SPEC-v3 §9.7](docs/spec/SPEC-v3.md#97-provides-stated-from-a-path)) +is a top-level addition made in place under the same draft-stage +latitude, approved by maintainer @cdupuis's instruction to implement it. +Publishing replaces the field with ordinary `provides` entries and +drops it, so published descriptors keep their shape: readers of +published Kits, runtimes included, need no change. A frontend or other +reader of authored descriptors built before the field refuses a +descriptor that uses it. `schemaVersion` and every capability version +are unchanged. + The exception ends with the first stable specification release. After that, changes to shipped capability meanings, defaults, permitted values, reconciliation, or persistent-state transitions require a new diff --git a/cmd/frontend/build.go b/cmd/frontend/build.go index b1c57efa..c39a2b16 100644 --- a/cmd/frontend/build.go +++ b/cmd/frontend/build.go @@ -106,6 +106,13 @@ func Build(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) { if err != nil { return nil, withYAMLSource(err, filename, src.descriptor) } + // providesFrom is an instruction to this build, kept on d until the + // filesystems it names exist. The published form never carries it + // (§9.7), so it leaves here, before anything judges that form. + published, err = spec.WithoutProvidesFrom(published) + if err != nil { + return nil, withYAMLSource(err, filename, src.descriptor) + } content := commentContent switch { @@ -243,6 +250,12 @@ func Build(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) { if err := spec.RequireAuthoredProvides(pd); err != nil { return nil, withYAMLSource(err, filename, src.descriptor) } + // On the expanded form for the same reason: a provides entry + // whose name came out of an arg can only now be seen to state + // a capability providesFrom states too. + if err := spec.RequireDistinctProvidesFrom(pd, d.ProvidesFrom); err != nil { + return nil, withYAMLSource(err, filename, src.descriptor) + } } res := gwclient.NewResult() @@ -308,6 +321,13 @@ func Build(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) { if err != nil { return nil, err } + // What the author asked for by name stands ahead of the + // inventory nobody asked for entry by entry. + stated, err := statedProvides(ctx, d.ProvidesFrom, builds) + if err != nil { + return nil, withYAMLSource(err, filename, src.descriptor) + } + derived = append(stated, derived...) if len(derived) > 0 { published, pd, err = withDerivedProvides(published, derived) if err != nil { diff --git a/cmd/frontend/packages.go b/cmd/frontend/packages.go index 2896379b..f0562419 100644 --- a/cmd/frontend/packages.go +++ b/cmd/frontend/packages.go @@ -4,7 +4,9 @@ import ( "bytes" "context" "fmt" + "os" + "github.com/containerd/platforms" gwclient "github.com/moby/buildkit/frontend/gateway/client" "github.com/docker/sandbox-kit-spec/v3/spec" @@ -53,3 +55,52 @@ func derivedProvides(ctx context.Context, refs []gwclient.Reference) ([]string, } return out, nil } + +// statedProvides resolves the kit's providesFrom declaration against +// every platform's filesystem (§9.7): each path to the installed package +// that owns it, and each name to that package's version. +func statedProvides(ctx context.Context, from map[string]string, builds []platformBuild) ([]string, error) { + if len(from) == 0 { + return nil, nil + } + files := make([]spec.PlatformFiles, 0, len(builds)) + for _, b := range builds { + p := spec.PlatformFiles{Platform: platforms.FormatAll(platformOrDefault(b.platform))} + if b.ref != nil { + p.Files = imageFiles{ref: b.ref} + } + files = append(files, p) + } + return spec.ProvidesFromPaths(ctx, from, files) +} + +// imageFiles reads a solved filesystem for the spec package, which +// knows what to look for and not what a BuildKit reference is. +type imageFiles struct{ ref gwclient.Reference } + +func (f imageFiles) ReadFile(ctx context.Context, name string) ([]byte, bool, error) { + return readImageFile(ctx, f.ref, name) +} + +// ReadDir lists a directory through the gateway, which walks it without +// following its entries: a symbolic link arrives with its target, which +// is the one thing StatFile cannot say — it resolves the whole path and +// answers for wherever that ends. +func (f imageFiles) ReadDir(ctx context.Context, name string) ([]spec.ImageEntry, bool, error) { + stats, err := f.ref.ReadDir(ctx, gwclient.ReadDirRequest{Path: name}) + if err != nil { + if isNotExist(err) { + return nil, false, nil + } + return nil, false, err + } + entries := make([]spec.ImageEntry, 0, len(stats)) + for _, st := range stats { + entries = append(entries, spec.ImageEntry{ + Name: st.Path, + Dir: os.FileMode(st.Mode).IsDir(), + Link: st.Linkname, + }) + } + return entries, true, nil +} diff --git a/cmd/frontend/provides_from_test.go b/cmd/frontend/provides_from_test.go new file mode 100644 index 00000000..4dcf0cf9 --- /dev/null +++ b/cmd/frontend/provides_from_test.go @@ -0,0 +1,118 @@ +package main + +import ( + "context" + "io/fs" + "os" + "path" + "strings" + "testing" + + gwclient "github.com/moby/buildkit/frontend/gateway/client" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/require" + fstypes "github.com/tonistiigi/fsutil/types" + + "github.com/docker/sandbox-kit-spec/v3/spec" +) + +// packagedReference is a solved filesystem as the gateway presents one: +// files read by name, and a directory listed with each entry's own +// stat, so a symbolic link arrives as a link with its target. +type packagedReference struct { + fakeReference + files map[string]string + links map[string]string +} + +func (r packagedReference) ReadFile(_ context.Context, req gwclient.ReadRequest) ([]byte, error) { + body, ok := r.files[req.Filename] + if !ok { + return nil, fs.ErrNotExist + } + return []byte(body), nil +} + +func (r packagedReference) ReadDir(_ context.Context, req gwclient.ReadDirRequest) ([]*fstypes.Stat, error) { + seen := map[string]*fstypes.Stat{} + for name := range r.files { + if path.Dir(name) == req.Path { + seen[path.Base(name)] = &fstypes.Stat{Path: path.Base(name), Mode: 0o755} + } else if rest, under := strings.CutPrefix(name, strings.TrimSuffix(req.Path, "/")+"/"); under { + top, _, _ := strings.Cut(rest, "/") + seen[top] = &fstypes.Stat{Path: top, Mode: uint32(os.ModeDir | 0o755)} + } + } + for name, target := range r.links { + if path.Dir(name) == req.Path { + seen[path.Base(name)] = &fstypes.Stat{Path: path.Base(name), Mode: uint32(os.ModeSymlink | 0o777), Linkname: target} + } + } + if len(seen) == 0 { + return nil, fs.ErrNotExist + } + out := make([]*fstypes.Stat, 0, len(seen)) + for _, st := range seen { + out = append(out, st) + } + return out, nil +} + +func debianReference(sshVersion string) packagedReference { + return packagedReference{ + files: map[string]string{ + spec.DpkgStatusPath: "Package: openssh-client\nStatus: install ok installed\nVersion: " + sshVersion + "\n\n" + + "Package: python-3.14-minimal\nStatus: install ok installed\nVersion: 3.14.10-6\n", + spec.DpkgInfoDir + "/openssh-client.list": "/usr/bin/ssh\n", + spec.DpkgInfoDir + "/python-3.14-minimal.list": "/usr/bin/python3.14\n/usr/bin/python3\n", + "/usr/bin/ssh": "elf", + "/usr/bin/python3.14": "elf", + }, + links: map[string]string{ + "/bin": "usr/bin", + "/usr/bin/python3": "python3.14", + }, + } +} + +// The spec package decides what a path resolves to; this is the half +// that only the frontend can get wrong — reading a directory through the +// gateway so that a link is seen as one, and naming each platform. +func TestStatedProvidesReadEveryPlatformThroughTheGateway(t *testing.T) { + from := map[string]string{"python": "/usr/bin/python3", "ssh": "/bin/ssh"} + arm := &ocispecs.Platform{OS: "linux", Architecture: "arm64"} + amd := &ocispecs.Platform{OS: "linux", Architecture: "amd64"} + + got, err := statedProvides(t.Context(), from, []platformBuild{ + {platform: arm, ref: debianReference("1:10.0p1-7+deb13u4")}, + {platform: amd, ref: debianReference("1:10.0p1-7+deb13u4")}, + }) + require.NoError(t, err) + require.Equal(t, []string{"python@3.14.10", "ssh@10.0"}, got) + + _, err = statedProvides(t.Context(), from, []platformBuild{ + {platform: arm, ref: debianReference("1:10.0p1-7+deb13u4")}, + {platform: amd, ref: debianReference("1:10.1p1-1")}, + }) + require.ErrorContains(t, err, "providesFrom.ssh: /bin/ssh is at 10.0 on linux/arm64 and 10.1 on linux/amd64") + + // Nothing asked for, nothing read: a kit without the field never + // pays for the file lists. + got, err = statedProvides(t.Context(), nil, []platformBuild{{platform: arm}}) + require.NoError(t, err) + require.Empty(t, got) +} + +// A directory that is not there is an absence, not a failure: the spec +// package turns it into the diagnostic that names the missing database. +func TestImageFilesReportAMissingDirectoryAsAbsent(t *testing.T) { + files := imageFiles{ref: packagedReference{}} + entries, present, err := files.ReadDir(context.Background(), spec.DpkgInfoDir) + require.NoError(t, err) + require.False(t, present) + require.Empty(t, entries) + + _, present, err = files.ReadFile(context.Background(), spec.DpkgStatusPath) + require.NoError(t, err) + require.False(t, present) +} diff --git a/docs/spec/SPEC-v3.md b/docs/spec/SPEC-v3.md index b7b47144..516d0eb0 100644 --- a/docs/spec/SPEC-v3.md +++ b/docs/spec/SPEC-v3.md @@ -252,6 +252,7 @@ iconUrl: https://… # optional. Image shown in catalogs and pickers. version: "1.4.2" # optional. Fallback version; see below. licenses: [Apache-2.0] # optional. SPDX identifiers. provides: [] # optional. §5. +providesFrom: {} # optional. §9.7. Authored form only. requires: [] # optional. §5. integrates: [] # optional. §5. conflicts: [] # optional. §5. @@ -273,6 +274,7 @@ kits: [] # optional. §3.4. Exclusive with build, dockerfile. | `iconUrl` | string | optional. Absolute **https** URL to an image representing the Kit in catalogs and pickers. Display metadata, self-asserted like `displayName`. The scheme is constrained because a consumer fetches and renders this rather than merely displaying it: `javascript:`, `data:`, and `file:` are refused, and plain `http:` is refused so a rendered icon is not attacker-swappable in transit. Referenced rather than staged into a layer, because the surfaces that want an icon hold the manifest and not the layers — an icon a consumer had to pull the Kit to see would arrive after the moment it existed to inform. No `org.opencontainers.image.*` annotation is emitted for it: OCI defines no icon key, and the descriptor annotation already carries the field. | | `version` | string | optional. Version-shaped ([§5.2](#52-versions)), or a build-phase arg reference expanded at publish. See below. | | `licenses` | list\ | optional. SHOULD be SPDX identifiers. | +| `providesFrom` | map\ | optional. Capability name to the absolute path of a file in the Kit's own filesystem; publishing states each name at the version of the package that owns the file. Workload only, and absent from the published descriptor. See [§9.7](#97-provides-stated-from-a-path). | | `build` | string | optional. Literal Dockerfile text. Exclusive with `dockerfile` and `kits`. | | `dockerfile` | string | optional. Relative path inside the descriptor's directory; MUST NOT escape it (that directory is the build context's root). Exclusive with `build` and `kits`. | | `kits` | list\ | optional. The Kits this Kit's content is merged from ([§3.4](#34-kit-set)). Exclusive with `build` and `dockerfile`. | @@ -315,6 +317,10 @@ conflicts: ["podman"] # must be absent from the resolv exception is not an author's claim at all — publishing derives an entry per installed distribution package, in namespaces reserved for it ([§9.6](#96-derived-provides)). +- A name whose version the author does not know — a tool the base image + ships — is stated with `providesFrom` instead, which names the file + and leaves the version to publishing + ([§9.7](#97-provides-stated-from-a-path)). - Arg references are permitted in `provides` (and `version`) only — they are expanded at publish ([§9.1](#91-expansion)). `requires`, `integrates`, and `conflicts` MUST stay literal: they are what the resolver judges, and a @@ -1288,6 +1294,101 @@ Derived entries do not vote on archive never agrees on one version, and counting it would leave every Kit that named no `version:` with no version annotation at all. +### 9.7 Provides stated from a path + +Derived entries are an inventory, and they carry the distribution's +names: a mixin that needs Python on a Debian workload requires +`deb/python-3.14`, a name that moves when the base image moves to 3.15 +and that no Alpine workload will ever provide. A name that stays put — +`python` — is an author's to choose, but its version is not an author's +to know, and [§9.2](#92-versioned-provides) will not publish it without +one. Written as `provides: [python]` it takes the `version:` fallback, +the statement [§9.6](#96-derived-provides) exists to stop. + +`providesFrom` states the name and leaves the version to the content: + +```yaml +kind: workload +provides: ["claude@${{ kit.args.version }}"] # the author's own +providesFrom: # the base image's + python: /usr/bin/python3 + ssh: /usr/bin/ssh + docker-engine: /usr/bin/dockerd +``` + +Each key is a capability name and each value the absolute path of a +file in the Kit's own filesystem. Publishing finds the installed package +that owns the file and states `name@version` in `provides`, with that +package's version as [§9.6](#96-derived-provides) renders it. The Kit +above publishes `python@3.14.10` beside `deb/python-3.14@3.14.10`, and +keeps publishing `python` when the package underneath is renamed. + +The path names the command rather than the package because the command +is what a requirer means, and because it is the name that survives: +`/usr/bin/python3` is there under every Python 3 a base image has +shipped. + +- A key **MUST** be a capability name outside the namespaces + [§5.1](#51-names-and-namespaces) reserves, and **MUST NOT** name a + capability the Kit's `provides` also states, in either spelling. A + capability has one version, and two entries for it leave a resolver to + pick. A value is a literal, absolute, canonical path: arg references + are not permitted, since the path names a file in the image and not an + input. +- `providesFrom` applies to a `kind: workload` Kit that builds its own + content, and **MUST** be refused on a mixin or a set. A mixin's layers + are a delta, where a package database is not an inventory + ([§9.6](#96-derived-provides)); a set builds nothing, and the workload + it lists already stated what it carries. +- Publishing **MUST** fail unless exactly one installed package lists + the path: it fails when the path does not exist, is a directory, is + listed by more than one package, or is listed by none. The lists are + the ones kept beside the databases [§9.6](#96-derived-provides) reads: + `/var/lib/dpkg/info/.list` for dpkg, the `F:` and `R:` + records of `/lib/apk/db/installed` for apk. Nothing is stated in + place of an entry that could not be resolved: the author asked for it + by name, so its absence is a build failure the author sees rather + than a capability a consumer finds missing. A file a recipe placed + has no package to take a version from, and is stated in `provides` + under a build-phase arg as before. +- Publishing **MUST NOT** follow a symbolic link that a package lists. + A link no package lists is followed to its target, and the target + judged the same way. A package that ships `python3` as a link is the + answer for that link, while a link nothing owns was made by a recipe + or by `update-alternatives`, and what it leads to is the thing + installed. Paths are compared through a top-level directory that is + itself a link, so `/bin/sh` and `/usr/bin/sh` name one file on a + merged-`/usr` system whichever of them the package recorded. +- The version **MUST** be the owner's version under + [§9.6](#96-derived-provides)'s rule, and publishing **MUST** fail + when that rule names none — a prerelease, or one package recorded at + two versions. It is never taken from the `version:` fallback. +- Every platform the Kit publishes **MUST** resolve every path, to one + version, or publishing fails. Derived entries are dropped where + platforms disagree because nobody asked for them one by one; these + were asked for. +- The published descriptor **MUST NOT** carry `providesFrom`. It is an + instruction to the build, and what it produced is in `provides`. This + is also what lets the field exist under `schemaVersion: "3"`: decoding + is strict, so a runtime built before this field would refuse a + published Kit that carried it. + +A stated entry is an ordinary `provides` entry once published. It is the +workload's to provide, so [§5.3](#53-resolution-semantics-consumer-contract)'s +one-provider rule applies to it: a workload that states `gh` no longer +composes with a mixin that provides `gh`. State the tools a mixin would +require of a base, and leave a name to the Kit whose purpose is to +bring it. A stated entry also votes on +`org.opencontainers.image.version` like any authored one +([§9.3](#93-annotations)), so a Kit that states several tools names its +own `version:` if it wants that annotation. + +`providesFrom` extends the unfinished schema-3 grammar in place. An +authored descriptor using it requires a frontend that knows the field; +an older one refuses it. Published descriptors do not change shape, so +readers of published Kits are unaffected, and it adds no capability +version. + --- ## 10. The OCI layout @@ -1405,10 +1506,14 @@ The spec library enforces, beyond per-field rules stated above: compiles. - **raw form**: size budget ([§9.4](#94-size-budget)); every `${{ kit.args.* }}` reference names a declared arg. +- **providesFrom**: workload only, never beside `kits`; keys are + capability names outside the reserved namespaces that `provides` does + not also state; values are literal, absolute, canonical paths + ([§9.7](#97-provides-stated-from-a-path)). - **published form**: no build-phase references remain; provides are - literal and ([§9.2](#92-versioned-provides)) versioned; `kind` is the - derived `workload` or `mixin`, never `set`; every listed Kit carries - a digest. + literal and ([§9.2](#92-versioned-provides)) versioned; `providesFrom` + is gone; `kind` is the derived `workload` or `mixin`, never `set`; + every listed Kit carries a digest. Validation collects independent failures in a `ValidationErrors` collection, preserving warnings alongside errors. A failed config decode diff --git a/examples/ubuntu-docker/ubuntu-docker.yaml b/examples/ubuntu-docker/ubuntu-docker.yaml index 1b0f63f6..552a29a9 100644 --- a/examples/ubuntu-docker/ubuntu-docker.yaml +++ b/examples/ubuntu-docker/ubuntu-docker.yaml @@ -10,6 +10,22 @@ version: "24.04" # Package provides are derived from the image's dpkg database. The Kit's # release version is not a claim about the installed Docker version. +# +# Those entries carry Ubuntu's package names, so the engine is +# deb/docker-ce here and something else on another base. The tools a +# mixin is likely to ask a base for are also stated under names that do +# not depend on the distribution: each line names the command, and +# publishing states the name at the version of the package that installed +# it (§9.7). The field itself is not published. +providesFrom: + bash: /usr/bin/bash + curl: /usr/bin/curl + docker: /usr/bin/docker + docker-buildx: /usr/libexec/docker/cli-plugins/docker-buildx + docker-compose: /usr/libexec/docker/cli-plugins/docker-compose + docker-engine: /usr/bin/dockerd + git: /usr/bin/git + capabilities: - type: com.docker.sandbox/sbx@1 diff --git a/schema/kit.schema.json b/schema/kit.schema.json index 34247ced..1b85bfbd 100644 --- a/schema/kit.schema.json +++ b/schema/kit.schema.json @@ -61,6 +61,22 @@ }, "description": "Capabilities this kit offers, optionally versioned as name@version. Nothing an author offers is provided implicitly. Bare names normalize into the default namespace (gh IS com.docker.kit/gh, like debian is docker.io/library/debian); third parties qualify under their own namespace, which never matches the bare form. The deb/ and apk/ namespaces are reserved: publishing fills them from the package databases in a workload's content, one entry per installed package, so do not author them." }, + "providesFrom": { + "type": "object", + "propertyNames": { + "not": { + "pattern": "^(deb|apk)/" + } + }, + "additionalProperties": { + "type": "string", + "pattern": "^/.", + "not": { + "pattern": "\\$\\{\\{" + } + }, + "description": "Authored form only, workload only. Capability name to the absolute path of a file in the kit's own filesystem. Publishing finds the installed package that owns the file, states name@ in provides, and drops this field, so a published descriptor never carries it. For a tool the base image ships at a version the author does not know: python: /usr/bin/python3. A path no installed package owns, or one whose platforms disagree on the version, fails the build. Paths are literal; arg references are not allowed." + }, "requires": { "type": "array", "items": { @@ -211,6 +227,26 @@ }, "description": "com.docker.sandbox/sbx@1 is workload-only: a mixin's image config never becomes the composed image's, so the identity it would promise is not the one a host reads." }, + { + "if": { + "required": [ + "providesFrom" + ] + }, + "then": { + "properties": { + "kind": { + "const": "workload" + } + }, + "not": { + "required": [ + "kits" + ] + } + }, + "description": "providesFrom reads the filesystem the kit builds: a mixin's layers are a delta, where a package database is no inventory, and a set builds nothing." + }, { "$ref": "#/definitions/noNul" } diff --git a/skills/create-kit-v3/SKILL.md b/skills/create-kit-v3/SKILL.md index 80cd29e5..95ef02bb 100644 --- a/skills/create-kit-v3/SKILL.md +++ b/skills/create-kit-v3/SKILL.md @@ -230,6 +230,32 @@ publish: `RequireVersionedProvides` fails the build. Pin the version, or drop the provide — a kit with no `provides` publishes fine, and offering nothing matchable is honest when the kit cannot know what it installed. +**A workload states the tools its base ships with `providesFrom`.** You do +not know what version of Python the base image carries, and `deb/python-3.14` +is a name that moves at the next minor and that no Alpine base will ever +provide. Name the command and let publishing find the version: + +```yaml +kind: workload +providesFrom: + python: /usr/bin/python3 + ssh: /usr/bin/ssh +``` + +Publishing looks up the installed package that owns each file and states +`python@3.14.10`, `ssh@10.0` in `provides`; the field itself is not published +(§9.7). It reads dpkg's and apk's file lists, so it is workload-only, and it +fails the build for a path that is absent, a directory, or owned by no +package — a binary your recipe downloaded still takes the build-phase +`version` arg above. Check a path before you list it: + +```sh +docker run --rm sh -c 'grep -lx /usr/bin/python3 /var/lib/dpkg/info/*.list' +``` + +Do not state a name a mixin exists to provide. One name has one provider +(§5.3), so a workload that states `gh` stops composing with the `gh` kit. + **`requires` is a closed-set check**: a name nothing in the composition provides makes your kit refuse to compose *anywhere*, which is worse than saying nothing. It constrains the composed **runtime** set, not your builder — @@ -478,7 +504,8 @@ varies, keep the rest exact and vary only that part: - [SPEC-v3.md](https://github.com/docker/sandbox-kit-spec/blob/main/docs/spec/SPEC-v3.md) — the grammar. §3 authoring forms, §5 provides/requires, §6 args, - §7 capabilities, §9.6 derived provides. + §7 capabilities, §9.6 derived provides, §9.7 provides stated from a + path. - [capability pages](https://github.com/docker/sandbox-kit-spec/tree/main/docs/spec/capabilities/com.docker.sandbox) — normative per type. - [examples](https://github.com/docker/sandbox-kit-spec/tree/main/examples) — diff --git a/skills/migrate-kit-to-v3/FIELD-MAPPING.md b/skills/migrate-kit-to-v3/FIELD-MAPPING.md index 8615c44e..6b15d41f 100644 --- a/skills/migrate-kit-to-v3/FIELD-MAPPING.md +++ b/skills/migrate-kit-to-v3/FIELD-MAPPING.md @@ -195,6 +195,12 @@ workload on `dhi.io/sbx-templates:*` offers `deb/apt`, `deb/jq`, `deb/docker-ce`, `deb/git` and hundreds more with nobody authoring them, and a mixin may require those names. +A `deb/` name is the distribution's, so it is only as stable as the package: +`deb/python-3.14` is gone when the base moves to 3.15, and no Alpine workload +provides it. Where the workload states a tool under a name of its own with +`providesFrom` (§9.7) — `python`, `node`, `docker-engine` — require that name +instead, and keep `deb/` for what is Debian by nature, such as `deb/apt`. + **On a multi-platform workload the set is an intersection, not a union.** §9.6 states an entry only where every published platform agrees on the normalized name *and* version, so an architecture-specific package, or one at different diff --git a/spec/provides_from.go b/spec/provides_from.go new file mode 100644 index 00000000..e5ac324f --- /dev/null +++ b/spec/provides_from.go @@ -0,0 +1,495 @@ +package spec + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "maps" + "path" + "slices" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +// DpkgInfoDir is where dpkg keeps one .list per installed +// package, naming every path the package put on disk. The status file +// says what is installed; this is the only place that says whose a file +// is. +const DpkgInfoDir = "/var/lib/dpkg/info" + +// maxProvidesFromLinks bounds how many symbolic links one path is +// followed through. Deeper than an alternatives chain ever is, and +// small enough that a link cycle ends in a diagnostic rather than a +// build that never does. +const maxProvidesFromLinks = 16 + +// validateProvidesFrom holds providesFrom to its grammar (§9.7). +// +// The kind rules are here rather than at resolution because they are +// knowable from the descriptor alone: a mixin's layers are a delta and +// a set builds nothing, so neither has a filesystem whose package +// database is an inventory, and saying so before a build runs is kinder +// than saying it after. +func validateProvidesFrom(d *Descriptor) error { + var errs ValidationErrors + if len(d.ProvidesFrom) == 0 { + return nil + } + switch { + case len(d.Kits) > 0: + errs.add(fieldErrorf("providesFrom", "providesFrom reads the filesystem this kit builds, and a set builds none: its content is the kits it lists, and the workload among them already states what it carries")) + case d.Kind != KindWorkload: + errs.add(fieldErrorf("providesFrom", "providesFrom is for kind: %s only: a %s's layers are a delta, where a package database is whatever the recipe happened to rewrite rather than an inventory", KindWorkload, d.Kind)) + } + + errs.add(RequireDistinctProvidesFrom(d, d.ProvidesFrom)) + seen := map[string]string{} + for _, name := range slices.Sorted(maps.Keys(d.ProvidesFrom)) { + at := "providesFrom." + name + file := d.ProvidesFrom[name] + if err := capabilityNameError(name); err != nil { + errs.add(fieldErrorf(at, "providesFrom key %q: %v", name, err)) + continue + } + if IsDerivedProvide(name) { + namespace, _, _ := strings.Cut(name, "/") + errs.add(fieldErrorf(at, "providesFrom key %q is in the %s/ namespace, which publishing fills from the image's package database; name the capability the file stands for instead", name, namespace)) + continue + } + normalized := NormalizeCapabilityName(name) + if prev, dup := seen[normalized]; dup { + errs.add(fieldErrorf(at, "providesFrom keys %q and %q are the same capability", prev, name)) + } + seen[normalized] = name + switch { + case ContainsKitPlaceholder(file): + errs.add(fieldErrorf(at, "providesFrom.%s: arg references are not allowed in providesFrom; the path names a file in the image, not an input", name)) + case !canonicalAbsPath(file) || file == "/": + errs.add(fieldErrorf(at, "providesFrom.%s: %q is not an absolute, canonical path to a file", name, file)) + } + } + return errs.err() +} + +// RequireDistinctProvidesFrom refuses a providesFrom key that names a +// capability the descriptor's provides already states: a capability has +// one version, and two entries for it would leave a resolver to pick. +// +// Separate from the grammar check, and exported, because the frontend +// asks it a second time on the build-expanded descriptor: an authored +// `${{ kit.args.tool }}@1.0.0` holds no name to compare until the +// build-phase value is in it. +func RequireDistinctProvidesFrom(d *Descriptor, from map[string]string) error { + var errs ValidationErrors + authored := map[string]int{} + for i, s := range d.Provides { + if p, err := ParseProvide(s); err == nil { + authored[p.Name] = i + } + } + for _, name := range slices.Sorted(maps.Keys(from)) { + if capabilityNameError(name) != nil { + continue + } + if i, dup := authored[NormalizeCapabilityName(name)]; dup { + errs.add(fieldErrorf("providesFrom."+name, "providesFrom key %q is already stated by provides[%d]; a capability has one version, so state it in one place", name, i)) + } + } + return errs.err() +} + +// WithoutProvidesFrom returns a descriptor with its providesFrom field +// removed, for the published form (§9.7). +// +// The node tree is edited rather than re-encoded from the struct, for +// the reason every other publish-time rewrite here is: the copy a +// reader finds staged in the image keeps the author's own comments and +// ordering. A descriptor without the field comes back byte for byte. +func WithoutProvidesFrom(raw []byte) ([]byte, error) { + var doc yaml.Node + if err := yaml.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("reparse descriptor: %w", err) + } + root := &doc + if root.Kind == yaml.DocumentNode && len(root.Content) > 0 { + root = root.Content[0] + } + if root.Kind != yaml.MappingNode { + return raw, nil + } + kept := make([]*yaml.Node, 0, len(root.Content)) + removed := false + for i := 0; i+1 < len(root.Content); i += 2 { + if root.Content[i].Value == "providesFrom" { + removed = true + continue + } + kept = append(kept, root.Content[i], root.Content[i+1]) + } + if !removed { + return raw, nil + } + root.Content = kept + out, err := yaml.Marshal(&doc) + if err != nil { + return nil, fmt.Errorf("re-encode descriptor without providesFrom: %w", err) + } + return out, nil +} + +// ImageEntry is one name in a directory of a kit's filesystem. Link is +// the target of a symbolic link exactly as stored, and empty for +// anything else. +type ImageEntry struct { + Name string + Dir bool + Link string +} + +// ImageFiles reads one platform's filesystem. It is the whole of what +// [ProvidesFromPaths] needs from a build, so a frontend adapts whatever +// holds its result — a BuildKit reference, a directory — without this +// package learning what that is. +// +// present is false, with a nil error, for a path that does not exist. +// ReadDir lists a directory's own entries without descending and +// without following a link in the last component of an entry, so a +// link is reported as a link. +type ImageFiles interface { + ReadFile(ctx context.Context, name string) (body []byte, present bool, err error) + ReadDir(ctx context.Context, name string) (entries []ImageEntry, present bool, err error) +} + +// PlatformFiles is one platform's filesystem beside the name +// diagnostics call it by. +type PlatformFiles struct { + Platform string + Files ImageFiles +} + +// ProvidesFromPaths resolves a providesFrom declaration against every +// platform a kit publishes and renders the provides entries it states, +// sorted (§9.7). +// +// Each path has to resolve on every platform, to one version. Derived +// entries are dropped where platforms disagree, because nobody asked +// for them one by one; these were asked for by name, so an entry that +// cannot be stated is a build failure the author can see rather than a +// capability a consumer discovers missing. +func ProvidesFromPaths(ctx context.Context, from map[string]string, platforms []PlatformFiles) ([]string, error) { + if len(from) == 0 { + return nil, nil + } + if len(platforms) == 0 { + return nil, fmt.Errorf("providesFrom: no filesystem to read; the kit built no content") + } + names := slices.Sorted(maps.Keys(from)) + versions := map[string]string{} + for i, p := range platforms { + if p.Files == nil { + return nil, fmt.Errorf("providesFrom: no filesystem to read%s; the kit built no content", onPlatform(p.Platform)) + } + owners, err := readFileOwners(ctx, p.Files) + if err != nil { + return nil, fmt.Errorf("providesFrom%s: %w", onPlatform(p.Platform), err) + } + for _, name := range names { + version, err := owners.version(ctx, from[name]) + if err != nil { + return nil, fmt.Errorf("providesFrom.%s%s: %w", name, onPlatform(p.Platform), err) + } + if i > 0 && versions[name] != version { + return nil, fmt.Errorf("providesFrom.%s: %s is at %s%s and %s%s; one descriptor serves every platform, so it can state only a version they share", + name, from[name], versions[name], onPlatform(platforms[0].Platform), version, onPlatform(p.Platform)) + } + versions[name] = version + } + } + out := make([]string, 0, len(names)) + for _, name := range names { + out = append(out, name+"@"+versions[name]) + } + sort.Strings(out) + return out, nil +} + +func onPlatform(platform string) string { + if platform == "" { + return "" + } + return " on " + platform +} + +// fileOwners is one filesystem's record of which installed package put +// each path on disk. +type fileOwners struct { + files ImageFiles + // owners maps a path to the packages that list it, each named + // /. More than one is ordinary for a directory + // and a fact worth refusing for a file. + owners map[string]map[string]bool + // versions maps a package to the one version this model can name + // for it, or "" when it can name none. + versions map[string]string + // aliases maps a top-level directory that is a symbolic link to + // where it leads: /bin to /usr/bin on a merged-/usr system. + aliases map[string]string + dirs map[string][]ImageEntry +} + +// readFileOwners reads every package database the filesystem carries. +func readFileOwners(ctx context.Context, files ImageFiles) (*fileOwners, error) { + o := &fileOwners{ + files: files, + owners: map[string]map[string]bool{}, + versions: map[string]string{}, + aliases: map[string]string{}, + dirs: map[string][]ImageEntry{}, + } + // The top-level links first, because every path read below is + // compared through them. A merged-/usr system reaches one file by + // two spellings — dpkg may record /bin/sh for what PATH finds at + // /usr/bin/sh — and a lookup that knew only one of them would call + // an installed command unowned. + root, _, err := files.ReadDir(ctx, "/") + if err != nil { + return nil, fmt.Errorf("read /: %w", err) + } + for _, e := range root { + if e.Link == "" { + continue + } + target := e.Link + if !path.IsAbs(target) { + target = path.Join("/", target) + } + o.aliases["/"+e.Name] = path.Clean(target) + } + + found := false + for _, db := range PackageDatabases() { + body, present, err := files.ReadFile(ctx, db.Path) + if err != nil { + return nil, fmt.Errorf("read %s: %w", db.Path, err) + } + if !present { + continue + } + found = true + packages, err := db.Read(bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("%s: %w", db.Path, err) + } + for _, p := range packages { + key := db.Namespace + "/" + p.Name + version := PackageVersion(p.Version) + // A multiarch database names one package once per + // architecture. Two records at two versions are a package + // this model cannot put one version on. + if known, ok := o.versions[key]; ok && known != version { + version = "" + } + o.versions[key] = version + } + switch db.Namespace { + case DebNamespace: + err = o.readDpkgLists(ctx) + case ApkNamespace: + err = o.readApkFiles(body) + } + if err != nil { + return nil, err + } + } + if !found { + return nil, fmt.Errorf("the image carries neither %s nor %s, so nothing records which package a file belongs to", DpkgStatusPath, ApkInstalledPath) + } + return o, nil +} + +// readDpkgLists reads dpkg's per-package file lists. +// +// Files rather than `dpkg -S`, because an image need not carry dpkg to +// carry its database: one assembled from packages keeps the lists +// without the tool that wrote them. +func (o *fileOwners) readDpkgLists(ctx context.Context) error { + entries, present, err := o.files.ReadDir(ctx, DpkgInfoDir) + if err != nil { + return fmt.Errorf("read %s: %w", DpkgInfoDir, err) + } + if !present { + return fmt.Errorf("the image has %s but no %s, so nothing records which package a file belongs to", DpkgStatusPath, DpkgInfoDir) + } + for _, e := range entries { + stem, isList := strings.CutSuffix(e.Name, ".list") + if !isList || e.Dir { + continue + } + // A multiarch list is :.list; the architecture + // is not part of the name the status file knows it by. + name, _, _ := strings.Cut(stem, ":") + key := DebNamespace + "/" + name + // A list outlives its package: a removed one keeps the + // conffiles it left behind. Only what the status file calls + // installed owns anything. + if _, installed := o.versions[key]; !installed { + continue + } + list := path.Join(DpkgInfoDir, e.Name) + body, present, err := o.files.ReadFile(ctx, list) + if err != nil { + return fmt.Errorf("read %s: %w", list, err) + } + if !present { + continue + } + if err := eachLine(bytes.NewReader(body), func(line string) { + o.own(line, key) + }); err != nil { + return fmt.Errorf("%s: %w", list, err) + } + } + return nil +} + +// readApkFiles reads the file lists apk keeps inside its installed +// database: within a package's stanza, an F: line names a directory and +// each R: line after it a file in that directory. +func (o *fileOwners) readApkFiles(body []byte) error { + key, dir := "", "" + return eachLine(bytes.NewReader(body), func(line string) { + field, value, found := strings.Cut(line, ":") + if !found || len(field) != 1 { + return + } + switch field { + case "P": + key, dir = ApkNamespace+"/"+value, "" + case "F": + dir = value + case "R": + if key != "" { + o.own(path.Join("/", dir, value), key) + } + } + }, func() { key, dir = "", "" }) +} + +// eachLine calls line for every non-blank line and blank, when given, +// for every blank one — the stanza boundary of both databases. +func eachLine(r io.Reader, line func(string), blank ...func()) error { + s := bufio.NewScanner(r) + s.Buffer(make([]byte, 0, 64*1024), maxPackageDatabaseLine) + for s.Scan() { + text := strings.TrimSuffix(s.Text(), "\r") + if strings.TrimSpace(text) == "" { + for _, f := range blank { + f() + } + continue + } + line(text) + } + if err := s.Err(); err != nil { + return fmt.Errorf("read package database: %w", err) + } + return nil +} + +func (o *fileOwners) own(file, key string) { + if !strings.HasPrefix(file, "/") { + return + } + file = o.canonical(file) + if o.owners[file] == nil { + o.owners[file] = map[string]bool{} + } + o.owners[file][key] = true +} + +// canonical rewrites a path through a top-level symbolic link, so the +// two spellings a merged-/usr system gives one file compare equal. +func (o *fileOwners) canonical(file string) string { + file = path.Clean(file) + rest := strings.TrimPrefix(file, "/") + top, tail, nested := strings.Cut(rest, "/") + if target, aliased := o.aliases["/"+top]; aliased && nested { + return path.Join(target, tail) + } + return file +} + +// version names the version of the package that owns file, following +// the file through symbolic links until something owns it. +// +// The link is followed only when the path itself is unowned. A package +// that ships a link — python3 pointing at python3.14 — is the answer +// for that link; a link nothing owns was made by a recipe or by +// update-alternatives, and what it leads to is the thing installed. +func (o *fileOwners) version(ctx context.Context, file string) (string, error) { + current := o.canonical(file) + for range maxProvidesFromLinks { + entry, present, err := o.entry(ctx, current) + if err != nil { + return "", err + } + if !present { + if current != file { + return "", fmt.Errorf("%s leads to %s, which does not exist in the image", file, current) + } + return "", fmt.Errorf("%s does not exist in the image", file) + } + if entry.Dir { + return "", fmt.Errorf("%s is a directory; name a file a package installs, such as the command itself", current) + } + if owners := o.owners[current]; len(owners) > 0 { + return o.ownerVersion(file, owners) + } + if entry.Link == "" { + return "", fmt.Errorf("%s belongs to no installed package; a file a recipe placed has no package to take a version from, so state its version in provides", current) + } + next := entry.Link + if !path.IsAbs(next) { + next = path.Join(path.Dir(current), next) + } + current = o.canonical(next) + } + return "", fmt.Errorf("%s leads through more than %d symbolic links", file, maxProvidesFromLinks) +} + +func (o *fileOwners) ownerVersion(file string, owners map[string]bool) (string, error) { + keys := slices.Sorted(maps.Keys(owners)) + if len(keys) > 1 { + return "", fmt.Errorf("%s is listed by %s; a version comes from one package, so name a file only one of them installs", file, strings.Join(keys, " and ")) + } + version := o.versions[keys[0]] + if version == "" { + return "", fmt.Errorf("%s belongs to %s, whose recorded version this model cannot name (§9.6); state the capability's version in provides", file, keys[0]) + } + return version, nil +} + +// entry reports what stands at file, read out of its directory's +// listing so that a symbolic link is seen as one. +func (o *fileOwners) entry(ctx context.Context, file string) (ImageEntry, bool, error) { + dir, base := path.Dir(file), path.Base(file) + entries, cached := o.dirs[dir] + if !cached { + var err error + entries, _, err = o.files.ReadDir(ctx, dir) + if err != nil { + return ImageEntry{}, false, fmt.Errorf("read %s: %w", dir, err) + } + o.dirs[dir] = entries + } + for _, e := range entries { + if e.Name == base { + return e, true, nil + } + } + return ImageEntry{}, false, nil +} diff --git a/spec/provides_from_test.go b/spec/provides_from_test.go new file mode 100644 index 00000000..21c246cf --- /dev/null +++ b/spec/provides_from_test.go @@ -0,0 +1,294 @@ +package spec + +import ( + "context" + "path" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// fakeImage is a filesystem held in a map: path to file body, with a +// symbolic link written as "-> target" and a directory implied by +// anything beneath it. +type fakeImage map[string]string + +func (f fakeImage) ReadFile(_ context.Context, name string) ([]byte, bool, error) { + body, ok := f[name] + if !ok || strings.HasPrefix(body, "-> ") { + return nil, false, nil + } + return []byte(body), true, nil +} + +func (f fakeImage) ReadDir(_ context.Context, name string) ([]ImageEntry, bool, error) { + seen := map[string]ImageEntry{} + for file, body := range f { + if path.Dir(file) == name { + e := ImageEntry{Name: path.Base(file)} + if target, link := strings.CutPrefix(body, "-> "); link { + e.Link = target + } + seen[e.Name] = e + continue + } + prefix := strings.TrimSuffix(name, "/") + "/" + if rest, under := strings.CutPrefix(file, prefix); under { + top, _, _ := strings.Cut(rest, "/") + if _, known := seen[top]; !known { + seen[top] = ImageEntry{Name: top, Dir: true} + } + } + } + out := make([]ImageEntry, 0, len(seen)) + for _, e := range seen { + out = append(out, e) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out, len(out) > 0, nil +} + +// debianImage is the shape of a merged-/usr Debian root: /bin is a link, +// python3 is a link its package ships, awk is an alternatives link no +// package owns, and one removed package still has a list. +func debianImage() fakeImage { + return fakeImage{ + "/bin": "-> usr/bin", + DpkgStatusPath: strings.Join([]string{ + "Package: python-3.14-minimal\nStatus: install ok installed\nVersion: 3.14.10-6\n", + "Package: openssh-client\nStatus: install ok installed\nVersion: 1:10.0p1-7+deb13u4+dhi3\n", + "Package: mawk\nStatus: install ok installed\nVersion: 1.3.4.20250131-1+deb13u0\n", + "Package: bash\nStatus: install ok installed\nVersion: 5.2.37-2+dhi1\n", + "Package: gone\nStatus: deinstall ok config-files\nVersion: 1.0-1\n", + "Package: pre\nStatus: install ok installed\nVersion: 2.0~rc1-1\n", + "Package: libfoo\nStatus: install ok installed\nVersion: 1.2.3-1\n", + "Package: libbar\nStatus: install ok installed\nVersion: 4.5.6-1\n", + }, "\n"), + DpkgInfoDir + "/python-3.14-minimal.list": "/.\n/usr\n/usr/bin\n/usr/bin/python3.14\n/usr/bin/python3\n", + DpkgInfoDir + "/openssh-client.list": "/.\n/usr\n/usr/bin\n/usr/bin/ssh\n", + DpkgInfoDir + "/mawk.list": "/.\n/usr\n/usr/bin\n/usr/bin/mawk\n", + // Recorded under /bin, as packages built before the /usr merge do. + DpkgInfoDir + "/bash.list": "/.\n/bin\n/bin/bash\n", + DpkgInfoDir + "/gone.list": "/etc/gone.conf\n", + DpkgInfoDir + "/pre.list": "/usr/bin/pre\n", + DpkgInfoDir + "/libfoo:arm64.list": "/usr/lib/shared.so\n", + DpkgInfoDir + "/libbar.list": "/usr/lib/shared.so\n", + DpkgInfoDir + "/bash.md5sums": "ignored\n", + "/usr/bin/python3": "-> python3.14", + "/usr/bin/python3.14": "elf", + "/usr/bin/ssh": "elf", + "/usr/bin/mawk": "elf", + "/usr/bin/awk": "-> /etc/alternatives/awk", + "/etc/alternatives/awk": "-> /usr/bin/mawk", + "/usr/bin/bash": "elf", + "/usr/bin/pre": "elf", + "/usr/lib/shared.so": "elf", + "/usr/local/bin/claude": "elf", + "/usr/local/bin/dangling": "-> /nowhere", + "/usr/local/bin/loop": "-> loop", + "/etc/gone.conf": "conf", + } +} + +func one(f ImageFiles) []PlatformFiles { + return []PlatformFiles{{Platform: "linux/arm64", Files: f}} +} + +func TestProvidesFromPaths(t *testing.T) { + got, err := ProvidesFromPaths(context.Background(), map[string]string{ + "python": "/usr/bin/python3", + "ssh": "/usr/bin/ssh", + "awk": "/usr/bin/awk", + "bash": "/usr/bin/bash", + "com.example/shell": "/bin/bash", + }, one(debianImage())) + require.NoError(t, err) + require.Equal(t, []string{ + "awk@1.3.4", + "bash@5.2.37", + "com.example/shell@5.2.37", + "python@3.14.10", + "ssh@10.0", + }, got) +} + +func TestProvidesFromPathsRefusals(t *testing.T) { + for name, tc := range map[string]struct { + file string + want string + }{ + "absent": {"/usr/bin/nope", "does not exist in the image"}, + "directory": {"/usr/bin", "is a directory"}, + "unowned": {"/usr/local/bin/claude", "belongs to no installed package"}, + "dangling link": {"/usr/local/bin/dangling", "leads to /nowhere, which does not exist"}, + "link cycle": {"/usr/local/bin/loop", "more than 16 symbolic links"}, + "removed package": {"/etc/gone.conf", "belongs to no installed package"}, + "prerelease": {"/usr/bin/pre", "cannot name"}, + "two packages": {"/usr/lib/shared.so", "deb/libbar and deb/libfoo"}, + "unversioned owner": {"/usr/bin/pre", "state the capability's version in provides"}, + } { + t.Run(name, func(t *testing.T) { + _, err := ProvidesFromPaths(context.Background(), map[string]string{"x": tc.file}, one(debianImage())) + require.ErrorContains(t, err, tc.want) + require.ErrorContains(t, err, "providesFrom.x on linux/arm64") + }) + } +} + +func TestProvidesFromPathsNeedsADatabase(t *testing.T) { + _, err := ProvidesFromPaths(context.Background(), map[string]string{"x": "/bin/x"}, one(fakeImage{"/bin/x": "elf"})) + require.ErrorContains(t, err, "neither /var/lib/dpkg/status nor /lib/apk/db/installed") + + noLists := fakeImage{"/bin/x": "elf", DpkgStatusPath: "Package: x\nStatus: install ok installed\nVersion: 1.0-1\n"} + _, err = ProvidesFromPaths(context.Background(), map[string]string{"x": "/bin/x"}, one(noLists)) + require.ErrorContains(t, err, "no /var/lib/dpkg/info") + + _, err = ProvidesFromPaths(context.Background(), map[string]string{"x": "/bin/x"}, nil) + require.ErrorContains(t, err, "no filesystem to read") + + _, err = ProvidesFromPaths(context.Background(), map[string]string{"x": "/bin/x"}, []PlatformFiles{{Platform: "linux/amd64"}}) + require.ErrorContains(t, err, "no filesystem to read on linux/amd64") + + got, err := ProvidesFromPaths(context.Background(), nil, nil) + require.NoError(t, err) + require.Empty(t, got) +} + +func TestProvidesFromPathsAcrossPlatforms(t *testing.T) { + arm, amd := debianImage(), debianImage() + platforms := []PlatformFiles{{Platform: "linux/arm64", Files: arm}, {Platform: "linux/amd64", Files: amd}} + from := map[string]string{"ssh": "/usr/bin/ssh"} + + got, err := ProvidesFromPaths(context.Background(), from, platforms) + require.NoError(t, err) + require.Equal(t, []string{"ssh@10.0"}, got) + + // A Debian revision is packaging bookkeeping (§9.6), so platforms + // that differ only there still share a version. + amd[DpkgStatusPath] = strings.Replace(amd[DpkgStatusPath], "1:10.0p1-7+deb13u4+dhi3", "1:10.0p1-8", 1) + got, err = ProvidesFromPaths(context.Background(), from, platforms) + require.NoError(t, err) + require.Equal(t, []string{"ssh@10.0"}, got) + + amd[DpkgStatusPath] = strings.Replace(amd[DpkgStatusPath], "1:10.0p1-8", "1:10.1p1-1", 1) + _, err = ProvidesFromPaths(context.Background(), from, platforms) + require.ErrorContains(t, err, "/usr/bin/ssh is at 10.0 on linux/arm64 and 10.1 on linux/amd64") + + delete(amd, "/usr/bin/ssh") + _, err = ProvidesFromPaths(context.Background(), from, platforms) + require.ErrorContains(t, err, "providesFrom.ssh on linux/amd64: /usr/bin/ssh does not exist") +} + +func TestProvidesFromPathsApk(t *testing.T) { + image := fakeImage{ + ApkInstalledPath: strings.Join([]string{ + "C:Q1abc=\nP:busybox\nV:1.37.0-r20\nA:aarch64\nF:bin\nR:busybox\na:0:0:755\nZ:Q1def=\nF:etc\nR:securetty\n", + "C:Q1ghi=\nP:python3\nV:3.12.12-r0\nF:usr\nF:usr/bin\nR:python3.12\nR:python3\na:0:0:777\n", + }, "\n"), + "/bin/busybox": "elf", + "/bin/sh": "-> /bin/busybox", + "/usr/bin/python3": "-> python3.12", + "/usr/bin/python3.12": "elf", + } + got, err := ProvidesFromPaths(context.Background(), map[string]string{ + "python": "/usr/bin/python3", + "sh": "/bin/sh", + }, one(image)) + require.NoError(t, err) + require.Equal(t, []string{"python@3.12.12", "sh@1.37.0"}, got) +} + +func TestValidateProvidesFrom(t *testing.T) { + base := func() *Descriptor { + return &Descriptor{SchemaVersion: "3", Kind: KindWorkload, Build: "FROM scratch\n"} + } + ok := base() + ok.Provides = []string{"claude@1.0.0"} + ok.ProvidesFrom = map[string]string{"python": "/usr/bin/python3", "com.example/tool": "/opt/tool/bin/tool"} + _, err := Validate(ok) + require.NoError(t, err) + + for name, tc := range map[string]struct { + mutate func(*Descriptor) + want string + }{ + "mixin": {func(d *Descriptor) { d.Kind = KindMixin }, "providesFrom is for kind: workload only"}, + "set": {func(d *Descriptor) { + d.Kind, d.Build = KindSet, "" + d.Kits = []Kit{{Ref: "docker.io/example/base:1.0.0"}} + }, "a set builds none"}, + "bad name": {func(d *Descriptor) { d.ProvidesFrom = map[string]string{"Python": "/usr/bin/python3"} }, "invalid capability name"}, + "derived space": {func(d *Descriptor) { d.ProvidesFrom = map[string]string{"deb/python": "/usr/bin/python3"} }, "deb/ namespace"}, + "same capability": {func(d *Descriptor) { + d.ProvidesFrom = map[string]string{"gh": "/usr/bin/gh", "com.docker.kit/gh": "/usr/bin/gh"} + }, "are the same capability"}, + "already provided": {func(d *Descriptor) { + d.Provides = []string{"python@3.0.0"} + d.ProvidesFrom = map[string]string{"python": "/usr/bin/python3"} + }, "already stated by provides[0]"}, + "relative": {func(d *Descriptor) { d.ProvidesFrom = map[string]string{"python": "usr/bin/python3"} }, "not an absolute, canonical path"}, + "not clean": {func(d *Descriptor) { d.ProvidesFrom = map[string]string{"python": "/usr/bin/../bin/python3"} }, "not an absolute, canonical path"}, + "root": {func(d *Descriptor) { d.ProvidesFrom = map[string]string{"python": "/"} }, "not an absolute, canonical path"}, + "empty": {func(d *Descriptor) { d.ProvidesFrom = map[string]string{"python": ""} }, "not an absolute, canonical path"}, + "arg in value": {func(d *Descriptor) { d.ProvidesFrom = map[string]string{"python": "/usr/bin/${{ kit.args.python }}"} }, "arg references are not allowed in providesFrom"}, + } { + t.Run(name, func(t *testing.T) { + d := base() + d.ProvidesFrom = map[string]string{"python": "/usr/bin/python3"} + tc.mutate(d) + _, err := Validate(d) + require.ErrorContains(t, err, tc.want) + }) + } +} + +func TestWithoutProvidesFrom(t *testing.T) { + authored := []byte(`schemaVersion: "3" +kind: workload +# what the agent is +provides: ["claude@1.0.0"] +# what the base ships +providesFrom: + python: /usr/bin/python3 + ssh: /usr/bin/ssh +version: "1.0.0" +build: | + FROM scratch +`) + out, err := WithoutProvidesFrom(authored) + require.NoError(t, err) + require.NotContains(t, string(out), "providesFrom") + require.NotContains(t, string(out), "/usr/bin/python3") + require.Contains(t, string(out), "# what the agent is") + + d, err := Decode(out) + require.NoError(t, err) + require.Empty(t, d.ProvidesFrom) + require.Equal(t, []string{"claude@1.0.0"}, d.Provides) + _, err = ValidatePublished(out, d) + require.NoError(t, err) + + // A descriptor without the field is returned untouched, comments, + // quoting and all. + plain := []byte("schemaVersion: \"3\"\nkind: mixin # overlay\n") + out, err = WithoutProvidesFrom(plain) + require.NoError(t, err) + require.Equal(t, string(plain), string(out)) +} + +func TestPublishedRefusesProvidesFrom(t *testing.T) { + raw := []byte(`schemaVersion: "3" +kind: workload +version: "1.0.0" +providesFrom: + python: /usr/bin/python3 +build: | + FROM scratch +`) + d, err := Decode(raw) + require.NoError(t, err) + _, err = ValidatePublished(raw, d) + require.ErrorContains(t, err, "published descriptor still carries providesFrom") +} diff --git a/spec/schema_test.go b/spec/schema_test.go index 1140f322..cb5a97a7 100644 --- a/spec/schema_test.go +++ b/spec/schema_test.go @@ -769,3 +769,32 @@ func TestAgentContextV2DeliverySchema(t *testing.T) { require.ElementsMatch(t, []any{AgentContextDeliveryProgressive, AgentContextDeliveryInline}, branches[0].(map[string]any)["enum"]) assertAcceptsKitArg(t, delivery, "bearing") } + +// The schema restates two providesFrom rules the validator owns: which +// kinds may use it, and which namespaces its keys may not be in. Pinned +// so an editor's verdict and the build's cannot drift. +func TestProvidesFromSchemaMatchesGrammar(t *testing.T) { + schema := loadSchema(t) + field := at(t, schema, "properties", "providesFrom") + require.Equal(t, "object", field["type"]) + require.Equal(t, "string", at(t, field, "additionalProperties")["type"]) + + reserved := regexp.MustCompile(at(t, field, "propertyNames", "not")["pattern"].(string)) + for namespace := range reservedNamespaces { + require.True(t, reserved.MatchString(namespace+"/bash"), "the schema admits a providesFrom key under %s/", namespace) + } + require.False(t, reserved.MatchString("bash")) + require.False(t, reserved.MatchString("com.example/bash")) + + var rule map[string]any + for _, r := range schema["allOf"].([]any) { + candidate := r.(map[string]any) + ifClause, _ := candidate["if"].(map[string]any) + if required, _ := ifClause["required"].([]any); len(required) == 1 && required[0] == "providesFrom" { + rule = candidate + } + } + require.NotNil(t, rule, "the schema states no providesFrom rule") + require.Equal(t, KindWorkload, at(t, rule, "then", "properties", "kind")["const"]) + require.Equal(t, []any{"kits"}, at(t, rule, "then", "not")["required"]) +} diff --git a/spec/types.go b/spec/types.go index 1daa910a..35b9ce3d 100644 --- a/spec/types.go +++ b/spec/types.go @@ -164,6 +164,21 @@ type Descriptor struct { // requirable by name states so here. Provides []string `json:"provides,omitempty" yaml:"provides,omitempty"` + // ProvidesFrom states provides entries whose version the author + // does not know: capability name to the absolute path of a file in + // the kit's own filesystem. Publishing finds the installed package + // that owns the file and states name@ in + // Provides (§9.7), so a workload can offer `python` at whatever its + // base image ships without a release number of its own standing in + // for one. + // + // Authored form only. It is an instruction to the build rather than + // a declaration, and the published descriptor carries its result and + // not the field: a runtime built before this field existed decodes + // strictly, and would refuse a published kit for a key that has + // nothing left to tell it. + ProvidesFrom map[string]string `json:"providesFrom,omitempty" yaml:"providesFrom,omitempty"` + // Requires lists capabilities that must be satisfied by the resolved // set, as "name" or "name version" (comma-AND'd constraints; // ops: >=, >, <=, <, =). Capability names resolve to nothing and diff --git a/spec/validate.go b/spec/validate.go index 5286dbf2..5707f2a4 100644 --- a/spec/validate.go +++ b/spec/validate.go @@ -61,6 +61,7 @@ func Validate(d *Descriptor) (warnings []string, err error) { errs.add(validateNoNUL(d)) errs.add(validateIconURL(d.IconURL)) errs.add(validateCapabilityNames(d)) + errs.add(validateProvidesFrom(d)) errs.add(validateCapabilityEntries(d)) errs.add(validateArgs(d.Args)) errs.add(validateRecipe(d)) @@ -353,6 +354,13 @@ func ValidatePublished(raw []byte, d *Descriptor) (warnings []string, err error) errs.add(fieldErrorf(fmt.Sprintf("provides[%d]", i), "published descriptor still references an arg in provides entry %q; build-phase expansion did not run", s)) } } + // providesFrom is an instruction to the build, and publishing + // replaces it with the entries it stated (§9.7). One that survives + // names files nobody resolved, in a field readers built before it + // existed refuse to decode. + if len(d.ProvidesFrom) > 0 { + errs.add(fieldErrorf("providesFrom", "published descriptor still carries providesFrom; publishing states those entries in provides and drops the field, so this artifact was never resolved")) + } // §9.2 is a property of every published descriptor, not only of the // frontend's build path, so third-party artifacts are held to it too. if d.Version == "" { diff --git a/tck/sandbox/coverage_test.go b/tck/sandbox/coverage_test.go index 832b0758..c5e7c459 100644 --- a/tck/sandbox/coverage_test.go +++ b/tck/sandbox/coverage_test.go @@ -146,6 +146,9 @@ var kitCovers = map[string][]string{ "SPEC-v3 §7.1.1/validate-declarations", "SPEC-v3 §9.2/versioned-provides", "SPEC-v3 §3.4/set-kind-never-published", + // ValidatePublished refuses the field, so an artifact that + // still carries it fails the same check. + "SPEC-v3 §9.7/not-published", }, // A merged kit's staged roots are counted against the kits its @@ -283,6 +286,17 @@ var waived = map[string]string{ "SPEC-v3 §5.1/derived-namespaces-reserved": "grammar rule, judged by the spec package's validators; a published descriptor cannot distinguish an authored deb/ entry from a derived one, and derived-provides refuses either if the image disagrees", "SPEC-v3 §5.1/namespace-reverse-dns": "grammar rule, judged by the spec package's validators and schema tests", + // Stated provides: publishing replaces providesFrom with ordinary + // provides entries and drops the field, so a published Kit keeps no + // record of which entry came from a path or which path that was. + // What the frontend did with one is judged where it happens. + "SPEC-v3 §9.7/name-is-authored": "grammar rule, judged by the spec package's validators and schema tests", + "SPEC-v3 §9.7/workload-only": "grammar rule, judged by the spec package's validators and schema tests", + "SPEC-v3 §9.7/owned-by-one-package": "build-time resolution, judged by spec/provides_from_test.go; the published descriptor does not record the path an entry was stated from", + "SPEC-v3 §9.7/links-followed-when-unowned": "build-time resolution, judged by spec/provides_from_test.go; the published descriptor does not record the path an entry was stated from", + "SPEC-v3 §9.7/version-is-the-owners": "build-time resolution, judged by spec/provides_from_test.go; the published descriptor does not record the path an entry was stated from", + "SPEC-v3 §9.7/agreed-across-platforms": "the frontend resolves every path on every target platform; one published manifest cannot show what another platform recorded", + // Grammar rules: the spec package's validators and schema tests are // where these are judged, statement by statement. "SPEC-v3 §4/schema-version-3": "grammar rule, judged by the spec package's validators and schema tests", From 3ea27942b8d5d09cf3b05d1bf02e4afa441fb978 Mon Sep 17 00:00:00 2001 From: Christian Dupuis Date: Fri, 20 Nov 2026 13:24:00 +0000 Subject: [PATCH 02/10] fix(spec): remove providesFrom wherever the root gets it, and refuse the key in any spelling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three ways the published form could still carry the field, or break while losing it, all found in review of the change that added it. WithoutProvidesFrom removed only a key written directly at the top level. A YAML merge key hands the root another mapping's keys, so `<<: {providesFrom: …}` decoded to the same descriptor and kept the field through publishing, and the build then refused its own published form. The field is now removed from every mapping the root merges in, inline or nested. A merge source reached through an alias is copied and the copy is edited, because the anchored mapping belongs to wherever it was defined. Removing the field also removed any anchor defined under it, leaving an alias elsewhere naming nothing and the document unparseable. Each alias that points into what is removed, or at an anchored mapping that is edited in place, is first replaced by its own copy. ValidatePublished judged the decoded map's length, so `providesFrom: {}` and a bare `providesFrom:` passed although the key is present and a reader built before the field refuses it. Presence is now read from the raw document, merge keys included. The rewrite parses its own output before returning it, so a case it cannot handle fails with its cause rather than as a decode error later in the build. Signed-off-by: Christian Dupuis Sandbox-Id: spiffe://sbx.docker.com/org/57b45934-74c0-11e4-bea4-0242ac11001b/user/1580b874-0d59-41d8-bd81-82a10abc4100/sandbox/74f8ada1-8c4b-4fd6-9802-da94ba278db2 Sandbox-Kit: sha256:dfcafd4d7957b8b6ac4574528241772465187a778e16bdf809d9c85f43e8fed7 Co-authored-by: sbx docker/cdupuis/74f8ada1 --- docs/spec/SPEC-v3.md | 5 +- spec/provides_from.go | 284 ++++++++++++++++++++++++++++++++++--- spec/provides_from_test.go | 150 ++++++++++++++++++++ spec/validate.go | 7 +- 4 files changed, 428 insertions(+), 18 deletions(-) diff --git a/docs/spec/SPEC-v3.md b/docs/spec/SPEC-v3.md index 516d0eb0..eefbeaec 100644 --- a/docs/spec/SPEC-v3.md +++ b/docs/spec/SPEC-v3.md @@ -1371,7 +1371,10 @@ shipped. instruction to the build, and what it produced is in `provides`. This is also what lets the field exist under `schemaVersion: "3"`: decoding is strict, so a runtime built before this field would refuse a - published Kit that carried it. + published Kit that carried it. The rule is about the key, so an empty + or null `providesFrom` is as much a violation as a full one, and + publishing removes the field wherever the document's root gets it + from — a YAML merge key included. A stated entry is an ordinary `provides` entry once published. It is the workload's to provide, so [§5.3](#53-resolution-semantics-consumer-contract)'s diff --git a/spec/provides_from.go b/spec/provides_from.go index e5ac324f..57bf7621 100644 --- a/spec/provides_from.go +++ b/spec/provides_from.go @@ -102,6 +102,14 @@ func RequireDistinctProvidesFrom(d *Descriptor, from map[string]string) error { return errs.err() } +// providesFromField is the descriptor key, as an author writes it. +const providesFromField = "providesFrom" + +// maxYAMLIndirection bounds how far a merge key or an alias is followed +// while looking for the field. YAML admits an anchor that contains its +// own alias, and a walk with no bound would not return from one. +const maxYAMLIndirection = 32 + // WithoutProvidesFrom returns a descriptor with its providesFrom field // removed, for the published form (§9.7). // @@ -109,36 +117,280 @@ func RequireDistinctProvidesFrom(d *Descriptor, from map[string]string) error { // the reason every other publish-time rewrite here is: the copy a // reader finds staged in the image keeps the author's own comments and // ordering. A descriptor without the field comes back byte for byte. +// +// The field is removed wherever the root mapping gets it from, not only +// where it is written at the top level. A YAML merge key hands the root +// every key of another mapping, so `<<: {providesFrom: …}` decodes to +// the same descriptor as the plain spelling and has to publish the same +// way. And the removal takes nothing else with it: an alias elsewhere +// in the document that points into what is removed is given its own +// copy first, since an anchor that disappears leaves every alias to it +// naming nothing and the document no longer parsing. func WithoutProvidesFrom(raw []byte) ([]byte, error) { var doc yaml.Node if err := yaml.Unmarshal(raw, &doc); err != nil { return nil, fmt.Errorf("reparse descriptor: %w", err) } - root := &doc + root := documentRoot(&doc) + if root == nil || !mappingCarries(root, providesFromField, 0) { + return raw, nil + } + + // Three passes, in this order. What is about to change is found + // first and nothing is touched, so that the second pass can copy an + // aliased node as the author wrote it; only then is anything removed. + edit := &fieldRemoval{field: providesFromField, removed: map[*yaml.Node]bool{}, edited: map[*yaml.Node]bool{}} + edit.plan(root, 0) + edit.detachAliases(&doc) + edit.strip(root, 0) + + out, err := yaml.Marshal(&doc) + if err != nil { + return nil, fmt.Errorf("re-encode descriptor without %s: %w", providesFromField, err) + } + // Parsed once more before it is handed back: this is the document + // that gets staged and signed, and a rewrite that left it unreadable + // should say so here rather than as a decode failure with no cause. + var check yaml.Node + if err := yaml.Unmarshal(out, &check); err != nil { + return nil, fmt.Errorf("descriptor no longer parses with %s removed: %w", providesFromField, err) + } + if mappingCarries(documentRoot(&check), providesFromField, 0) { + return nil, fmt.Errorf("%s could not be removed from the descriptor; write it as a plain top-level field", providesFromField) + } + return out, nil +} + +// carriesProvidesFrom reports whether a raw descriptor has the field at +// all — written out or inherited through a merge key, with entries or +// without. Presence is the question for the published form, where a +// reader built before the field refuses the key whatever is under it, +// and the decoded struct cannot answer it: an empty map and a null +// decode exactly like a field that was never there. +func carriesProvidesFrom(raw []byte) bool { + var doc yaml.Node + if err := yaml.Unmarshal(raw, &doc); err != nil { + // Unparseable bytes are another rule's to report. + return false + } + return mappingCarries(documentRoot(&doc), providesFromField, 0) +} + +// documentRoot returns the mapping a descriptor document consists of, +// or nil when it is not one. +func documentRoot(doc *yaml.Node) *yaml.Node { + root := doc if root.Kind == yaml.DocumentNode && len(root.Content) > 0 { root = root.Content[0] } - if root.Kind != yaml.MappingNode { - return raw, nil + root = aliasTarget(root) + if root == nil || root.Kind != yaml.MappingNode { + return nil + } + return root +} + +// aliasTarget follows an alias to the node it names. +func aliasTarget(n *yaml.Node) *yaml.Node { + for range maxYAMLIndirection { + if n == nil || n.Kind != yaml.AliasNode { + return n + } + n = n.Alias + } + return nil +} + +// isMergeKey reports whether a mapping key is YAML's merge key. Only +// the plain scalar is: a quoted "<<" is an ordinary string key, which +// is the distinction the decoder draws too. +func isMergeKey(n *yaml.Node) bool { + return n.Kind == yaml.ScalarNode && n.Value == "<<" && n.Tag == "!!merge" +} + +// mergeSources returns the mappings a merge value contributes: the +// value itself, or each element when it is a sequence of them. +func mergeSources(value *yaml.Node) []*yaml.Node { + if value.Kind == yaml.SequenceNode { + return value.Content } - kept := make([]*yaml.Node, 0, len(root.Content)) - removed := false - for i := 0; i+1 < len(root.Content); i += 2 { - if root.Content[i].Value == "providesFrom" { - removed = true + return []*yaml.Node{value} +} + +// mappingCarries reports whether a mapping has key, directly or through +// any mapping it merges in. +func mappingCarries(m *yaml.Node, key string, depth int) bool { + m = aliasTarget(m) + if m == nil || m.Kind != yaml.MappingNode || depth > maxYAMLIndirection { + return false + } + for i := 0; i+1 < len(m.Content); i += 2 { + k, v := m.Content[i], m.Content[i+1] + if isMergeKey(k) { + for _, source := range mergeSources(v) { + if mappingCarries(source, key, depth+1) { + return true + } + } continue } - kept = append(kept, root.Content[i], root.Content[i+1]) + if k.Value == key { + return true + } } - if !removed { - return raw, nil + return false +} + +// fieldRemoval takes one key out of a mapping and everything the +// mapping merges in. +type fieldRemoval struct { + field string + // removed holds every node that leaves the document: the field's + // key and value, and all that is under the value. + removed map[*yaml.Node]bool + // edited holds the anchored mappings the field is taken out of in + // place. An alias to one of these would see the mapping change + // under it. + edited map[*yaml.Node]bool +} + +// plan records what strip will remove from m and which shared mappings +// it will edit, without changing anything. +// +// A merge source reached through an alias is not recorded: strip works +// on a copy of it, so the anchored original stays as it was for +// whatever else refers to it. +func (e *fieldRemoval) plan(m *yaml.Node, depth int) { + if depth > maxYAMLIndirection { + return } - root.Content = kept - out, err := yaml.Marshal(&doc) - if err != nil { - return nil, fmt.Errorf("re-encode descriptor without providesFrom: %w", err) + for i := 0; i+1 < len(m.Content); i += 2 { + k, v := m.Content[i], m.Content[i+1] + if isMergeKey(k) { + for _, source := range mergeSources(v) { + if source.Kind != yaml.MappingNode || !mappingCarries(source, e.field, depth+1) { + continue + } + if source.Anchor != "" { + e.edited[source] = true + } + e.plan(source, depth+1) + } + continue + } + if k.Value == e.field { + e.markRemoved(k) + e.markRemoved(v) + } } - return out, nil +} + +func (e *fieldRemoval) markRemoved(n *yaml.Node) { + if n == nil || e.removed[n] { + return + } + e.removed[n] = true + for _, child := range n.Content { + e.markRemoved(child) + } +} + +// detachAliases gives every alias that survives the removal its own +// copy of a node the removal is about to take away or change. +func (e *fieldRemoval) detachAliases(n *yaml.Node) { + if n == nil || e.removed[n] { + return + } + if n.Kind == yaml.AliasNode { + if e.removed[n.Alias] || e.edited[n.Alias] { + // Onto the alias node itself, so whatever holds it now + // holds the copy. + *n = *e.detachedCopy(n.Alias, 0) + } + return + } + for _, child := range n.Content { + e.detachAliases(child) + } +} + +// detachedCopy copies a subtree so that it stands on its own: no anchor +// is copied, since the original keeps its name or is going away, and an +// alias to a node that is going away is replaced by a copy of that +// node. An alias to anything that stays is kept as an alias. +// +// hops counts the aliases followed to get here, not how deep the tree +// is: only following an alias can lead back to where the copy started. +func (e *fieldRemoval) detachedCopy(n *yaml.Node, hops int) *yaml.Node { + if n.Kind == yaml.AliasNode && n.Alias != nil && e.removed[n.Alias] && hops < maxYAMLIndirection { + return e.detachedCopy(n.Alias, hops+1) + } + c := *n + c.Anchor = "" + c.Content = nil + for _, child := range n.Content { + c.Content = append(c.Content, e.detachedCopy(child, hops)) + } + return &c +} + +// strip removes the field from m and from every mapping m merges in. +func (e *fieldRemoval) strip(m *yaml.Node, depth int) { + if depth > maxYAMLIndirection { + return + } + kept := make([]*yaml.Node, 0, len(m.Content)) + for i := 0; i+1 < len(m.Content); i += 2 { + k, v := m.Content[i], m.Content[i+1] + switch { + case isMergeKey(k) && v.Kind == yaml.SequenceNode: + sources := make([]*yaml.Node, 0, len(v.Content)) + for _, source := range v.Content { + if source = e.stripSource(source, depth); source != nil { + sources = append(sources, source) + } + } + v.Content = sources + // A merge of nothing is noise the author did not write. + if len(sources) == 0 && v.Anchor == "" { + continue + } + case isMergeKey(k): + if v = e.stripSource(v, depth); v == nil { + continue + } + case k.Value == e.field: + continue + } + kept = append(kept, k, v) + } + m.Content = kept +} + +// stripSource returns one merge source with the field taken out, or nil +// when nothing is left of it to merge. +func (e *fieldRemoval) stripSource(source *yaml.Node, depth int) *yaml.Node { + if !mappingCarries(source, e.field, depth+1) { + return source + } + if source.Kind == yaml.AliasNode { + // The anchored mapping belongs to wherever it was defined, and + // may be merged or aliased there for reasons of its own, so + // the root merges a copy without the field instead. + target := aliasTarget(source) + if target == nil { + return source + } + source = e.detachedCopy(target, 0) + } + if source.Kind != yaml.MappingNode { + return source + } + e.strip(source, depth+1) + if len(source.Content) == 0 && source.Anchor == "" { + return nil + } + return source } // ImageEntry is one name in a directory of a kit's filesystem. Link is diff --git a/spec/provides_from_test.go b/spec/provides_from_test.go index 21c246cf..63ae6dcd 100644 --- a/spec/provides_from_test.go +++ b/spec/provides_from_test.go @@ -292,3 +292,153 @@ build: | _, err = ValidatePublished(raw, d) require.ErrorContains(t, err, "published descriptor still carries providesFrom") } + +// A merge key hands the root mapping another mapping's keys, so the +// field can reach a descriptor without being written at the top level. +// Publishing has to remove it from wherever it came, and leave the rest +// of what was merged in place. +func TestWithoutProvidesFromReachesMergedMappings(t *testing.T) { + const head = "schemaVersion: \"3\"\nkind: workload\nversion: \"1.0.0\"\n" + const tail = "build: |\n FROM scratch\n" + for name, tc := range map[string]struct { + body string + displayName string + keeps string + }{ + "inline": { + body: "<<: {providesFrom: {python: /usr/bin/python3}, displayName: Merged}\n", + displayName: "Merged", + }, + "inline with nothing else": { + body: "<<: {providesFrom: {python: /usr/bin/python3}}\n", + }, + "sequence": { + body: "<<: [{providesFrom: {python: /usr/bin/python3}}, {displayName: Second}]\n", + displayName: "Second", + }, + "nested": { + body: "<<: {displayName: Outer, <<: {providesFrom: {python: /usr/bin/python3}}}\n", + displayName: "Outer", + }, + "beside the plain spelling": { + body: "providesFrom: {ssh: /usr/bin/ssh}\n<<: {providesFrom: {python: /usr/bin/python3}, displayName: Merged}\n", + displayName: "Merged", + }, + // The anchored mapping is a capability's config, which is not + // this rewrite's to change: the root merges a copy without the + // field and the config keeps every key it had. + "alias to a mapping defined elsewhere": { + body: "capabilities:\n - type: com.example/thing@1\n config: &shared\n providesFrom: {python: /usr/bin/python3}\n displayName: Shared\n" + + "<<: *shared\n", + displayName: "Shared", + keeps: "/usr/bin/python3", + }, + } { + t.Run(name, func(t *testing.T) { + authored := []byte(head + tc.body + tail) + d, err := Decode(authored) + require.NoError(t, err) + require.NotEmpty(t, d.ProvidesFrom, "the case must reach the field through the merge") + + out, err := WithoutProvidesFrom(authored) + require.NoError(t, err) + pd, err := Decode(out) + require.NoError(t, err, "%s", out) + require.Empty(t, pd.ProvidesFrom, "%s", out) + require.Equal(t, tc.displayName, pd.DisplayName, "what was merged beside the field stays merged") + require.Equal(t, "FROM scratch\n", pd.Build) + _, err = ValidatePublished(out, pd) + require.NoError(t, err, "%s", out) + if tc.keeps != "" { + require.Contains(t, string(out), tc.keeps) + require.Len(t, pd.Capabilities, 1) + } + }) + } +} + +// Removing the field removes any anchor defined under it. An alias +// elsewhere would then name nothing and the document would stop +// parsing, so each one is given a copy of what it pointed at. +func TestWithoutProvidesFromKeepsAliasesIntoWhatItRemoves(t *testing.T) { + const head = "schemaVersion: \"3\"\nkind: workload\nversion: \"1.0.0\"\n" + const tail = "build: |\n FROM scratch\n" + + scalar := []byte(head + "providesFrom: {python: &command /usr/bin/python3}\ndescription: *command\n" + tail) + out, err := WithoutProvidesFrom(scalar) + require.NoError(t, err) + d, err := Decode(out) + require.NoError(t, err, "%s", out) + require.Empty(t, d.ProvidesFrom) + require.Equal(t, "/usr/bin/python3", d.Description) + require.NotContains(t, string(out), "*command") + _, err = ValidatePublished(out, d) + require.NoError(t, err) + + // The whole value anchored, and used twice more: once as a config, + // once through an alias nested in another config. + whole := []byte(head + "providesFrom: &paths\n python: &py /usr/bin/python3\n ssh: /usr/bin/ssh\n" + + "capabilities:\n - type: com.example/a@1\n config: *paths\n - type: com.example/b@1\n config: {interpreter: *py, all: *paths}\n" + tail) + out, err = WithoutProvidesFrom(whole) + require.NoError(t, err) + d, err = Decode(out) + require.NoError(t, err, "%s", out) + require.Empty(t, d.ProvidesFrom) + require.Len(t, d.Capabilities, 2) + paths := map[string]any{"python": "/usr/bin/python3", "ssh": "/usr/bin/ssh"} + require.Equal(t, paths, d.Capabilities[0].Config) + require.Equal(t, map[string]any{"interpreter": "/usr/bin/python3", "all": paths}, d.Capabilities[1].Config) + + // The other direction needs nothing: an alias inside the field, + // to an anchor that stays, simply goes with the field. + inward := []byte(head + "description: &text /usr/bin/python3\nprovidesFrom: {python: *text}\n" + tail) + out, err = WithoutProvidesFrom(inward) + require.NoError(t, err) + d, err = Decode(out) + require.NoError(t, err, "%s", out) + require.Empty(t, d.ProvidesFrom) + require.Equal(t, "/usr/bin/python3", d.Description) +} + +// The published rule is that the key is absent, not that it is empty: +// a reader built before the field refuses the key whatever is under it, +// and an empty map or a null decodes exactly like no field at all. +func TestPublishedRefusesProvidesFromInAnySpelling(t *testing.T) { + const yamlHead = "schemaVersion: \"3\"\nkind: workload\nversion: \"1.0.0\"\n" + for name, raw := range map[string]string{ + "entries": yamlHead + "providesFrom: {python: /usr/bin/python3}\n", + "empty map": yamlHead + "providesFrom: {}\n", + "null": yamlHead + "providesFrom:\n", + "merged": yamlHead + "<<: {providesFrom: {python: /usr/bin/python3}}\n", + "merged empty": yamlHead + "<<: {providesFrom: {}}\n", + "json": `{"schemaVersion":"3","kind":"workload","version":"1.0.0","providesFrom":{}}`, + "json null": `{"schemaVersion":"3","kind":"workload","version":"1.0.0","providesFrom":null}`, + "quoted the key": yamlHead + "\"providesFrom\": {}\n", + } { + t.Run(name, func(t *testing.T) { + d, err := Decode([]byte(raw)) + require.NoError(t, err) + _, err = ValidatePublished([]byte(raw), d) + require.ErrorContains(t, err, "published descriptor still carries providesFrom") + + // And publishing's own rewrite clears every one of them. + out, err := WithoutProvidesFrom([]byte(raw)) + require.NoError(t, err) + pd, err := Decode(out) + require.NoError(t, err, "%s", out) + _, err = ValidatePublished(out, pd) + require.NoError(t, err, "%s", out) + }) + } + + // A descriptor that never had the field is not caught by the word + // appearing somewhere else in it. + elsewhere := []byte(yamlHead + "description: see providesFrom in the docs\ncapabilities:\n - type: com.example/a@1\n config: {providesFrom: {x: /y}}\n") + d, err := Decode(elsewhere) + require.NoError(t, err) + _, err = ValidatePublished(elsewhere, d) + require.NoError(t, err) + out, err := WithoutProvidesFrom(elsewhere) + require.NoError(t, err) + require.Equal(t, string(elsewhere), string(out)) +} diff --git a/spec/validate.go b/spec/validate.go index 5707f2a4..9af534f3 100644 --- a/spec/validate.go +++ b/spec/validate.go @@ -358,7 +358,12 @@ func ValidatePublished(raw []byte, d *Descriptor) (warnings []string, err error) // replaces it with the entries it stated (§9.7). One that survives // names files nobody resolved, in a field readers built before it // existed refuse to decode. - if len(d.ProvidesFrom) > 0 { + // + // Judged on the bytes as well as the struct, because the rule is + // about the key being there: `providesFrom: {}` and a bare + // `providesFrom:` decode to no entries at all, and a strict reader + // that does not know the field refuses them just the same. + if len(d.ProvidesFrom) > 0 || carriesProvidesFrom(raw) { errs.add(fieldErrorf("providesFrom", "published descriptor still carries providesFrom; publishing states those entries in provides and drops the field, so this artifact was never resolved")) } // §9.2 is a property of every published descriptor, not only of the From 04f66bacce7a98171cde015d6d3d4afdf408e607 Mon Sep 17 00:00:00 2001 From: Christian Dupuis Date: Fri, 20 Nov 2026 13:37:22 +0000 Subject: [PATCH 03/10] fix(frontend): keep provides a merge key supplies when entries are added at publish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit withDerivedProvides looked for provides among the root mapping's own keys and created the field when it found none. A merge key can hand the root a provides list from another mapping, and a key created on the root overrides a merged one, so `<<: {provides: [agent@1.0.0]}` published without agent@1.0.0 as soon as a single derived entry was added. This predates providesFrom: the deb/ entries §9.6 derives are enough to trigger it on any workload with a package database. It was raised in review of the providesFrom change, which adds a second source of entries to the same list. The created list now starts from the entries the descriptor decodes to. The merged mapping is left as it is, since it may be anchored and shared. Signed-off-by: Christian Dupuis Sandbox-Id: spiffe://sbx.docker.com/org/57b45934-74c0-11e4-bea4-0242ac11001b/user/1580b874-0d59-41d8-bd81-82a10abc4100/sandbox/74f8ada1-8c4b-4fd6-9802-da94ba278db2 Sandbox-Kit: sha256:dfcafd4d7957b8b6ac4574528241772465187a778e16bdf809d9c85f43e8fed7 Co-authored-by: sbx docker/cdupuis/74f8ada1 --- cmd/frontend/build.go | 15 +++++++++++++++ cmd/frontend/rewrite_test.go | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/cmd/frontend/build.go b/cmd/frontend/build.go index c39a2b16..612eb9be 100644 --- a/cmd/frontend/build.go +++ b/cmd/frontend/build.go @@ -1049,6 +1049,21 @@ func withDerivedProvides(published []byte, derived []string) ([]byte, *spec.Desc // A kit that declared none still carries what its filesystem // holds, so the field is created rather than skipped. provides = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + // No key on the root is not the same as no entries: a merge + // key can hand the root a provides list from another mapping, + // and a key created here would override that list rather than + // extend it. So the new list starts from what the descriptor + // decodes to, which is the author's entries wherever they were + // written — and leaves the merged mapping alone, since + // something else may share it. + inherited, err := spec.Decode(published) + if err != nil { + return nil, nil, fmt.Errorf("published descriptor no longer decodes: %w", err) + } + for _, entry := range inherited.Provides { + provides.Content = append(provides.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: entry}) + } root.Content = append(root.Content, &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "provides"}, provides) case provides.Kind == yaml.ScalarNode && provides.Tag == "!!null": diff --git a/cmd/frontend/rewrite_test.go b/cmd/frontend/rewrite_test.go index cdffc396..aa45832a 100644 --- a/cmd/frontend/rewrite_test.go +++ b/cmd/frontend/rewrite_test.go @@ -172,3 +172,40 @@ capabilities: require.Equal(t, "./always.md", d.Capabilities[0].Config["contentFile"]) require.Equal(t, "/staged/maybe.md", d.Capabilities[1].Group.Capabilities[0].Config["contentFile"]) } + +// A merge key can supply provides, and then the root mapping has no +// provides key of its own. Creating one for the derived entries would +// override the inherited list, so the author's entries are carried into +// the list that is created. +func TestDerivedProvidesKeepWhatAMergeKeyProvides(t *testing.T) { + published := []byte(`schemaVersion: "3" +kind: workload +version: "1.0.0" +<<: {provides: ["agent@1.0.0"], displayName: Merged} +`) + + out, pd, err := withDerivedProvides(published, []string{"python@3.14.10", "deb/bash@5.2.37"}) + require.NoError(t, err) + require.Equal(t, []string{"agent@1.0.0", "python@3.14.10", "deb/bash@5.2.37"}, pd.Provides) + require.Equal(t, "Merged", pd.DisplayName, "the rest of what was merged is untouched") + + fromBytes, err := spec.Decode(out) + require.NoError(t, err) + require.Equal(t, pd.Provides, fromBytes.Provides) + + // An anchored mapping merged by alias is not edited: whatever else + // refers to it still sees the list it had. + shared := []byte(`schemaVersion: "3" +kind: workload +version: "1.0.0" +capabilities: + - type: com.example/thing@1 + config: &shared + provides: ["agent@1.0.0"] +<<: *shared +`) + _, pd, err = withDerivedProvides(shared, []string{"deb/bash@5.2.37"}) + require.NoError(t, err) + require.Equal(t, []string{"agent@1.0.0", "deb/bash@5.2.37"}, pd.Provides) + require.Equal(t, map[string]any{"provides": []any{"agent@1.0.0"}}, pd.Capabilities[0].Config) +} From ba7743483d8e0badefe961f56090cdc6177216c9 Mon Sep 17 00:00:00 2001 From: Christian Dupuis Date: Fri, 20 Nov 2026 13:57:35 +0000 Subject: [PATCH 04/10] fix(spec): judge a dpkg file list by its own architecture, and follow chained top-level links Two ways the ownership lookup behind providesFrom gave a wrong answer, both found in review. A co-installable package has one file list per architecture, :.list, and one architecture can be removed while another stays installed. The lookup dropped the architecture before asking whether the package was installed, so the removed architecture's list, which still holds the conffiles it left behind, passed on the strength of the installed one's name. A leftover conffile could then be stated at the version of a package that does not own it. The status file is now read for which architectures each package is installed for, and an architecture-qualified list owns something only when that architecture is among them. Top-level symbolic links were followed one step. With /bin pointing at compat/bin and /compat at usr, a recorded /bin/tool was indexed as /compat/bin/tool, where a requested /usr/bin/tool never looked, and an installed command was reported as unowned. The links are now followed until the result no longer starts at one, with a bound so that links leading in a circle end as a path that does not exist. ReadDpkgStatus's field splitting and installed test are named so the new reader shares them; what it returns is unchanged. Signed-off-by: Christian Dupuis Sandbox-Id: spiffe://sbx.docker.com/org/57b45934-74c0-11e4-bea4-0242ac11001b/user/1580b874-0d59-41d8-bd81-82a10abc4100/sandbox/74f8ada1-8c4b-4fd6-9802-da94ba278db2 Sandbox-Kit: sha256:dfcafd4d7957b8b6ac4574528241772465187a778e16bdf809d9c85f43e8fed7 Co-authored-by: sbx docker/cdupuis/74f8ada1 --- docs/spec/SPEC-v3.md | 22 +++++----- spec/packages.go | 41 +++++++++++-------- spec/provides_from.go | 71 +++++++++++++++++++++++++------- spec/provides_from_test.go | 83 +++++++++++++++++++++++++++++++++++++- 4 files changed, 177 insertions(+), 40 deletions(-) diff --git a/docs/spec/SPEC-v3.md b/docs/spec/SPEC-v3.md index eefbeaec..0d199736 100644 --- a/docs/spec/SPEC-v3.md +++ b/docs/spec/SPEC-v3.md @@ -1345,20 +1345,24 @@ shipped. listed by more than one package, or is listed by none. The lists are the ones kept beside the databases [§9.6](#96-derived-provides) reads: `/var/lib/dpkg/info/.list` for dpkg, the `F:` and `R:` - records of `/lib/apk/db/installed` for apk. Nothing is stated in - place of an entry that could not be resolved: the author asked for it - by name, so its absence is a build failure the author sees rather - than a capability a consumer finds missing. A file a recipe placed - has no package to take a version from, and is stated in `provides` - under a build-phase arg as before. + records of `/lib/apk/db/installed` for apk. A list can outlive its + package — a removed package keeps the configuration files it left, + and so does one removed architecture of a package another + architecture still has installed — and such a list owns nothing. + Nothing is stated in place of an entry that could not be resolved: + the author asked for it by name, so its absence is a build failure + the author sees rather than a capability a consumer finds missing. A + file a recipe placed has no package to take a version from, and is + stated in `provides` under a build-phase arg as before. - Publishing **MUST NOT** follow a symbolic link that a package lists. A link no package lists is followed to its target, and the target judged the same way. A package that ships `python3` as a link is the answer for that link, while a link nothing owns was made by a recipe or by `update-alternatives`, and what it leads to is the thing - installed. Paths are compared through a top-level directory that is - itself a link, so `/bin/sh` and `/usr/bin/sh` name one file on a - merged-`/usr` system whichever of them the package recorded. + installed. Paths are compared through top-level directories that are + themselves links, each followed to wherever it finally leads, so + `/bin/sh` and `/usr/bin/sh` name one file on a merged-`/usr` system + whichever of them the package recorded. - The version **MUST** be the owner's version under [§9.6](#96-derived-provides)'s rule, and publishing **MUST** fail when that rule names none — a prerelease, or one package recorded at diff --git a/spec/packages.go b/spec/packages.go index 6bc55ac1..c22b121d 100644 --- a/spec/packages.go +++ b/spec/packages.go @@ -63,30 +63,39 @@ func PackageDatabases() []PackageDatabase { func ReadDpkgStatus(r io.Reader) ([]Package, error) { var out []Package err := eachStanza(r, func(fields map[string]string) { - // The third word of dpkg's want/flag/status triple is the one - // that says whether the files are on disk. - parts := strings.Fields(fields["status"]) - if len(parts) == 0 || parts[len(parts)-1] != "installed" { + if !dpkgInstalled(fields) { return } if name := fields["package"]; name != "" { out = append(out, Package{Name: name, Version: fields["version"]}) } - }, func(line string) (string, string, bool) { - // A continuation line belongs to the field above it, and none of - // the three fields read here spans lines, so it is skipped. - if strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") { - return "", "", false - } - key, value, found := strings.Cut(line, ":") - if !found { - return "", "", false - } - return strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value), true - }) + }, dpkgField) return out, err } +// dpkgInstalled reports whether a status stanza records a package whose +// files are on disk. The third word of dpkg's want/flag/status triple is +// the one that says so. +func dpkgInstalled(fields map[string]string) bool { + parts := strings.Fields(fields["status"]) + return len(parts) > 0 && parts[len(parts)-1] == "installed" +} + +// dpkgField splits one line of a dpkg status stanza into a lowercased +// field name and its value. +func dpkgField(line string) (string, string, bool) { + // A continuation line belongs to the field above it, and none of + // the fields read here spans lines, so it is skipped. + if strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") { + return "", "", false + } + key, value, found := strings.Cut(line, ":") + if !found { + return "", "", false + } + return strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value), true +} + // ReadApkInstalled lists the packages an apk installed database records. // // apk keeps no removed-package records, so every stanza counts; the diff --git a/spec/provides_from.go b/spec/provides_from.go index 57bf7621..038384db 100644 --- a/spec/provides_from.go +++ b/spec/provides_from.go @@ -549,7 +549,7 @@ func readFileOwners(ctx context.Context, files ImageFiles) (*fileOwners, error) } switch db.Namespace { case DebNamespace: - err = o.readDpkgLists(ctx) + err = o.readDpkgLists(ctx, body) case ApkNamespace: err = o.readApkFiles(body) } @@ -568,7 +568,11 @@ func readFileOwners(ctx context.Context, files ImageFiles) (*fileOwners, error) // Files rather than `dpkg -S`, because an image need not carry dpkg to // carry its database: one assembled from packages keeps the lists // without the tool that wrote them. -func (o *fileOwners) readDpkgLists(ctx context.Context) error { +func (o *fileOwners) readDpkgLists(ctx context.Context, status []byte) error { + installed, err := dpkgInstalledArchitectures(status) + if err != nil { + return fmt.Errorf("%s: %w", DpkgStatusPath, err) + } entries, present, err := o.files.ReadDir(ctx, DpkgInfoDir) if err != nil { return fmt.Errorf("read %s: %w", DpkgInfoDir, err) @@ -581,16 +585,22 @@ func (o *fileOwners) readDpkgLists(ctx context.Context) error { if !isList || e.Dir { continue } - // A multiarch list is :.list; the architecture - // is not part of the name the status file knows it by. - name, _, _ := strings.Cut(stem, ":") - key := DebNamespace + "/" + name // A list outlives its package: a removed one keeps the // conffiles it left behind. Only what the status file calls - // installed owns anything. - if _, installed := o.versions[key]; !installed { + // installed owns anything, and that is judged for the package + // the list is actually named for. A co-installable package has + // one list per architecture, :.list, and one of + // them can be removed while another stays: the list of the + // removed one must not pass for the installed one's because + // they share a name. + name, arch, qualified := strings.Cut(stem, ":") + architectures, known := installed[name] + if !known || (qualified && !architectures[arch]) { continue } + // The architecture is part of which package this is, not of + // the name a capability is stated under. + key := DebNamespace + "/" + name list := path.Join(DpkgInfoDir, e.Name) body, present, err := o.files.ReadFile(ctx, list) if err != nil { @@ -608,6 +618,25 @@ func (o *fileOwners) readDpkgLists(ctx context.Context) error { return nil } +// dpkgInstalledArchitectures maps each installed package name to the +// architectures it is installed for. [ReadDpkgStatus] folds those into +// one name, which is all a derived entry needs; choosing a file list +// needs to know which of them a list belongs to. +func dpkgInstalledArchitectures(status []byte) (map[string]map[string]bool, error) { + installed := map[string]map[string]bool{} + err := eachStanza(bytes.NewReader(status), func(fields map[string]string) { + name := fields["package"] + if name == "" || !dpkgInstalled(fields) { + return + } + if installed[name] == nil { + installed[name] = map[string]bool{} + } + installed[name][fields["architecture"]] = true + }, dpkgField) + return installed, err +} + // readApkFiles reads the file lists apk keeps inside its installed // database: within a package's stanza, an F: line names a directory and // each R: line after it a file in that directory. @@ -663,15 +692,29 @@ func (o *fileOwners) own(file, key string) { o.owners[file][key] = true } -// canonical rewrites a path through a top-level symbolic link, so the -// two spellings a merged-/usr system gives one file compare equal. +// canonical rewrites a path through top-level symbolic links, so the +// spellings a merged-/usr system gives one file compare equal. +// +// For as long as the result starts at another such link, since one can +// lead to the next: with /bin pointing at compat/bin and /compat at +// usr, /bin/sh is /usr/bin/sh, and stopping after the first step would +// index a recorded /bin/sh where a requested /usr/bin/sh never looks. +// Recorded and requested paths both come through here, so whatever it +// settles on, they settle on the same thing. func (o *fileOwners) canonical(file string) string { file = path.Clean(file) - rest := strings.TrimPrefix(file, "/") - top, tail, nested := strings.Cut(rest, "/") - if target, aliased := o.aliases["/"+top]; aliased && nested { - return path.Join(target, tail) + current := file + for range maxProvidesFromLinks { + rest := strings.TrimPrefix(current, "/") + top, tail, nested := strings.Cut(rest, "/") + target, aliased := o.aliases["/"+top] + if !aliased || !nested { + return current + } + current = path.Join(target, tail) } + // Links that lead in a circle name no file. The path is left as it + // was written, and the lookup reports it as one nothing owns. return file } diff --git a/spec/provides_from_test.go b/spec/provides_from_test.go index 63ae6dcd..fc4e5ff4 100644 --- a/spec/provides_from_test.go +++ b/spec/provides_from_test.go @@ -63,7 +63,7 @@ func debianImage() fakeImage { "Package: bash\nStatus: install ok installed\nVersion: 5.2.37-2+dhi1\n", "Package: gone\nStatus: deinstall ok config-files\nVersion: 1.0-1\n", "Package: pre\nStatus: install ok installed\nVersion: 2.0~rc1-1\n", - "Package: libfoo\nStatus: install ok installed\nVersion: 1.2.3-1\n", + "Package: libfoo\nStatus: install ok installed\nArchitecture: arm64\nMulti-Arch: same\nVersion: 1.2.3-1\n", "Package: libbar\nStatus: install ok installed\nVersion: 4.5.6-1\n", }, "\n"), DpkgInfoDir + "/python-3.14-minimal.list": "/.\n/usr\n/usr/bin\n/usr/bin/python3.14\n/usr/bin/python3\n", @@ -442,3 +442,84 @@ func TestPublishedRefusesProvidesFromInAnySpelling(t *testing.T) { require.NoError(t, err) require.Equal(t, string(elsewhere), string(out)) } + +// A co-installable package keeps one file list per architecture, and +// one architecture can be removed while another stays installed. The +// removed one's list still holds the conffiles it left behind; it must +// not own them on the strength of the installed one's name. +func TestProvidesFromPathsJudgesEachArchitecturesList(t *testing.T) { + image := fakeImage{ + DpkgStatusPath: strings.Join([]string{ + "Package: libfoo\nStatus: install ok installed\nArchitecture: amd64\nMulti-Arch: same\nVersion: 2.0-1\n", + "Package: libfoo\nStatus: deinstall ok config-files\nArchitecture: i386\nMulti-Arch: same\nVersion: 1.0-1\n", + "Package: libbar\nStatus: install ok installed\nArchitecture: amd64\nMulti-Arch: same\nVersion: 3.1-2\n", + "Package: libbar\nStatus: install ok installed\nArchitecture: i386\nMulti-Arch: same\nVersion: 3.1-2\n", + "Package: plain\nStatus: install ok installed\nArchitecture: amd64\nVersion: 4.2-1\n", + }, "\n"), + DpkgInfoDir + "/libfoo:amd64.list": "/usr/lib/x86_64-linux-gnu/libfoo.so.2\n", + DpkgInfoDir + "/libfoo:i386.list": "/etc/libfoo-i386.conf\n", + DpkgInfoDir + "/libbar:amd64.list": "/usr/lib/x86_64-linux-gnu/libbar.so.3\n/usr/share/doc/libbar/copyright\n", + DpkgInfoDir + "/libbar:i386.list": "/usr/lib/i386-linux-gnu/libbar.so.3\n/usr/share/doc/libbar/copyright\n", + // A list named without an architecture belongs to the package + // of that name, whatever it is installed for. + DpkgInfoDir + "/plain.list": "/usr/bin/plain\n", + // Named for an architecture the package is not installed for. + DpkgInfoDir + "/plain:i386.list": "/usr/bin/plain32\n", + "/usr/lib/x86_64-linux-gnu/libfoo.so.2": "elf", + "/etc/libfoo-i386.conf": "conf", + "/usr/lib/x86_64-linux-gnu/libbar.so.3": "elf", + "/usr/lib/i386-linux-gnu/libbar.so.3": "elf", + "/usr/share/doc/libbar/copyright": "text", + "/usr/bin/plain": "elf", + "/usr/bin/plain32": "elf", + } + + got, err := ProvidesFromPaths(context.Background(), map[string]string{ + "foo": "/usr/lib/x86_64-linux-gnu/libfoo.so.2", + "bar": "/usr/lib/i386-linux-gnu/libbar.so.3", + "bar-doc": "/usr/share/doc/libbar/copyright", + "plain": "/usr/bin/plain", + }, one(image)) + require.NoError(t, err) + require.Equal(t, []string{"bar-doc@3.1", "bar@3.1", "foo@2.0", "plain@4.2"}, got, + "a file both architectures of one package list has one owner") + + for _, file := range []string{"/etc/libfoo-i386.conf", "/usr/bin/plain32"} { + _, err = ProvidesFromPaths(context.Background(), map[string]string{"x": file}, one(image)) + require.ErrorContains(t, err, file+" belongs to no installed package") + } +} + +// One top-level link can lead to another. A path a package recorded and +// a path an author wrote have to end up spelled the same way however +// many links stand between them and the file. +func TestProvidesFromPathsFollowsChainedTopLevelLinks(t *testing.T) { + image := fakeImage{ + "/bin": "-> compat/bin", + "/compat": "-> usr", + "/sbin": "-> /bin", + DpkgStatusPath: "Package: tool\nStatus: install ok installed\nArchitecture: amd64\nVersion: 1.2.3-1\n", + DpkgInfoDir + "/tool.list": "/bin/tool\n/sbin/admin\n", + "/usr/bin/tool": "elf", + "/usr/bin/admin": "elf", + } + got, err := ProvidesFromPaths(context.Background(), map[string]string{ + "direct": "/usr/bin/tool", + "recorded": "/bin/tool", + "middle": "/compat/bin/tool", + "twice": "/sbin/admin", + }, one(image)) + require.NoError(t, err) + require.Equal(t, []string{"direct@1.2.3", "middle@1.2.3", "recorded@1.2.3", "twice@1.2.3"}, got) + + // Links that lead in a circle reach no file. The lookup ends, and + // says the path does not exist rather than never returning. + circle := fakeImage{ + "/a": "-> b", + "/b": "-> a", + DpkgStatusPath: "Package: tool\nStatus: install ok installed\nArchitecture: amd64\nVersion: 1.2.3-1\n", + DpkgInfoDir + "/tool.list": "/a/tool\n", + } + _, err = ProvidesFromPaths(context.Background(), map[string]string{"x": "/a/tool"}, one(circle)) + require.ErrorContains(t, err, "/a/tool does not exist in the image") +} From 49800d798a5425bfcf929819b812608948a219a9 Mon Sep 17 00:00:00 2001 From: Christian Dupuis Date: Fri, 20 Nov 2026 14:15:29 +0000 Subject: [PATCH 05/10] fix(spec): resolve directory links on the way to a providesFrom path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A directory in the middle of a requested path can be a symbolic link: a default JDK, a current release. The lookup compared the path as written with what packages recorded, and only followed a link when the file itself was one, so /usr/lib/jvm/default/bin/java found no owner although a package installed that file as /usr/lib/jvm/java-21/bin/java. The build failed with "belongs to no installed package" for a file a package installed. The directories of a requested path are now resolved before the file lists are consulted, relative and absolute targets and targets that climb included. The path is still tried as written first, since that is how a package that ships into a linked directory records it. A relative link at the end of the path is resolved against the directory it really is in. Recorded paths are not resolved this way. That would cost a directory read for every directory in the image; they stay compared as recorded, through top-level links, and §9.7 now says which side gets which treatment. Signed-off-by: Christian Dupuis Sandbox-Id: spiffe://sbx.docker.com/org/57b45934-74c0-11e4-bea4-0242ac11001b/user/1580b874-0d59-41d8-bd81-82a10abc4100/sandbox/74f8ada1-8c4b-4fd6-9802-da94ba278db2 Sandbox-Kit: sha256:dfcafd4d7957b8b6ac4574528241772465187a778e16bdf809d9c85f43e8fed7 Co-authored-by: sbx docker/cdupuis/74f8ada1 --- docs/spec/SPEC-v3.md | 11 +++-- spec/provides_from.go | 82 ++++++++++++++++++++++++++++++++++---- spec/provides_from_test.go | 72 +++++++++++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 12 deletions(-) diff --git a/docs/spec/SPEC-v3.md b/docs/spec/SPEC-v3.md index 0d199736..8910c500 100644 --- a/docs/spec/SPEC-v3.md +++ b/docs/spec/SPEC-v3.md @@ -1359,10 +1359,13 @@ shipped. judged the same way. A package that ships `python3` as a link is the answer for that link, while a link nothing owns was made by a recipe or by `update-alternatives`, and what it leads to is the thing - installed. Paths are compared through top-level directories that are - themselves links, each followed to wherever it finally leads, so - `/bin/sh` and `/usr/bin/sh` name one file on a merged-`/usr` system - whichever of them the package recorded. + installed. A directory on the way to the file that is a link is + followed too, so `/usr/lib/jvm/default/bin/java` finds the package + that installed the file where it really is. What a package recorded + is compared as recorded, through top-level directories that are + themselves links, each followed to wherever it finally leads: `/bin/sh` + and `/usr/bin/sh` name one file on a merged-`/usr` system whichever + of them the package recorded. - The version **MUST** be the owner's version under [§9.6](#96-derived-provides)'s rule, and publishing **MUST** fail when that rule names none — a prerelease, or one package recorded at diff --git a/spec/provides_from.go b/spec/provides_from.go index 038384db..05cfc524 100644 --- a/spec/provides_from.go +++ b/spec/provides_from.go @@ -725,15 +725,25 @@ func (o *fileOwners) canonical(file string) string { // that ships a link — python3 pointing at python3.14 — is the answer // for that link; a link nothing owns was made by a recipe or by // update-alternatives, and what it leads to is the thing installed. +// +// A path is looked up under two spellings. As written is how a package +// may have recorded it. With every directory on the way resolved is how +// a package recorded the file it really is: /usr/lib/jvm/default/bin/java +// is installed as /usr/lib/jvm/java-21/bin/java when default is a link, +// and asking for the first must find the owner of the second. func (o *fileOwners) version(ctx context.Context, file string) (string, error) { - current := o.canonical(file) + current := path.Clean(file) for range maxProvidesFromLinks { - entry, present, err := o.entry(ctx, current) + resolved, err := o.resolveDirectories(ctx, current) + if err != nil { + return "", err + } + entry, present, err := o.entry(ctx, resolved) if err != nil { return "", err } if !present { - if current != file { + if current != path.Clean(file) { return "", fmt.Errorf("%s leads to %s, which does not exist in the image", file, current) } return "", fmt.Errorf("%s does not exist in the image", file) @@ -741,21 +751,77 @@ func (o *fileOwners) version(ctx context.Context, file string) (string, error) { if entry.Dir { return "", fmt.Errorf("%s is a directory; name a file a package installs, such as the command itself", current) } - if owners := o.owners[current]; len(owners) > 0 { + owners := o.owners[o.canonical(current)] + if len(owners) == 0 { + owners = o.owners[resolved] + } + if len(owners) > 0 { return o.ownerVersion(file, owners) } if entry.Link == "" { return "", fmt.Errorf("%s belongs to no installed package; a file a recipe placed has no package to take a version from, so state its version in provides", current) } - next := entry.Link - if !path.IsAbs(next) { - next = path.Join(path.Dir(current), next) + // Relative to where the link really is, which is what the + // kernel resolves it against. + current = entry.Link + if !path.IsAbs(current) { + current = path.Join(path.Dir(resolved), current) } - current = o.canonical(next) + current = path.Clean(current) } return "", fmt.Errorf("%s leads through more than %d symbolic links", file, maxProvidesFromLinks) } +// resolveDirectories returns file with every directory on the way to it +// that is a symbolic link replaced by where the link leads. The last +// component is left alone: whether the file itself is a link is the +// caller's question. +// +// Only requested paths come through here. Doing the same for every path +// a package recorded would mean a directory read per directory in the +// image, so recorded paths are compared as recorded, through top-level +// links only — which is why version tries a path as written first. +func (o *fileOwners) resolveDirectories(ctx context.Context, file string) (string, error) { + file = path.Clean(file) + dir, base := path.Split(file) + pending := strings.FieldsFunc(dir, func(r rune) bool { return r == '/' }) + resolved := "/" + links := 0 + for len(pending) > 0 { + part := pending[0] + pending = pending[1:] + if part == ".." { + // A link target may climb; the path itself is clean. + resolved = path.Dir(resolved) + continue + } + candidate := path.Join(resolved, part) + entry, present, err := o.entry(ctx, candidate) + if err != nil { + return "", err + } + if !present { + // Nothing further to resolve. The caller finds the file + // missing and says so. + return path.Join(append([]string{candidate}, append(pending, base)...)...), nil + } + if entry.Link == "" { + resolved = candidate + continue + } + if links++; links > maxProvidesFromLinks { + // Directory links that lead in a circle reach no file. + return file, nil + } + target := strings.FieldsFunc(entry.Link, func(r rune) bool { return r == '/' }) + if path.IsAbs(entry.Link) { + resolved = "/" + } + pending = append(target, pending...) + } + return path.Join(resolved, base), nil +} + func (o *fileOwners) ownerVersion(file string, owners map[string]bool) (string, error) { keys := slices.Sorted(maps.Keys(owners)) if len(keys) > 1 { diff --git a/spec/provides_from_test.go b/spec/provides_from_test.go index fc4e5ff4..0e7bb2ea 100644 --- a/spec/provides_from_test.go +++ b/spec/provides_from_test.go @@ -523,3 +523,75 @@ func TestProvidesFromPathsFollowsChainedTopLevelLinks(t *testing.T) { _, err = ProvidesFromPaths(context.Background(), map[string]string{"x": "/a/tool"}, one(circle)) require.ErrorContains(t, err, "/a/tool does not exist in the image") } + +// A directory on the way to a file can be a symbolic link — a default +// JDK, a current release. The package recorded the file where it really +// is, so the directories of a requested path are resolved before the +// file lists are consulted. +func TestProvidesFromPathsResolvesDirectoryLinks(t *testing.T) { + image := fakeImage{ + "/bin": "-> usr/bin", + DpkgStatusPath: strings.Join([]string{ + "Package: jdk\nStatus: install ok installed\nArchitecture: arm64\nVersion: 21.0.9-1\n", + "Package: tool\nStatus: install ok installed\nArchitecture: arm64\nVersion: 1.2.3-1\n", + "Package: odd\nStatus: install ok installed\nArchitecture: arm64\nVersion: 7.0-1\n", + }, "\n"), + DpkgInfoDir + "/jdk.list": "/usr/lib/jvm/java-21/bin/java\n", + DpkgInfoDir + "/tool.list": "/opt/tool-1.2/bin/tool\n", + // Recorded through a directory link, as a package that ships + // into another package's link would be. + DpkgInfoDir + "/odd.list": "/usr/lib/jvm/default/bin/odd\n", + "/usr/lib/jvm/java-21/bin/java": "elf", + "/usr/lib/jvm/java-21/bin/odd": "elf", + "/usr/lib/jvm/default": "-> java-21", + "/usr/lib/jvm/absolute": "-> /usr/lib/jvm/default", + "/opt/tool-1.2/bin/tool": "elf", + "/opt/tool/current": "-> ../tool-1.2", + "/opt/tool/README": "text", + // An alternatives-style link in a directory reached through a + // link: its relative target is relative to the real directory. + "/usr/lib/jvm/java-21/bin/jvm": "-> java", + "/usr/bin/java": "-> /usr/lib/jvm/absolute/bin/jvm", + "/opt/loop/a": "-> b", + "/opt/loop/b": "-> a", + } + _, err := ProvidesFromPaths(context.Background(), map[string]string{ + "real": "/usr/lib/jvm/java-21/bin/java", + "default": "/usr/lib/jvm/default/bin/java", + "chained": "/usr/lib/jvm/absolute/bin/java", + "climbing": "/opt/tool/current/bin/tool", + "command": "/bin/java", + "as-listed": "/usr/lib/jvm/default/bin/odd", + "listed": "/usr/lib/jvm/java-21/bin/odd", + }, one(image)) + require.ErrorContains(t, err, "providesFrom.listed on linux/arm64: /usr/lib/jvm/java-21/bin/odd belongs to no installed package", + "a recorded path is compared as recorded: only the spelling the package used finds it") + + got, err := ProvidesFromPaths(context.Background(), map[string]string{ + "real": "/usr/lib/jvm/java-21/bin/java", + "default": "/usr/lib/jvm/default/bin/java", + "chained": "/usr/lib/jvm/absolute/bin/java", + "climbing": "/opt/tool/current/bin/tool", + "command": "/bin/java", + "as-listed": "/usr/lib/jvm/default/bin/odd", + }, one(image)) + require.NoError(t, err) + require.Equal(t, []string{ + "as-listed@7.0", + "chained@21.0.9", + "climbing@1.2.3", + "command@21.0.9", + "default@21.0.9", + "real@21.0.9", + }, got) + + for file, want := range map[string]string{ + "/usr/lib/jvm/default/bin/nope": "/usr/lib/jvm/default/bin/nope does not exist in the image", + "/usr/lib/jvm/missing/bin/java": "/usr/lib/jvm/missing/bin/java does not exist in the image", + "/usr/lib/jvm/default/bin": "/usr/lib/jvm/default/bin is a directory", + "/opt/loop/a/tool": "/opt/loop/a/tool does not exist in the image", + } { + _, err := ProvidesFromPaths(context.Background(), map[string]string{"x": file}, one(image)) + require.ErrorContains(t, err, want) + } +} From 5b1fb1b70d1c1b4bd46a2c88ba436c105f12e783 Mon Sep 17 00:00:00 2001 From: Christian Dupuis Date: Fri, 20 Nov 2026 14:30:30 +0000 Subject: [PATCH 06/10] fix(spec): end providesFrom resolution at the file that is really there Two readings of a path as text could end at a different file from the one in the image, and state the version of a package that does not own it. The previous change looked a requested path up as written before looking it up with its directories resolved. A file list can hold a spelling from before a directory on it became a link, so /opt/app/current/bin/tool found a stale record under that name although current now leads to release-2, where another package installed the file. The resolved spelling is now looked up first and the written one only when nothing owns the real file, which keeps the case it was for: a package that ships into a linked directory and records the path that way. A link's target was joined to its directory and cleaned, which collapses sub/../tool to tool beside the link. In the image, ".." there is the parent of wherever sub leads. Targets are now walked a component at a time through the same directory resolution, for a link at the end of a path and for a directory link alike. Top-level links are resolved by that same walk when the file lists are read, so an alias names the directory its link finally reaches and a recorded path needs one substitution rather than a loop. Signed-off-by: Christian Dupuis Sandbox-Id: spiffe://sbx.docker.com/org/57b45934-74c0-11e4-bea4-0242ac11001b/user/1580b874-0d59-41d8-bd81-82a10abc4100/sandbox/74f8ada1-8c4b-4fd6-9802-da94ba278db2 Sandbox-Kit: sha256:dfcafd4d7957b8b6ac4574528241772465187a778e16bdf809d9c85f43e8fed7 Co-authored-by: sbx docker/cdupuis/74f8ada1 --- docs/spec/SPEC-v3.md | 15 ++-- spec/provides_from.go | 138 +++++++++++++++++++++++-------------- spec/provides_from_test.go | 84 ++++++++++++++++++++++ 3 files changed, 180 insertions(+), 57 deletions(-) diff --git a/docs/spec/SPEC-v3.md b/docs/spec/SPEC-v3.md index 8910c500..2f9de75a 100644 --- a/docs/spec/SPEC-v3.md +++ b/docs/spec/SPEC-v3.md @@ -1361,11 +1361,16 @@ shipped. or by `update-alternatives`, and what it leads to is the thing installed. A directory on the way to the file that is a link is followed too, so `/usr/lib/jvm/default/bin/java` finds the package - that installed the file where it really is. What a package recorded - is compared as recorded, through top-level directories that are - themselves links, each followed to wherever it finally leads: `/bin/sh` - and `/usr/bin/sh` name one file on a merged-`/usr` system whichever - of them the package recorded. + that installed the file where it really is. Paths are resolved the + way the system resolves them, a component at a time, so `..` in a + link's target is the parent of the directory reached and not of the + one written. The owner of the file where it really is comes first; + the path as written is consulted only when nothing owns that, since + a list can hold a spelling from before a directory became a link. + What a package recorded is compared as recorded, through top-level + directories that are themselves links, each followed to wherever it + finally leads: `/bin/sh` and `/usr/bin/sh` name one file on a + merged-`/usr` system whichever of them the package recorded. - The version **MUST** be the owner's version under [§9.6](#96-derived-provides)'s rule, and publishing **MUST** fail when that rule names none — a prerelease, or one package recorded at diff --git a/spec/provides_from.go b/spec/provides_from.go index 05cfc524..5014f547 100644 --- a/spec/provides_from.go +++ b/spec/provides_from.go @@ -511,15 +511,21 @@ func readFileOwners(ctx context.Context, files ImageFiles) (*fileOwners, error) if err != nil { return nil, fmt.Errorf("read /: %w", err) } + o.dirs["/"] = root for _, e := range root { if e.Link == "" { continue } - target := e.Link - if !path.IsAbs(target) { - target = path.Join("/", target) + // To the directory the link really leads to, however many + // links that takes, so one substitution is the whole answer. A + // link that leads nowhere is no alias for anything. + target, reached, err := o.realDirectory(ctx, []string{e.Name}) + if err != nil { + return nil, err + } + if reached { + o.aliases["/"+e.Name] = target } - o.aliases["/"+e.Name] = path.Clean(target) } found := false @@ -692,29 +698,19 @@ func (o *fileOwners) own(file, key string) { o.owners[file][key] = true } -// canonical rewrites a path through top-level symbolic links, so the -// spellings a merged-/usr system gives one file compare equal. +// canonical rewrites a recorded path through a top-level symbolic link, +// so the spellings a merged-/usr system gives one file compare equal. // -// For as long as the result starts at another such link, since one can -// lead to the next: with /bin pointing at compat/bin and /compat at -// usr, /bin/sh is /usr/bin/sh, and stopping after the first step would -// index a recorded /bin/sh where a requested /usr/bin/sh never looks. -// Recorded and requested paths both come through here, so whatever it -// settles on, they settle on the same thing. +// One substitution is enough, because an alias already names the +// directory its link finally reaches: with /bin pointing at compat/bin +// and /compat at usr, the alias for /bin is /usr/bin. func (o *fileOwners) canonical(file string) string { file = path.Clean(file) - current := file - for range maxProvidesFromLinks { - rest := strings.TrimPrefix(current, "/") - top, tail, nested := strings.Cut(rest, "/") - target, aliased := o.aliases["/"+top] - if !aliased || !nested { - return current - } - current = path.Join(target, tail) + rest := strings.TrimPrefix(file, "/") + top, tail, nested := strings.Cut(rest, "/") + if target, aliased := o.aliases["/"+top]; aliased && nested { + return path.Join(target, tail) } - // Links that lead in a circle name no file. The path is left as it - // was written, and the lookup reports it as one nothing owns. return file } @@ -726,13 +722,17 @@ func (o *fileOwners) canonical(file string) string { // for that link; a link nothing owns was made by a recipe or by // update-alternatives, and what it leads to is the thing installed. // -// A path is looked up under two spellings. As written is how a package -// may have recorded it. With every directory on the way resolved is how -// a package recorded the file it really is: /usr/lib/jvm/default/bin/java -// is installed as /usr/lib/jvm/java-21/bin/java when default is a link, -// and asking for the first must find the owner of the second. +// The owner is looked up where the file really is: with every directory +// on the way resolved, /usr/lib/jvm/default/bin/java is the file a +// package installed as /usr/lib/jvm/java-21/bin/java. That spelling +// comes first because it names the file that is actually there. A list +// can still hold the path as it was written — a record from before the +// directory became a link — and taking that one would state the version +// of a package whose file this no longer is. The written spelling is +// the fallback, for a package that ships into a linked directory and +// records the path that way. func (o *fileOwners) version(ctx context.Context, file string) (string, error) { - current := path.Clean(file) + current := file for range maxProvidesFromLinks { resolved, err := o.resolveDirectories(ctx, current) if err != nil { @@ -743,7 +743,7 @@ func (o *fileOwners) version(ctx context.Context, file string) (string, error) { return "", err } if !present { - if current != path.Clean(file) { + if current != file { return "", fmt.Errorf("%s leads to %s, which does not exist in the image", file, current) } return "", fmt.Errorf("%s does not exist in the image", file) @@ -751,9 +751,12 @@ func (o *fileOwners) version(ctx context.Context, file string) (string, error) { if entry.Dir { return "", fmt.Errorf("%s is a directory; name a file a package installs, such as the command itself", current) } - owners := o.owners[o.canonical(current)] - if len(owners) == 0 { - owners = o.owners[resolved] + owners := o.owners[resolved] + // Only a spelling with nothing to collapse can be compared as + // text: a link target may climb with "..", and what that means + // depends on the directories before it. + if len(owners) == 0 && current == path.Clean(current) { + owners = o.owners[o.canonical(current)] } if len(owners) > 0 { return o.ownerVersion(file, owners) @@ -761,13 +764,14 @@ func (o *fileOwners) version(ctx context.Context, file string) (string, error) { if entry.Link == "" { return "", fmt.Errorf("%s belongs to no installed package; a file a recipe placed has no package to take a version from, so state its version in provides", current) } - // Relative to where the link really is, which is what the - // kernel resolves it against. + // Relative to where the link really is, and left as written: + // resolveDirectories walks it a component at a time, which is + // the only reading under which sub/../tool means the parent of + // wherever sub leads rather than the directory sub is in. current = entry.Link if !path.IsAbs(current) { - current = path.Join(path.Dir(resolved), current) + current = path.Dir(resolved) + "/" + current } - current = path.Clean(current) } return "", fmt.Errorf("%s leads through more than %d symbolic links", file, maxProvidesFromLinks) } @@ -777,49 +781,79 @@ func (o *fileOwners) version(ctx context.Context, file string) (string, error) { // component is left alone: whether the file itself is a link is the // caller's question. // +// file is read a component at a time and never cleaned as text, so a +// ".." is the parent of the directory actually reached. +// // Only requested paths come through here. Doing the same for every path // a package recorded would mean a directory read per directory in the // image, so recorded paths are compared as recorded, through top-level -// links only — which is why version tries a path as written first. +// links only. func (o *fileOwners) resolveDirectories(ctx context.Context, file string) (string, error) { - file = path.Clean(file) - dir, base := path.Split(file) - pending := strings.FieldsFunc(dir, func(r rune) bool { return r == '/' }) + parts := strings.Split(file, "/") + for len(parts) > 0 && (parts[len(parts)-1] == "" || parts[len(parts)-1] == ".") { + parts = parts[:len(parts)-1] + } + if len(parts) == 0 { + return "/", nil + } + base := parts[len(parts)-1] + if base == ".." { + // The path names a directory. Resolve all of it; the caller + // reports what it finds there. + parts, base = append(parts, "."), "" + } + // When a component is not there, or directory links lead in a + // circle, no file is reached: dir is then as far as the walk got, + // and the caller finds nothing at the path and says so. + dir, _, err := o.realDirectory(ctx, parts[:len(parts)-1]) + if err != nil { + return "", err + } + return path.Join(dir, base), nil +} + +// realDirectory walks components from the root and returns the +// directory they name with every symbolic link on the way followed, +// the last one included. +// +// reached is false when a component does not exist or the links lead in +// a circle; the path returned then is as far as the walk got with the +// rest appended, which names nothing in the image. +func (o *fileOwners) realDirectory(ctx context.Context, parts []string) (dir string, reached bool, err error) { + pending := append([]string(nil), parts...) resolved := "/" links := 0 for len(pending) > 0 { part := pending[0] pending = pending[1:] - if part == ".." { - // A link target may climb; the path itself is clean. + switch part { + case "", ".": + continue + case "..": resolved = path.Dir(resolved) continue } candidate := path.Join(resolved, part) entry, present, err := o.entry(ctx, candidate) if err != nil { - return "", err + return "", false, err } if !present { - // Nothing further to resolve. The caller finds the file - // missing and says so. - return path.Join(append([]string{candidate}, append(pending, base)...)...), nil + return path.Join(append([]string{candidate}, pending...)...), false, nil } if entry.Link == "" { resolved = candidate continue } if links++; links > maxProvidesFromLinks { - // Directory links that lead in a circle reach no file. - return file, nil + return path.Join(append([]string{candidate}, pending...)...), false, nil } - target := strings.FieldsFunc(entry.Link, func(r rune) bool { return r == '/' }) if path.IsAbs(entry.Link) { resolved = "/" } - pending = append(target, pending...) + pending = append(strings.Split(entry.Link, "/"), pending...) } - return path.Join(resolved, base), nil + return resolved, true, nil } func (o *fileOwners) ownerVersion(file string, owners map[string]bool) (string, error) { diff --git a/spec/provides_from_test.go b/spec/provides_from_test.go index 0e7bb2ea..5bebb8e7 100644 --- a/spec/provides_from_test.go +++ b/spec/provides_from_test.go @@ -595,3 +595,87 @@ func TestProvidesFromPathsResolvesDirectoryLinks(t *testing.T) { require.ErrorContains(t, err, want) } } + +// Resolution has to end at the file that is actually there. Two ways of +// reading a path as text end somewhere else, and each of them states +// the version of a package that does not own the file. +func TestProvidesFromPathsEndsAtTheRealFile(t *testing.T) { + status := func(packages ...string) string { + var out []string + for i := 0; i+1 < len(packages); i += 2 { + out = append(out, "Package: "+packages[i]+"\nStatus: install ok installed\nArchitecture: arm64\nVersion: "+packages[i+1]+"\n") + } + return strings.Join(out, "\n") + } + + // A list can hold a path from before a directory on it became a + // link. The file there now is another package's. + stale := fakeImage{ + DpkgStatusPath: status("old", "1.0-1", "new", "2.0-1"), + DpkgInfoDir + "/old.list": "/opt/app/current/bin/tool\n", + DpkgInfoDir + "/new.list": "/opt/app/release-2/bin/tool\n", + "/opt/app/release-2/bin/tool": "elf", + "/opt/app/current": "-> release-2", + } + got, err := ProvidesFromPaths(context.Background(), map[string]string{"tool": "/opt/app/current/bin/tool"}, one(stale)) + require.NoError(t, err) + require.Equal(t, []string{"tool@2.0"}, got, "the owner of the real file, not of the spelling") + + // ".." after a linked directory is that directory's parent. Read as + // text, sub/../real-tool is the neighbour of sub; in the image it + // is the neighbour of wherever sub leads. + climb := fakeImage{ + DpkgStatusPath: status("near", "1.0-1", "far", "2.0-1"), + DpkgInfoDir + "/near.list": "/opt/app/bin/real-tool\n", + DpkgInfoDir + "/far.list": "/srv/real-tool\n", + "/opt/app/bin/real-tool": "elf", + "/srv/real-tool": "elf", + "/srv/other/keep": "x", + "/opt/app/bin/sub": "-> /srv/other", + "/opt/app/bin/tool": "-> sub/../real-tool", + "/opt/app/bin/dotted": "-> ./sub/./../real-tool", + // The same shape in a directory link rather than a file link. + "/opt/app/via": "-> bin/sub/..", + "/opt/app/bin/up": "-> ..", + "/opt/app/bin/updir": "-> sub/..", + "/opt/app/bin/straight": "-> ../bin/real-tool", + } + got, err = ProvidesFromPaths(context.Background(), map[string]string{ + "tool": "/opt/app/bin/tool", + "dotted": "/opt/app/bin/dotted", + "via": "/opt/app/via/real-tool", + "straight": "/opt/app/bin/straight", + }, one(climb)) + require.NoError(t, err) + require.Equal(t, []string{"dotted@2.0", "straight@1.0", "tool@2.0", "via@2.0"}, got) + + for _, file := range []string{"/opt/app/bin/up", "/opt/app/bin/updir"} { + _, err = ProvidesFromPaths(context.Background(), map[string]string{"x": file}, one(climb)) + require.ErrorContains(t, err, "is a directory", "%s leads to a directory", file) + } +} + +// A top-level link is an alias for the directory it finally reaches, +// worked out the same way: through as many links as it takes, with ".." +// meaning the parent of what was reached. +func TestProvidesFromPathsResolvesTopLevelAliasesFully(t *testing.T) { + image := fakeImage{ + "/bin": "-> compat/../usr/bin", + "/compat": "-> srv/elsewhere", + "/srv/elsewhere/keep": "x", + "/srv/usr/bin/tool": "elf", + "/usr/bin/tool": "elf", + "/dangling": "-> nowhere", + DpkgStatusPath: "Package: tool\nStatus: install ok installed\nArchitecture: arm64\nVersion: 1.2.3-1\n\nPackage: other\nStatus: install ok installed\nArchitecture: arm64\nVersion: 9.9-1\n", + DpkgInfoDir + "/tool.list": "/bin/tool\n", + // What reading /bin's target as text would have reached. + DpkgInfoDir + "/other.list": "/usr/bin/tool\n/dangling/thing\n", + } + got, err := ProvidesFromPaths(context.Background(), map[string]string{ + "recorded": "/bin/tool", + "real": "/srv/usr/bin/tool", + "textual": "/usr/bin/tool", + }, one(image)) + require.NoError(t, err) + require.Equal(t, []string{"real@1.2.3", "recorded@1.2.3", "textual@9.9"}, got) +} From de74933ef055b7ccec0cfb3d0e93b309a10a09c9 Mon Sep 17 00:00:00 2001 From: Christian Dupuis Date: Fri, 20 Nov 2026 14:41:35 +0000 Subject: [PATCH 07/10] fix(spec): refuse a providesFrom path whose directories cannot be walked The directory walk reported whether it arrived and the caller ignored the answer, using whatever path came back. When a component was missing the remaining components were appended and the result cleaned, so a ".." after the missing one stepped back out of it: /opt/link pointing at missing/../tool resolved to /opt/tool and published its owner's version for a path the system cannot open. The walk also accepted any component that was not a link, so a regular file in the middle of a path was walked through as if it were a directory, with the same result for plain/../tool. A walk that does not arrive now returns no path at all, and nothing is looked up for it. A component has to be a directory or a link to be passed through. A path that ends in a slash, "." or ".." is walked whole as a directory, so a file with a slash after it is refused as well. Signed-off-by: Christian Dupuis Sandbox-Id: spiffe://sbx.docker.com/org/57b45934-74c0-11e4-bea4-0242ac11001b/user/1580b874-0d59-41d8-bd81-82a10abc4100/sandbox/74f8ada1-8c4b-4fd6-9802-da94ba278db2 Sandbox-Kit: sha256:dfcafd4d7957b8b6ac4574528241772465187a778e16bdf809d9c85f43e8fed7 Co-authored-by: sbx docker/cdupuis/74f8ada1 --- docs/spec/SPEC-v3.md | 17 +++++--- spec/provides_from.go | 87 +++++++++++++++++++++----------------- spec/provides_from_test.go | 45 ++++++++++++++++++++ 3 files changed, 103 insertions(+), 46 deletions(-) diff --git a/docs/spec/SPEC-v3.md b/docs/spec/SPEC-v3.md index 2f9de75a..d27f45eb 100644 --- a/docs/spec/SPEC-v3.md +++ b/docs/spec/SPEC-v3.md @@ -1364,13 +1364,16 @@ shipped. that installed the file where it really is. Paths are resolved the way the system resolves them, a component at a time, so `..` in a link's target is the parent of the directory reached and not of the - one written. The owner of the file where it really is comes first; - the path as written is consulted only when nothing owns that, since - a list can hold a spelling from before a directory became a link. - What a package recorded is compared as recorded, through top-level - directories that are themselves links, each followed to wherever it - finally leads: `/bin/sh` and `/usr/bin/sh` name one file on a - merged-`/usr` system whichever of them the package recorded. + one written, and a path the system could not open — through a + component that is missing, or a file where a directory has to be — + names no file whatever follows. The owner of the file where it really + is comes first; the path as written is consulted only when nothing + owns that, since a list can hold a spelling from before a directory + became a link. What a package recorded is compared as recorded, + through top-level directories that are themselves links, each + followed to wherever it finally leads: `/bin/sh` and `/usr/bin/sh` + name one file on a merged-`/usr` system whichever of them the package + recorded. - The version **MUST** be the owner's version under [§9.6](#96-derived-provides)'s rule, and publishing **MUST** fail when that rule names none — a prerelease, or one package recorded at diff --git a/spec/provides_from.go b/spec/provides_from.go index 5014f547..36dce93d 100644 --- a/spec/provides_from.go +++ b/spec/provides_from.go @@ -734,13 +734,20 @@ func (o *fileOwners) canonical(file string) string { func (o *fileOwners) version(ctx context.Context, file string) (string, error) { current := file for range maxProvidesFromLinks { - resolved, err := o.resolveDirectories(ctx, current) + resolved, reached, err := o.resolveDirectories(ctx, current) if err != nil { return "", err } - entry, present, err := o.entry(ctx, resolved) - if err != nil { - return "", err + // A walk that did not get there names no file, and nothing is + // looked up for it. What is left of the path is not a path: a + // ".." after a component that is missing would otherwise step + // back out of the hole and land on a file that is there. + var entry ImageEntry + present := false + if reached { + if entry, present, err = o.entry(ctx, resolved); err != nil { + return "", err + } } if !present { if current != file { @@ -784,41 +791,39 @@ func (o *fileOwners) version(ctx context.Context, file string) (string, error) { // file is read a component at a time and never cleaned as text, so a // ".." is the parent of the directory actually reached. // +// reached is false when the directories do not lead anywhere, and the +// path returned then means nothing. +// // Only requested paths come through here. Doing the same for every path // a package recorded would mean a directory read per directory in the // image, so recorded paths are compared as recorded, through top-level // links only. -func (o *fileOwners) resolveDirectories(ctx context.Context, file string) (string, error) { +func (o *fileOwners) resolveDirectories(ctx context.Context, file string) (resolved string, reached bool, err error) { parts := strings.Split(file, "/") - for len(parts) > 0 && (parts[len(parts)-1] == "" || parts[len(parts)-1] == ".") { - parts = parts[:len(parts)-1] - } - if len(parts) == 0 { - return "/", nil - } - base := parts[len(parts)-1] - if base == ".." { - // The path names a directory. Resolve all of it; the caller - // reports what it finds there. - parts, base = append(parts, "."), "" - } - // When a component is not there, or directory links lead in a - // circle, no file is reached: dir is then as far as the walk got, - // and the caller finds nothing at the path and says so. - dir, _, err := o.realDirectory(ctx, parts[:len(parts)-1]) - if err != nil { - return "", err + // A path that ends in a slash, a "." or a ".." can only name a + // directory, so all of it is walked as one: a file with a slash + // after it is refused, as the system refuses it, and a directory + // is handed back for the caller to say that it is one. + if last := parts[len(parts)-1]; last == "" || last == "." || last == ".." { + return o.realDirectory(ctx, parts) } - return path.Join(dir, base), nil + dir, reached, err := o.realDirectory(ctx, parts[:len(parts)-1]) + if err != nil || !reached { + return "", false, err + } + return path.Join(dir, parts[len(parts)-1]), true, nil } // realDirectory walks components from the root and returns the // directory they name with every symbolic link on the way followed, // the last one included. // -// reached is false when a component does not exist or the links lead in -// a circle; the path returned then is as far as the walk got with the -// rest appended, which names nothing in the image. +// reached is false when the walk cannot be completed the way the +// system would complete it: a component does not exist, a component is +// a file where a directory has to be, or the links lead in a circle. +// Nothing is returned for the path then. Stopping is the point — the +// components still pending are not a remainder to append, since a ".." +// among them would be taken from somewhere the walk never got to. func (o *fileOwners) realDirectory(ctx context.Context, parts []string) (dir string, reached bool, err error) { pending := append([]string(nil), parts...) resolved := "/" @@ -838,20 +843,24 @@ func (o *fileOwners) realDirectory(ctx context.Context, parts []string) (dir str if err != nil { return "", false, err } - if !present { - return path.Join(append([]string{candidate}, pending...)...), false, nil - } - if entry.Link == "" { + switch { + case !present: + return "", false, nil + case entry.Link != "": + if links++; links > maxProvidesFromLinks { + return "", false, nil + } + if path.IsAbs(entry.Link) { + resolved = "/" + } + pending = append(strings.Split(entry.Link, "/"), pending...) + case !entry.Dir: + // A file cannot be walked through, and the system refuses + // the path whatever comes after it. + return "", false, nil + default: resolved = candidate - continue - } - if links++; links > maxProvidesFromLinks { - return path.Join(append([]string{candidate}, pending...)...), false, nil - } - if path.IsAbs(entry.Link) { - resolved = "/" } - pending = append(strings.Split(entry.Link, "/"), pending...) } return resolved, true, nil } diff --git a/spec/provides_from_test.go b/spec/provides_from_test.go index 5bebb8e7..37fc19fa 100644 --- a/spec/provides_from_test.go +++ b/spec/provides_from_test.go @@ -679,3 +679,48 @@ func TestProvidesFromPathsResolvesTopLevelAliasesFully(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{"real@1.2.3", "recorded@1.2.3", "textual@9.9"}, got) } + +// A walk that cannot be completed names no file. The components left +// over are not a path to fall back on: a ".." among them would step out +// of a directory that was never entered and land on a file that exists. +func TestProvidesFromPathsRefusesAWalkThatDoesNotArrive(t *testing.T) { + image := fakeImage{ + DpkgStatusPath: "Package: tool\nStatus: install ok installed\nArchitecture: arm64\nVersion: 1.2.3-1\n", + DpkgInfoDir + "/tool.list": "/opt/tool\n/opt/dir/inner\n", + "/opt/tool": "elf", + "/opt/dir/inner": "elf", + "/opt/plain": "a regular file", + "/opt/missing-then-up": "-> missing/../tool", + "/opt/file-then-up": "-> plain/../tool", + "/opt/file-then-down": "-> plain/tool", + "/opt/deep-missing": "-> missing/also/../../tool", + "/opt/dirlink-missing": "-> missing/..", + "/opt/dirlink-file": "-> plain/..", + "/opt/good": "-> dir/../tool", + "/opt/file-with-slash": "-> tool/", + "/opt/file-with-dot": "-> tool/.", + "/opt/dir-with-slash": "-> dir/", + } + for file, want := range map[string]string{ + "/opt/missing-then-up": "/opt/missing-then-up leads to /opt/missing/../tool, which does not exist in the image", + "/opt/file-then-up": "/opt/file-then-up leads to /opt/plain/../tool, which does not exist in the image", + "/opt/file-then-down": "/opt/file-then-down leads to /opt/plain/tool, which does not exist in the image", + "/opt/deep-missing": "which does not exist in the image", + "/opt/dirlink-missing/tool": "/opt/dirlink-missing/tool does not exist in the image", + "/opt/dirlink-file/tool": "/opt/dirlink-file/tool does not exist in the image", + "/opt/plain/tool": "/opt/plain/tool does not exist in the image", + "/opt/missing/tool": "/opt/missing/tool does not exist in the image", + // A slash after a file asks for a directory that is not one. + "/opt/file-with-slash": "/opt/file-with-slash leads to /opt/tool/, which does not exist in the image", + "/opt/file-with-dot": "/opt/file-with-dot leads to /opt/tool/., which does not exist in the image", + "/opt/dir-with-slash": "/opt/dir/ is a directory", + } { + _, err := ProvidesFromPaths(context.Background(), map[string]string{"x": file}, one(image)) + require.ErrorContains(t, err, want, file) + } + + // The same shape through a directory that is there still resolves. + got, err := ProvidesFromPaths(context.Background(), map[string]string{"x": "/opt/good"}, one(image)) + require.NoError(t, err) + require.Equal(t, []string{"x@1.2.3"}, got) +} From 5043867a1dd69f748dd4dfda84aa1edcaf151b01 Mon Sep 17 00:00:00 2001 From: Christian Dupuis Date: Fri, 20 Nov 2026 14:47:35 +0000 Subject: [PATCH 08/10] fix(spec): leave an anchored merge sequence as it was for whatever else aliases it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removing providesFrom edits a merge key's list of sources in place: an element is edited, swapped for a copy, or dropped. The removal already gave aliases their own copy of an anchored mapping before changing it, but not of the sequence holding the sources. With `<<: &shared [{providesFrom: …}, {…}]` and a capability config using *shared, the config lost an element when the kit was published, with nothing said. An anchored sequence is now recorded among the nodes the removal changes, so an alias to it is detached with the sequence as the author wrote it. The test walks every node the removal touches in place and checks that a config aliasing it decodes to what it decoded to before. Signed-off-by: Christian Dupuis Sandbox-Id: spiffe://sbx.docker.com/org/57b45934-74c0-11e4-bea4-0242ac11001b/user/1580b874-0d59-41d8-bd81-82a10abc4100/sandbox/74f8ada1-8c4b-4fd6-9802-da94ba278db2 Sandbox-Kit: sha256:dfcafd4d7957b8b6ac4574528241772465187a778e16bdf809d9c85f43e8fed7 Co-authored-by: sbx docker/cdupuis/74f8ada1 --- spec/provides_from.go | 25 +++++++++---- spec/provides_from_test.go | 73 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 7 deletions(-) diff --git a/spec/provides_from.go b/spec/provides_from.go index 36dce93d..d6a3b7fd 100644 --- a/spec/provides_from.go +++ b/spec/provides_from.go @@ -248,18 +248,19 @@ type fieldRemoval struct { // removed holds every node that leaves the document: the field's // key and value, and all that is under the value. removed map[*yaml.Node]bool - // edited holds the anchored mappings the field is taken out of in - // place. An alias to one of these would see the mapping change - // under it. + // edited holds the anchored nodes that are changed in place: a + // mapping the field is taken out of, or a merge key's sequence that + // loses or swaps an element. An alias to one of these would see it + // change underneath. edited map[*yaml.Node]bool } // plan records what strip will remove from m and which shared mappings // it will edit, without changing anything. // -// A merge source reached through an alias is not recorded: strip works -// on a copy of it, so the anchored original stays as it was for -// whatever else refers to it. +// A merge source reached through an alias is not recorded for itself: +// strip works on a copy of it, so the anchored original stays as it was +// for whatever else refers to it. func (e *fieldRemoval) plan(m *yaml.Node, depth int) { if depth > maxYAMLIndirection { return @@ -268,7 +269,17 @@ func (e *fieldRemoval) plan(m *yaml.Node, depth int) { k, v := m.Content[i], m.Content[i+1] if isMergeKey(k) { for _, source := range mergeSources(v) { - if source.Kind != yaml.MappingNode || !mappingCarries(source, e.field, depth+1) { + if !mappingCarries(source, e.field, depth+1) { + continue + } + // The list of sources is itself a node an alias can + // name. Whatever happens to this source — edited where + // it stands, replaced by a copy, or dropped — happens + // to that list. + if v.Kind == yaml.SequenceNode && v.Anchor != "" { + e.edited[v] = true + } + if source.Kind != yaml.MappingNode { continue } if source.Anchor != "" { diff --git a/spec/provides_from_test.go b/spec/provides_from_test.go index 37fc19fa..47eb4213 100644 --- a/spec/provides_from_test.go +++ b/spec/provides_from_test.go @@ -724,3 +724,76 @@ func TestProvidesFromPathsRefusesAWalkThatDoesNotArrive(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{"x@1.2.3"}, got) } + +// Everything the removal changes in place is something an alias can +// name. Whatever else uses that alias — here a capability's config, +// which is opaque to this rewrite — has to read what it read before. +func TestWithoutProvidesFromLeavesSharedNodesAsTheyWere(t *testing.T) { + const head = "schemaVersion: \"3\"\nkind: workload\nversion: \"1.0.0\"\n" + const tail = "build: |\n FROM scratch\n" + uses := func(alias string) string { + return "capabilities:\n - type: com.example/thing@1\n config: {uses: *" + alias + "}\n" + } + paths := map[string]any{"python": "/usr/bin/python3"} + + for name, tc := range map[string]struct { + body string + config any + displayName string + }{ + // The sequence a merge key lists its sources in. + "anchored merge sequence": { + body: "<<: &shared [{providesFrom: {python: /usr/bin/python3}}, {displayName: Second}]\n" + uses("shared"), + config: []any{map[string]any{"providesFrom": paths}, map[string]any{"displayName": "Second"}}, + displayName: "Second", + }, + // A source inside an unanchored sequence. + "anchored element of a merge sequence": { + body: "<<: [&shared {providesFrom: {python: /usr/bin/python3}, displayName: First}, {description: Second}]\n" + uses("shared"), + config: map[string]any{"providesFrom": paths, "displayName": "First"}, + displayName: "First", + }, + // A single anchored source. + "anchored merge mapping": { + body: "<<: &shared {providesFrom: {python: /usr/bin/python3}, displayName: Only}\n" + uses("shared"), + config: map[string]any{"providesFrom": paths, "displayName": "Only"}, + displayName: "Only", + }, + // The field reaches the anchored mapping through a merge of its + // own, so the mapping is edited one level further in. + "anchored mapping with a nested merge": { + body: "<<: &shared {displayName: Outer, <<: {providesFrom: {python: /usr/bin/python3}}}\n" + uses("shared"), + config: map[string]any{"providesFrom": paths, "displayName": "Outer"}, + displayName: "Outer", + }, + // An anchored sequence whose only source goes away entirely. + "anchored sequence left empty": { + body: "<<: &shared [{providesFrom: {python: /usr/bin/python3}}]\n" + uses("shared"), + config: []any{map[string]any{"providesFrom": paths}}, + }, + // The field's own value. + "anchored value": { + body: "providesFrom: &shared {python: /usr/bin/python3}\n" + uses("shared"), + config: paths, + }, + } { + t.Run(name, func(t *testing.T) { + authored := []byte(head + tc.body + tail) + before, err := Decode(authored) + require.NoError(t, err) + require.NotEmpty(t, before.ProvidesFrom) + require.Equal(t, map[string]any{"uses": tc.config}, before.Capabilities[0].Config, "the case is not what it says") + + out, err := WithoutProvidesFrom(authored) + require.NoError(t, err) + after, err := Decode(out) + require.NoError(t, err, "%s", out) + require.Empty(t, after.ProvidesFrom, "%s", out) + require.Equal(t, before.Capabilities[0].Config, after.Capabilities[0].Config, + "the config reads what it read before:\n%s", out) + require.Equal(t, tc.displayName, after.DisplayName) + _, err = ValidatePublished(out, after) + require.NoError(t, err, "%s", out) + }) + } +} From 8eeb7d5e7896e1fde868bbf53be151f6a08fab9e Mon Sep 17 00:00:00 2001 From: Christian Dupuis Date: Fri, 20 Nov 2026 14:54:08 +0000 Subject: [PATCH 09/10] fix(spec): apply the providesFrom kind rule to a field that is written but empty The schema refuses providesFrom on a mixin or a set by the key being present. Validation applied the rule only when the map had entries, so `providesFrom: {}` or a bare `providesFrom:` on a mixin passed, and the frontend then dropped the field without a word: three answers for one descriptor, and an editor marking wrong what the build accepted. The rule is now about the field being written. An empty map is visible on the decoded struct. A null is not, since it decodes like no field at all, so ValidateRaw reads presence from the bytes for that spelling, merge keys included. On a workload an empty field still asks for nothing and is dropped at publish. Signed-off-by: Christian Dupuis Sandbox-Id: spiffe://sbx.docker.com/org/57b45934-74c0-11e4-bea4-0242ac11001b/user/1580b874-0d59-41d8-bd81-82a10abc4100/sandbox/74f8ada1-8c4b-4fd6-9802-da94ba278db2 Sandbox-Kit: sha256:dfcafd4d7957b8b6ac4574528241772465187a778e16bdf809d9c85f43e8fed7 Co-authored-by: sbx docker/cdupuis/74f8ada1 --- docs/spec/SPEC-v3.md | 3 ++- spec/provides_from.go | 38 +++++++++++++++++++++++------- spec/provides_from_test.go | 48 ++++++++++++++++++++++++++++++++++++++ spec/validate.go | 1 + 4 files changed, 81 insertions(+), 9 deletions(-) diff --git a/docs/spec/SPEC-v3.md b/docs/spec/SPEC-v3.md index d27f45eb..f6fdd958 100644 --- a/docs/spec/SPEC-v3.md +++ b/docs/spec/SPEC-v3.md @@ -1339,7 +1339,8 @@ shipped. content, and **MUST** be refused on a mixin or a set. A mixin's layers are a delta, where a package database is not an inventory ([§9.6](#96-derived-provides)); a set builds nothing, and the workload - it lists already stated what it carries. + it lists already stated what it carries. The field is refused there + for being written, empty or not. - Publishing **MUST** fail unless exactly one installed package lists the path: it fails when the path does not exist, is a directory, is listed by more than one package, or is listed by none. The lists are diff --git a/spec/provides_from.go b/spec/provides_from.go index d6a3b7fd..097e15ba 100644 --- a/spec/provides_from.go +++ b/spec/provides_from.go @@ -36,16 +36,15 @@ const maxProvidesFromLinks = 16 // than saying it after. func validateProvidesFrom(d *Descriptor) error { var errs ValidationErrors - if len(d.ProvidesFrom) == 0 { + // Present, not populated: `providesFrom: {}` on a mixin asks the + // same thing of a filesystem that cannot answer it, and the schema + // refuses the key there whatever is under it. A nil map is a field + // that was not written, or was written with nothing after it; + // ValidateRaw tells those two apart from the bytes. + if d.ProvidesFrom == nil { return nil } - switch { - case len(d.Kits) > 0: - errs.add(fieldErrorf("providesFrom", "providesFrom reads the filesystem this kit builds, and a set builds none: its content is the kits it lists, and the workload among them already states what it carries")) - case d.Kind != KindWorkload: - errs.add(fieldErrorf("providesFrom", "providesFrom is for kind: %s only: a %s's layers are a delta, where a package database is whatever the recipe happened to rewrite rather than an inventory", KindWorkload, d.Kind)) - } - + errs.add(validateProvidesFromKind(d)) errs.add(RequireDistinctProvidesFrom(d, d.ProvidesFrom)) seen := map[string]string{} for _, name := range slices.Sorted(maps.Keys(d.ProvidesFrom)) { @@ -75,6 +74,29 @@ func validateProvidesFrom(d *Descriptor) error { return errs.err() } +// validateProvidesFromKind holds the field to the kinds that have a +// filesystem to read: a workload that builds its own content. +func validateProvidesFromKind(d *Descriptor) error { + switch { + case len(d.Kits) > 0: + return fieldErrorf("providesFrom", "providesFrom reads the filesystem this kit builds, and a set builds none: its content is the kits it lists, and the workload among them already states what it carries") + case d.Kind != KindWorkload: + return fieldErrorf("providesFrom", "providesFrom is for kind: %s only: a %s's layers are a delta, where a package database is whatever the recipe happened to rewrite rather than an inventory", KindWorkload, d.Kind) + } + return nil +} + +// validateRawProvidesFrom applies the kind rule to a field the struct +// cannot show: a bare `providesFrom:` decodes to a nil map, exactly +// like no field at all, and only the bytes say it was written. With +// any other spelling Validate has already judged it. +func validateRawProvidesFrom(raw []byte, d *Descriptor) error { + if d.ProvidesFrom != nil || !carriesProvidesFrom(raw) { + return nil + } + return validateProvidesFromKind(d) +} + // RequireDistinctProvidesFrom refuses a providesFrom key that names a // capability the descriptor's provides already states: a capability has // one version, and two entries for it would leave a resolver to pick. diff --git a/spec/provides_from_test.go b/spec/provides_from_test.go index 47eb4213..beda1537 100644 --- a/spec/provides_from_test.go +++ b/spec/provides_from_test.go @@ -797,3 +797,51 @@ func TestWithoutProvidesFromLeavesSharedNodesAsTheyWere(t *testing.T) { }) } } + +// The kind rule is about the field being written, not about it having +// entries: the schema refuses the key on a mixin or a set whatever is +// under it, and validation must not accept a descriptor an editor +// already marked wrong — nor let the build drop the field in silence. +func TestProvidesFromKindRuleAppliesToAnEmptyField(t *testing.T) { + const mixin = "schemaVersion: \"3\"\nkind: mixin\nversion: \"1.0.0\"\n" + const set = "schemaVersion: \"3\"\nkind: set\nkits:\n - ref: docker.io/example/base:1.0.0\n" + const workload = "schemaVersion: \"3\"\nkind: workload\nversion: \"1.0.0\"\nbuild: |\n FROM scratch\n" + + for name, tc := range map[string]struct{ raw, want string }{ + "mixin, empty map": {mixin + "providesFrom: {}\n", "providesFrom is for kind: workload only"}, + "mixin, null": {mixin + "providesFrom:\n", "providesFrom is for kind: workload only"}, + "mixin, merged": {mixin + "<<: {providesFrom: {}}\n", "providesFrom is for kind: workload only"}, + "mixin, merged null": {mixin + "<<: {providesFrom: }\n", "providesFrom is for kind: workload only"}, + "set, empty map": {set + "providesFrom: {}\n", "a set builds none"}, + "set, null": {set + "providesFrom:\n", "a set builds none"}, + } { + t.Run(name, func(t *testing.T) { + d, err := Decode([]byte(tc.raw)) + require.NoError(t, err) + _, err = ValidateDeclarations([]byte(tc.raw), d) + require.ErrorContains(t, err, tc.want) + // Once, however the field was spelled. + require.Equal(t, 1, strings.Count(err.Error(), tc.want), "%v", err) + }) + } + + // On a workload an empty field asks for nothing, and is dropped at + // publish like a full one. + for _, body := range []string{"providesFrom: {}\n", "providesFrom:\n"} { + raw := []byte(workload + body) + d, err := Decode(raw) + require.NoError(t, err) + _, err = ValidateDeclarations(raw, d) + require.NoError(t, err) + out, err := WithoutProvidesFrom(raw) + require.NoError(t, err) + require.NotContains(t, string(out), "providesFrom") + } + + // A descriptor with no such field is untouched by any of this. + plain := []byte(mixin) + d, err := Decode(plain) + require.NoError(t, err) + _, err = ValidateDeclarations(plain, d) + require.NoError(t, err) +} diff --git a/spec/validate.go b/spec/validate.go index 9af534f3..cb3234ab 100644 --- a/spec/validate.go +++ b/spec/validate.go @@ -245,6 +245,7 @@ func ValidateRaw(raw []byte, d *Descriptor) (warnings []string, err error) { warnings, err = Validate(d) errs.add(err) errs.add(validateRawBuildSecrets(raw, d)) + errs.add(validateRawProvidesFrom(raw, d)) sizeWarnings, sizeErr := validateSize(raw) warnings = append(warnings, sizeWarnings...) From d43d0a5aafbff87d6681904fe432358b4fe7c2b5 Mon Sep 17 00:00:00 2001 From: Christian Dupuis Date: Fri, 20 Nov 2026 15:03:31 +0000 Subject: [PATCH 10/10] fix(spec): follow merge keys to any depth when removing or detecting providesFrom The walk that finds providesFrom through YAML merge keys stopped at 32 levels, a bound put there to keep an anchor that aliases itself from recursing forever. The decoder has no such bound: it follows merges as deep as they are nested. A field 33 merges down was therefore neither removed at publish nor seen by the published-form check, and with an empty value nothing else caught it either, so a kit could publish with a key that readers built before the field refuse. The bound is gone. A walk that would not end is recognized by the same mapping coming round again: presence is judged once per mapping and remembered, which also keeps a mapping merged in many places from being judged many times, and a mapping whose copy is already being stripped is not expanded again beneath itself. Copying is the one thing that can still grow without limit, when aliases fan out and each level merges the one below it twice. The decoder refuses that document for excessive aliasing before a build reaches the removal; a direct caller now gets an error after a million copied nodes. Signed-off-by: Christian Dupuis Sandbox-Id: spiffe://sbx.docker.com/org/57b45934-74c0-11e4-bea4-0242ac11001b/user/1580b874-0d59-41d8-bd81-82a10abc4100/sandbox/74f8ada1-8c4b-4fd6-9802-da94ba278db2 Sandbox-Kit: sha256:dfcafd4d7957b8b6ac4574528241772465187a778e16bdf809d9c85f43e8fed7 Co-authored-by: sbx docker/cdupuis/74f8ada1 --- spec/provides_from.go | 225 ++++++++++++++++++++++++------------- spec/provides_from_test.go | 109 ++++++++++++++++++ 2 files changed, 258 insertions(+), 76 deletions(-) diff --git a/spec/provides_from.go b/spec/provides_from.go index 097e15ba..89d14134 100644 --- a/spec/provides_from.go +++ b/spec/provides_from.go @@ -127,10 +127,14 @@ func RequireDistinctProvidesFrom(d *Descriptor, from map[string]string) error { // providesFromField is the descriptor key, as an author writes it. const providesFromField = "providesFrom" -// maxYAMLIndirection bounds how far a merge key or an alias is followed -// while looking for the field. YAML admits an anchor that contains its -// own alias, and a walk with no bound would not return from one. -const maxYAMLIndirection = 32 +// maxDetachedNodes bounds how many nodes the removal may copy. A merge +// source reached through an alias is copied before it is edited, and +// aliases that fan out — each level merging the one below it twice — +// make that copy double per level. The decoder refuses such a document +// for excessive aliasing before any build gets this far; this is the +// same refusal for a caller that did not decode first, in place of +// memory that runs out. +const maxDetachedNodes = 1 << 20 // WithoutProvidesFrom returns a descriptor with its providesFrom field // removed, for the published form (§9.7). @@ -144,27 +148,32 @@ const maxYAMLIndirection = 32 // where it is written at the top level. A YAML merge key hands the root // every key of another mapping, so `<<: {providesFrom: …}` decodes to // the same descriptor as the plain spelling and has to publish the same -// way. And the removal takes nothing else with it: an alias elsewhere -// in the document that points into what is removed is given its own -// copy first, since an anchor that disappears leaves every alias to it -// naming nothing and the document no longer parsing. +// way — at any depth: merges nest, and the decoder follows them as far +// as they go, so this does too. And the removal takes nothing else with +// it: an alias elsewhere in the document that points into what is +// removed is given its own copy first, since an anchor that disappears +// leaves every alias to it naming nothing and the document no longer +// parsing. func WithoutProvidesFrom(raw []byte) ([]byte, error) { var doc yaml.Node if err := yaml.Unmarshal(raw, &doc); err != nil { return nil, fmt.Errorf("reparse descriptor: %w", err) } root := documentRoot(&doc) - if root == nil || !mappingCarries(root, providesFromField, 0) { + edit := newFieldRemoval(providesFromField) + if root == nil || !edit.carries(root) { return raw, nil } // Three passes, in this order. What is about to change is found // first and nothing is touched, so that the second pass can copy an // aliased node as the author wrote it; only then is anything removed. - edit := &fieldRemoval{field: providesFromField, removed: map[*yaml.Node]bool{}, edited: map[*yaml.Node]bool{}} - edit.plan(root, 0) + edit.plan(root) edit.detachAliases(&doc) - edit.strip(root, 0) + edit.strip(root) + if edit.copied > maxDetachedNodes { + return nil, fmt.Errorf("%s is merged in through too many aliases to be removed; write it as a plain top-level field", providesFromField) + } out, err := yaml.Marshal(&doc) if err != nil { @@ -172,12 +181,13 @@ func WithoutProvidesFrom(raw []byte) ([]byte, error) { } // Parsed once more before it is handed back: this is the document // that gets staged and signed, and a rewrite that left it unreadable - // should say so here rather than as a decode failure with no cause. + // or left the field in should say so here, with its cause, rather + // than as a failure somewhere later in the build. var check yaml.Node if err := yaml.Unmarshal(out, &check); err != nil { return nil, fmt.Errorf("descriptor no longer parses with %s removed: %w", providesFromField, err) } - if mappingCarries(documentRoot(&check), providesFromField, 0) { + if newFieldRemoval(providesFromField).carries(documentRoot(&check)) { return nil, fmt.Errorf("%s could not be removed from the descriptor; write it as a plain top-level field", providesFromField) } return out, nil @@ -195,7 +205,7 @@ func carriesProvidesFrom(raw []byte) bool { // Unparseable bytes are another rule's to report. return false } - return mappingCarries(documentRoot(&doc), providesFromField, 0) + return newFieldRemoval(providesFromField).carries(documentRoot(&doc)) } // documentRoot returns the mapping a descriptor document consists of, @@ -212,15 +222,14 @@ func documentRoot(doc *yaml.Node) *yaml.Node { return root } -// aliasTarget follows an alias to the node it names. +// aliasTarget follows an alias to the node it names. One step is all +// there is: an alias cannot carry an anchor, so nothing an alias names +// is itself an alias. func aliasTarget(n *yaml.Node) *yaml.Node { - for range maxYAMLIndirection { - if n == nil || n.Kind != yaml.AliasNode { - return n - } - n = n.Alias + if n != nil && n.Kind == yaml.AliasNode { + return n.Alias } - return nil + return n } // isMergeKey reports whether a mapping key is YAML's merge key. Only @@ -239,32 +248,15 @@ func mergeSources(value *yaml.Node) []*yaml.Node { return []*yaml.Node{value} } -// mappingCarries reports whether a mapping has key, directly or through -// any mapping it merges in. -func mappingCarries(m *yaml.Node, key string, depth int) bool { - m = aliasTarget(m) - if m == nil || m.Kind != yaml.MappingNode || depth > maxYAMLIndirection { - return false - } - for i := 0; i+1 < len(m.Content); i += 2 { - k, v := m.Content[i], m.Content[i+1] - if isMergeKey(k) { - for _, source := range mergeSources(v) { - if mappingCarries(source, key, depth+1) { - return true - } - } - continue - } - if k.Value == key { - return true - } - } - return false -} - // fieldRemoval takes one key out of a mapping and everything the // mapping merges in. +// +// Nothing here stops at a depth. Merges nest as far as an author nests +// them and the decoder follows all of it, so a walk that gave up early +// would leave a field in place that every reader still sees. The one +// thing that has to be guarded against is a walk that does not end: an +// anchor may contain an alias to itself, and that is caught by noticing +// the same node come round again, not by counting. type fieldRemoval struct { field string // removed holds every node that leaves the document: the field's @@ -275,23 +267,91 @@ type fieldRemoval struct { // loses or swaps an element. An alias to one of these would see it // change underneath. edited map[*yaml.Node]bool + // carried remembers, for each mapping asked about, whether it has + // the field. A mapping merged in many places is judged once, and + // one still being judged when it comes round again is a cycle. + carried map[*yaml.Node]carryState + // expanding holds the anchored mappings whose copies are being + // stripped right now, and copying the removed nodes being copied. + // Meeting one again means the document refers to itself. + expanding map[*yaml.Node]bool + copying map[*yaml.Node]bool + // copied counts the nodes detachedCopy has made, against + // maxDetachedNodes. + copied int +} + +type carryState uint8 + +const ( + carryUnknown carryState = iota + carryJudging + carryYes + carryNo +) + +func newFieldRemoval(field string) *fieldRemoval { + return &fieldRemoval{ + field: field, + removed: map[*yaml.Node]bool{}, + edited: map[*yaml.Node]bool{}, + carried: map[*yaml.Node]carryState{}, + expanding: map[*yaml.Node]bool{}, + copying: map[*yaml.Node]bool{}, + } +} + +// carries reports whether a mapping has the field, directly or through +// any mapping it merges in, however far down. +func (e *fieldRemoval) carries(m *yaml.Node) bool { + m = aliasTarget(m) + if m == nil || m.Kind != yaml.MappingNode { + return false + } + switch e.carried[m] { + case carryYes: + return true + case carryNo, carryJudging: + // Still being judged means this mapping merges itself in. It + // contributes nothing it has not already been asked for. + return false + } + e.carried[m] = carryJudging + found := false + for i := 0; i+1 < len(m.Content) && !found; i += 2 { + k, v := m.Content[i], m.Content[i+1] + if isMergeKey(k) { + for _, source := range mergeSources(v) { + if e.carries(source) { + found = true + break + } + } + continue + } + found = k.Value == e.field + } + if found { + e.carried[m] = carryYes + } else { + e.carried[m] = carryNo + } + return found } -// plan records what strip will remove from m and which shared mappings -// it will edit, without changing anything. +// plan records what strip will remove from m and which shared nodes it +// will edit, without changing anything. // // A merge source reached through an alias is not recorded for itself: // strip works on a copy of it, so the anchored original stays as it was -// for whatever else refers to it. -func (e *fieldRemoval) plan(m *yaml.Node, depth int) { - if depth > maxYAMLIndirection { - return - } +// for whatever else refers to it. Only mappings written in place are +// descended into, and those form a tree, so this ends. +func (e *fieldRemoval) plan(m *yaml.Node) { for i := 0; i+1 < len(m.Content); i += 2 { k, v := m.Content[i], m.Content[i+1] if isMergeKey(k) { for _, source := range mergeSources(v) { - if !mappingCarries(source, e.field, depth+1) { + if !e.carries(source) { continue } // The list of sources is itself a node an alias can @@ -307,7 +367,7 @@ func (e *fieldRemoval) plan(m *yaml.Node, depth int) { if source.Anchor != "" { e.edited[source] = true } - e.plan(source, depth+1) + e.plan(source) } continue } @@ -338,7 +398,7 @@ func (e *fieldRemoval) detachAliases(n *yaml.Node) { if e.removed[n.Alias] || e.edited[n.Alias] { // Onto the alias node itself, so whatever holds it now // holds the copy. - *n = *e.detachedCopy(n.Alias, 0) + *n = *e.detachedCopy(n.Alias) } return } @@ -351,27 +411,32 @@ func (e *fieldRemoval) detachAliases(n *yaml.Node) { // is copied, since the original keeps its name or is going away, and an // alias to a node that is going away is replaced by a copy of that // node. An alias to anything that stays is kept as an alias. -// -// hops counts the aliases followed to get here, not how deep the tree -// is: only following an alias can lead back to where the copy started. -func (e *fieldRemoval) detachedCopy(n *yaml.Node, hops int) *yaml.Node { - if n.Kind == yaml.AliasNode && n.Alias != nil && e.removed[n.Alias] && hops < maxYAMLIndirection { - return e.detachedCopy(n.Alias, hops+1) - } +func (e *fieldRemoval) detachedCopy(n *yaml.Node) *yaml.Node { + if n.Kind == yaml.AliasNode && n.Alias != nil && e.removed[n.Alias] && !e.copying[n.Alias] { + // Not while that node is itself being copied: a removed anchor + // that aliases itself has no finite copy. The alias is left, + // and the caller's last parse reports the document for it. + return e.detachedCopy(n.Alias) + } + e.copying[n] = true + defer delete(e.copying, n) c := *n c.Anchor = "" c.Content = nil + if e.copied++; e.copied > maxDetachedNodes { + // Over budget: stop descending. What comes back is incomplete, + // and WithoutProvidesFrom refuses the document rather than + // return it. + return &c + } for _, child := range n.Content { - c.Content = append(c.Content, e.detachedCopy(child, hops)) + c.Content = append(c.Content, e.detachedCopy(child)) } return &c } // strip removes the field from m and from every mapping m merges in. -func (e *fieldRemoval) strip(m *yaml.Node, depth int) { - if depth > maxYAMLIndirection { - return - } +func (e *fieldRemoval) strip(m *yaml.Node) { kept := make([]*yaml.Node, 0, len(m.Content)) for i := 0; i+1 < len(m.Content); i += 2 { k, v := m.Content[i], m.Content[i+1] @@ -379,7 +444,7 @@ func (e *fieldRemoval) strip(m *yaml.Node, depth int) { case isMergeKey(k) && v.Kind == yaml.SequenceNode: sources := make([]*yaml.Node, 0, len(v.Content)) for _, source := range v.Content { - if source = e.stripSource(source, depth); source != nil { + if source = e.stripSource(source); source != nil { sources = append(sources, source) } } @@ -389,7 +454,7 @@ func (e *fieldRemoval) strip(m *yaml.Node, depth int) { continue } case isMergeKey(k): - if v = e.stripSource(v, depth); v == nil { + if v = e.stripSource(v); v == nil { continue } case k.Value == e.field: @@ -402,24 +467,32 @@ func (e *fieldRemoval) strip(m *yaml.Node, depth int) { // stripSource returns one merge source with the field taken out, or nil // when nothing is left of it to merge. -func (e *fieldRemoval) stripSource(source *yaml.Node, depth int) *yaml.Node { - if !mappingCarries(source, e.field, depth+1) { +func (e *fieldRemoval) stripSource(source *yaml.Node) *yaml.Node { + if !e.carries(source) { return source } if source.Kind == yaml.AliasNode { // The anchored mapping belongs to wherever it was defined, and // may be merged or aliased there for reasons of its own, so // the root merges a copy without the field instead. - target := aliasTarget(source) - if target == nil { + target := source.Alias + if target == nil || e.expanding[target] { + // A mapping that merges itself in. Its copy is already + // being stripped further up, and expanding it again here + // would never finish. + return source + } + if e.copied > maxDetachedNodes { return source } - source = e.detachedCopy(target, 0) + e.expanding[target] = true + defer delete(e.expanding, target) + source = e.detachedCopy(target) } if source.Kind != yaml.MappingNode { return source } - e.strip(source, depth+1) + e.strip(source) if len(source.Content) == 0 && source.Anchor == "" { return nil } diff --git a/spec/provides_from_test.go b/spec/provides_from_test.go index beda1537..51c47865 100644 --- a/spec/provides_from_test.go +++ b/spec/provides_from_test.go @@ -4,8 +4,10 @@ import ( "context" "path" "sort" + "strconv" "strings" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -845,3 +847,110 @@ func TestProvidesFromKindRuleAppliesToAnEmptyField(t *testing.T) { _, err = ValidateDeclarations(plain, d) require.NoError(t, err) } + +// Merges nest, and the decoder follows them however deep they go. The +// removal and the published-form check have to follow just as far: a +// walk that gave up at some depth would publish a field every reader +// still sees. +func TestProvidesFromIsFoundAtAnyMergeDepth(t *testing.T) { + const head = "schemaVersion: \"3\"\nkind: workload\nversion: \"1.0.0\"\n" + const tail = "\nbuild: |\n FROM scratch\n" + nested := func(inner string, depth int) string { + return strings.Repeat("<<: {", depth) + inner + strings.Repeat("}", depth) + } + for _, depth := range []int{1, 31, 32, 33, 40, 300} { + for name, inner := range map[string]string{ + "entries": "providesFrom: {python: /usr/bin/python3}, displayName: Deep", + "empty": "providesFrom: {}, displayName: Deep", + "null": "providesFrom: , displayName: Deep", + } { + raw := []byte(head + nested(inner, depth) + tail) + d, err := Decode(raw) + require.NoError(t, err, "%s at depth %d", name, depth) + require.Equal(t, "Deep", d.DisplayName) + + _, err = ValidatePublished(raw, d) + require.ErrorContains(t, err, "published descriptor still carries providesFrom", "%s at depth %d", name, depth) + + out, err := WithoutProvidesFrom(raw) + require.NoError(t, err, "%s at depth %d", name, depth) + require.NotContains(t, string(out), "providesFrom", "%s at depth %d", name, depth) + pd, err := Decode(out) + require.NoError(t, err, "%s at depth %d", name, depth) + require.Equal(t, "Deep", pd.DisplayName, "what was merged beside the field stays merged") + _, err = ValidatePublished(out, pd) + require.NoError(t, err, "%s at depth %d", name, depth) + } + } + + // The same through a chain of aliases, each mapping merging the one + // defined before it. A capability's config is the one place a + // descriptor has room to define them. + var chain strings.Builder + chain.WriteString("capabilities:\n - type: com.example/thing@1\n config:\n m0: &m0 {providesFrom: {python: /usr/bin/python3}, displayName: Chained}\n") + const links = 60 + for i := 1; i <= links; i++ { + chain.WriteString(" m" + strconv.Itoa(i) + ": &m" + strconv.Itoa(i) + " {<<: *m" + strconv.Itoa(i-1) + "}\n") + } + chain.WriteString("<<: *m" + strconv.Itoa(links) + "\n") + raw := []byte(head + chain.String() + tail) + d, err := Decode(raw) + require.NoError(t, err) + require.NotEmpty(t, d.ProvidesFrom) + out, err := WithoutProvidesFrom(raw) + require.NoError(t, err) + pd, err := Decode(out) + require.NoError(t, err, "%s", out) + require.Empty(t, pd.ProvidesFrom) + require.Equal(t, "Chained", pd.DisplayName) + require.Equal(t, d.Capabilities[0].Config, pd.Capabilities[0].Config, "the config the anchors live in is untouched") + _, err = ValidatePublished(out, pd) + require.NoError(t, err) +} + +// What does have to be guarded against is a walk that never ends, and +// one that copies without limit. Neither document can be decoded, so no +// build reaches the removal with one; a caller that did not decode +// first gets an answer instead of a hang. +func TestWithoutProvidesFromEndsOnDocumentsThatReferToThemselves(t *testing.T) { + const head = "schemaVersion: \"3\"\nkind: workload\nversion: \"1.0.0\"\n" + + // An anchor that merges itself in. + cyclic := []byte(head + "capabilities:\n - type: com.example/thing@1\n config: &self {providesFrom: {python: /usr/bin/python3}, <<: *self}\n<<: *self\n") + _, err := Decode(cyclic) + require.Error(t, err, "the decoder refuses an anchor that contains itself") + done := make(chan struct{}) + go func() { + defer close(done) + _, _ = WithoutProvidesFrom(cyclic) + _ = carriesProvidesFrom(cyclic) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("removal did not return for a mapping that merges itself in") + } + + // Aliases that fan out: each level merges the one below it twice, + // so a copy of the top doubles per level. + var fan strings.Builder + fan.WriteString("capabilities:\n - type: com.example/thing@1\n config:\n m0: &m0 {providesFrom: {python: /usr/bin/python3}}\n") + const levels = 40 + for i := 1; i <= levels; i++ { + fan.WriteString(" m" + strconv.Itoa(i) + ": &m" + strconv.Itoa(i) + " {<<: [*m" + strconv.Itoa(i-1) + ", *m" + strconv.Itoa(i-1) + "]}\n") + } + fan.WriteString("<<: *m" + strconv.Itoa(levels) + "\n") + wide := []byte(head + fan.String()) + require.True(t, carriesProvidesFrom(wide), "presence is judged once per mapping, however often it is merged") + done = make(chan struct{}) + go func() { + defer close(done) + _, err = WithoutProvidesFrom(wide) + }() + select { + case <-done: + require.ErrorContains(t, err, "too many aliases") + case <-time.After(60 * time.Second): + t.Fatal("removal did not return for aliases that fan out") + } +}