diff --git a/RELEASES.md b/RELEASES.md index c60fc867..16e51c1a 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -142,6 +142,17 @@ new field; runtimes that previously loaded all `@2` mixin guidance inline need the matching draft implementation, even when the field is omitted. `agent-context@1` and `schemaVersion` are unchanged. +The authored-only `providesFrom` field +([SPEC-v3 §9.7](docs/spec/SPEC-v3.md#97-provides-stated-from-a-path)) +is a top-level addition made in place under the same draft-stage +latitude, approved by maintainer @cdupuis's instruction to implement it. +Publishing replaces the field with ordinary `provides` entries and +drops it, so published descriptors keep their shape: readers of +published Kits, runtimes included, need no change. A frontend or other +reader of authored descriptors built before the field refuses a +descriptor that uses it. `schemaVersion` and every capability version +are unchanged. + The exception ends with the first stable specification release. After that, changes to shipped capability meanings, defaults, permitted values, reconciliation, or persistent-state transitions require a new diff --git a/cmd/frontend/build.go b/cmd/frontend/build.go index b1c57efa..612eb9be 100644 --- a/cmd/frontend/build.go +++ b/cmd/frontend/build.go @@ -106,6 +106,13 @@ func Build(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) { if err != nil { return nil, withYAMLSource(err, filename, src.descriptor) } + // providesFrom is an instruction to this build, kept on d until the + // filesystems it names exist. The published form never carries it + // (§9.7), so it leaves here, before anything judges that form. + published, err = spec.WithoutProvidesFrom(published) + if err != nil { + return nil, withYAMLSource(err, filename, src.descriptor) + } content := commentContent switch { @@ -243,6 +250,12 @@ func Build(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) { if err := spec.RequireAuthoredProvides(pd); err != nil { return nil, withYAMLSource(err, filename, src.descriptor) } + // On the expanded form for the same reason: a provides entry + // whose name came out of an arg can only now be seen to state + // a capability providesFrom states too. + if err := spec.RequireDistinctProvidesFrom(pd, d.ProvidesFrom); err != nil { + return nil, withYAMLSource(err, filename, src.descriptor) + } } res := gwclient.NewResult() @@ -308,6 +321,13 @@ func Build(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) { if err != nil { return nil, err } + // What the author asked for by name stands ahead of the + // inventory nobody asked for entry by entry. + stated, err := statedProvides(ctx, d.ProvidesFrom, builds) + if err != nil { + return nil, withYAMLSource(err, filename, src.descriptor) + } + derived = append(stated, derived...) if len(derived) > 0 { published, pd, err = withDerivedProvides(published, derived) if err != nil { @@ -1029,6 +1049,21 @@ func withDerivedProvides(published []byte, derived []string) ([]byte, *spec.Desc // A kit that declared none still carries what its filesystem // holds, so the field is created rather than skipped. provides = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + // No key on the root is not the same as no entries: a merge + // key can hand the root a provides list from another mapping, + // and a key created here would override that list rather than + // extend it. So the new list starts from what the descriptor + // decodes to, which is the author's entries wherever they were + // written — and leaves the merged mapping alone, since + // something else may share it. + inherited, err := spec.Decode(published) + if err != nil { + return nil, nil, fmt.Errorf("published descriptor no longer decodes: %w", err) + } + for _, entry := range inherited.Provides { + provides.Content = append(provides.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: entry}) + } root.Content = append(root.Content, &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "provides"}, provides) case provides.Kind == yaml.ScalarNode && provides.Tag == "!!null": diff --git a/cmd/frontend/packages.go b/cmd/frontend/packages.go index 2896379b..f0562419 100644 --- a/cmd/frontend/packages.go +++ b/cmd/frontend/packages.go @@ -4,7 +4,9 @@ import ( "bytes" "context" "fmt" + "os" + "github.com/containerd/platforms" gwclient "github.com/moby/buildkit/frontend/gateway/client" "github.com/docker/sandbox-kit-spec/v3/spec" @@ -53,3 +55,52 @@ func derivedProvides(ctx context.Context, refs []gwclient.Reference) ([]string, } return out, nil } + +// statedProvides resolves the kit's providesFrom declaration against +// every platform's filesystem (§9.7): each path to the installed package +// that owns it, and each name to that package's version. +func statedProvides(ctx context.Context, from map[string]string, builds []platformBuild) ([]string, error) { + if len(from) == 0 { + return nil, nil + } + files := make([]spec.PlatformFiles, 0, len(builds)) + for _, b := range builds { + p := spec.PlatformFiles{Platform: platforms.FormatAll(platformOrDefault(b.platform))} + if b.ref != nil { + p.Files = imageFiles{ref: b.ref} + } + files = append(files, p) + } + return spec.ProvidesFromPaths(ctx, from, files) +} + +// imageFiles reads a solved filesystem for the spec package, which +// knows what to look for and not what a BuildKit reference is. +type imageFiles struct{ ref gwclient.Reference } + +func (f imageFiles) ReadFile(ctx context.Context, name string) ([]byte, bool, error) { + return readImageFile(ctx, f.ref, name) +} + +// ReadDir lists a directory through the gateway, which walks it without +// following its entries: a symbolic link arrives with its target, which +// is the one thing StatFile cannot say — it resolves the whole path and +// answers for wherever that ends. +func (f imageFiles) ReadDir(ctx context.Context, name string) ([]spec.ImageEntry, bool, error) { + stats, err := f.ref.ReadDir(ctx, gwclient.ReadDirRequest{Path: name}) + if err != nil { + if isNotExist(err) { + return nil, false, nil + } + return nil, false, err + } + entries := make([]spec.ImageEntry, 0, len(stats)) + for _, st := range stats { + entries = append(entries, spec.ImageEntry{ + Name: st.Path, + Dir: os.FileMode(st.Mode).IsDir(), + Link: st.Linkname, + }) + } + return entries, true, nil +} diff --git a/cmd/frontend/provides_from_test.go b/cmd/frontend/provides_from_test.go new file mode 100644 index 00000000..4dcf0cf9 --- /dev/null +++ b/cmd/frontend/provides_from_test.go @@ -0,0 +1,118 @@ +package main + +import ( + "context" + "io/fs" + "os" + "path" + "strings" + "testing" + + gwclient "github.com/moby/buildkit/frontend/gateway/client" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/require" + fstypes "github.com/tonistiigi/fsutil/types" + + "github.com/docker/sandbox-kit-spec/v3/spec" +) + +// packagedReference is a solved filesystem as the gateway presents one: +// files read by name, and a directory listed with each entry's own +// stat, so a symbolic link arrives as a link with its target. +type packagedReference struct { + fakeReference + files map[string]string + links map[string]string +} + +func (r packagedReference) ReadFile(_ context.Context, req gwclient.ReadRequest) ([]byte, error) { + body, ok := r.files[req.Filename] + if !ok { + return nil, fs.ErrNotExist + } + return []byte(body), nil +} + +func (r packagedReference) ReadDir(_ context.Context, req gwclient.ReadDirRequest) ([]*fstypes.Stat, error) { + seen := map[string]*fstypes.Stat{} + for name := range r.files { + if path.Dir(name) == req.Path { + seen[path.Base(name)] = &fstypes.Stat{Path: path.Base(name), Mode: 0o755} + } else if rest, under := strings.CutPrefix(name, strings.TrimSuffix(req.Path, "/")+"/"); under { + top, _, _ := strings.Cut(rest, "/") + seen[top] = &fstypes.Stat{Path: top, Mode: uint32(os.ModeDir | 0o755)} + } + } + for name, target := range r.links { + if path.Dir(name) == req.Path { + seen[path.Base(name)] = &fstypes.Stat{Path: path.Base(name), Mode: uint32(os.ModeSymlink | 0o777), Linkname: target} + } + } + if len(seen) == 0 { + return nil, fs.ErrNotExist + } + out := make([]*fstypes.Stat, 0, len(seen)) + for _, st := range seen { + out = append(out, st) + } + return out, nil +} + +func debianReference(sshVersion string) packagedReference { + return packagedReference{ + files: map[string]string{ + spec.DpkgStatusPath: "Package: openssh-client\nStatus: install ok installed\nVersion: " + sshVersion + "\n\n" + + "Package: python-3.14-minimal\nStatus: install ok installed\nVersion: 3.14.10-6\n", + spec.DpkgInfoDir + "/openssh-client.list": "/usr/bin/ssh\n", + spec.DpkgInfoDir + "/python-3.14-minimal.list": "/usr/bin/python3.14\n/usr/bin/python3\n", + "/usr/bin/ssh": "elf", + "/usr/bin/python3.14": "elf", + }, + links: map[string]string{ + "/bin": "usr/bin", + "/usr/bin/python3": "python3.14", + }, + } +} + +// The spec package decides what a path resolves to; this is the half +// that only the frontend can get wrong — reading a directory through the +// gateway so that a link is seen as one, and naming each platform. +func TestStatedProvidesReadEveryPlatformThroughTheGateway(t *testing.T) { + from := map[string]string{"python": "/usr/bin/python3", "ssh": "/bin/ssh"} + arm := &ocispecs.Platform{OS: "linux", Architecture: "arm64"} + amd := &ocispecs.Platform{OS: "linux", Architecture: "amd64"} + + got, err := statedProvides(t.Context(), from, []platformBuild{ + {platform: arm, ref: debianReference("1:10.0p1-7+deb13u4")}, + {platform: amd, ref: debianReference("1:10.0p1-7+deb13u4")}, + }) + require.NoError(t, err) + require.Equal(t, []string{"python@3.14.10", "ssh@10.0"}, got) + + _, err = statedProvides(t.Context(), from, []platformBuild{ + {platform: arm, ref: debianReference("1:10.0p1-7+deb13u4")}, + {platform: amd, ref: debianReference("1:10.1p1-1")}, + }) + require.ErrorContains(t, err, "providesFrom.ssh: /bin/ssh is at 10.0 on linux/arm64 and 10.1 on linux/amd64") + + // Nothing asked for, nothing read: a kit without the field never + // pays for the file lists. + got, err = statedProvides(t.Context(), nil, []platformBuild{{platform: arm}}) + require.NoError(t, err) + require.Empty(t, got) +} + +// A directory that is not there is an absence, not a failure: the spec +// package turns it into the diagnostic that names the missing database. +func TestImageFilesReportAMissingDirectoryAsAbsent(t *testing.T) { + files := imageFiles{ref: packagedReference{}} + entries, present, err := files.ReadDir(context.Background(), spec.DpkgInfoDir) + require.NoError(t, err) + require.False(t, present) + require.Empty(t, entries) + + _, present, err = files.ReadFile(context.Background(), spec.DpkgStatusPath) + require.NoError(t, err) + require.False(t, present) +} diff --git a/cmd/frontend/rewrite_test.go b/cmd/frontend/rewrite_test.go index cdffc396..aa45832a 100644 --- a/cmd/frontend/rewrite_test.go +++ b/cmd/frontend/rewrite_test.go @@ -172,3 +172,40 @@ capabilities: require.Equal(t, "./always.md", d.Capabilities[0].Config["contentFile"]) require.Equal(t, "/staged/maybe.md", d.Capabilities[1].Group.Capabilities[0].Config["contentFile"]) } + +// A merge key can supply provides, and then the root mapping has no +// provides key of its own. Creating one for the derived entries would +// override the inherited list, so the author's entries are carried into +// the list that is created. +func TestDerivedProvidesKeepWhatAMergeKeyProvides(t *testing.T) { + published := []byte(`schemaVersion: "3" +kind: workload +version: "1.0.0" +<<: {provides: ["agent@1.0.0"], displayName: Merged} +`) + + out, pd, err := withDerivedProvides(published, []string{"python@3.14.10", "deb/bash@5.2.37"}) + require.NoError(t, err) + require.Equal(t, []string{"agent@1.0.0", "python@3.14.10", "deb/bash@5.2.37"}, pd.Provides) + require.Equal(t, "Merged", pd.DisplayName, "the rest of what was merged is untouched") + + fromBytes, err := spec.Decode(out) + require.NoError(t, err) + require.Equal(t, pd.Provides, fromBytes.Provides) + + // An anchored mapping merged by alias is not edited: whatever else + // refers to it still sees the list it had. + shared := []byte(`schemaVersion: "3" +kind: workload +version: "1.0.0" +capabilities: + - type: com.example/thing@1 + config: &shared + provides: ["agent@1.0.0"] +<<: *shared +`) + _, pd, err = withDerivedProvides(shared, []string{"deb/bash@5.2.37"}) + require.NoError(t, err) + require.Equal(t, []string{"agent@1.0.0", "deb/bash@5.2.37"}, pd.Provides) + require.Equal(t, map[string]any{"provides": []any{"agent@1.0.0"}}, pd.Capabilities[0].Config) +} diff --git a/docs/spec/SPEC-v3.md b/docs/spec/SPEC-v3.md index b7b47144..f6fdd958 100644 --- a/docs/spec/SPEC-v3.md +++ b/docs/spec/SPEC-v3.md @@ -252,6 +252,7 @@ iconUrl: https://… # optional. Image shown in catalogs and pickers. version: "1.4.2" # optional. Fallback version; see below. licenses: [Apache-2.0] # optional. SPDX identifiers. provides: [] # optional. §5. +providesFrom: {} # optional. §9.7. Authored form only. requires: [] # optional. §5. integrates: [] # optional. §5. conflicts: [] # optional. §5. @@ -273,6 +274,7 @@ kits: [] # optional. §3.4. Exclusive with build, dockerfile. | `iconUrl` | string | optional. Absolute **https** URL to an image representing the Kit in catalogs and pickers. Display metadata, self-asserted like `displayName`. The scheme is constrained because a consumer fetches and renders this rather than merely displaying it: `javascript:`, `data:`, and `file:` are refused, and plain `http:` is refused so a rendered icon is not attacker-swappable in transit. Referenced rather than staged into a layer, because the surfaces that want an icon hold the manifest and not the layers — an icon a consumer had to pull the Kit to see would arrive after the moment it existed to inform. No `org.opencontainers.image.*` annotation is emitted for it: OCI defines no icon key, and the descriptor annotation already carries the field. | | `version` | string | optional. Version-shaped ([§5.2](#52-versions)), or a build-phase arg reference expanded at publish. See below. | | `licenses` | list\ | optional. SHOULD be SPDX identifiers. | +| `providesFrom` | map\ | optional. Capability name to the absolute path of a file in the Kit's own filesystem; publishing states each name at the version of the package that owns the file. Workload only, and absent from the published descriptor. See [§9.7](#97-provides-stated-from-a-path). | | `build` | string | optional. Literal Dockerfile text. Exclusive with `dockerfile` and `kits`. | | `dockerfile` | string | optional. Relative path inside the descriptor's directory; MUST NOT escape it (that directory is the build context's root). Exclusive with `build` and `kits`. | | `kits` | list\ | optional. The Kits this Kit's content is merged from ([§3.4](#34-kit-set)). Exclusive with `build` and `dockerfile`. | @@ -315,6 +317,10 @@ conflicts: ["podman"] # must be absent from the resolv exception is not an author's claim at all — publishing derives an entry per installed distribution package, in namespaces reserved for it ([§9.6](#96-derived-provides)). +- A name whose version the author does not know — a tool the base image + ships — is stated with `providesFrom` instead, which names the file + and leaves the version to publishing + ([§9.7](#97-provides-stated-from-a-path)). - Arg references are permitted in `provides` (and `version`) only — they are expanded at publish ([§9.1](#91-expansion)). `requires`, `integrates`, and `conflicts` MUST stay literal: they are what the resolver judges, and a @@ -1288,6 +1294,120 @@ Derived entries do not vote on archive never agrees on one version, and counting it would leave every Kit that named no `version:` with no version annotation at all. +### 9.7 Provides stated from a path + +Derived entries are an inventory, and they carry the distribution's +names: a mixin that needs Python on a Debian workload requires +`deb/python-3.14`, a name that moves when the base image moves to 3.15 +and that no Alpine workload will ever provide. A name that stays put — +`python` — is an author's to choose, but its version is not an author's +to know, and [§9.2](#92-versioned-provides) will not publish it without +one. Written as `provides: [python]` it takes the `version:` fallback, +the statement [§9.6](#96-derived-provides) exists to stop. + +`providesFrom` states the name and leaves the version to the content: + +```yaml +kind: workload +provides: ["claude@${{ kit.args.version }}"] # the author's own +providesFrom: # the base image's + python: /usr/bin/python3 + ssh: /usr/bin/ssh + docker-engine: /usr/bin/dockerd +``` + +Each key is a capability name and each value the absolute path of a +file in the Kit's own filesystem. Publishing finds the installed package +that owns the file and states `name@version` in `provides`, with that +package's version as [§9.6](#96-derived-provides) renders it. The Kit +above publishes `python@3.14.10` beside `deb/python-3.14@3.14.10`, and +keeps publishing `python` when the package underneath is renamed. + +The path names the command rather than the package because the command +is what a requirer means, and because it is the name that survives: +`/usr/bin/python3` is there under every Python 3 a base image has +shipped. + +- A key **MUST** be a capability name outside the namespaces + [§5.1](#51-names-and-namespaces) reserves, and **MUST NOT** name a + capability the Kit's `provides` also states, in either spelling. A + capability has one version, and two entries for it leave a resolver to + pick. A value is a literal, absolute, canonical path: arg references + are not permitted, since the path names a file in the image and not an + input. +- `providesFrom` applies to a `kind: workload` Kit that builds its own + content, and **MUST** be refused on a mixin or a set. A mixin's layers + are a delta, where a package database is not an inventory + ([§9.6](#96-derived-provides)); a set builds nothing, and the workload + it lists already stated what it carries. The field is refused there + for being written, empty or not. +- Publishing **MUST** fail unless exactly one installed package lists + the path: it fails when the path does not exist, is a directory, is + listed by more than one package, or is listed by none. The lists are + the ones kept beside the databases [§9.6](#96-derived-provides) reads: + `/var/lib/dpkg/info/.list` for dpkg, the `F:` and `R:` + records of `/lib/apk/db/installed` for apk. A list can outlive its + package — a removed package keeps the configuration files it left, + and so does one removed architecture of a package another + architecture still has installed — and such a list owns nothing. + Nothing is stated in place of an entry that could not be resolved: + the author asked for it by name, so its absence is a build failure + the author sees rather than a capability a consumer finds missing. A + file a recipe placed has no package to take a version from, and is + stated in `provides` under a build-phase arg as before. +- Publishing **MUST NOT** follow a symbolic link that a package lists. + A link no package lists is followed to its target, and the target + judged the same way. A package that ships `python3` as a link is the + answer for that link, while a link nothing owns was made by a recipe + or by `update-alternatives`, and what it leads to is the thing + installed. A directory on the way to the file that is a link is + followed too, so `/usr/lib/jvm/default/bin/java` finds the package + that installed the file where it really is. Paths are resolved the + way the system resolves them, a component at a time, so `..` in a + link's target is the parent of the directory reached and not of the + one written, and a path the system could not open — through a + component that is missing, or a file where a directory has to be — + names no file whatever follows. The owner of the file where it really + is comes first; the path as written is consulted only when nothing + owns that, since a list can hold a spelling from before a directory + became a link. What a package recorded is compared as recorded, + through top-level directories that are themselves links, each + followed to wherever it finally leads: `/bin/sh` and `/usr/bin/sh` + name one file on a merged-`/usr` system whichever of them the package + recorded. +- The version **MUST** be the owner's version under + [§9.6](#96-derived-provides)'s rule, and publishing **MUST** fail + when that rule names none — a prerelease, or one package recorded at + two versions. It is never taken from the `version:` fallback. +- Every platform the Kit publishes **MUST** resolve every path, to one + version, or publishing fails. Derived entries are dropped where + platforms disagree because nobody asked for them one by one; these + were asked for. +- The published descriptor **MUST NOT** carry `providesFrom`. It is an + instruction to the build, and what it produced is in `provides`. This + is also what lets the field exist under `schemaVersion: "3"`: decoding + is strict, so a runtime built before this field would refuse a + published Kit that carried it. The rule is about the key, so an empty + or null `providesFrom` is as much a violation as a full one, and + publishing removes the field wherever the document's root gets it + from — a YAML merge key included. + +A stated entry is an ordinary `provides` entry once published. It is the +workload's to provide, so [§5.3](#53-resolution-semantics-consumer-contract)'s +one-provider rule applies to it: a workload that states `gh` no longer +composes with a mixin that provides `gh`. State the tools a mixin would +require of a base, and leave a name to the Kit whose purpose is to +bring it. A stated entry also votes on +`org.opencontainers.image.version` like any authored one +([§9.3](#93-annotations)), so a Kit that states several tools names its +own `version:` if it wants that annotation. + +`providesFrom` extends the unfinished schema-3 grammar in place. An +authored descriptor using it requires a frontend that knows the field; +an older one refuses it. Published descriptors do not change shape, so +readers of published Kits are unaffected, and it adds no capability +version. + --- ## 10. The OCI layout @@ -1405,10 +1525,14 @@ The spec library enforces, beyond per-field rules stated above: compiles. - **raw form**: size budget ([§9.4](#94-size-budget)); every `${{ kit.args.* }}` reference names a declared arg. +- **providesFrom**: workload only, never beside `kits`; keys are + capability names outside the reserved namespaces that `provides` does + not also state; values are literal, absolute, canonical paths + ([§9.7](#97-provides-stated-from-a-path)). - **published form**: no build-phase references remain; provides are - literal and ([§9.2](#92-versioned-provides)) versioned; `kind` is the - derived `workload` or `mixin`, never `set`; every listed Kit carries - a digest. + literal and ([§9.2](#92-versioned-provides)) versioned; `providesFrom` + is gone; `kind` is the derived `workload` or `mixin`, never `set`; + every listed Kit carries a digest. Validation collects independent failures in a `ValidationErrors` collection, preserving warnings alongside errors. A failed config decode diff --git a/examples/ubuntu-docker/ubuntu-docker.yaml b/examples/ubuntu-docker/ubuntu-docker.yaml index 1b0f63f6..552a29a9 100644 --- a/examples/ubuntu-docker/ubuntu-docker.yaml +++ b/examples/ubuntu-docker/ubuntu-docker.yaml @@ -10,6 +10,22 @@ version: "24.04" # Package provides are derived from the image's dpkg database. The Kit's # release version is not a claim about the installed Docker version. +# +# Those entries carry Ubuntu's package names, so the engine is +# deb/docker-ce here and something else on another base. The tools a +# mixin is likely to ask a base for are also stated under names that do +# not depend on the distribution: each line names the command, and +# publishing states the name at the version of the package that installed +# it (§9.7). The field itself is not published. +providesFrom: + bash: /usr/bin/bash + curl: /usr/bin/curl + docker: /usr/bin/docker + docker-buildx: /usr/libexec/docker/cli-plugins/docker-buildx + docker-compose: /usr/libexec/docker/cli-plugins/docker-compose + docker-engine: /usr/bin/dockerd + git: /usr/bin/git + capabilities: - type: com.docker.sandbox/sbx@1 diff --git a/schema/kit.schema.json b/schema/kit.schema.json index 34247ced..1b85bfbd 100644 --- a/schema/kit.schema.json +++ b/schema/kit.schema.json @@ -61,6 +61,22 @@ }, "description": "Capabilities this kit offers, optionally versioned as name@version. Nothing an author offers is provided implicitly. Bare names normalize into the default namespace (gh IS com.docker.kit/gh, like debian is docker.io/library/debian); third parties qualify under their own namespace, which never matches the bare form. The deb/ and apk/ namespaces are reserved: publishing fills them from the package databases in a workload's content, one entry per installed package, so do not author them." }, + "providesFrom": { + "type": "object", + "propertyNames": { + "not": { + "pattern": "^(deb|apk)/" + } + }, + "additionalProperties": { + "type": "string", + "pattern": "^/.", + "not": { + "pattern": "\\$\\{\\{" + } + }, + "description": "Authored form only, workload only. Capability name to the absolute path of a file in the kit's own filesystem. Publishing finds the installed package that owns the file, states name@ in provides, and drops this field, so a published descriptor never carries it. For a tool the base image ships at a version the author does not know: python: /usr/bin/python3. A path no installed package owns, or one whose platforms disagree on the version, fails the build. Paths are literal; arg references are not allowed." + }, "requires": { "type": "array", "items": { @@ -211,6 +227,26 @@ }, "description": "com.docker.sandbox/sbx@1 is workload-only: a mixin's image config never becomes the composed image's, so the identity it would promise is not the one a host reads." }, + { + "if": { + "required": [ + "providesFrom" + ] + }, + "then": { + "properties": { + "kind": { + "const": "workload" + } + }, + "not": { + "required": [ + "kits" + ] + } + }, + "description": "providesFrom reads the filesystem the kit builds: a mixin's layers are a delta, where a package database is no inventory, and a set builds nothing." + }, { "$ref": "#/definitions/noNul" } diff --git a/skills/create-kit-v3/SKILL.md b/skills/create-kit-v3/SKILL.md index 80cd29e5..95ef02bb 100644 --- a/skills/create-kit-v3/SKILL.md +++ b/skills/create-kit-v3/SKILL.md @@ -230,6 +230,32 @@ publish: `RequireVersionedProvides` fails the build. Pin the version, or drop the provide — a kit with no `provides` publishes fine, and offering nothing matchable is honest when the kit cannot know what it installed. +**A workload states the tools its base ships with `providesFrom`.** You do +not know what version of Python the base image carries, and `deb/python-3.14` +is a name that moves at the next minor and that no Alpine base will ever +provide. Name the command and let publishing find the version: + +```yaml +kind: workload +providesFrom: + python: /usr/bin/python3 + ssh: /usr/bin/ssh +``` + +Publishing looks up the installed package that owns each file and states +`python@3.14.10`, `ssh@10.0` in `provides`; the field itself is not published +(§9.7). It reads dpkg's and apk's file lists, so it is workload-only, and it +fails the build for a path that is absent, a directory, or owned by no +package — a binary your recipe downloaded still takes the build-phase +`version` arg above. Check a path before you list it: + +```sh +docker run --rm sh -c 'grep -lx /usr/bin/python3 /var/lib/dpkg/info/*.list' +``` + +Do not state a name a mixin exists to provide. One name has one provider +(§5.3), so a workload that states `gh` stops composing with the `gh` kit. + **`requires` is a closed-set check**: a name nothing in the composition provides makes your kit refuse to compose *anywhere*, which is worse than saying nothing. It constrains the composed **runtime** set, not your builder — @@ -478,7 +504,8 @@ varies, keep the rest exact and vary only that part: - [SPEC-v3.md](https://github.com/docker/sandbox-kit-spec/blob/main/docs/spec/SPEC-v3.md) — the grammar. §3 authoring forms, §5 provides/requires, §6 args, - §7 capabilities, §9.6 derived provides. + §7 capabilities, §9.6 derived provides, §9.7 provides stated from a + path. - [capability pages](https://github.com/docker/sandbox-kit-spec/tree/main/docs/spec/capabilities/com.docker.sandbox) — normative per type. - [examples](https://github.com/docker/sandbox-kit-spec/tree/main/examples) — diff --git a/skills/migrate-kit-to-v3/FIELD-MAPPING.md b/skills/migrate-kit-to-v3/FIELD-MAPPING.md index 8615c44e..6b15d41f 100644 --- a/skills/migrate-kit-to-v3/FIELD-MAPPING.md +++ b/skills/migrate-kit-to-v3/FIELD-MAPPING.md @@ -195,6 +195,12 @@ workload on `dhi.io/sbx-templates:*` offers `deb/apt`, `deb/jq`, `deb/docker-ce`, `deb/git` and hundreds more with nobody authoring them, and a mixin may require those names. +A `deb/` name is the distribution's, so it is only as stable as the package: +`deb/python-3.14` is gone when the base moves to 3.15, and no Alpine workload +provides it. Where the workload states a tool under a name of its own with +`providesFrom` (§9.7) — `python`, `node`, `docker-engine` — require that name +instead, and keep `deb/` for what is Debian by nature, such as `deb/apt`. + **On a multi-platform workload the set is an intersection, not a union.** §9.6 states an entry only where every published platform agrees on the normalized name *and* version, so an architecture-specific package, or one at different diff --git a/spec/packages.go b/spec/packages.go index 6bc55ac1..c22b121d 100644 --- a/spec/packages.go +++ b/spec/packages.go @@ -63,30 +63,39 @@ func PackageDatabases() []PackageDatabase { func ReadDpkgStatus(r io.Reader) ([]Package, error) { var out []Package err := eachStanza(r, func(fields map[string]string) { - // The third word of dpkg's want/flag/status triple is the one - // that says whether the files are on disk. - parts := strings.Fields(fields["status"]) - if len(parts) == 0 || parts[len(parts)-1] != "installed" { + if !dpkgInstalled(fields) { return } if name := fields["package"]; name != "" { out = append(out, Package{Name: name, Version: fields["version"]}) } - }, func(line string) (string, string, bool) { - // A continuation line belongs to the field above it, and none of - // the three fields read here spans lines, so it is skipped. - if strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") { - return "", "", false - } - key, value, found := strings.Cut(line, ":") - if !found { - return "", "", false - } - return strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value), true - }) + }, dpkgField) return out, err } +// dpkgInstalled reports whether a status stanza records a package whose +// files are on disk. The third word of dpkg's want/flag/status triple is +// the one that says so. +func dpkgInstalled(fields map[string]string) bool { + parts := strings.Fields(fields["status"]) + return len(parts) > 0 && parts[len(parts)-1] == "installed" +} + +// dpkgField splits one line of a dpkg status stanza into a lowercased +// field name and its value. +func dpkgField(line string) (string, string, bool) { + // A continuation line belongs to the field above it, and none of + // the fields read here spans lines, so it is skipped. + if strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") { + return "", "", false + } + key, value, found := strings.Cut(line, ":") + if !found { + return "", "", false + } + return strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value), true +} + // ReadApkInstalled lists the packages an apk installed database records. // // apk keeps no removed-package records, so every stanza counts; the diff --git a/spec/provides_from.go b/spec/provides_from.go new file mode 100644 index 00000000..89d14134 --- /dev/null +++ b/spec/provides_from.go @@ -0,0 +1,1005 @@ +package spec + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "maps" + "path" + "slices" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +// DpkgInfoDir is where dpkg keeps one .list per installed +// package, naming every path the package put on disk. The status file +// says what is installed; this is the only place that says whose a file +// is. +const DpkgInfoDir = "/var/lib/dpkg/info" + +// maxProvidesFromLinks bounds how many symbolic links one path is +// followed through. Deeper than an alternatives chain ever is, and +// small enough that a link cycle ends in a diagnostic rather than a +// build that never does. +const maxProvidesFromLinks = 16 + +// validateProvidesFrom holds providesFrom to its grammar (§9.7). +// +// The kind rules are here rather than at resolution because they are +// knowable from the descriptor alone: a mixin's layers are a delta and +// a set builds nothing, so neither has a filesystem whose package +// database is an inventory, and saying so before a build runs is kinder +// than saying it after. +func validateProvidesFrom(d *Descriptor) error { + var errs ValidationErrors + // Present, not populated: `providesFrom: {}` on a mixin asks the + // same thing of a filesystem that cannot answer it, and the schema + // refuses the key there whatever is under it. A nil map is a field + // that was not written, or was written with nothing after it; + // ValidateRaw tells those two apart from the bytes. + if d.ProvidesFrom == nil { + return nil + } + errs.add(validateProvidesFromKind(d)) + errs.add(RequireDistinctProvidesFrom(d, d.ProvidesFrom)) + seen := map[string]string{} + for _, name := range slices.Sorted(maps.Keys(d.ProvidesFrom)) { + at := "providesFrom." + name + file := d.ProvidesFrom[name] + if err := capabilityNameError(name); err != nil { + errs.add(fieldErrorf(at, "providesFrom key %q: %v", name, err)) + continue + } + if IsDerivedProvide(name) { + namespace, _, _ := strings.Cut(name, "/") + errs.add(fieldErrorf(at, "providesFrom key %q is in the %s/ namespace, which publishing fills from the image's package database; name the capability the file stands for instead", name, namespace)) + continue + } + normalized := NormalizeCapabilityName(name) + if prev, dup := seen[normalized]; dup { + errs.add(fieldErrorf(at, "providesFrom keys %q and %q are the same capability", prev, name)) + } + seen[normalized] = name + switch { + case ContainsKitPlaceholder(file): + errs.add(fieldErrorf(at, "providesFrom.%s: arg references are not allowed in providesFrom; the path names a file in the image, not an input", name)) + case !canonicalAbsPath(file) || file == "/": + errs.add(fieldErrorf(at, "providesFrom.%s: %q is not an absolute, canonical path to a file", name, file)) + } + } + return errs.err() +} + +// validateProvidesFromKind holds the field to the kinds that have a +// filesystem to read: a workload that builds its own content. +func validateProvidesFromKind(d *Descriptor) error { + switch { + case len(d.Kits) > 0: + return fieldErrorf("providesFrom", "providesFrom reads the filesystem this kit builds, and a set builds none: its content is the kits it lists, and the workload among them already states what it carries") + case d.Kind != KindWorkload: + return fieldErrorf("providesFrom", "providesFrom is for kind: %s only: a %s's layers are a delta, where a package database is whatever the recipe happened to rewrite rather than an inventory", KindWorkload, d.Kind) + } + return nil +} + +// validateRawProvidesFrom applies the kind rule to a field the struct +// cannot show: a bare `providesFrom:` decodes to a nil map, exactly +// like no field at all, and only the bytes say it was written. With +// any other spelling Validate has already judged it. +func validateRawProvidesFrom(raw []byte, d *Descriptor) error { + if d.ProvidesFrom != nil || !carriesProvidesFrom(raw) { + return nil + } + return validateProvidesFromKind(d) +} + +// RequireDistinctProvidesFrom refuses a providesFrom key that names a +// capability the descriptor's provides already states: a capability has +// one version, and two entries for it would leave a resolver to pick. +// +// Separate from the grammar check, and exported, because the frontend +// asks it a second time on the build-expanded descriptor: an authored +// `${{ kit.args.tool }}@1.0.0` holds no name to compare until the +// build-phase value is in it. +func RequireDistinctProvidesFrom(d *Descriptor, from map[string]string) error { + var errs ValidationErrors + authored := map[string]int{} + for i, s := range d.Provides { + if p, err := ParseProvide(s); err == nil { + authored[p.Name] = i + } + } + for _, name := range slices.Sorted(maps.Keys(from)) { + if capabilityNameError(name) != nil { + continue + } + if i, dup := authored[NormalizeCapabilityName(name)]; dup { + errs.add(fieldErrorf("providesFrom."+name, "providesFrom key %q is already stated by provides[%d]; a capability has one version, so state it in one place", name, i)) + } + } + return errs.err() +} + +// providesFromField is the descriptor key, as an author writes it. +const providesFromField = "providesFrom" + +// maxDetachedNodes bounds how many nodes the removal may copy. A merge +// source reached through an alias is copied before it is edited, and +// aliases that fan out — each level merging the one below it twice — +// make that copy double per level. The decoder refuses such a document +// for excessive aliasing before any build gets this far; this is the +// same refusal for a caller that did not decode first, in place of +// memory that runs out. +const maxDetachedNodes = 1 << 20 + +// WithoutProvidesFrom returns a descriptor with its providesFrom field +// removed, for the published form (§9.7). +// +// The node tree is edited rather than re-encoded from the struct, for +// the reason every other publish-time rewrite here is: the copy a +// reader finds staged in the image keeps the author's own comments and +// ordering. A descriptor without the field comes back byte for byte. +// +// The field is removed wherever the root mapping gets it from, not only +// where it is written at the top level. A YAML merge key hands the root +// every key of another mapping, so `<<: {providesFrom: …}` decodes to +// the same descriptor as the plain spelling and has to publish the same +// way — at any depth: merges nest, and the decoder follows them as far +// as they go, so this does too. And the removal takes nothing else with +// it: an alias elsewhere in the document that points into what is +// removed is given its own copy first, since an anchor that disappears +// leaves every alias to it naming nothing and the document no longer +// parsing. +func WithoutProvidesFrom(raw []byte) ([]byte, error) { + var doc yaml.Node + if err := yaml.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("reparse descriptor: %w", err) + } + root := documentRoot(&doc) + edit := newFieldRemoval(providesFromField) + if root == nil || !edit.carries(root) { + return raw, nil + } + + // Three passes, in this order. What is about to change is found + // first and nothing is touched, so that the second pass can copy an + // aliased node as the author wrote it; only then is anything removed. + edit.plan(root) + edit.detachAliases(&doc) + edit.strip(root) + if edit.copied > maxDetachedNodes { + return nil, fmt.Errorf("%s is merged in through too many aliases to be removed; write it as a plain top-level field", providesFromField) + } + + out, err := yaml.Marshal(&doc) + if err != nil { + return nil, fmt.Errorf("re-encode descriptor without %s: %w", providesFromField, err) + } + // Parsed once more before it is handed back: this is the document + // that gets staged and signed, and a rewrite that left it unreadable + // or left the field in should say so here, with its cause, rather + // than as a failure somewhere later in the build. + var check yaml.Node + if err := yaml.Unmarshal(out, &check); err != nil { + return nil, fmt.Errorf("descriptor no longer parses with %s removed: %w", providesFromField, err) + } + if newFieldRemoval(providesFromField).carries(documentRoot(&check)) { + return nil, fmt.Errorf("%s could not be removed from the descriptor; write it as a plain top-level field", providesFromField) + } + return out, nil +} + +// carriesProvidesFrom reports whether a raw descriptor has the field at +// all — written out or inherited through a merge key, with entries or +// without. Presence is the question for the published form, where a +// reader built before the field refuses the key whatever is under it, +// and the decoded struct cannot answer it: an empty map and a null +// decode exactly like a field that was never there. +func carriesProvidesFrom(raw []byte) bool { + var doc yaml.Node + if err := yaml.Unmarshal(raw, &doc); err != nil { + // Unparseable bytes are another rule's to report. + return false + } + return newFieldRemoval(providesFromField).carries(documentRoot(&doc)) +} + +// documentRoot returns the mapping a descriptor document consists of, +// or nil when it is not one. +func documentRoot(doc *yaml.Node) *yaml.Node { + root := doc + if root.Kind == yaml.DocumentNode && len(root.Content) > 0 { + root = root.Content[0] + } + root = aliasTarget(root) + if root == nil || root.Kind != yaml.MappingNode { + return nil + } + return root +} + +// aliasTarget follows an alias to the node it names. One step is all +// there is: an alias cannot carry an anchor, so nothing an alias names +// is itself an alias. +func aliasTarget(n *yaml.Node) *yaml.Node { + if n != nil && n.Kind == yaml.AliasNode { + return n.Alias + } + return n +} + +// isMergeKey reports whether a mapping key is YAML's merge key. Only +// the plain scalar is: a quoted "<<" is an ordinary string key, which +// is the distinction the decoder draws too. +func isMergeKey(n *yaml.Node) bool { + return n.Kind == yaml.ScalarNode && n.Value == "<<" && n.Tag == "!!merge" +} + +// mergeSources returns the mappings a merge value contributes: the +// value itself, or each element when it is a sequence of them. +func mergeSources(value *yaml.Node) []*yaml.Node { + if value.Kind == yaml.SequenceNode { + return value.Content + } + return []*yaml.Node{value} +} + +// fieldRemoval takes one key out of a mapping and everything the +// mapping merges in. +// +// Nothing here stops at a depth. Merges nest as far as an author nests +// them and the decoder follows all of it, so a walk that gave up early +// would leave a field in place that every reader still sees. The one +// thing that has to be guarded against is a walk that does not end: an +// anchor may contain an alias to itself, and that is caught by noticing +// the same node come round again, not by counting. +type fieldRemoval struct { + field string + // removed holds every node that leaves the document: the field's + // key and value, and all that is under the value. + removed map[*yaml.Node]bool + // edited holds the anchored nodes that are changed in place: a + // mapping the field is taken out of, or a merge key's sequence that + // loses or swaps an element. An alias to one of these would see it + // change underneath. + edited map[*yaml.Node]bool + // carried remembers, for each mapping asked about, whether it has + // the field. A mapping merged in many places is judged once, and + // one still being judged when it comes round again is a cycle. + carried map[*yaml.Node]carryState + // expanding holds the anchored mappings whose copies are being + // stripped right now, and copying the removed nodes being copied. + // Meeting one again means the document refers to itself. + expanding map[*yaml.Node]bool + copying map[*yaml.Node]bool + // copied counts the nodes detachedCopy has made, against + // maxDetachedNodes. + copied int +} + +type carryState uint8 + +const ( + carryUnknown carryState = iota + carryJudging + carryYes + carryNo +) + +func newFieldRemoval(field string) *fieldRemoval { + return &fieldRemoval{ + field: field, + removed: map[*yaml.Node]bool{}, + edited: map[*yaml.Node]bool{}, + carried: map[*yaml.Node]carryState{}, + expanding: map[*yaml.Node]bool{}, + copying: map[*yaml.Node]bool{}, + } +} + +// carries reports whether a mapping has the field, directly or through +// any mapping it merges in, however far down. +func (e *fieldRemoval) carries(m *yaml.Node) bool { + m = aliasTarget(m) + if m == nil || m.Kind != yaml.MappingNode { + return false + } + switch e.carried[m] { + case carryYes: + return true + case carryNo, carryJudging: + // Still being judged means this mapping merges itself in. It + // contributes nothing it has not already been asked for. + return false + } + e.carried[m] = carryJudging + found := false + for i := 0; i+1 < len(m.Content) && !found; i += 2 { + k, v := m.Content[i], m.Content[i+1] + if isMergeKey(k) { + for _, source := range mergeSources(v) { + if e.carries(source) { + found = true + break + } + } + continue + } + found = k.Value == e.field + } + if found { + e.carried[m] = carryYes + } else { + e.carried[m] = carryNo + } + return found +} + +// plan records what strip will remove from m and which shared nodes it +// will edit, without changing anything. +// +// A merge source reached through an alias is not recorded for itself: +// strip works on a copy of it, so the anchored original stays as it was +// for whatever else refers to it. Only mappings written in place are +// descended into, and those form a tree, so this ends. +func (e *fieldRemoval) plan(m *yaml.Node) { + for i := 0; i+1 < len(m.Content); i += 2 { + k, v := m.Content[i], m.Content[i+1] + if isMergeKey(k) { + for _, source := range mergeSources(v) { + if !e.carries(source) { + continue + } + // The list of sources is itself a node an alias can + // name. Whatever happens to this source — edited where + // it stands, replaced by a copy, or dropped — happens + // to that list. + if v.Kind == yaml.SequenceNode && v.Anchor != "" { + e.edited[v] = true + } + if source.Kind != yaml.MappingNode { + continue + } + if source.Anchor != "" { + e.edited[source] = true + } + e.plan(source) + } + continue + } + if k.Value == e.field { + e.markRemoved(k) + e.markRemoved(v) + } + } +} + +func (e *fieldRemoval) markRemoved(n *yaml.Node) { + if n == nil || e.removed[n] { + return + } + e.removed[n] = true + for _, child := range n.Content { + e.markRemoved(child) + } +} + +// detachAliases gives every alias that survives the removal its own +// copy of a node the removal is about to take away or change. +func (e *fieldRemoval) detachAliases(n *yaml.Node) { + if n == nil || e.removed[n] { + return + } + if n.Kind == yaml.AliasNode { + if e.removed[n.Alias] || e.edited[n.Alias] { + // Onto the alias node itself, so whatever holds it now + // holds the copy. + *n = *e.detachedCopy(n.Alias) + } + return + } + for _, child := range n.Content { + e.detachAliases(child) + } +} + +// detachedCopy copies a subtree so that it stands on its own: no anchor +// is copied, since the original keeps its name or is going away, and an +// alias to a node that is going away is replaced by a copy of that +// node. An alias to anything that stays is kept as an alias. +func (e *fieldRemoval) detachedCopy(n *yaml.Node) *yaml.Node { + if n.Kind == yaml.AliasNode && n.Alias != nil && e.removed[n.Alias] && !e.copying[n.Alias] { + // Not while that node is itself being copied: a removed anchor + // that aliases itself has no finite copy. The alias is left, + // and the caller's last parse reports the document for it. + return e.detachedCopy(n.Alias) + } + e.copying[n] = true + defer delete(e.copying, n) + c := *n + c.Anchor = "" + c.Content = nil + if e.copied++; e.copied > maxDetachedNodes { + // Over budget: stop descending. What comes back is incomplete, + // and WithoutProvidesFrom refuses the document rather than + // return it. + return &c + } + for _, child := range n.Content { + c.Content = append(c.Content, e.detachedCopy(child)) + } + return &c +} + +// strip removes the field from m and from every mapping m merges in. +func (e *fieldRemoval) strip(m *yaml.Node) { + kept := make([]*yaml.Node, 0, len(m.Content)) + for i := 0; i+1 < len(m.Content); i += 2 { + k, v := m.Content[i], m.Content[i+1] + switch { + case isMergeKey(k) && v.Kind == yaml.SequenceNode: + sources := make([]*yaml.Node, 0, len(v.Content)) + for _, source := range v.Content { + if source = e.stripSource(source); source != nil { + sources = append(sources, source) + } + } + v.Content = sources + // A merge of nothing is noise the author did not write. + if len(sources) == 0 && v.Anchor == "" { + continue + } + case isMergeKey(k): + if v = e.stripSource(v); v == nil { + continue + } + case k.Value == e.field: + continue + } + kept = append(kept, k, v) + } + m.Content = kept +} + +// stripSource returns one merge source with the field taken out, or nil +// when nothing is left of it to merge. +func (e *fieldRemoval) stripSource(source *yaml.Node) *yaml.Node { + if !e.carries(source) { + return source + } + if source.Kind == yaml.AliasNode { + // The anchored mapping belongs to wherever it was defined, and + // may be merged or aliased there for reasons of its own, so + // the root merges a copy without the field instead. + target := source.Alias + if target == nil || e.expanding[target] { + // A mapping that merges itself in. Its copy is already + // being stripped further up, and expanding it again here + // would never finish. + return source + } + if e.copied > maxDetachedNodes { + return source + } + e.expanding[target] = true + defer delete(e.expanding, target) + source = e.detachedCopy(target) + } + if source.Kind != yaml.MappingNode { + return source + } + e.strip(source) + if len(source.Content) == 0 && source.Anchor == "" { + return nil + } + return source +} + +// ImageEntry is one name in a directory of a kit's filesystem. Link is +// the target of a symbolic link exactly as stored, and empty for +// anything else. +type ImageEntry struct { + Name string + Dir bool + Link string +} + +// ImageFiles reads one platform's filesystem. It is the whole of what +// [ProvidesFromPaths] needs from a build, so a frontend adapts whatever +// holds its result — a BuildKit reference, a directory — without this +// package learning what that is. +// +// present is false, with a nil error, for a path that does not exist. +// ReadDir lists a directory's own entries without descending and +// without following a link in the last component of an entry, so a +// link is reported as a link. +type ImageFiles interface { + ReadFile(ctx context.Context, name string) (body []byte, present bool, err error) + ReadDir(ctx context.Context, name string) (entries []ImageEntry, present bool, err error) +} + +// PlatformFiles is one platform's filesystem beside the name +// diagnostics call it by. +type PlatformFiles struct { + Platform string + Files ImageFiles +} + +// ProvidesFromPaths resolves a providesFrom declaration against every +// platform a kit publishes and renders the provides entries it states, +// sorted (§9.7). +// +// Each path has to resolve on every platform, to one version. Derived +// entries are dropped where platforms disagree, because nobody asked +// for them one by one; these were asked for by name, so an entry that +// cannot be stated is a build failure the author can see rather than a +// capability a consumer discovers missing. +func ProvidesFromPaths(ctx context.Context, from map[string]string, platforms []PlatformFiles) ([]string, error) { + if len(from) == 0 { + return nil, nil + } + if len(platforms) == 0 { + return nil, fmt.Errorf("providesFrom: no filesystem to read; the kit built no content") + } + names := slices.Sorted(maps.Keys(from)) + versions := map[string]string{} + for i, p := range platforms { + if p.Files == nil { + return nil, fmt.Errorf("providesFrom: no filesystem to read%s; the kit built no content", onPlatform(p.Platform)) + } + owners, err := readFileOwners(ctx, p.Files) + if err != nil { + return nil, fmt.Errorf("providesFrom%s: %w", onPlatform(p.Platform), err) + } + for _, name := range names { + version, err := owners.version(ctx, from[name]) + if err != nil { + return nil, fmt.Errorf("providesFrom.%s%s: %w", name, onPlatform(p.Platform), err) + } + if i > 0 && versions[name] != version { + return nil, fmt.Errorf("providesFrom.%s: %s is at %s%s and %s%s; one descriptor serves every platform, so it can state only a version they share", + name, from[name], versions[name], onPlatform(platforms[0].Platform), version, onPlatform(p.Platform)) + } + versions[name] = version + } + } + out := make([]string, 0, len(names)) + for _, name := range names { + out = append(out, name+"@"+versions[name]) + } + sort.Strings(out) + return out, nil +} + +func onPlatform(platform string) string { + if platform == "" { + return "" + } + return " on " + platform +} + +// fileOwners is one filesystem's record of which installed package put +// each path on disk. +type fileOwners struct { + files ImageFiles + // owners maps a path to the packages that list it, each named + // /. More than one is ordinary for a directory + // and a fact worth refusing for a file. + owners map[string]map[string]bool + // versions maps a package to the one version this model can name + // for it, or "" when it can name none. + versions map[string]string + // aliases maps a top-level directory that is a symbolic link to + // where it leads: /bin to /usr/bin on a merged-/usr system. + aliases map[string]string + dirs map[string][]ImageEntry +} + +// readFileOwners reads every package database the filesystem carries. +func readFileOwners(ctx context.Context, files ImageFiles) (*fileOwners, error) { + o := &fileOwners{ + files: files, + owners: map[string]map[string]bool{}, + versions: map[string]string{}, + aliases: map[string]string{}, + dirs: map[string][]ImageEntry{}, + } + // The top-level links first, because every path read below is + // compared through them. A merged-/usr system reaches one file by + // two spellings — dpkg may record /bin/sh for what PATH finds at + // /usr/bin/sh — and a lookup that knew only one of them would call + // an installed command unowned. + root, _, err := files.ReadDir(ctx, "/") + if err != nil { + return nil, fmt.Errorf("read /: %w", err) + } + o.dirs["/"] = root + for _, e := range root { + if e.Link == "" { + continue + } + // To the directory the link really leads to, however many + // links that takes, so one substitution is the whole answer. A + // link that leads nowhere is no alias for anything. + target, reached, err := o.realDirectory(ctx, []string{e.Name}) + if err != nil { + return nil, err + } + if reached { + o.aliases["/"+e.Name] = target + } + } + + found := false + for _, db := range PackageDatabases() { + body, present, err := files.ReadFile(ctx, db.Path) + if err != nil { + return nil, fmt.Errorf("read %s: %w", db.Path, err) + } + if !present { + continue + } + found = true + packages, err := db.Read(bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("%s: %w", db.Path, err) + } + for _, p := range packages { + key := db.Namespace + "/" + p.Name + version := PackageVersion(p.Version) + // A multiarch database names one package once per + // architecture. Two records at two versions are a package + // this model cannot put one version on. + if known, ok := o.versions[key]; ok && known != version { + version = "" + } + o.versions[key] = version + } + switch db.Namespace { + case DebNamespace: + err = o.readDpkgLists(ctx, body) + case ApkNamespace: + err = o.readApkFiles(body) + } + if err != nil { + return nil, err + } + } + if !found { + return nil, fmt.Errorf("the image carries neither %s nor %s, so nothing records which package a file belongs to", DpkgStatusPath, ApkInstalledPath) + } + return o, nil +} + +// readDpkgLists reads dpkg's per-package file lists. +// +// Files rather than `dpkg -S`, because an image need not carry dpkg to +// carry its database: one assembled from packages keeps the lists +// without the tool that wrote them. +func (o *fileOwners) readDpkgLists(ctx context.Context, status []byte) error { + installed, err := dpkgInstalledArchitectures(status) + if err != nil { + return fmt.Errorf("%s: %w", DpkgStatusPath, err) + } + entries, present, err := o.files.ReadDir(ctx, DpkgInfoDir) + if err != nil { + return fmt.Errorf("read %s: %w", DpkgInfoDir, err) + } + if !present { + return fmt.Errorf("the image has %s but no %s, so nothing records which package a file belongs to", DpkgStatusPath, DpkgInfoDir) + } + for _, e := range entries { + stem, isList := strings.CutSuffix(e.Name, ".list") + if !isList || e.Dir { + continue + } + // A list outlives its package: a removed one keeps the + // conffiles it left behind. Only what the status file calls + // installed owns anything, and that is judged for the package + // the list is actually named for. A co-installable package has + // one list per architecture, :.list, and one of + // them can be removed while another stays: the list of the + // removed one must not pass for the installed one's because + // they share a name. + name, arch, qualified := strings.Cut(stem, ":") + architectures, known := installed[name] + if !known || (qualified && !architectures[arch]) { + continue + } + // The architecture is part of which package this is, not of + // the name a capability is stated under. + key := DebNamespace + "/" + name + list := path.Join(DpkgInfoDir, e.Name) + body, present, err := o.files.ReadFile(ctx, list) + if err != nil { + return fmt.Errorf("read %s: %w", list, err) + } + if !present { + continue + } + if err := eachLine(bytes.NewReader(body), func(line string) { + o.own(line, key) + }); err != nil { + return fmt.Errorf("%s: %w", list, err) + } + } + return nil +} + +// dpkgInstalledArchitectures maps each installed package name to the +// architectures it is installed for. [ReadDpkgStatus] folds those into +// one name, which is all a derived entry needs; choosing a file list +// needs to know which of them a list belongs to. +func dpkgInstalledArchitectures(status []byte) (map[string]map[string]bool, error) { + installed := map[string]map[string]bool{} + err := eachStanza(bytes.NewReader(status), func(fields map[string]string) { + name := fields["package"] + if name == "" || !dpkgInstalled(fields) { + return + } + if installed[name] == nil { + installed[name] = map[string]bool{} + } + installed[name][fields["architecture"]] = true + }, dpkgField) + return installed, err +} + +// readApkFiles reads the file lists apk keeps inside its installed +// database: within a package's stanza, an F: line names a directory and +// each R: line after it a file in that directory. +func (o *fileOwners) readApkFiles(body []byte) error { + key, dir := "", "" + return eachLine(bytes.NewReader(body), func(line string) { + field, value, found := strings.Cut(line, ":") + if !found || len(field) != 1 { + return + } + switch field { + case "P": + key, dir = ApkNamespace+"/"+value, "" + case "F": + dir = value + case "R": + if key != "" { + o.own(path.Join("/", dir, value), key) + } + } + }, func() { key, dir = "", "" }) +} + +// eachLine calls line for every non-blank line and blank, when given, +// for every blank one — the stanza boundary of both databases. +func eachLine(r io.Reader, line func(string), blank ...func()) error { + s := bufio.NewScanner(r) + s.Buffer(make([]byte, 0, 64*1024), maxPackageDatabaseLine) + for s.Scan() { + text := strings.TrimSuffix(s.Text(), "\r") + if strings.TrimSpace(text) == "" { + for _, f := range blank { + f() + } + continue + } + line(text) + } + if err := s.Err(); err != nil { + return fmt.Errorf("read package database: %w", err) + } + return nil +} + +func (o *fileOwners) own(file, key string) { + if !strings.HasPrefix(file, "/") { + return + } + file = o.canonical(file) + if o.owners[file] == nil { + o.owners[file] = map[string]bool{} + } + o.owners[file][key] = true +} + +// canonical rewrites a recorded path through a top-level symbolic link, +// so the spellings a merged-/usr system gives one file compare equal. +// +// One substitution is enough, because an alias already names the +// directory its link finally reaches: with /bin pointing at compat/bin +// and /compat at usr, the alias for /bin is /usr/bin. +func (o *fileOwners) canonical(file string) string { + file = path.Clean(file) + rest := strings.TrimPrefix(file, "/") + top, tail, nested := strings.Cut(rest, "/") + if target, aliased := o.aliases["/"+top]; aliased && nested { + return path.Join(target, tail) + } + return file +} + +// version names the version of the package that owns file, following +// the file through symbolic links until something owns it. +// +// The link is followed only when the path itself is unowned. A package +// that ships a link — python3 pointing at python3.14 — is the answer +// for that link; a link nothing owns was made by a recipe or by +// update-alternatives, and what it leads to is the thing installed. +// +// The owner is looked up where the file really is: with every directory +// on the way resolved, /usr/lib/jvm/default/bin/java is the file a +// package installed as /usr/lib/jvm/java-21/bin/java. That spelling +// comes first because it names the file that is actually there. A list +// can still hold the path as it was written — a record from before the +// directory became a link — and taking that one would state the version +// of a package whose file this no longer is. The written spelling is +// the fallback, for a package that ships into a linked directory and +// records the path that way. +func (o *fileOwners) version(ctx context.Context, file string) (string, error) { + current := file + for range maxProvidesFromLinks { + resolved, reached, err := o.resolveDirectories(ctx, current) + if err != nil { + return "", err + } + // A walk that did not get there names no file, and nothing is + // looked up for it. What is left of the path is not a path: a + // ".." after a component that is missing would otherwise step + // back out of the hole and land on a file that is there. + var entry ImageEntry + present := false + if reached { + if entry, present, err = o.entry(ctx, resolved); err != nil { + return "", err + } + } + if !present { + if current != file { + return "", fmt.Errorf("%s leads to %s, which does not exist in the image", file, current) + } + return "", fmt.Errorf("%s does not exist in the image", file) + } + if entry.Dir { + return "", fmt.Errorf("%s is a directory; name a file a package installs, such as the command itself", current) + } + owners := o.owners[resolved] + // Only a spelling with nothing to collapse can be compared as + // text: a link target may climb with "..", and what that means + // depends on the directories before it. + if len(owners) == 0 && current == path.Clean(current) { + owners = o.owners[o.canonical(current)] + } + if len(owners) > 0 { + return o.ownerVersion(file, owners) + } + if entry.Link == "" { + return "", fmt.Errorf("%s belongs to no installed package; a file a recipe placed has no package to take a version from, so state its version in provides", current) + } + // Relative to where the link really is, and left as written: + // resolveDirectories walks it a component at a time, which is + // the only reading under which sub/../tool means the parent of + // wherever sub leads rather than the directory sub is in. + current = entry.Link + if !path.IsAbs(current) { + current = path.Dir(resolved) + "/" + current + } + } + return "", fmt.Errorf("%s leads through more than %d symbolic links", file, maxProvidesFromLinks) +} + +// resolveDirectories returns file with every directory on the way to it +// that is a symbolic link replaced by where the link leads. The last +// component is left alone: whether the file itself is a link is the +// caller's question. +// +// file is read a component at a time and never cleaned as text, so a +// ".." is the parent of the directory actually reached. +// +// reached is false when the directories do not lead anywhere, and the +// path returned then means nothing. +// +// Only requested paths come through here. Doing the same for every path +// a package recorded would mean a directory read per directory in the +// image, so recorded paths are compared as recorded, through top-level +// links only. +func (o *fileOwners) resolveDirectories(ctx context.Context, file string) (resolved string, reached bool, err error) { + parts := strings.Split(file, "/") + // A path that ends in a slash, a "." or a ".." can only name a + // directory, so all of it is walked as one: a file with a slash + // after it is refused, as the system refuses it, and a directory + // is handed back for the caller to say that it is one. + if last := parts[len(parts)-1]; last == "" || last == "." || last == ".." { + return o.realDirectory(ctx, parts) + } + dir, reached, err := o.realDirectory(ctx, parts[:len(parts)-1]) + if err != nil || !reached { + return "", false, err + } + return path.Join(dir, parts[len(parts)-1]), true, nil +} + +// realDirectory walks components from the root and returns the +// directory they name with every symbolic link on the way followed, +// the last one included. +// +// reached is false when the walk cannot be completed the way the +// system would complete it: a component does not exist, a component is +// a file where a directory has to be, or the links lead in a circle. +// Nothing is returned for the path then. Stopping is the point — the +// components still pending are not a remainder to append, since a ".." +// among them would be taken from somewhere the walk never got to. +func (o *fileOwners) realDirectory(ctx context.Context, parts []string) (dir string, reached bool, err error) { + pending := append([]string(nil), parts...) + resolved := "/" + links := 0 + for len(pending) > 0 { + part := pending[0] + pending = pending[1:] + switch part { + case "", ".": + continue + case "..": + resolved = path.Dir(resolved) + continue + } + candidate := path.Join(resolved, part) + entry, present, err := o.entry(ctx, candidate) + if err != nil { + return "", false, err + } + switch { + case !present: + return "", false, nil + case entry.Link != "": + if links++; links > maxProvidesFromLinks { + return "", false, nil + } + if path.IsAbs(entry.Link) { + resolved = "/" + } + pending = append(strings.Split(entry.Link, "/"), pending...) + case !entry.Dir: + // A file cannot be walked through, and the system refuses + // the path whatever comes after it. + return "", false, nil + default: + resolved = candidate + } + } + return resolved, true, nil +} + +func (o *fileOwners) ownerVersion(file string, owners map[string]bool) (string, error) { + keys := slices.Sorted(maps.Keys(owners)) + if len(keys) > 1 { + return "", fmt.Errorf("%s is listed by %s; a version comes from one package, so name a file only one of them installs", file, strings.Join(keys, " and ")) + } + version := o.versions[keys[0]] + if version == "" { + return "", fmt.Errorf("%s belongs to %s, whose recorded version this model cannot name (§9.6); state the capability's version in provides", file, keys[0]) + } + return version, nil +} + +// entry reports what stands at file, read out of its directory's +// listing so that a symbolic link is seen as one. +func (o *fileOwners) entry(ctx context.Context, file string) (ImageEntry, bool, error) { + dir, base := path.Dir(file), path.Base(file) + entries, cached := o.dirs[dir] + if !cached { + var err error + entries, _, err = o.files.ReadDir(ctx, dir) + if err != nil { + return ImageEntry{}, false, fmt.Errorf("read %s: %w", dir, err) + } + o.dirs[dir] = entries + } + for _, e := range entries { + if e.Name == base { + return e, true, nil + } + } + return ImageEntry{}, false, nil +} diff --git a/spec/provides_from_test.go b/spec/provides_from_test.go new file mode 100644 index 00000000..51c47865 --- /dev/null +++ b/spec/provides_from_test.go @@ -0,0 +1,956 @@ +package spec + +import ( + "context" + "path" + "sort" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// fakeImage is a filesystem held in a map: path to file body, with a +// symbolic link written as "-> target" and a directory implied by +// anything beneath it. +type fakeImage map[string]string + +func (f fakeImage) ReadFile(_ context.Context, name string) ([]byte, bool, error) { + body, ok := f[name] + if !ok || strings.HasPrefix(body, "-> ") { + return nil, false, nil + } + return []byte(body), true, nil +} + +func (f fakeImage) ReadDir(_ context.Context, name string) ([]ImageEntry, bool, error) { + seen := map[string]ImageEntry{} + for file, body := range f { + if path.Dir(file) == name { + e := ImageEntry{Name: path.Base(file)} + if target, link := strings.CutPrefix(body, "-> "); link { + e.Link = target + } + seen[e.Name] = e + continue + } + prefix := strings.TrimSuffix(name, "/") + "/" + if rest, under := strings.CutPrefix(file, prefix); under { + top, _, _ := strings.Cut(rest, "/") + if _, known := seen[top]; !known { + seen[top] = ImageEntry{Name: top, Dir: true} + } + } + } + out := make([]ImageEntry, 0, len(seen)) + for _, e := range seen { + out = append(out, e) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out, len(out) > 0, nil +} + +// debianImage is the shape of a merged-/usr Debian root: /bin is a link, +// python3 is a link its package ships, awk is an alternatives link no +// package owns, and one removed package still has a list. +func debianImage() fakeImage { + return fakeImage{ + "/bin": "-> usr/bin", + DpkgStatusPath: strings.Join([]string{ + "Package: python-3.14-minimal\nStatus: install ok installed\nVersion: 3.14.10-6\n", + "Package: openssh-client\nStatus: install ok installed\nVersion: 1:10.0p1-7+deb13u4+dhi3\n", + "Package: mawk\nStatus: install ok installed\nVersion: 1.3.4.20250131-1+deb13u0\n", + "Package: bash\nStatus: install ok installed\nVersion: 5.2.37-2+dhi1\n", + "Package: gone\nStatus: deinstall ok config-files\nVersion: 1.0-1\n", + "Package: pre\nStatus: install ok installed\nVersion: 2.0~rc1-1\n", + "Package: libfoo\nStatus: install ok installed\nArchitecture: arm64\nMulti-Arch: same\nVersion: 1.2.3-1\n", + "Package: libbar\nStatus: install ok installed\nVersion: 4.5.6-1\n", + }, "\n"), + DpkgInfoDir + "/python-3.14-minimal.list": "/.\n/usr\n/usr/bin\n/usr/bin/python3.14\n/usr/bin/python3\n", + DpkgInfoDir + "/openssh-client.list": "/.\n/usr\n/usr/bin\n/usr/bin/ssh\n", + DpkgInfoDir + "/mawk.list": "/.\n/usr\n/usr/bin\n/usr/bin/mawk\n", + // Recorded under /bin, as packages built before the /usr merge do. + DpkgInfoDir + "/bash.list": "/.\n/bin\n/bin/bash\n", + DpkgInfoDir + "/gone.list": "/etc/gone.conf\n", + DpkgInfoDir + "/pre.list": "/usr/bin/pre\n", + DpkgInfoDir + "/libfoo:arm64.list": "/usr/lib/shared.so\n", + DpkgInfoDir + "/libbar.list": "/usr/lib/shared.so\n", + DpkgInfoDir + "/bash.md5sums": "ignored\n", + "/usr/bin/python3": "-> python3.14", + "/usr/bin/python3.14": "elf", + "/usr/bin/ssh": "elf", + "/usr/bin/mawk": "elf", + "/usr/bin/awk": "-> /etc/alternatives/awk", + "/etc/alternatives/awk": "-> /usr/bin/mawk", + "/usr/bin/bash": "elf", + "/usr/bin/pre": "elf", + "/usr/lib/shared.so": "elf", + "/usr/local/bin/claude": "elf", + "/usr/local/bin/dangling": "-> /nowhere", + "/usr/local/bin/loop": "-> loop", + "/etc/gone.conf": "conf", + } +} + +func one(f ImageFiles) []PlatformFiles { + return []PlatformFiles{{Platform: "linux/arm64", Files: f}} +} + +func TestProvidesFromPaths(t *testing.T) { + got, err := ProvidesFromPaths(context.Background(), map[string]string{ + "python": "/usr/bin/python3", + "ssh": "/usr/bin/ssh", + "awk": "/usr/bin/awk", + "bash": "/usr/bin/bash", + "com.example/shell": "/bin/bash", + }, one(debianImage())) + require.NoError(t, err) + require.Equal(t, []string{ + "awk@1.3.4", + "bash@5.2.37", + "com.example/shell@5.2.37", + "python@3.14.10", + "ssh@10.0", + }, got) +} + +func TestProvidesFromPathsRefusals(t *testing.T) { + for name, tc := range map[string]struct { + file string + want string + }{ + "absent": {"/usr/bin/nope", "does not exist in the image"}, + "directory": {"/usr/bin", "is a directory"}, + "unowned": {"/usr/local/bin/claude", "belongs to no installed package"}, + "dangling link": {"/usr/local/bin/dangling", "leads to /nowhere, which does not exist"}, + "link cycle": {"/usr/local/bin/loop", "more than 16 symbolic links"}, + "removed package": {"/etc/gone.conf", "belongs to no installed package"}, + "prerelease": {"/usr/bin/pre", "cannot name"}, + "two packages": {"/usr/lib/shared.so", "deb/libbar and deb/libfoo"}, + "unversioned owner": {"/usr/bin/pre", "state the capability's version in provides"}, + } { + t.Run(name, func(t *testing.T) { + _, err := ProvidesFromPaths(context.Background(), map[string]string{"x": tc.file}, one(debianImage())) + require.ErrorContains(t, err, tc.want) + require.ErrorContains(t, err, "providesFrom.x on linux/arm64") + }) + } +} + +func TestProvidesFromPathsNeedsADatabase(t *testing.T) { + _, err := ProvidesFromPaths(context.Background(), map[string]string{"x": "/bin/x"}, one(fakeImage{"/bin/x": "elf"})) + require.ErrorContains(t, err, "neither /var/lib/dpkg/status nor /lib/apk/db/installed") + + noLists := fakeImage{"/bin/x": "elf", DpkgStatusPath: "Package: x\nStatus: install ok installed\nVersion: 1.0-1\n"} + _, err = ProvidesFromPaths(context.Background(), map[string]string{"x": "/bin/x"}, one(noLists)) + require.ErrorContains(t, err, "no /var/lib/dpkg/info") + + _, err = ProvidesFromPaths(context.Background(), map[string]string{"x": "/bin/x"}, nil) + require.ErrorContains(t, err, "no filesystem to read") + + _, err = ProvidesFromPaths(context.Background(), map[string]string{"x": "/bin/x"}, []PlatformFiles{{Platform: "linux/amd64"}}) + require.ErrorContains(t, err, "no filesystem to read on linux/amd64") + + got, err := ProvidesFromPaths(context.Background(), nil, nil) + require.NoError(t, err) + require.Empty(t, got) +} + +func TestProvidesFromPathsAcrossPlatforms(t *testing.T) { + arm, amd := debianImage(), debianImage() + platforms := []PlatformFiles{{Platform: "linux/arm64", Files: arm}, {Platform: "linux/amd64", Files: amd}} + from := map[string]string{"ssh": "/usr/bin/ssh"} + + got, err := ProvidesFromPaths(context.Background(), from, platforms) + require.NoError(t, err) + require.Equal(t, []string{"ssh@10.0"}, got) + + // A Debian revision is packaging bookkeeping (§9.6), so platforms + // that differ only there still share a version. + amd[DpkgStatusPath] = strings.Replace(amd[DpkgStatusPath], "1:10.0p1-7+deb13u4+dhi3", "1:10.0p1-8", 1) + got, err = ProvidesFromPaths(context.Background(), from, platforms) + require.NoError(t, err) + require.Equal(t, []string{"ssh@10.0"}, got) + + amd[DpkgStatusPath] = strings.Replace(amd[DpkgStatusPath], "1:10.0p1-8", "1:10.1p1-1", 1) + _, err = ProvidesFromPaths(context.Background(), from, platforms) + require.ErrorContains(t, err, "/usr/bin/ssh is at 10.0 on linux/arm64 and 10.1 on linux/amd64") + + delete(amd, "/usr/bin/ssh") + _, err = ProvidesFromPaths(context.Background(), from, platforms) + require.ErrorContains(t, err, "providesFrom.ssh on linux/amd64: /usr/bin/ssh does not exist") +} + +func TestProvidesFromPathsApk(t *testing.T) { + image := fakeImage{ + ApkInstalledPath: strings.Join([]string{ + "C:Q1abc=\nP:busybox\nV:1.37.0-r20\nA:aarch64\nF:bin\nR:busybox\na:0:0:755\nZ:Q1def=\nF:etc\nR:securetty\n", + "C:Q1ghi=\nP:python3\nV:3.12.12-r0\nF:usr\nF:usr/bin\nR:python3.12\nR:python3\na:0:0:777\n", + }, "\n"), + "/bin/busybox": "elf", + "/bin/sh": "-> /bin/busybox", + "/usr/bin/python3": "-> python3.12", + "/usr/bin/python3.12": "elf", + } + got, err := ProvidesFromPaths(context.Background(), map[string]string{ + "python": "/usr/bin/python3", + "sh": "/bin/sh", + }, one(image)) + require.NoError(t, err) + require.Equal(t, []string{"python@3.12.12", "sh@1.37.0"}, got) +} + +func TestValidateProvidesFrom(t *testing.T) { + base := func() *Descriptor { + return &Descriptor{SchemaVersion: "3", Kind: KindWorkload, Build: "FROM scratch\n"} + } + ok := base() + ok.Provides = []string{"claude@1.0.0"} + ok.ProvidesFrom = map[string]string{"python": "/usr/bin/python3", "com.example/tool": "/opt/tool/bin/tool"} + _, err := Validate(ok) + require.NoError(t, err) + + for name, tc := range map[string]struct { + mutate func(*Descriptor) + want string + }{ + "mixin": {func(d *Descriptor) { d.Kind = KindMixin }, "providesFrom is for kind: workload only"}, + "set": {func(d *Descriptor) { + d.Kind, d.Build = KindSet, "" + d.Kits = []Kit{{Ref: "docker.io/example/base:1.0.0"}} + }, "a set builds none"}, + "bad name": {func(d *Descriptor) { d.ProvidesFrom = map[string]string{"Python": "/usr/bin/python3"} }, "invalid capability name"}, + "derived space": {func(d *Descriptor) { d.ProvidesFrom = map[string]string{"deb/python": "/usr/bin/python3"} }, "deb/ namespace"}, + "same capability": {func(d *Descriptor) { + d.ProvidesFrom = map[string]string{"gh": "/usr/bin/gh", "com.docker.kit/gh": "/usr/bin/gh"} + }, "are the same capability"}, + "already provided": {func(d *Descriptor) { + d.Provides = []string{"python@3.0.0"} + d.ProvidesFrom = map[string]string{"python": "/usr/bin/python3"} + }, "already stated by provides[0]"}, + "relative": {func(d *Descriptor) { d.ProvidesFrom = map[string]string{"python": "usr/bin/python3"} }, "not an absolute, canonical path"}, + "not clean": {func(d *Descriptor) { d.ProvidesFrom = map[string]string{"python": "/usr/bin/../bin/python3"} }, "not an absolute, canonical path"}, + "root": {func(d *Descriptor) { d.ProvidesFrom = map[string]string{"python": "/"} }, "not an absolute, canonical path"}, + "empty": {func(d *Descriptor) { d.ProvidesFrom = map[string]string{"python": ""} }, "not an absolute, canonical path"}, + "arg in value": {func(d *Descriptor) { d.ProvidesFrom = map[string]string{"python": "/usr/bin/${{ kit.args.python }}"} }, "arg references are not allowed in providesFrom"}, + } { + t.Run(name, func(t *testing.T) { + d := base() + d.ProvidesFrom = map[string]string{"python": "/usr/bin/python3"} + tc.mutate(d) + _, err := Validate(d) + require.ErrorContains(t, err, tc.want) + }) + } +} + +func TestWithoutProvidesFrom(t *testing.T) { + authored := []byte(`schemaVersion: "3" +kind: workload +# what the agent is +provides: ["claude@1.0.0"] +# what the base ships +providesFrom: + python: /usr/bin/python3 + ssh: /usr/bin/ssh +version: "1.0.0" +build: | + FROM scratch +`) + out, err := WithoutProvidesFrom(authored) + require.NoError(t, err) + require.NotContains(t, string(out), "providesFrom") + require.NotContains(t, string(out), "/usr/bin/python3") + require.Contains(t, string(out), "# what the agent is") + + d, err := Decode(out) + require.NoError(t, err) + require.Empty(t, d.ProvidesFrom) + require.Equal(t, []string{"claude@1.0.0"}, d.Provides) + _, err = ValidatePublished(out, d) + require.NoError(t, err) + + // A descriptor without the field is returned untouched, comments, + // quoting and all. + plain := []byte("schemaVersion: \"3\"\nkind: mixin # overlay\n") + out, err = WithoutProvidesFrom(plain) + require.NoError(t, err) + require.Equal(t, string(plain), string(out)) +} + +func TestPublishedRefusesProvidesFrom(t *testing.T) { + raw := []byte(`schemaVersion: "3" +kind: workload +version: "1.0.0" +providesFrom: + python: /usr/bin/python3 +build: | + FROM scratch +`) + d, err := Decode(raw) + require.NoError(t, err) + _, err = ValidatePublished(raw, d) + require.ErrorContains(t, err, "published descriptor still carries providesFrom") +} + +// A merge key hands the root mapping another mapping's keys, so the +// field can reach a descriptor without being written at the top level. +// Publishing has to remove it from wherever it came, and leave the rest +// of what was merged in place. +func TestWithoutProvidesFromReachesMergedMappings(t *testing.T) { + const head = "schemaVersion: \"3\"\nkind: workload\nversion: \"1.0.0\"\n" + const tail = "build: |\n FROM scratch\n" + for name, tc := range map[string]struct { + body string + displayName string + keeps string + }{ + "inline": { + body: "<<: {providesFrom: {python: /usr/bin/python3}, displayName: Merged}\n", + displayName: "Merged", + }, + "inline with nothing else": { + body: "<<: {providesFrom: {python: /usr/bin/python3}}\n", + }, + "sequence": { + body: "<<: [{providesFrom: {python: /usr/bin/python3}}, {displayName: Second}]\n", + displayName: "Second", + }, + "nested": { + body: "<<: {displayName: Outer, <<: {providesFrom: {python: /usr/bin/python3}}}\n", + displayName: "Outer", + }, + "beside the plain spelling": { + body: "providesFrom: {ssh: /usr/bin/ssh}\n<<: {providesFrom: {python: /usr/bin/python3}, displayName: Merged}\n", + displayName: "Merged", + }, + // The anchored mapping is a capability's config, which is not + // this rewrite's to change: the root merges a copy without the + // field and the config keeps every key it had. + "alias to a mapping defined elsewhere": { + body: "capabilities:\n - type: com.example/thing@1\n config: &shared\n providesFrom: {python: /usr/bin/python3}\n displayName: Shared\n" + + "<<: *shared\n", + displayName: "Shared", + keeps: "/usr/bin/python3", + }, + } { + t.Run(name, func(t *testing.T) { + authored := []byte(head + tc.body + tail) + d, err := Decode(authored) + require.NoError(t, err) + require.NotEmpty(t, d.ProvidesFrom, "the case must reach the field through the merge") + + out, err := WithoutProvidesFrom(authored) + require.NoError(t, err) + pd, err := Decode(out) + require.NoError(t, err, "%s", out) + require.Empty(t, pd.ProvidesFrom, "%s", out) + require.Equal(t, tc.displayName, pd.DisplayName, "what was merged beside the field stays merged") + require.Equal(t, "FROM scratch\n", pd.Build) + _, err = ValidatePublished(out, pd) + require.NoError(t, err, "%s", out) + if tc.keeps != "" { + require.Contains(t, string(out), tc.keeps) + require.Len(t, pd.Capabilities, 1) + } + }) + } +} + +// Removing the field removes any anchor defined under it. An alias +// elsewhere would then name nothing and the document would stop +// parsing, so each one is given a copy of what it pointed at. +func TestWithoutProvidesFromKeepsAliasesIntoWhatItRemoves(t *testing.T) { + const head = "schemaVersion: \"3\"\nkind: workload\nversion: \"1.0.0\"\n" + const tail = "build: |\n FROM scratch\n" + + scalar := []byte(head + "providesFrom: {python: &command /usr/bin/python3}\ndescription: *command\n" + tail) + out, err := WithoutProvidesFrom(scalar) + require.NoError(t, err) + d, err := Decode(out) + require.NoError(t, err, "%s", out) + require.Empty(t, d.ProvidesFrom) + require.Equal(t, "/usr/bin/python3", d.Description) + require.NotContains(t, string(out), "*command") + _, err = ValidatePublished(out, d) + require.NoError(t, err) + + // The whole value anchored, and used twice more: once as a config, + // once through an alias nested in another config. + whole := []byte(head + "providesFrom: &paths\n python: &py /usr/bin/python3\n ssh: /usr/bin/ssh\n" + + "capabilities:\n - type: com.example/a@1\n config: *paths\n - type: com.example/b@1\n config: {interpreter: *py, all: *paths}\n" + tail) + out, err = WithoutProvidesFrom(whole) + require.NoError(t, err) + d, err = Decode(out) + require.NoError(t, err, "%s", out) + require.Empty(t, d.ProvidesFrom) + require.Len(t, d.Capabilities, 2) + paths := map[string]any{"python": "/usr/bin/python3", "ssh": "/usr/bin/ssh"} + require.Equal(t, paths, d.Capabilities[0].Config) + require.Equal(t, map[string]any{"interpreter": "/usr/bin/python3", "all": paths}, d.Capabilities[1].Config) + + // The other direction needs nothing: an alias inside the field, + // to an anchor that stays, simply goes with the field. + inward := []byte(head + "description: &text /usr/bin/python3\nprovidesFrom: {python: *text}\n" + tail) + out, err = WithoutProvidesFrom(inward) + require.NoError(t, err) + d, err = Decode(out) + require.NoError(t, err, "%s", out) + require.Empty(t, d.ProvidesFrom) + require.Equal(t, "/usr/bin/python3", d.Description) +} + +// The published rule is that the key is absent, not that it is empty: +// a reader built before the field refuses the key whatever is under it, +// and an empty map or a null decodes exactly like no field at all. +func TestPublishedRefusesProvidesFromInAnySpelling(t *testing.T) { + const yamlHead = "schemaVersion: \"3\"\nkind: workload\nversion: \"1.0.0\"\n" + for name, raw := range map[string]string{ + "entries": yamlHead + "providesFrom: {python: /usr/bin/python3}\n", + "empty map": yamlHead + "providesFrom: {}\n", + "null": yamlHead + "providesFrom:\n", + "merged": yamlHead + "<<: {providesFrom: {python: /usr/bin/python3}}\n", + "merged empty": yamlHead + "<<: {providesFrom: {}}\n", + "json": `{"schemaVersion":"3","kind":"workload","version":"1.0.0","providesFrom":{}}`, + "json null": `{"schemaVersion":"3","kind":"workload","version":"1.0.0","providesFrom":null}`, + "quoted the key": yamlHead + "\"providesFrom\": {}\n", + } { + t.Run(name, func(t *testing.T) { + d, err := Decode([]byte(raw)) + require.NoError(t, err) + _, err = ValidatePublished([]byte(raw), d) + require.ErrorContains(t, err, "published descriptor still carries providesFrom") + + // And publishing's own rewrite clears every one of them. + out, err := WithoutProvidesFrom([]byte(raw)) + require.NoError(t, err) + pd, err := Decode(out) + require.NoError(t, err, "%s", out) + _, err = ValidatePublished(out, pd) + require.NoError(t, err, "%s", out) + }) + } + + // A descriptor that never had the field is not caught by the word + // appearing somewhere else in it. + elsewhere := []byte(yamlHead + "description: see providesFrom in the docs\ncapabilities:\n - type: com.example/a@1\n config: {providesFrom: {x: /y}}\n") + d, err := Decode(elsewhere) + require.NoError(t, err) + _, err = ValidatePublished(elsewhere, d) + require.NoError(t, err) + out, err := WithoutProvidesFrom(elsewhere) + require.NoError(t, err) + require.Equal(t, string(elsewhere), string(out)) +} + +// A co-installable package keeps one file list per architecture, and +// one architecture can be removed while another stays installed. The +// removed one's list still holds the conffiles it left behind; it must +// not own them on the strength of the installed one's name. +func TestProvidesFromPathsJudgesEachArchitecturesList(t *testing.T) { + image := fakeImage{ + DpkgStatusPath: strings.Join([]string{ + "Package: libfoo\nStatus: install ok installed\nArchitecture: amd64\nMulti-Arch: same\nVersion: 2.0-1\n", + "Package: libfoo\nStatus: deinstall ok config-files\nArchitecture: i386\nMulti-Arch: same\nVersion: 1.0-1\n", + "Package: libbar\nStatus: install ok installed\nArchitecture: amd64\nMulti-Arch: same\nVersion: 3.1-2\n", + "Package: libbar\nStatus: install ok installed\nArchitecture: i386\nMulti-Arch: same\nVersion: 3.1-2\n", + "Package: plain\nStatus: install ok installed\nArchitecture: amd64\nVersion: 4.2-1\n", + }, "\n"), + DpkgInfoDir + "/libfoo:amd64.list": "/usr/lib/x86_64-linux-gnu/libfoo.so.2\n", + DpkgInfoDir + "/libfoo:i386.list": "/etc/libfoo-i386.conf\n", + DpkgInfoDir + "/libbar:amd64.list": "/usr/lib/x86_64-linux-gnu/libbar.so.3\n/usr/share/doc/libbar/copyright\n", + DpkgInfoDir + "/libbar:i386.list": "/usr/lib/i386-linux-gnu/libbar.so.3\n/usr/share/doc/libbar/copyright\n", + // A list named without an architecture belongs to the package + // of that name, whatever it is installed for. + DpkgInfoDir + "/plain.list": "/usr/bin/plain\n", + // Named for an architecture the package is not installed for. + DpkgInfoDir + "/plain:i386.list": "/usr/bin/plain32\n", + "/usr/lib/x86_64-linux-gnu/libfoo.so.2": "elf", + "/etc/libfoo-i386.conf": "conf", + "/usr/lib/x86_64-linux-gnu/libbar.so.3": "elf", + "/usr/lib/i386-linux-gnu/libbar.so.3": "elf", + "/usr/share/doc/libbar/copyright": "text", + "/usr/bin/plain": "elf", + "/usr/bin/plain32": "elf", + } + + got, err := ProvidesFromPaths(context.Background(), map[string]string{ + "foo": "/usr/lib/x86_64-linux-gnu/libfoo.so.2", + "bar": "/usr/lib/i386-linux-gnu/libbar.so.3", + "bar-doc": "/usr/share/doc/libbar/copyright", + "plain": "/usr/bin/plain", + }, one(image)) + require.NoError(t, err) + require.Equal(t, []string{"bar-doc@3.1", "bar@3.1", "foo@2.0", "plain@4.2"}, got, + "a file both architectures of one package list has one owner") + + for _, file := range []string{"/etc/libfoo-i386.conf", "/usr/bin/plain32"} { + _, err = ProvidesFromPaths(context.Background(), map[string]string{"x": file}, one(image)) + require.ErrorContains(t, err, file+" belongs to no installed package") + } +} + +// One top-level link can lead to another. A path a package recorded and +// a path an author wrote have to end up spelled the same way however +// many links stand between them and the file. +func TestProvidesFromPathsFollowsChainedTopLevelLinks(t *testing.T) { + image := fakeImage{ + "/bin": "-> compat/bin", + "/compat": "-> usr", + "/sbin": "-> /bin", + DpkgStatusPath: "Package: tool\nStatus: install ok installed\nArchitecture: amd64\nVersion: 1.2.3-1\n", + DpkgInfoDir + "/tool.list": "/bin/tool\n/sbin/admin\n", + "/usr/bin/tool": "elf", + "/usr/bin/admin": "elf", + } + got, err := ProvidesFromPaths(context.Background(), map[string]string{ + "direct": "/usr/bin/tool", + "recorded": "/bin/tool", + "middle": "/compat/bin/tool", + "twice": "/sbin/admin", + }, one(image)) + require.NoError(t, err) + require.Equal(t, []string{"direct@1.2.3", "middle@1.2.3", "recorded@1.2.3", "twice@1.2.3"}, got) + + // Links that lead in a circle reach no file. The lookup ends, and + // says the path does not exist rather than never returning. + circle := fakeImage{ + "/a": "-> b", + "/b": "-> a", + DpkgStatusPath: "Package: tool\nStatus: install ok installed\nArchitecture: amd64\nVersion: 1.2.3-1\n", + DpkgInfoDir + "/tool.list": "/a/tool\n", + } + _, err = ProvidesFromPaths(context.Background(), map[string]string{"x": "/a/tool"}, one(circle)) + require.ErrorContains(t, err, "/a/tool does not exist in the image") +} + +// A directory on the way to a file can be a symbolic link — a default +// JDK, a current release. The package recorded the file where it really +// is, so the directories of a requested path are resolved before the +// file lists are consulted. +func TestProvidesFromPathsResolvesDirectoryLinks(t *testing.T) { + image := fakeImage{ + "/bin": "-> usr/bin", + DpkgStatusPath: strings.Join([]string{ + "Package: jdk\nStatus: install ok installed\nArchitecture: arm64\nVersion: 21.0.9-1\n", + "Package: tool\nStatus: install ok installed\nArchitecture: arm64\nVersion: 1.2.3-1\n", + "Package: odd\nStatus: install ok installed\nArchitecture: arm64\nVersion: 7.0-1\n", + }, "\n"), + DpkgInfoDir + "/jdk.list": "/usr/lib/jvm/java-21/bin/java\n", + DpkgInfoDir + "/tool.list": "/opt/tool-1.2/bin/tool\n", + // Recorded through a directory link, as a package that ships + // into another package's link would be. + DpkgInfoDir + "/odd.list": "/usr/lib/jvm/default/bin/odd\n", + "/usr/lib/jvm/java-21/bin/java": "elf", + "/usr/lib/jvm/java-21/bin/odd": "elf", + "/usr/lib/jvm/default": "-> java-21", + "/usr/lib/jvm/absolute": "-> /usr/lib/jvm/default", + "/opt/tool-1.2/bin/tool": "elf", + "/opt/tool/current": "-> ../tool-1.2", + "/opt/tool/README": "text", + // An alternatives-style link in a directory reached through a + // link: its relative target is relative to the real directory. + "/usr/lib/jvm/java-21/bin/jvm": "-> java", + "/usr/bin/java": "-> /usr/lib/jvm/absolute/bin/jvm", + "/opt/loop/a": "-> b", + "/opt/loop/b": "-> a", + } + _, err := ProvidesFromPaths(context.Background(), map[string]string{ + "real": "/usr/lib/jvm/java-21/bin/java", + "default": "/usr/lib/jvm/default/bin/java", + "chained": "/usr/lib/jvm/absolute/bin/java", + "climbing": "/opt/tool/current/bin/tool", + "command": "/bin/java", + "as-listed": "/usr/lib/jvm/default/bin/odd", + "listed": "/usr/lib/jvm/java-21/bin/odd", + }, one(image)) + require.ErrorContains(t, err, "providesFrom.listed on linux/arm64: /usr/lib/jvm/java-21/bin/odd belongs to no installed package", + "a recorded path is compared as recorded: only the spelling the package used finds it") + + got, err := ProvidesFromPaths(context.Background(), map[string]string{ + "real": "/usr/lib/jvm/java-21/bin/java", + "default": "/usr/lib/jvm/default/bin/java", + "chained": "/usr/lib/jvm/absolute/bin/java", + "climbing": "/opt/tool/current/bin/tool", + "command": "/bin/java", + "as-listed": "/usr/lib/jvm/default/bin/odd", + }, one(image)) + require.NoError(t, err) + require.Equal(t, []string{ + "as-listed@7.0", + "chained@21.0.9", + "climbing@1.2.3", + "command@21.0.9", + "default@21.0.9", + "real@21.0.9", + }, got) + + for file, want := range map[string]string{ + "/usr/lib/jvm/default/bin/nope": "/usr/lib/jvm/default/bin/nope does not exist in the image", + "/usr/lib/jvm/missing/bin/java": "/usr/lib/jvm/missing/bin/java does not exist in the image", + "/usr/lib/jvm/default/bin": "/usr/lib/jvm/default/bin is a directory", + "/opt/loop/a/tool": "/opt/loop/a/tool does not exist in the image", + } { + _, err := ProvidesFromPaths(context.Background(), map[string]string{"x": file}, one(image)) + require.ErrorContains(t, err, want) + } +} + +// Resolution has to end at the file that is actually there. Two ways of +// reading a path as text end somewhere else, and each of them states +// the version of a package that does not own the file. +func TestProvidesFromPathsEndsAtTheRealFile(t *testing.T) { + status := func(packages ...string) string { + var out []string + for i := 0; i+1 < len(packages); i += 2 { + out = append(out, "Package: "+packages[i]+"\nStatus: install ok installed\nArchitecture: arm64\nVersion: "+packages[i+1]+"\n") + } + return strings.Join(out, "\n") + } + + // A list can hold a path from before a directory on it became a + // link. The file there now is another package's. + stale := fakeImage{ + DpkgStatusPath: status("old", "1.0-1", "new", "2.0-1"), + DpkgInfoDir + "/old.list": "/opt/app/current/bin/tool\n", + DpkgInfoDir + "/new.list": "/opt/app/release-2/bin/tool\n", + "/opt/app/release-2/bin/tool": "elf", + "/opt/app/current": "-> release-2", + } + got, err := ProvidesFromPaths(context.Background(), map[string]string{"tool": "/opt/app/current/bin/tool"}, one(stale)) + require.NoError(t, err) + require.Equal(t, []string{"tool@2.0"}, got, "the owner of the real file, not of the spelling") + + // ".." after a linked directory is that directory's parent. Read as + // text, sub/../real-tool is the neighbour of sub; in the image it + // is the neighbour of wherever sub leads. + climb := fakeImage{ + DpkgStatusPath: status("near", "1.0-1", "far", "2.0-1"), + DpkgInfoDir + "/near.list": "/opt/app/bin/real-tool\n", + DpkgInfoDir + "/far.list": "/srv/real-tool\n", + "/opt/app/bin/real-tool": "elf", + "/srv/real-tool": "elf", + "/srv/other/keep": "x", + "/opt/app/bin/sub": "-> /srv/other", + "/opt/app/bin/tool": "-> sub/../real-tool", + "/opt/app/bin/dotted": "-> ./sub/./../real-tool", + // The same shape in a directory link rather than a file link. + "/opt/app/via": "-> bin/sub/..", + "/opt/app/bin/up": "-> ..", + "/opt/app/bin/updir": "-> sub/..", + "/opt/app/bin/straight": "-> ../bin/real-tool", + } + got, err = ProvidesFromPaths(context.Background(), map[string]string{ + "tool": "/opt/app/bin/tool", + "dotted": "/opt/app/bin/dotted", + "via": "/opt/app/via/real-tool", + "straight": "/opt/app/bin/straight", + }, one(climb)) + require.NoError(t, err) + require.Equal(t, []string{"dotted@2.0", "straight@1.0", "tool@2.0", "via@2.0"}, got) + + for _, file := range []string{"/opt/app/bin/up", "/opt/app/bin/updir"} { + _, err = ProvidesFromPaths(context.Background(), map[string]string{"x": file}, one(climb)) + require.ErrorContains(t, err, "is a directory", "%s leads to a directory", file) + } +} + +// A top-level link is an alias for the directory it finally reaches, +// worked out the same way: through as many links as it takes, with ".." +// meaning the parent of what was reached. +func TestProvidesFromPathsResolvesTopLevelAliasesFully(t *testing.T) { + image := fakeImage{ + "/bin": "-> compat/../usr/bin", + "/compat": "-> srv/elsewhere", + "/srv/elsewhere/keep": "x", + "/srv/usr/bin/tool": "elf", + "/usr/bin/tool": "elf", + "/dangling": "-> nowhere", + DpkgStatusPath: "Package: tool\nStatus: install ok installed\nArchitecture: arm64\nVersion: 1.2.3-1\n\nPackage: other\nStatus: install ok installed\nArchitecture: arm64\nVersion: 9.9-1\n", + DpkgInfoDir + "/tool.list": "/bin/tool\n", + // What reading /bin's target as text would have reached. + DpkgInfoDir + "/other.list": "/usr/bin/tool\n/dangling/thing\n", + } + got, err := ProvidesFromPaths(context.Background(), map[string]string{ + "recorded": "/bin/tool", + "real": "/srv/usr/bin/tool", + "textual": "/usr/bin/tool", + }, one(image)) + require.NoError(t, err) + require.Equal(t, []string{"real@1.2.3", "recorded@1.2.3", "textual@9.9"}, got) +} + +// A walk that cannot be completed names no file. The components left +// over are not a path to fall back on: a ".." among them would step out +// of a directory that was never entered and land on a file that exists. +func TestProvidesFromPathsRefusesAWalkThatDoesNotArrive(t *testing.T) { + image := fakeImage{ + DpkgStatusPath: "Package: tool\nStatus: install ok installed\nArchitecture: arm64\nVersion: 1.2.3-1\n", + DpkgInfoDir + "/tool.list": "/opt/tool\n/opt/dir/inner\n", + "/opt/tool": "elf", + "/opt/dir/inner": "elf", + "/opt/plain": "a regular file", + "/opt/missing-then-up": "-> missing/../tool", + "/opt/file-then-up": "-> plain/../tool", + "/opt/file-then-down": "-> plain/tool", + "/opt/deep-missing": "-> missing/also/../../tool", + "/opt/dirlink-missing": "-> missing/..", + "/opt/dirlink-file": "-> plain/..", + "/opt/good": "-> dir/../tool", + "/opt/file-with-slash": "-> tool/", + "/opt/file-with-dot": "-> tool/.", + "/opt/dir-with-slash": "-> dir/", + } + for file, want := range map[string]string{ + "/opt/missing-then-up": "/opt/missing-then-up leads to /opt/missing/../tool, which does not exist in the image", + "/opt/file-then-up": "/opt/file-then-up leads to /opt/plain/../tool, which does not exist in the image", + "/opt/file-then-down": "/opt/file-then-down leads to /opt/plain/tool, which does not exist in the image", + "/opt/deep-missing": "which does not exist in the image", + "/opt/dirlink-missing/tool": "/opt/dirlink-missing/tool does not exist in the image", + "/opt/dirlink-file/tool": "/opt/dirlink-file/tool does not exist in the image", + "/opt/plain/tool": "/opt/plain/tool does not exist in the image", + "/opt/missing/tool": "/opt/missing/tool does not exist in the image", + // A slash after a file asks for a directory that is not one. + "/opt/file-with-slash": "/opt/file-with-slash leads to /opt/tool/, which does not exist in the image", + "/opt/file-with-dot": "/opt/file-with-dot leads to /opt/tool/., which does not exist in the image", + "/opt/dir-with-slash": "/opt/dir/ is a directory", + } { + _, err := ProvidesFromPaths(context.Background(), map[string]string{"x": file}, one(image)) + require.ErrorContains(t, err, want, file) + } + + // The same shape through a directory that is there still resolves. + got, err := ProvidesFromPaths(context.Background(), map[string]string{"x": "/opt/good"}, one(image)) + require.NoError(t, err) + require.Equal(t, []string{"x@1.2.3"}, got) +} + +// Everything the removal changes in place is something an alias can +// name. Whatever else uses that alias — here a capability's config, +// which is opaque to this rewrite — has to read what it read before. +func TestWithoutProvidesFromLeavesSharedNodesAsTheyWere(t *testing.T) { + const head = "schemaVersion: \"3\"\nkind: workload\nversion: \"1.0.0\"\n" + const tail = "build: |\n FROM scratch\n" + uses := func(alias string) string { + return "capabilities:\n - type: com.example/thing@1\n config: {uses: *" + alias + "}\n" + } + paths := map[string]any{"python": "/usr/bin/python3"} + + for name, tc := range map[string]struct { + body string + config any + displayName string + }{ + // The sequence a merge key lists its sources in. + "anchored merge sequence": { + body: "<<: &shared [{providesFrom: {python: /usr/bin/python3}}, {displayName: Second}]\n" + uses("shared"), + config: []any{map[string]any{"providesFrom": paths}, map[string]any{"displayName": "Second"}}, + displayName: "Second", + }, + // A source inside an unanchored sequence. + "anchored element of a merge sequence": { + body: "<<: [&shared {providesFrom: {python: /usr/bin/python3}, displayName: First}, {description: Second}]\n" + uses("shared"), + config: map[string]any{"providesFrom": paths, "displayName": "First"}, + displayName: "First", + }, + // A single anchored source. + "anchored merge mapping": { + body: "<<: &shared {providesFrom: {python: /usr/bin/python3}, displayName: Only}\n" + uses("shared"), + config: map[string]any{"providesFrom": paths, "displayName": "Only"}, + displayName: "Only", + }, + // The field reaches the anchored mapping through a merge of its + // own, so the mapping is edited one level further in. + "anchored mapping with a nested merge": { + body: "<<: &shared {displayName: Outer, <<: {providesFrom: {python: /usr/bin/python3}}}\n" + uses("shared"), + config: map[string]any{"providesFrom": paths, "displayName": "Outer"}, + displayName: "Outer", + }, + // An anchored sequence whose only source goes away entirely. + "anchored sequence left empty": { + body: "<<: &shared [{providesFrom: {python: /usr/bin/python3}}]\n" + uses("shared"), + config: []any{map[string]any{"providesFrom": paths}}, + }, + // The field's own value. + "anchored value": { + body: "providesFrom: &shared {python: /usr/bin/python3}\n" + uses("shared"), + config: paths, + }, + } { + t.Run(name, func(t *testing.T) { + authored := []byte(head + tc.body + tail) + before, err := Decode(authored) + require.NoError(t, err) + require.NotEmpty(t, before.ProvidesFrom) + require.Equal(t, map[string]any{"uses": tc.config}, before.Capabilities[0].Config, "the case is not what it says") + + out, err := WithoutProvidesFrom(authored) + require.NoError(t, err) + after, err := Decode(out) + require.NoError(t, err, "%s", out) + require.Empty(t, after.ProvidesFrom, "%s", out) + require.Equal(t, before.Capabilities[0].Config, after.Capabilities[0].Config, + "the config reads what it read before:\n%s", out) + require.Equal(t, tc.displayName, after.DisplayName) + _, err = ValidatePublished(out, after) + require.NoError(t, err, "%s", out) + }) + } +} + +// The kind rule is about the field being written, not about it having +// entries: the schema refuses the key on a mixin or a set whatever is +// under it, and validation must not accept a descriptor an editor +// already marked wrong — nor let the build drop the field in silence. +func TestProvidesFromKindRuleAppliesToAnEmptyField(t *testing.T) { + const mixin = "schemaVersion: \"3\"\nkind: mixin\nversion: \"1.0.0\"\n" + const set = "schemaVersion: \"3\"\nkind: set\nkits:\n - ref: docker.io/example/base:1.0.0\n" + const workload = "schemaVersion: \"3\"\nkind: workload\nversion: \"1.0.0\"\nbuild: |\n FROM scratch\n" + + for name, tc := range map[string]struct{ raw, want string }{ + "mixin, empty map": {mixin + "providesFrom: {}\n", "providesFrom is for kind: workload only"}, + "mixin, null": {mixin + "providesFrom:\n", "providesFrom is for kind: workload only"}, + "mixin, merged": {mixin + "<<: {providesFrom: {}}\n", "providesFrom is for kind: workload only"}, + "mixin, merged null": {mixin + "<<: {providesFrom: }\n", "providesFrom is for kind: workload only"}, + "set, empty map": {set + "providesFrom: {}\n", "a set builds none"}, + "set, null": {set + "providesFrom:\n", "a set builds none"}, + } { + t.Run(name, func(t *testing.T) { + d, err := Decode([]byte(tc.raw)) + require.NoError(t, err) + _, err = ValidateDeclarations([]byte(tc.raw), d) + require.ErrorContains(t, err, tc.want) + // Once, however the field was spelled. + require.Equal(t, 1, strings.Count(err.Error(), tc.want), "%v", err) + }) + } + + // On a workload an empty field asks for nothing, and is dropped at + // publish like a full one. + for _, body := range []string{"providesFrom: {}\n", "providesFrom:\n"} { + raw := []byte(workload + body) + d, err := Decode(raw) + require.NoError(t, err) + _, err = ValidateDeclarations(raw, d) + require.NoError(t, err) + out, err := WithoutProvidesFrom(raw) + require.NoError(t, err) + require.NotContains(t, string(out), "providesFrom") + } + + // A descriptor with no such field is untouched by any of this. + plain := []byte(mixin) + d, err := Decode(plain) + require.NoError(t, err) + _, err = ValidateDeclarations(plain, d) + require.NoError(t, err) +} + +// Merges nest, and the decoder follows them however deep they go. The +// removal and the published-form check have to follow just as far: a +// walk that gave up at some depth would publish a field every reader +// still sees. +func TestProvidesFromIsFoundAtAnyMergeDepth(t *testing.T) { + const head = "schemaVersion: \"3\"\nkind: workload\nversion: \"1.0.0\"\n" + const tail = "\nbuild: |\n FROM scratch\n" + nested := func(inner string, depth int) string { + return strings.Repeat("<<: {", depth) + inner + strings.Repeat("}", depth) + } + for _, depth := range []int{1, 31, 32, 33, 40, 300} { + for name, inner := range map[string]string{ + "entries": "providesFrom: {python: /usr/bin/python3}, displayName: Deep", + "empty": "providesFrom: {}, displayName: Deep", + "null": "providesFrom: , displayName: Deep", + } { + raw := []byte(head + nested(inner, depth) + tail) + d, err := Decode(raw) + require.NoError(t, err, "%s at depth %d", name, depth) + require.Equal(t, "Deep", d.DisplayName) + + _, err = ValidatePublished(raw, d) + require.ErrorContains(t, err, "published descriptor still carries providesFrom", "%s at depth %d", name, depth) + + out, err := WithoutProvidesFrom(raw) + require.NoError(t, err, "%s at depth %d", name, depth) + require.NotContains(t, string(out), "providesFrom", "%s at depth %d", name, depth) + pd, err := Decode(out) + require.NoError(t, err, "%s at depth %d", name, depth) + require.Equal(t, "Deep", pd.DisplayName, "what was merged beside the field stays merged") + _, err = ValidatePublished(out, pd) + require.NoError(t, err, "%s at depth %d", name, depth) + } + } + + // The same through a chain of aliases, each mapping merging the one + // defined before it. A capability's config is the one place a + // descriptor has room to define them. + var chain strings.Builder + chain.WriteString("capabilities:\n - type: com.example/thing@1\n config:\n m0: &m0 {providesFrom: {python: /usr/bin/python3}, displayName: Chained}\n") + const links = 60 + for i := 1; i <= links; i++ { + chain.WriteString(" m" + strconv.Itoa(i) + ": &m" + strconv.Itoa(i) + " {<<: *m" + strconv.Itoa(i-1) + "}\n") + } + chain.WriteString("<<: *m" + strconv.Itoa(links) + "\n") + raw := []byte(head + chain.String() + tail) + d, err := Decode(raw) + require.NoError(t, err) + require.NotEmpty(t, d.ProvidesFrom) + out, err := WithoutProvidesFrom(raw) + require.NoError(t, err) + pd, err := Decode(out) + require.NoError(t, err, "%s", out) + require.Empty(t, pd.ProvidesFrom) + require.Equal(t, "Chained", pd.DisplayName) + require.Equal(t, d.Capabilities[0].Config, pd.Capabilities[0].Config, "the config the anchors live in is untouched") + _, err = ValidatePublished(out, pd) + require.NoError(t, err) +} + +// What does have to be guarded against is a walk that never ends, and +// one that copies without limit. Neither document can be decoded, so no +// build reaches the removal with one; a caller that did not decode +// first gets an answer instead of a hang. +func TestWithoutProvidesFromEndsOnDocumentsThatReferToThemselves(t *testing.T) { + const head = "schemaVersion: \"3\"\nkind: workload\nversion: \"1.0.0\"\n" + + // An anchor that merges itself in. + cyclic := []byte(head + "capabilities:\n - type: com.example/thing@1\n config: &self {providesFrom: {python: /usr/bin/python3}, <<: *self}\n<<: *self\n") + _, err := Decode(cyclic) + require.Error(t, err, "the decoder refuses an anchor that contains itself") + done := make(chan struct{}) + go func() { + defer close(done) + _, _ = WithoutProvidesFrom(cyclic) + _ = carriesProvidesFrom(cyclic) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("removal did not return for a mapping that merges itself in") + } + + // Aliases that fan out: each level merges the one below it twice, + // so a copy of the top doubles per level. + var fan strings.Builder + fan.WriteString("capabilities:\n - type: com.example/thing@1\n config:\n m0: &m0 {providesFrom: {python: /usr/bin/python3}}\n") + const levels = 40 + for i := 1; i <= levels; i++ { + fan.WriteString(" m" + strconv.Itoa(i) + ": &m" + strconv.Itoa(i) + " {<<: [*m" + strconv.Itoa(i-1) + ", *m" + strconv.Itoa(i-1) + "]}\n") + } + fan.WriteString("<<: *m" + strconv.Itoa(levels) + "\n") + wide := []byte(head + fan.String()) + require.True(t, carriesProvidesFrom(wide), "presence is judged once per mapping, however often it is merged") + done = make(chan struct{}) + go func() { + defer close(done) + _, err = WithoutProvidesFrom(wide) + }() + select { + case <-done: + require.ErrorContains(t, err, "too many aliases") + case <-time.After(60 * time.Second): + t.Fatal("removal did not return for aliases that fan out") + } +} diff --git a/spec/schema_test.go b/spec/schema_test.go index 1140f322..cb5a97a7 100644 --- a/spec/schema_test.go +++ b/spec/schema_test.go @@ -769,3 +769,32 @@ func TestAgentContextV2DeliverySchema(t *testing.T) { require.ElementsMatch(t, []any{AgentContextDeliveryProgressive, AgentContextDeliveryInline}, branches[0].(map[string]any)["enum"]) assertAcceptsKitArg(t, delivery, "bearing") } + +// The schema restates two providesFrom rules the validator owns: which +// kinds may use it, and which namespaces its keys may not be in. Pinned +// so an editor's verdict and the build's cannot drift. +func TestProvidesFromSchemaMatchesGrammar(t *testing.T) { + schema := loadSchema(t) + field := at(t, schema, "properties", "providesFrom") + require.Equal(t, "object", field["type"]) + require.Equal(t, "string", at(t, field, "additionalProperties")["type"]) + + reserved := regexp.MustCompile(at(t, field, "propertyNames", "not")["pattern"].(string)) + for namespace := range reservedNamespaces { + require.True(t, reserved.MatchString(namespace+"/bash"), "the schema admits a providesFrom key under %s/", namespace) + } + require.False(t, reserved.MatchString("bash")) + require.False(t, reserved.MatchString("com.example/bash")) + + var rule map[string]any + for _, r := range schema["allOf"].([]any) { + candidate := r.(map[string]any) + ifClause, _ := candidate["if"].(map[string]any) + if required, _ := ifClause["required"].([]any); len(required) == 1 && required[0] == "providesFrom" { + rule = candidate + } + } + require.NotNil(t, rule, "the schema states no providesFrom rule") + require.Equal(t, KindWorkload, at(t, rule, "then", "properties", "kind")["const"]) + require.Equal(t, []any{"kits"}, at(t, rule, "then", "not")["required"]) +} diff --git a/spec/types.go b/spec/types.go index 1daa910a..35b9ce3d 100644 --- a/spec/types.go +++ b/spec/types.go @@ -164,6 +164,21 @@ type Descriptor struct { // requirable by name states so here. Provides []string `json:"provides,omitempty" yaml:"provides,omitempty"` + // ProvidesFrom states provides entries whose version the author + // does not know: capability name to the absolute path of a file in + // the kit's own filesystem. Publishing finds the installed package + // that owns the file and states name@ in + // Provides (§9.7), so a workload can offer `python` at whatever its + // base image ships without a release number of its own standing in + // for one. + // + // Authored form only. It is an instruction to the build rather than + // a declaration, and the published descriptor carries its result and + // not the field: a runtime built before this field existed decodes + // strictly, and would refuse a published kit for a key that has + // nothing left to tell it. + ProvidesFrom map[string]string `json:"providesFrom,omitempty" yaml:"providesFrom,omitempty"` + // Requires lists capabilities that must be satisfied by the resolved // set, as "name" or "name version" (comma-AND'd constraints; // ops: >=, >, <=, <, =). Capability names resolve to nothing and diff --git a/spec/validate.go b/spec/validate.go index 5286dbf2..cb3234ab 100644 --- a/spec/validate.go +++ b/spec/validate.go @@ -61,6 +61,7 @@ func Validate(d *Descriptor) (warnings []string, err error) { errs.add(validateNoNUL(d)) errs.add(validateIconURL(d.IconURL)) errs.add(validateCapabilityNames(d)) + errs.add(validateProvidesFrom(d)) errs.add(validateCapabilityEntries(d)) errs.add(validateArgs(d.Args)) errs.add(validateRecipe(d)) @@ -244,6 +245,7 @@ func ValidateRaw(raw []byte, d *Descriptor) (warnings []string, err error) { warnings, err = Validate(d) errs.add(err) errs.add(validateRawBuildSecrets(raw, d)) + errs.add(validateRawProvidesFrom(raw, d)) sizeWarnings, sizeErr := validateSize(raw) warnings = append(warnings, sizeWarnings...) @@ -353,6 +355,18 @@ func ValidatePublished(raw []byte, d *Descriptor) (warnings []string, err error) errs.add(fieldErrorf(fmt.Sprintf("provides[%d]", i), "published descriptor still references an arg in provides entry %q; build-phase expansion did not run", s)) } } + // providesFrom is an instruction to the build, and publishing + // replaces it with the entries it stated (§9.7). One that survives + // names files nobody resolved, in a field readers built before it + // existed refuse to decode. + // + // Judged on the bytes as well as the struct, because the rule is + // about the key being there: `providesFrom: {}` and a bare + // `providesFrom:` decode to no entries at all, and a strict reader + // that does not know the field refuses them just the same. + if len(d.ProvidesFrom) > 0 || carriesProvidesFrom(raw) { + errs.add(fieldErrorf("providesFrom", "published descriptor still carries providesFrom; publishing states those entries in provides and drops the field, so this artifact was never resolved")) + } // §9.2 is a property of every published descriptor, not only of the // frontend's build path, so third-party artifacts are held to it too. if d.Version == "" { diff --git a/tck/sandbox/coverage_test.go b/tck/sandbox/coverage_test.go index 832b0758..c5e7c459 100644 --- a/tck/sandbox/coverage_test.go +++ b/tck/sandbox/coverage_test.go @@ -146,6 +146,9 @@ var kitCovers = map[string][]string{ "SPEC-v3 §7.1.1/validate-declarations", "SPEC-v3 §9.2/versioned-provides", "SPEC-v3 §3.4/set-kind-never-published", + // ValidatePublished refuses the field, so an artifact that + // still carries it fails the same check. + "SPEC-v3 §9.7/not-published", }, // A merged kit's staged roots are counted against the kits its @@ -283,6 +286,17 @@ var waived = map[string]string{ "SPEC-v3 §5.1/derived-namespaces-reserved": "grammar rule, judged by the spec package's validators; a published descriptor cannot distinguish an authored deb/ entry from a derived one, and derived-provides refuses either if the image disagrees", "SPEC-v3 §5.1/namespace-reverse-dns": "grammar rule, judged by the spec package's validators and schema tests", + // Stated provides: publishing replaces providesFrom with ordinary + // provides entries and drops the field, so a published Kit keeps no + // record of which entry came from a path or which path that was. + // What the frontend did with one is judged where it happens. + "SPEC-v3 §9.7/name-is-authored": "grammar rule, judged by the spec package's validators and schema tests", + "SPEC-v3 §9.7/workload-only": "grammar rule, judged by the spec package's validators and schema tests", + "SPEC-v3 §9.7/owned-by-one-package": "build-time resolution, judged by spec/provides_from_test.go; the published descriptor does not record the path an entry was stated from", + "SPEC-v3 §9.7/links-followed-when-unowned": "build-time resolution, judged by spec/provides_from_test.go; the published descriptor does not record the path an entry was stated from", + "SPEC-v3 §9.7/version-is-the-owners": "build-time resolution, judged by spec/provides_from_test.go; the published descriptor does not record the path an entry was stated from", + "SPEC-v3 §9.7/agreed-across-platforms": "the frontend resolves every path on every target platform; one published manifest cannot show what another platform recorded", + // Grammar rules: the spec package's validators and schema tests are // where these are judged, statement by statement. "SPEC-v3 §4/schema-version-3": "grammar rule, judged by the spec package's validators and schema tests",